Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

121 KB
Raw Blame
mod common;

/// Poll until `last_used_at` is set for the given PAK, or panic after ~4s.
async fn poll_for_pak_last_used_at(
    context: &feature_flags::utils::test_utils::TestContext,
    pak_id: &str,
    message: &str,
) {
    use tokio::time::{sleep, Duration};

    let mut conn = context.get_non_persons_connection().await.unwrap();
    for _ in 0..80 {
        let count: (i64,) = sqlx::query_as(
            "SELECT COUNT(*) FROM posthog_personalapikey WHERE id = $1 AND last_used_at IS NOT NULL",
        )
        .bind(pak_id)
        .fetch_one(&mut *conn)
        .await
        .unwrap();
        if count.0 > 0 {
            return;
        }
        sleep(Duration::from_millis(50)).await;
    }
    panic!("{message}");
}

#[tokio::test]
async fn test_hypercache_config_generation() {
    use common_hypercache::{HyperCacheConfig, KeyType};
    use common_types::TeamIdentifier;

    // Create a test team that implements TeamIdentifier
    #[derive(Debug, Clone)]
    struct TestTeam {
        id: i32,
        token: String,
    }

    impl TeamIdentifier for TestTeam {
        fn team_id(&self) -> i32 {
            self.id
        }

        fn api_token(&self) -> &str {
            &self.token
        }
    }

    let test_team = TestTeam {
        id: 123,
        token: "test_token".to_string(),
    };

    // Test that HyperCache configurations generate correct cache keys matching Django format
    let config_with_cohorts = HyperCacheConfig::new(
        "feature_flags".to_string(),
        "flags_with_cohorts.json".to_string(),
        "us-east-1".to_string(),
        "test-bucket".to_string(),
    );

    let config_without_cohorts = HyperCacheConfig::new(
        "feature_flags".to_string(),
        "flags_without_cohorts.json".to_string(),
        "us-east-1".to_string(),
        "test-bucket".to_string(),
    );

    let team_key = KeyType::team(test_team.clone());

    // Test Redis cache key generation (includes posthog:1: prefix)
    let redis_key_with_cohorts = config_with_cohorts.get_redis_cache_key(&team_key);
    assert_eq!(
        redis_key_with_cohorts,
        "posthog:1:cache/teams/123/feature_flags/flags_with_cohorts.json"
    );

    let redis_key_without_cohorts = config_without_cohorts.get_redis_cache_key(&team_key);
    assert_eq!(
        redis_key_without_cohorts,
        "posthog:1:cache/teams/123/feature_flags/flags_without_cohorts.json"
    );

    // Test S3 cache key generation (no prefix, matches Django object_storage)
    let s3_key_with_cohorts = config_with_cohorts.get_s3_cache_key(&team_key);
    assert_eq!(
        s3_key_with_cohorts,
        "cache/teams/123/feature_flags/flags_with_cohorts.json"
    );

    let s3_key_without_cohorts = config_without_cohorts.get_s3_cache_key(&team_key);
    assert_eq!(
        s3_key_without_cohorts,
        "cache/teams/123/feature_flags/flags_without_cohorts.json"
    );
}

#[tokio::test]
async fn test_flag_definitions_endpoint_exists() {
    use feature_flags::config::Config;
    use reqwest;

    let config = Config::default_test_config();
    let server = common::ServerHandle::for_config(config).await;
    let client = reqwest::Client::new();

    // Test that the endpoint exists and returns some response (even if unauthorized)
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token=any_token",
            server.addr
        ))
        .send()
        .await
        .unwrap();

    // Should return 401 Unauthorized when no authentication is provided
    assert_eq!(response.status(), 401);
}

#[tokio::test]
async fn test_personal_api_key_authentication_no_key() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;
    use serde_json::Value;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create a real team so we pass the token validation
    let team = context.insert_new_team(None).await.unwrap();

    let server = common::ServerHandle::for_config(config).await;
    let client = reqwest::Client::new();

    // Test that the endpoint returns 401 when no API key is provided
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .send()
        .await
        .unwrap();

    let status = response.status();
    let body_text = response.text().await.unwrap();

    assert_eq!(status, 401);

    // Verify the response body matches Django's format
    let body: Value = serde_json::from_str(&body_text).unwrap();
    assert_eq!(body["type"], "authentication_error");
    assert_eq!(body["code"], "not_authenticated");
    assert_eq!(
        body["detail"],
        "Authentication credentials were not provided."
    );
    assert_eq!(body["attr"], Value::Null);
}

#[tokio::test]
async fn test_personal_api_key_authentication_invalid_key() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;
    use serde_json::Value;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create a real team so we pass the token validation
    let team = context.insert_new_team(None).await.unwrap();

    let server = common::ServerHandle::for_config(config).await;
    let client = reqwest::Client::new();

    // Test that the endpoint returns 401 when an invalid API key is provided
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", "Bearer phx_invalid_key_12345")
        .send()
        .await
        .unwrap();

    let status = response.status();
    let body_text = response.text().await.unwrap();

    assert_eq!(status, 401);

    // Verify the response body matches Django's format
    let body: Value = serde_json::from_str(&body_text).unwrap();
    assert_eq!(body["type"], "authentication_error");
    assert_eq!(body["code"], "authentication_failed");
    assert_eq!(body["detail"], "Personal API key is invalid.");
    assert_eq!(body["attr"], Value::Null);
}

#[tokio::test]
async fn test_personal_api_key_authentication_valid_key_with_scopes() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create team, user, and personal API key using helpers
    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    let user_email = feature_flags::utils::test_utils::TestContext::generate_test_email("test_pak");
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();
    context
        .add_user_to_organization(user_id, &org_id, 15)
        .await
        .unwrap();

    let (_pak_id, api_key_value) = context
        .create_personal_api_key(user_id, "Test PAK", vec!["feature_flag:read"], None, None)
        .await
        .unwrap();

    // Populate cache to avoid 503 errors
    context.populate_cache_for_team(team.id).await.unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Test that the endpoint returns 200 when a valid API key with proper scopes is provided
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "Response body: {}",
        response.text().await.unwrap()
    );
}

#[tokio::test]
async fn test_personal_api_key_authentication_without_feature_flag_scopes() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create team, user, and personal API key with different scope
    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    let user_email =
        feature_flags::utils::test_utils::TestContext::generate_test_email("test_no_scopes");
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();

    let (_pak_id, api_key_value) = context
        .create_personal_api_key(
            user_id,
            "Test PAK No Scopes",
            vec!["insight:read"],
            None,
            None,
        )
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Test that the endpoint returns 403 when API key is valid but lacks feature_flag scopes
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();

    let status = response.status();
    let body_text = response.text().await.unwrap();
    assert_eq!(status, 403);

    let body: serde_json::Value = serde_json::from_str(&body_text).unwrap();
    assert_eq!(body["type"], "authentication_error");
    assert_eq!(body["code"], "permission_denied");
    assert_eq!(
        body["detail"],
        "Personal API key lacks required scopes (feature_flag:read or feature_flag:write)."
    );
    assert_eq!(body["attr"], serde_json::Value::Null);
}

#[tokio::test]
async fn test_personal_api_key_authentication_inactive_user() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create team
    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    // Create INACTIVE user using helper (is_active = false)
    let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
    let user_email = format!("test_inactive_{test_uuid}@posthog.com");
    let user_id = context
        .create_user_with_options(&user_email, &org_id, Some(team.id), false)
        .await
        .unwrap();

    let (_pak_id, api_key_value) = context
        .create_personal_api_key(
            user_id,
            "Test PAK Inactive",
            vec!["feature_flag:read"],
            None,
            None,
        )
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Test that the endpoint returns 401 when user is inactive
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();

    assert_eq!(response.status(), 401);
}

#[tokio::test]
async fn test_secret_api_token_authentication_valid_token() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create team with secret API token using helper
    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    // Populate cache to avoid 503 errors
    context.populate_cache_for_team(team.id).await.unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Test that the endpoint returns 200 when a valid secret API token is provided
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "Response body: {}",
        response.text().await.unwrap()
    );
}

#[tokio::test]
async fn test_secret_api_token_authentication_invalid_token() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;
    use serde_json::Value;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create a real team so we pass the token validation
    let team = context.insert_new_team(None).await.unwrap();

    let server = common::ServerHandle::for_config(config).await;
    let client = reqwest::Client::new();

    // Test that the endpoint returns 401 when an invalid secret token is provided
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", "Bearer phs_invalid_secret_token_xyz")
        .send()
        .await
        .unwrap();

    let status = response.status();
    let body_text = response.text().await.unwrap();

    assert_eq!(status, 401);

    // Verify the response body matches Django's format
    let body: Value = serde_json::from_str(&body_text).unwrap();
    assert_eq!(body["type"], "authentication_error");
    assert_eq!(body["code"], "authentication_failed");
    assert_eq!(body["detail"], "Secret API token is invalid.");
    assert_eq!(body["attr"], Value::Null);
}

/// ACCEPT_TEAM_SECRET_TOKENS is the kill switch for legacy team secret tokens (#66179).
/// When it is false, the lookup skips posthog_team.
/// A legacy token then authenticates only through a backfilled PSAK mirror row with the same string.
/// `test_secret_api_token_authentication_valid_token` covers the default (true) path.
#[rstest::rstest]
#[case::with_mirror_row(true, 200)]
#[case::without_mirror_row(false, 401)]
#[tokio::test]
async fn test_kill_switch_off_legacy_token_only_matches_mirror_row(
    #[case] mirror_row: bool,
    #[case] expected_status: u16,
) {
    use feature_flags::{
        config::{Config, FlexBool},
        utils::test_utils::TestContext,
    };
    use reqwest;

    let mut config = Config::default_test_config();
    config.accept_team_secret_tokens = FlexBool(false);
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();
    if mirror_row {
        context
            .create_project_secret_api_key_with_value(
                team.id,
                "Migrated legacy secret API key",
                Some(vec!["feature_flag:read", "support_ticket:read"]),
                &secret_token,
            )
            .await
            .unwrap();
    }
    context.populate_cache_for_team(team.id).await.unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    let status = response.status();
    let body_text = response.text().await.unwrap();
    assert_eq!(status, expected_status, "Response body: {body_text}");
}

/// Token param absent with a valid team-scoped secret — team is derived from the token.
#[rstest::rstest]
#[case::team_secret_token("secret")]
#[case::project_secret_key("project_secret")]
#[tokio::test]
async fn test_missing_token_param_success(#[case] auth_type: &str) {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let (team, bearer_value) = match auth_type {
        "secret" => {
            let (team, secret, _) = context
                .create_team_with_secret_token(None, None, None)
                .await
                .unwrap();
            (team, secret)
        }
        "project_secret" => {
            let team = context.insert_new_team(None).await.unwrap();
            let (_, key) = context
                .create_project_secret_api_key(team.id, "Test Key", Some(vec!["feature_flag:read"]))
                .await
                .unwrap();
            (team, key)
        }
        _ => unreachable!(),
    };
    context.populate_cache_for_team(team.id).await.unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    let response = client
        .get(format!("http://{}/flags/definitions", server.addr))
        .header("Authorization", format!("Bearer {bearer_value}"))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "Response body: {}",
        response.text().await.unwrap()
    );
}

/// Token param absent — error cases that don't need DB setup.
#[rstest::rstest]
#[case::no_auth(None, 401)]
#[case::invalid_phs_token(Some("Bearer phs_invalid_token_xyz"), 401)]
#[tokio::test]
async fn test_missing_token_param_error(
    #[case] auth_header: Option<&str>,
    #[case] expected_status: u16,
) {
    use feature_flags::config::Config;
    use reqwest;

    let config = Config::default_test_config();
    let server = common::ServerHandle::for_config(config).await;
    let client = reqwest::Client::new();

    let mut req = client.get(format!("http://{}/flags/definitions", server.addr));
    if let Some(header) = auth_header {
        req = req.header("Authorization", header);
    }
    let response = req.send().await.unwrap();

    assert_eq!(response.status(), expected_status);
}

/// When token param is missing, the response should contain flags for the team
/// that owns the phs_ token — not a different team's flags.
#[tokio::test]
async fn test_missing_token_param_returns_correct_team_flags() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;
    use serde_json::{json, Value};

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create two teams with different secret tokens
    let (team1, secret1, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();
    let (team2, secret2, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    // Seed each team's cache with distinguishable flags so we can verify isolation
    context
        .populate_cache_for_team_with_flags(
            team1.id,
            json!({"flags": [{"key": "team1-flag", "active": true}], "group_type_mapping": {}, "cohorts": {}}),
        )
        .await
        .unwrap();
    context
        .populate_cache_for_team_with_flags(
            team2.id,
            json!({"flags": [{"key": "team2-flag", "active": true}], "group_type_mapping": {}, "cohorts": {}}),
        )
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Request with team1's secret token (no ?token= param)
    let resp1 = client
        .get(format!("http://{}/flags/definitions", server.addr))
        .header("Authorization", format!("Bearer {secret1}"))
        .send()
        .await
        .unwrap();
    assert_eq!(resp1.status(), 200);
    let body1: Value = serde_json::from_str(&resp1.text().await.unwrap()).unwrap();

    // Request with team2's secret token (no ?token= param)
    let resp2 = client
        .get(format!("http://{}/flags/definitions", server.addr))
        .header("Authorization", format!("Bearer {secret2}"))
        .send()
        .await
        .unwrap();
    assert_eq!(resp2.status(), 200);
    let body2: Value = serde_json::from_str(&resp2.text().await.unwrap()).unwrap();

    // Each team should get its own flags, not the other's
    assert_eq!(body1["flags"][0]["key"], "team1-flag");
    assert_eq!(body2["flags"][0]["key"], "team2-flag");
    assert_ne!(
        body1.get("flags"),
        body2.get("flags"),
        "Different teams' phs_ tokens should resolve to different flags"
    );
}

/// When token param is missing and auth is a personal API key (multi-team),
/// should return 400 since we can't derive the team.
#[tokio::test]
async fn test_missing_token_param_with_personal_api_key_returns_400() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();
    let user_email = format!("test-pak-{}@posthog.com", uuid::Uuid::new_v4());
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();
    let (_, pak_token) = context
        .create_personal_api_key(user_id, "Test PAK", vec!["feature_flag:read"], None, None)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    let response = client
        .get(format!("http://{}/flags/definitions", server.addr))
        .header("Authorization", format!("Bearer {pak_token}"))
        .send()
        .await
        .unwrap();

    assert_eq!(response.status(), 400);
    let body = response.text().await.unwrap();
    assert!(
        body.contains("token"),
        "Error should mention the token parameter: {body}"
    );
}

#[tokio::test]
async fn test_token_mismatch_secret_token_for_different_team() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create two teams with secret tokens
    let (team1, _secret1, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let (_team2, secret2, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Test: Use team1's public token but team2's secret token - should fail
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team1.api_token
        ))
        .header("Authorization", format!("Bearer {secret2}"))
        .send()
        .await
        .unwrap();

    // Should return 401 because the secret token doesn't match the team
    assert_eq!(response.status(), 401);
}

#[tokio::test]
async fn test_personal_api_key_with_scoped_teams_allowed() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create team using helper (creates org, project, and team)
    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
    let user_email = format!("test_scoped_teams_{test_uuid}@posthog.com");
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();
    context
        .add_user_to_organization(user_id, &org_id, 15)
        .await
        .unwrap();

    // Create personal API key with scoped_teams restriction that includes our team
    let (_pak_id, api_key_value) = context
        .create_personal_api_key(
            user_id,
            "Test PAK Scoped Teams",
            vec!["feature_flag:read"],
            Some(vec![team.id]),
            None,
        )
        .await
        .unwrap();

    // Populate cache to avoid 503 errors
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    context
        .populate_flag_definitions_cache(redis_client, team.id)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Should succeed because the team is in the scoped_teams list
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();

    let status = response.status();
    let body = response.text().await.unwrap();
    assert_eq!(
        status, 200,
        "Should authenticate when team is in scoped_teams. Response body: {body}"
    );
}

#[tokio::test]
async fn test_personal_api_key_with_scoped_teams_denied() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
    let user_email = format!("test_denied_{test_uuid}@posthog.com");
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();

    // Create personal API key with scoped_teams restriction that excludes our team
    let (_pak_id, api_key_value) = context
        .create_personal_api_key(
            user_id,
            "Test PAK Denied",
            vec!["feature_flag:read"],
            Some(vec![99999]), // Different team ID
            None,
        )
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Should fail because the team is not in the scoped_teams list
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        401,
        "Should deny access when team is not in scoped_teams"
    );
}

#[tokio::test]
async fn test_personal_api_key_with_scoped_organizations_allowed() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
    let user_email = format!("test_org_allowed_{test_uuid}@posthog.com");
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();
    context
        .add_user_to_organization(user_id, &org_id, 15)
        .await
        .unwrap();

    // Create personal API key with scoped_organizations restriction that includes our org
    let org_id_str = org_id.to_string();
    let (_pak_id, api_key_value) = context
        .create_personal_api_key(
            user_id,
            "Test PAK Org Allowed",
            vec!["feature_flag:read"],
            None,
            Some(vec![org_id_str]),
        )
        .await
        .unwrap();

    // Populate cache to avoid 503 errors
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    context
        .populate_flag_definitions_cache(redis_client, team.id)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Should succeed because the organization is in the scoped_organizations list
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();

    let status = response.status();
    let body = response.text().await.unwrap();
    assert_eq!(
        status, 200,
        "Should authenticate when organization is in scoped_organizations. Response body: {body}"
    );
}

#[tokio::test]
async fn test_personal_api_key_with_scoped_organizations_denied() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create team using helper (creates org, project, and team)
    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
    let user_email = format!("test_org_denied_{test_uuid}@posthog.com");
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();

    // Create personal API key with scoped_organizations restriction that excludes our org
    let different_org_id = uuid::Uuid::new_v4().to_string();
    let (_pak_id, api_key_value) = context
        .create_personal_api_key(
            user_id,
            "Test PAK Org Denied",
            vec!["feature_flag:read"],
            None,
            Some(vec![different_org_id]), // Different organization ID
        )
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Should fail because the organization is not in the scoped_organizations list
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        401,
        "Should deny access when organization is not in scoped_organizations"
    );
}

#[tokio::test]
async fn test_personal_api_key_with_scoped_organizations_removed_member() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
    let user_email = format!("test_org_removed_{test_uuid}@posthog.com");
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();
    context
        .add_user_to_organization(user_id, &org_id, 15)
        .await
        .unwrap();

    let org_id_str = org_id.to_string();
    let (_pak_id, api_key_value) = context
        .create_personal_api_key(
            user_id,
            "Test PAK Org Removed",
            vec!["feature_flag:read"],
            None,
            Some(vec![org_id_str]),
        )
        .await
        .unwrap();

    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    context
        .populate_flag_definitions_cache(redis_client, team.id)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Verify access works while user is a member
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();
    assert_eq!(
        response.status(),
        200,
        "Should authenticate while user is an org member"
    );

    // Remove user from organization
    context
        .remove_user_from_organization(user_id, &org_id)
        .await
        .unwrap();
    // Simulate Django signal-based cache invalidation (Python handles this in production)
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    feature_flags::utils::test_utils::invalidate_personal_api_key_auth_cache(
        redis_client,
        &api_key_value,
    )
    .await
    .unwrap();

    // Should now fail because the user is no longer an org member
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();
    assert_eq!(
        response.status(),
        401,
        "Should deny access after user is removed from organization"
    );
}

#[tokio::test]
async fn test_personal_api_key_unscoped_removed_member() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
    let user_email = format!("test_unscoped_removed_{test_uuid}@posthog.com");
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();
    context
        .add_user_to_organization(user_id, &org_id, 15)
        .await
        .unwrap();

    // Create PAK without scoped_organizations to confirm the mandatory membership
    // check works for all PAK configurations, not just those with explicit org scoping.
    let (_pak_id, api_key_value) = context
        .create_personal_api_key(
            user_id,
            "Test PAK Unscoped",
            vec!["feature_flag:read"],
            None,
            None,
        )
        .await
        .unwrap();

    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    context
        .populate_flag_definitions_cache(redis_client, team.id)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Verify access works while user is a member
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();
    assert_eq!(
        response.status(),
        200,
        "Should authenticate while user is an org member"
    );

    // Remove user from organization
    context
        .remove_user_from_organization(user_id, &org_id)
        .await
        .unwrap();
    // Simulate Django signal-based cache invalidation (Python handles this in production)
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    feature_flags::utils::test_utils::invalidate_personal_api_key_auth_cache(
        redis_client,
        &api_key_value,
    )
    .await
    .unwrap();

    // Should fail because the user is no longer an org member, even without scoped_organizations
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();
    assert_eq!(
        response.status(),
        401,
        "Should deny access after user is removed from organization, even without scoped_organizations"
    );
}

#[tokio::test]
async fn test_cache_miss_returns_503() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;
    use serde_json::Value;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create team with secret API token
    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    // DO NOT populate cache - we want to test cache miss behavior

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Request should return 503 when cache is empty
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    let status = response.status();
    let body_text = response.text().await.unwrap();

    assert_eq!(
        status, 503,
        "Should return 503 on cache miss. Body: {body_text}"
    );

    // Verify the response body has proper error format (if JSON)
    if let Ok(body) = serde_json::from_str::<Value>(&body_text) {
        assert_eq!(body["type"], "server_error");
        assert_eq!(body["code"], "service_unavailable");
        assert!(
            body["detail"]
                .as_str()
                .unwrap()
                .contains("A service dependency is temporarily unavailable"),
            "Error message should mention cache miss"
        );
    } else {
        // If not JSON, verify the error message mentions cache
        assert!(
            body_text.contains("A service dependency is temporarily unavailable"),
            "Body should mention cache miss. Got: {body_text}"
        );
    }
}

#[tokio::test]
async fn test_personal_api_key_with_all_access_scopes() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create team, user, and personal API key with {*} (all access) scopes
    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
    let user_email = format!("test_all_access_{test_uuid}@posthog.com");
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();
    context
        .add_user_to_organization(user_id, &org_id, 15)
        .await
        .unwrap();

    // Create PAK with {*} scope (all access)
    let (_pak_id, api_key_value) = context
        .create_personal_api_key(user_id, "Test PAK All Access", vec!["*"], None, None)
        .await
        .unwrap();

    // Populate cache to avoid 503 errors
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    context
        .populate_flag_definitions_cache(redis_client, team.id)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Test that the endpoint returns 200 when PAK has all access
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "Response body: {}",
        response.text().await.unwrap()
    );
}

#[tokio::test]
async fn test_personal_api_key_with_feature_flag_write_scope() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create team, user, and personal API key with feature_flag:write scope
    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
    let user_email = format!("test_write_scope_{test_uuid}@posthog.com");
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();
    context
        .add_user_to_organization(user_id, &org_id, 15)
        .await
        .unwrap();

    // Create PAK with feature_flag:write scope (write includes read access)
    let (_pak_id, api_key_value) = context
        .create_personal_api_key(
            user_id,
            "Test PAK Write Scope",
            vec!["feature_flag:write"],
            None,
            None,
        )
        .await
        .unwrap();

    // Populate cache to avoid 503 errors
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    context
        .populate_flag_definitions_cache(redis_client, team.id)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Test that the endpoint returns 200 with feature_flag:write scope
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "Should succeed with feature_flag:write scope. Response body: {}",
        response.text().await.unwrap()
    );
}

#[tokio::test]
async fn test_backup_secret_api_token_authentication() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create a team with both secret_api_token and secret_api_token_backup using helper
    let backup_token_value = feature_flags::utils::test_utils::random_string("phs_backup_", 12);
    let (team, _secret_token, backup_secret_token) = context
        .create_team_with_secret_token(None, None, Some(&backup_token_value))
        .await
        .unwrap();

    // Populate cache to avoid 503 errors
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    context
        .populate_flag_definitions_cache(redis_client, team.id)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Test that the backup secret token works
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header(
            "Authorization",
            format!("Bearer {}", backup_secret_token.unwrap()),
        )
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "Should authenticate with backup secret token. Response body: {}",
        response.text().await.unwrap()
    );
}

#[tokio::test]
async fn test_invalid_project_api_key() {
    use feature_flags::config::Config;
    use reqwest;
    use serde_json::Value;

    let config = Config::default_test_config();
    let server = common::ServerHandle::for_config(config).await;
    let client = reqwest::Client::new();

    // Test with an explicitly invalid token (not just missing, but actually wrong)
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token=phc_definitely_invalid_token_12345",
            server.addr
        ))
        .send()
        .await
        .unwrap();

    let status = response.status();
    let body_text = response.text().await.unwrap();

    assert_eq!(
        status, 401,
        "Should return 401 for invalid token. Body: {body_text}"
    );

    // An invalid project token must return the structured DRF-style JSON envelope,
    // matching the secret/personal API key paths.
    let body: Value = serde_json::from_str(&body_text)
        .unwrap_or_else(|_| panic!("Body should be JSON. Got: {body_text}"));
    assert_eq!(body["type"], "authentication_error");
    assert_eq!(body["code"], "authentication_failed");
    assert_eq!(
        body["detail"],
        "The provided API key is invalid or has expired. Please check your API key and try again."
    );
    assert_eq!(body["attr"], Value::Null);
}

#[tokio::test]
async fn test_secret_token_takes_priority_over_personal_api_key() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create two teams with different data
    let (team1, secret_token1, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let team2 = context.insert_new_team(None).await.unwrap();
    let org_id2 = context.get_organization_id_for_team(&team2).await.unwrap();

    // Create PAK for team2
    let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
    let user_email = format!("test_priority_{test_uuid}@posthog.com");
    let user_id = context
        .create_user(&user_email, &org_id2, team2.id)
        .await
        .unwrap();

    let (_pak_id, _personal_key) = context
        .create_personal_api_key(
            user_id,
            "Test PAK Priority",
            vec!["feature_flag:read"],
            None,
            None,
        )
        .await
        .unwrap();

    // Populate cache for team1 only
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    context
        .populate_flag_definitions_cache(redis_client, team1.id)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Provide BOTH authentication methods:
    // - Secret token for team1 (in Authorization header - phs_ prefix)
    // - Personal API key for team2 (phx_ prefix)
    // Expected: Secret token takes priority, so we get team1's data
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team1.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token1}"))
        .send()
        .await
        .unwrap();

    // Should succeed with team1's secret token (200 status)
    // If PAK was tried first, it would fail because PAK belongs to team2
    assert_eq!(
        response.status(),
        200,
        "Should authenticate with secret token when both auth methods provided. Response body: {}",
        response.text().await.unwrap()
    );
}

#[tokio::test]
async fn test_flag_definitions_rate_limit_enforced() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let context = TestContext::new(None).await;

    // Create team with secret token
    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    // `/minute` (not `/second`) throughout this file: governor's QuantaClock (TSC-backed)
    // can replenish sub-second windows too fast on non-Depot runners. Don't revert.
    let mut config = Config::default_test_config();
    config.flag_definitions_rate_limits =
        format!(r#"{{"{}": "1/minute"}}"#, team.id).parse().unwrap();

    // Populate cache to avoid 503 errors
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    context
        .populate_flag_definitions_cache(redis_client, team.id)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // First request should succeed (within rate limit)
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "First request should succeed. Response body: {}",
        response.text().await.unwrap()
    );

    // Second request immediately after should be rate limited (429)
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    let status = response.status();
    let body = response.text().await.unwrap();

    assert_eq!(
        status, 429,
        "Second request should be rate limited. Response body: {body}"
    );
    assert!(
        body.contains("Rate limit exceeded"),
        "Response should mention rate limit. Got: {body}"
    );
}

#[tokio::test]
async fn test_flag_definitions_custom_rate_limit_overrides_default() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let context = TestContext::new(None).await;

    // Create teams with secret tokens
    let (custom_team, custom_secret, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();
    let (default_team, default_secret, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    // Create config with custom rate limit for custom_team (2 requests per minute)
    // Default is 600/minute, so custom should be more restrictive
    let mut config = Config::default_test_config();
    config.flag_definitions_rate_limits = format!(r#"{{"{}": "2/minute"}}"#, custom_team.id)
        .parse()
        .unwrap();

    // Populate cache for both teams
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    context
        .populate_flag_definitions_cache(redis_client.clone(), custom_team.id)
        .await
        .unwrap();
    context
        .populate_flag_definitions_cache(redis_client, default_team.id)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Test custom rate limit (2/minute)
    // Send all 3 requests concurrently to ensure they hit the rate limiter simultaneously
    let requests = (0..3).map(|_| {
        client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, custom_team.api_token
            ))
            .header("Authorization", format!("Bearer {custom_secret}"))
            .send()
    });

    let responses = futures::future::join_all(requests).await;
    let success_count = responses
        .into_iter()
        .filter(|r| r.as_ref().unwrap().status() == 200)
        .count();

    assert!(
        success_count <= 2,
        "Should allow at most 2 requests per minute for custom team. Got: {success_count}"
    );

    // Test default rate limit (600/minute)
    // Should allow more requests than custom limit
    // Send all 5 requests concurrently to ensure they hit the rate limiter simultaneously
    let requests = (0..5).map(|_| {
        client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, default_team.api_token
            ))
            .header("Authorization", format!("Bearer {default_secret}"))
            .send()
    });

    let responses = futures::future::join_all(requests).await;
    let default_success_count = responses
        .into_iter()
        .filter(|r| r.as_ref().unwrap().status() == 200)
        .count();

    assert!(
        default_success_count > 2,
        "Default team should allow more than 2 requests per minute. Got: {default_success_count}"
    );
}

#[tokio::test]
async fn test_flag_definitions_rate_limit_metrics_incremented() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let context = TestContext::new(None).await;

    // Create team with secret token
    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    // Create config with very low rate limit for this specific team
    let mut config = Config::default_test_config();
    config.flag_definitions_rate_limits =
        format!(r#"{{"{}": "1/minute"}}"#, team.id).parse().unwrap();
    config.flag_definitions_conditional_rate_per_minute = 1;
    config.enable_metrics = true; // Enable metrics collection

    // Populate cache
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    context
        .populate_flag_definitions_cache(redis_client.clone(), team.id)
        .await
        .unwrap();

    // The rate-limited team's `:etag` value cannot be decoded. Deleting `etag_read_failure_label`
    // left `redis_error` with no coverage, and this is the only test that can scrape /metrics.
    // No other test writes an undecodable `:etag`, so other tests running in the same process
    // cannot add to the `redis_error` count asserted below.
    redis_client
        .set_bytes(
            format!(
                "posthog:1:cache/teams/{}/feature_flags/flags_with_cohorts.json:etag",
                team.id
            ),
            b"not-a-pickle".to_vec(),
            None,
        )
        .await
        .unwrap();

    let (other_team, other_secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();
    context
        .populate_flag_definitions_cache(redis_client.clone(), other_team.id)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    let url = format!(
        "http://{}/flags/definitions?token={}",
        server.addr, team.api_token
    );

    let response = get_definitions(&client, &url, &secret_token, None).await;
    assert_eq!(response.status(), 200);

    let response = get_definitions(&client, &url, &secret_token, None).await;
    assert_eq!(response.status(), 429);

    let response = get_definitions(&client, &url, &secret_token, Some("W/\"stale\"")).await;
    assert_eq!(
        response.status(),
        429,
        "A stale ETag should pass the conditional budget and be refused by the spent full-response budget"
    );
    let response = get_definitions(&client, &url, &secret_token, Some("W/\"stale\"")).await;
    assert_eq!(
        response.status(),
        429,
        "The spent conditional budget should refuse a stale ETag before the ETag read"
    );

    // The second team is not rate limited, so this request reaches the ETag read.
    let other_url = format!(
        "http://{}/flags/definitions?token={}",
        server.addr, other_team.api_token
    );
    let response = get_definitions(&client, &other_url, &other_secret_token, None).await;
    assert_eq!(response.status(), 200);

    // Fetch metrics from /metrics endpoint
    let metrics_response = client
        .get(format!("http://{}/metrics", server.addr))
        .send()
        .await
        .unwrap();

    let metrics_text = metrics_response.text().await.unwrap();
    let metric_value = |prefix: &str, labels: &[&str]| {
        metrics_text
            .lines()
            .find(|line| {
                line.starts_with(prefix) && labels.iter().all(|label| line.contains(label))
            })
            .and_then(|line| line.rsplit(' ').next())
            .and_then(|value| value.parse::<f64>().ok())
    };

    // Both limiters share metric names and differ only in the `budget` label, so a dashboard
    // that sums a counter sees every request. A stale ETag passes both limiters but counts
    // once as a request. The full-response limiter still counts the 429 it returns for it.
    let team_key = format!(r#"key="{}""#, team.id);
    let team_count = |metric: &str, budget: &str| {
        metric_value(
            &format!("{metric}{{"),
            &[&team_key, &format!(r#"budget="{budget}""#)],
        )
    };
    assert_eq!(
        team_count("flags_flag_definitions_requests_total", "full"),
        Some(2.0),
        "A stale ETag should not count as a second request. Metrics: {metrics_text}"
    );
    assert_eq!(
        team_count("flags_flag_definitions_requests_total", "conditional"),
        Some(2.0),
        "Requests with an ETag in If-None-Match should count under the conditional budget. Metrics: {metrics_text}"
    );
    assert_eq!(
        team_count("flags_flag_definitions_rate_limited_total", "full"),
        Some(2.0),
        "The full-response limiter should count the 429 it returns for a stale ETag. Metrics: {metrics_text}"
    );
    assert_eq!(
        team_count("flags_flag_definitions_rate_limited_total", "conditional"),
        Some(1.0),
        "429s from the conditional limiter should count under the conditional budget. Metrics: {metrics_text}"
    );

    // Runbook step 2 picks which cluster to query from this gauge, so assert the series and
    // its value and not just the metric name. This config leaves the toggle off.
    let gauge_value = metric_value(
        "flags_flag_definitions_reads_dedicated_redis{",
        &[r#"reason="disabled""#],
    );
    assert_eq!(
        gauge_value,
        Some(0.0),
        "A reader left on shared Redis should report the gauge at 0 with reason=\"disabled\". Metrics: {metrics_text}"
    );

    // An absent ETag writes no log record, so this counter is the only trace that it
    // happened. populate_flag_definitions_cache writes no `:etag` key for the second team.
    assert!(
        metric_value(
            "flags_flag_definitions_etag_total{",
            &[r#"result="redis_missing""#]
        )
        .is_some(),
        "An absent ETag should count as redis_missing. Metrics: {metrics_text}"
    );

    // Routing a decode failure back to Ok(None) would revert this to redis_missing, which sends
    // the on-call to rebuild a cache tier when the fault is corrupt data. Only the rate-limited
    // team's 200 and its first stale ETag read its undecodable ETag. A limiter check that runs
    // after the ETag read raises the count, because the refused requests then read Redis too.
    let redis_error = metric_value(
        "flags_flag_definitions_etag_total{",
        &[r#"result="redis_error""#],
    );
    assert_eq!(
        redis_error,
        Some(2.0),
        "An undecodable ETag should count as redis_error, and only the 200 and the first stale ETag should read it. Metrics: {metrics_text}"
    );
}

async fn get_definitions(
    client: &reqwest::Client,
    url: &str,
    secret_token: &str,
    if_none_match: Option<&str>,
) -> reqwest::Response {
    let request = client
        .get(url)
        .header("Authorization", format!("Bearer {secret_token}"));
    match if_none_match {
        Some(etag) => request.header("If-None-Match", etag),
        None => request,
    }
    .send()
    .await
    .unwrap()
}

#[tokio::test]
async fn test_etag_304_does_not_consume_full_response_rate_limit() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let mut config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    config.flag_definitions_rate_limits =
        format!(r#"{{"{}": "1/minute"}}"#, team.id).parse().unwrap();

    let etag_value = "a1b2c3d4e5f6g7h8";
    context
        .populate_cache_for_team_with_etag(team.id, etag_value)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();
    let url = format!(
        "http://{}/flags/definitions?token={}",
        server.addr, team.api_token
    );

    let matching_etag = format!("W/\"{etag_value}\"");

    let response = get_definitions(&client, &url, &secret_token, None).await;
    assert_eq!(response.status(), 200, "First full response should succeed");

    // The full-response budget (1/minute) is now spent, but revalidation polls
    // with a matching ETag draw from the separate conditional budget.
    for _ in 0..3 {
        let response = get_definitions(&client, &url, &secret_token, Some(&matching_etag)).await;
        assert_eq!(
            response.status(),
            304,
            "Matching ETag should return 304 even when the full-response budget is spent"
        );
    }

    let response = get_definitions(&client, &url, &secret_token, None).await;
    assert_eq!(
        response.status(),
        429,
        "Full response should still be rate limited"
    );
}

#[tokio::test]
async fn test_etag_304_rate_limit_enforced() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let mut config = Config::default_test_config();
    config.flag_definitions_conditional_rate_per_minute = 1;
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let etag_value = "a1b2c3d4e5f6g7h8";
    context
        .populate_cache_for_team_with_etag(team.id, etag_value)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();
    let url = format!(
        "http://{}/flags/definitions?token={}",
        server.addr, team.api_token
    );

    let matching_etag = format!("W/\"{etag_value}\"");

    let response = get_definitions(&client, &url, &secret_token, Some(&matching_etag)).await;
    assert_eq!(response.status(), 304, "First 304 should succeed");

    let response = get_definitions(&client, &url, &secret_token, Some(&matching_etag)).await;
    let status = response.status();
    let body = response.text().await.unwrap();
    assert_eq!(
        status, 429,
        "Second 304 should be rate limited. Response body: {body}"
    );

    // The limiter runs before the ETag read. It cannot tell a stale ETag from a matching one.
    let response = get_definitions(&client, &url, &secret_token, Some("W/\"stale\"")).await;
    assert_eq!(
        response.status(),
        429,
        "A stale ETag should spend the conditional budget"
    );

    let response = get_definitions(&client, &url, &secret_token, None).await;
    assert_eq!(
        response.status(),
        200,
        "Full response should not be limited by the conditional budget"
    );
}

#[tokio::test]
async fn test_stale_etag_200_spends_both_budgets() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let mut config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let one_per_minute = format!(r#"{{"{}": "1/minute"}}"#, team.id);
    config.flag_definitions_rate_limits = one_per_minute.parse().unwrap();
    config.flag_definitions_conditional_rate_limits = one_per_minute.parse().unwrap();

    let etag_value = "a1b2c3d4e5f6g7h8";
    context
        .populate_cache_for_team_with_etag(team.id, etag_value)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();
    let url = format!(
        "http://{}/flags/definitions?token={}",
        server.addr, team.api_token
    );

    let matching_etag = format!("W/\"{etag_value}\"");

    let response = get_definitions(&client, &url, &secret_token, Some("W/\"stale\"")).await;
    assert_eq!(
        response.status(),
        200,
        "A stale ETag should get a full response"
    );

    let response = get_definitions(&client, &url, &secret_token, Some(&matching_etag)).await;
    assert_eq!(
        response.status(),
        429,
        "The stale ETag should have spent the team's 1/minute conditional override"
    );

    let response = get_definitions(&client, &url, &secret_token, None).await;
    assert_eq!(
        response.status(),
        429,
        "The stale ETag should have spent the team's 1/minute full-response override"
    );
}

#[tokio::test]
async fn test_etag_returns_304_when_matching() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let etag_value = "a1b2c3d4e5f6g7h8";
    context
        .populate_cache_for_team_with_etag(team.id, etag_value)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .header("If-None-Match", format!("W/\"{etag_value}\""))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        304,
        "Should return 304 when ETag matches. Body: {}",
        response.text().await.unwrap_or_default()
    );
}

#[tokio::test]
async fn test_etag_304_includes_etag_and_cache_control_headers() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let etag_value = "abcdef1234567890";
    context
        .populate_cache_for_team_with_etag(team.id, etag_value)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .header("If-None-Match", format!("W/\"{etag_value}\""))
        .send()
        .await
        .unwrap();

    assert_eq!(response.status(), 304);
    assert_eq!(
        response.headers().get("etag").unwrap().to_str().unwrap(),
        format!("W/\"{etag_value}\"")
    );
    assert_eq!(
        response
            .headers()
            .get("cache-control")
            .unwrap()
            .to_str()
            .unwrap(),
        "private, must-revalidate"
    );
}

#[tokio::test]
async fn test_etag_returns_200_when_not_matching() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    context
        .populate_cache_for_team_with_etag(team.id, "current_etag_value")
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .header("If-None-Match", "W/\"stale_etag_value\"")
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "Should return 200 when ETag does not match"
    );

    assert_eq!(
        response.headers().get("etag").unwrap().to_str().unwrap(),
        "W/\"current_etag_value\""
    );
    assert_eq!(
        response
            .headers()
            .get("cache-control")
            .unwrap()
            .to_str()
            .unwrap(),
        "private, must-revalidate"
    );
}

#[tokio::test]
async fn test_etag_200_includes_etag_header_without_if_none_match() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let etag_value = "freshdata12345678";
    context
        .populate_cache_for_team_with_etag(team.id, etag_value)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Request WITHOUT If-None-Match header
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    assert_eq!(response.status(), 200);
    assert_eq!(
        response.headers().get("etag").unwrap().to_str().unwrap(),
        format!("W/\"{etag_value}\""),
        "200 response should include ETag header even without If-None-Match"
    );
}

#[tokio::test]
async fn test_etag_graceful_degradation_without_stored_etag() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    // Populate cache WITHOUT an ETag
    context.populate_cache_for_team(team.id).await.unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    // Send If-None-Match even though no ETag is stored — should get 200 (graceful degradation)
    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .header("If-None-Match", "W/\"some_stale_etag\"")
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "Should return 200 when no ETag is stored (graceful degradation)"
    );
    assert!(
        response.headers().get("etag").is_none(),
        "Should not include ETag header when no ETag is stored"
    );
}

/// Stands in for the dedicated cluster. The shared cache is database 0, so keys cannot collide.
const DEDICATED_REDIS_URL: &str = "redis://localhost:6379/1";

/// Setup both dedicated-cluster tests need: the reader pointed at the dedicated database with
/// the toggle on, plus a team to request.
async fn dedicated_context_and_team() -> (
    feature_flags::config::Config,
    feature_flags::utils::test_utils::TestContext,
    feature_flags::team::team_models::Team,
    String,
) {
    use feature_flags::config::{Config, FlexBool};
    use feature_flags::utils::test_utils::TestContext;

    let config = Config {
        flags_redis_url: DEDICATED_REDIS_URL.to_string(),
        flag_definitions_dedicated_redis_enabled: FlexBool(true),
        ..Config::default_test_config()
    };
    let context = TestContext::new(Some(&config)).await;
    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    (config, context, team, secret_token)
}

/// Seeds only the dedicated database: a reader left on shared 503s, and a payload that moved
/// without its ETag loses the header.
#[tokio::test]
async fn test_dedicated_redis_serves_payload_and_etag() {
    use feature_flags::utils::test_utils::setup_redis_client;

    let (config, context, team, secret_token) = dedicated_context_and_team().await;

    let etag_value = "dedicated_etag_01";
    let dedicated = setup_redis_client(Some(DEDICATED_REDIS_URL.to_string())).await;
    context
        .populate_cache_for_team_with_etag_on(dedicated, team.id, etag_value)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    let status = response.status();
    let etag_header = response
        .headers()
        .get("etag")
        .map(|v| v.to_str().unwrap().to_string());
    let body_text = response.text().await.unwrap();

    assert_eq!(
        status, 200,
        "Should read the payload from the dedicated Redis. Body: {body_text}"
    );
    let body: serde_json::Value = serde_json::from_str(&body_text).unwrap();
    assert!(
        body.get("flags").is_some(),
        "Body should carry the dedicated payload. Body: {body_text}"
    );
    assert_eq!(
        etag_header.as_deref(),
        Some(format!("W/\"{etag_value}\"").as_str()),
        "Should serve the dedicated ETag"
    );
}

/// The split-brain 304, which is silent in production: a shared ETag matching while the
/// dedicated cluster holds a newer payload pins the SDK to stale definitions.
#[tokio::test]
async fn test_dedicated_redis_ignores_shared_etag() {
    use feature_flags::utils::test_utils::setup_redis_client;

    let (config, context, team, secret_token) = dedicated_context_and_team().await;

    let shared_etag = "shared_etag_0001";
    let dedicated_etag = "dedicated_etag_1";
    let shared = setup_redis_client(Some(config.redis_url.clone())).await;
    let dedicated = setup_redis_client(Some(DEDICATED_REDIS_URL.to_string())).await;
    context
        .populate_cache_for_team_with_etag_on(shared, team.id, shared_etag)
        .await
        .unwrap();
    context
        .populate_cache_for_team_with_etag_on(dedicated, team.id, dedicated_etag)
        .await
        .unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    assert_eq!(response.status(), 200);
    assert_eq!(
        response.headers().get("etag").unwrap().to_str().unwrap(),
        format!("W/\"{dedicated_etag}\""),
        "Should serve the dedicated ETag, not the shared one"
    );

    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .header("If-None-Match", format!("W/\"{shared_etag}\""))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "The shared cluster's ETag must not produce a 304 once reads are dedicated"
    );
}

#[tokio::test]
async fn test_flag_definitions_billing_limited_returns_402() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // Create team with secret token in real PG (needed for auth)
    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    // Start server with mock Redis where the team's token is billing-limited
    let server = common::ServerHandle::for_config_with_mock_redis(
        config,
        vec![team.api_token.clone()],            // billing-limited
        vec![(team.api_token.clone(), team.id)], // valid for team lookup
    )
    .await;

    let client = reqwest::Client::new();

    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    let status = response.status();
    let body_text = response.text().await.unwrap();

    assert_eq!(
        status, 402,
        "Should return 402 when billing quota is exceeded. Body: {body_text}"
    );
    // Response body matches Django's JSON format for SDK compatibility
    let body: serde_json::Value = serde_json::from_str(&body_text).unwrap();
    assert_eq!(body["type"], "quota_limited");
    assert_eq!(body["code"], "payment_required");
}

#[rstest::rstest]
#[case::read_scope(Some(vec!["feature_flag:read"]), true, 200)]
#[case::write_scope(Some(vec!["feature_flag:write"]), true, 200)]
#[case::null_scopes_full_access(None, true, 200)]
#[case::wildcard_scope(Some(vec!["*"]), true, 200)]
#[case::wrong_scope(Some(vec!["insight:read"]), true, 401)]
#[case::wrong_team(Some(vec!["feature_flag:read"]), false, 401)]
#[tokio::test]
async fn test_flag_definitions_project_secret_api_key(
    #[case] scopes: Option<Vec<&str>>,
    #[case] same_team: bool,
    #[case] expected_status: u16,
) {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let team = context.insert_new_team(None).await.unwrap();

    let key_team_id = if same_team {
        team.id
    } else {
        let other_team = context.insert_new_team(None).await.unwrap();
        other_team.id
    };

    let (_, raw_key) = context
        .create_project_secret_api_key(key_team_id, "Test Key", scopes)
        .await
        .unwrap();

    if expected_status == 200 {
        context.populate_cache_for_team(team.id).await.unwrap();
    }

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {raw_key}"))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        expected_status,
        "Response body: {}",
        response.text().await.unwrap()
    );
}

#[tokio::test]
async fn test_valid_pak_used_to_authenticate_from_cache_updates_last_used_at() {
    use feature_flags::{
        api::api_key_usage::{debounce_key, ApiKeyKind},
        config::Config,
        utils::test_utils::TestContext,
    };
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let team = context.insert_new_team(None).await.unwrap();
    let org_id = context.get_organization_id_for_team(&team).await.unwrap();

    let user_email = TestContext::generate_test_email("pak_cache_last_used");
    let user_id = context
        .create_user(&user_email, &org_id, team.id)
        .await
        .unwrap();
    context
        .add_user_to_organization(user_id, &org_id, 15)
        .await
        .unwrap();

    let (pak_id, api_key_value) = context
        .create_personal_api_key(
            user_id,
            "Test PAK Cache",
            vec!["feature_flag:read"],
            None,
            None,
        )
        .await
        .unwrap();

    context.populate_cache_for_team(team.id).await.unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    server.wait_until_ready().await;
    let client = reqwest::Client::new();
    let url = format!(
        "http://{}/flags/definitions?token={}",
        server.addr, team.api_token
    );

    // First request: populates the auth token cache and triggers last_used_at update
    let response = client
        .get(&url)
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();
    assert_eq!(response.status(), 200);

    // Poll for the spawned background task to complete the DB write
    poll_for_pak_last_used_at(
        &context,
        &pak_id,
        "Timed out waiting for background task to set last_used_at for PAK",
    )
    .await;

    // Clear the Redis debounce key so the next request can write to DB again
    let redis_client =
        feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
    redis_client
        .del(debounce_key(ApiKeyKind::Personal, &pak_id))
        .await
        .expect("Failed to delete debounce key");

    // Reset last_used_at to NULL so we can verify the second request sets it
    let mut conn = context.get_non_persons_connection().await.unwrap();
    sqlx::query("UPDATE posthog_personalapikey SET last_used_at = NULL WHERE id = $1")
        .bind(&pak_id)
        .execute(&mut *conn)
        .await
        .unwrap();

    // Second request: auth comes from the token cache (no DB query for auth),
    // but should still trigger the last_used_at update via record_api_key_last_used
    let response = client
        .get(&url)
        .header("Authorization", format!("Bearer {api_key_value}"))
        .send()
        .await
        .unwrap();
    assert_eq!(response.status(), 200);

    // Poll for the spawned background task to complete the DB write
    poll_for_pak_last_used_at(
        &context,
        &pak_id,
        "last_used_at should be set after authenticating from the auth token cache",
    )
    .await;
}

#[rstest::rstest]
#[case::with_token_param(true)]
#[case::bearer_only(false)]
#[tokio::test]
async fn test_project_secret_api_key_updates_last_used_at(#[case] with_token_param: bool) {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let team = context.insert_new_team(None).await.unwrap();
    let (key_id, key) = context
        .create_project_secret_api_key(team.id, "PSAK LastUsed", Some(vec!["feature_flag:read"]))
        .await
        .unwrap();
    context.populate_cache_for_team(team.id).await.unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    server.wait_until_ready().await;
    let url = if with_token_param {
        format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        )
    } else {
        format!("http://{}/flags/definitions", server.addr)
    };
    let response = reqwest::Client::new()
        .get(url)
        .header("Authorization", format!("Bearer {key}"))
        .send()
        .await
        .unwrap();
    assert_eq!(response.status(), 200);

    common::poll_for_psak_last_used_at(
        &context,
        &key_id,
        "Timed out waiting for last_used_at to be set for the project secret API key",
    )
    .await;
}

#[tokio::test]
async fn test_flag_definitions_with_legacy_secret_token_fallback() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};
    use reqwest;

    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    // create_team_with_secret_token creates a legacy phs_ token on posthog_team
    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    // Do NOT insert a project secret API key — the phs_ token should fall back to legacy
    context.populate_cache_for_team(team.id).await.unwrap();

    let server = common::ServerHandle::for_config(config.clone()).await;
    let client = reqwest::Client::new();

    let response = client
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "Response body: {}",
        response.text().await.unwrap()
    );
}

/// Tests the DB-backed rate limit allowlist end-to-end.
/// Combined into a single test to avoid DB-level interference — all scenarios
/// share the same `posthog_instancesetting` row.
///
/// Scenarios tested:
/// 1. Allowlisted team bypasses both rate limits (200 or 304 instead of 429)
/// 2. Non-allowlisted team gets rate limited (429)
/// 3. With two teams, only the listed one bypasses
/// 4. Env var allowlist preserved when DB row is missing
/// 5. null DB value treated as empty allowlist
/// 6. Invalid team IDs skipped, valid ones kept
/// 7. Django-format JSON-encoded string (json.dumps wrapping)
#[tokio::test]
async fn test_db_rate_limit_allowlist() {
    use feature_flags::{config::Config, utils::test_utils::TestContext};

    let context = TestContext::new(None).await;

    // Clean up any leftover rows from a previous failed run
    context
        .delete_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS")
        .await
        .unwrap();

    let (team1, secret1, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let (team2, secret2, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    // --- Scenario 1: allowlisted team bypasses rate limit ---
    {
        let mut config = Config::default_test_config();
        config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
            .parse()
            .unwrap();
        config.flag_definitions_conditional_rate_per_minute = 1;

        context
            .set_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS", &team1.id.to_string())
            .await
            .unwrap();

        let etag_value = "a1b2c3d4e5f6g7h8";
        let redis_client =
            feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
                .await;
        context
            .populate_cache_for_team_with_etag_on(redis_client, team1.id, etag_value)
            .await
            .unwrap();

        let server = common::ServerHandle::for_config(config).await;
        let client = reqwest::Client::new();

        // First request triggers DB refresh
        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(resp.status(), 200);

        // Second request would be rate limited, but allowlist bypasses it
        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(
            resp.status(),
            200,
            "Allowlisted team should bypass rate limit"
        );

        // The conditional budget (1/minute) would reject the second poll, but the
        // DB-refreshed allowlist applies to the conditional limiter too
        let url = format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team1.api_token
        );
        let matching_etag = format!("W/\"{etag_value}\"");
        for i in 0..2 {
            let resp = get_definitions(&client, &url, &secret1, Some(&matching_etag)).await;
            assert_eq!(
                resp.status(),
                304,
                "Allowlisted team should bypass the conditional rate limit (poll {i})"
            );
        }
    }

    // --- Scenario 2: non-allowlisted team gets rate limited ---
    {
        let mut config = Config::default_test_config();
        config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
            .parse()
            .unwrap();

        context
            .delete_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS")
            .await
            .unwrap();

        let redis_client =
            feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
                .await;
        context
            .populate_flag_definitions_cache(redis_client, team1.id)
            .await
            .unwrap();

        let server = common::ServerHandle::for_config(config).await;
        let client = reqwest::Client::new();

        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(resp.status(), 200);

        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(
            resp.status(),
            429,
            "Non-allowlisted team should be rate limited"
        );
    }

    // --- Scenario 3: only listed team bypasses, other team gets limited ---
    {
        let mut config = Config::default_test_config();
        config.flag_definitions_rate_limits = format!(
            r#"{{"{}": "1/minute", "{}": "1/minute"}}"#,
            team1.id, team2.id
        )
        .parse()
        .unwrap();

        context
            .set_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS", &team1.id.to_string())
            .await
            .unwrap();

        let redis_client =
            feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
                .await;
        context
            .populate_flag_definitions_cache(redis_client.clone(), team1.id)
            .await
            .unwrap();
        context
            .populate_flag_definitions_cache(redis_client, team2.id)
            .await
            .unwrap();

        let server = common::ServerHandle::for_config(config).await;
        let client = reqwest::Client::new();

        // Team1 (allowlisted) can make many requests
        for i in 0..5 {
            let resp = client
                .get(format!(
                    "http://{}/flags/definitions?token={}",
                    server.addr, team1.api_token
                ))
                .header("Authorization", format!("Bearer {secret1}"))
                .send()
                .await
                .unwrap();
            assert_eq!(
                resp.status(),
                200,
                "Allowlisted team1 request {i} should succeed"
            );
        }

        // Team2 (not allowlisted) should eventually be rate limited
        let mut saw_rate_limit = false;
        for _ in 0..10 {
            let resp = client
                .get(format!(
                    "http://{}/flags/definitions?token={}",
                    server.addr, team2.api_token
                ))
                .header("Authorization", format!("Bearer {secret2}"))
                .send()
                .await
                .unwrap();
            if resp.status() == 429 {
                saw_rate_limit = true;
                break;
            }
        }
        assert!(
            saw_rate_limit,
            "Non-allowlisted team2 should eventually be rate limited"
        );
    }

    // --- Scenario 4: env var allowlist preserved when DB row is missing ---
    {
        // Configure team1 as allowlisted via the env var (set at server startup)
        let mut config = Config::default_test_config();
        config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
            .parse()
            .unwrap();
        config.rate_limiting_allow_list_teams = team1.id.to_string().parse().unwrap();
        config.flag_definitions_conditional_rate_per_minute = 1;

        // Ensure no DB row exists — env var default should be kept
        context
            .delete_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS")
            .await
            .unwrap();

        let redis_client =
            feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
                .await;
        let etag_value = "a1b2c3d4e5f6g7h8";
        context
            .populate_cache_for_team_with_etag_on(redis_client, team1.id, etag_value)
            .await
            .unwrap();

        let server = common::ServerHandle::for_config(config).await;
        let client = reqwest::Client::new();

        // First request triggers DB refresh — row missing, so env var default is kept
        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(resp.status(), 200);

        // Second request: team1 should still bypass via env var allowlist
        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(
            resp.status(),
            200,
            "Env var allowlist should be preserved when DB row is missing"
        );

        let url = format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team1.api_token
        );
        let matching_etag = format!("W/\"{etag_value}\"");
        for _ in 0..2 {
            let resp = get_definitions(&client, &url, &secret1, Some(&matching_etag)).await;
            assert_eq!(
                resp.status(),
                304,
                "Env var allowlist should bypass the conditional rate limit"
            );
        }
    }

    // --- Scenario 5: null DB value treated as empty allowlist ---
    {
        let mut config = Config::default_test_config();
        config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
            .parse()
            .unwrap();

        context
            .set_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS", "null")
            .await
            .unwrap();

        let redis_client =
            feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
                .await;
        context
            .populate_flag_definitions_cache(redis_client, team1.id)
            .await
            .unwrap();

        let server = common::ServerHandle::for_config(config).await;
        let client = reqwest::Client::new();

        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(resp.status(), 200);

        // null means empty allowlist — team should be rate limited
        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(
            resp.status(),
            429,
            "null DB value should be treated as empty allowlist"
        );
    }

    // --- Scenario 6: invalid team IDs skipped, valid ones kept ---
    {
        let mut config = Config::default_test_config();
        config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
            .parse()
            .unwrap();

        // Mix of valid and invalid IDs — team1 should still be allowlisted
        context
            .set_instance_setting(
                "RATE_LIMITING_ALLOW_LIST_TEAMS",
                &format!("{},abc,xyz", team1.id),
            )
            .await
            .unwrap();

        let redis_client =
            feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
                .await;
        context
            .populate_flag_definitions_cache(redis_client, team1.id)
            .await
            .unwrap();

        let server = common::ServerHandle::for_config(config).await;
        let client = reqwest::Client::new();

        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(resp.status(), 200);

        // team1 is in the valid portion of the allowlist — should bypass
        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(
            resp.status(),
            200,
            "Valid team IDs should be kept even when invalid ones are present"
        );
    }

    // --- Scenario 7: Django-format JSON-encoded string ---
    {
        let mut config = Config::default_test_config();
        config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
            .parse()
            .unwrap();

        // Django's set_instance_setting calls json.dumps(value), so "23047"
        // is stored as "\"23047\"" in raw_value
        context
            .set_instance_setting(
                "RATE_LIMITING_ALLOW_LIST_TEAMS",
                &format!("\"{}\"", team1.id),
            )
            .await
            .unwrap();

        let redis_client =
            feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
                .await;
        context
            .populate_flag_definitions_cache(redis_client, team1.id)
            .await
            .unwrap();

        let server = common::ServerHandle::for_config(config).await;
        let client = reqwest::Client::new();

        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(resp.status(), 200);

        // team1 should bypass via JSON-encoded allowlist
        let resp = client
            .get(format!(
                "http://{}/flags/definitions?token={}",
                server.addr, team1.api_token
            ))
            .header("Authorization", format!("Bearer {secret1}"))
            .send()
            .await
            .unwrap();
        assert_eq!(
            resp.status(),
            200,
            "Django-format JSON-encoded allowlist should work"
        );
    }

    // Clean up
    context
        .delete_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS")
        .await
        .unwrap();
}

/// Verifies the billing path writes (or, when `skip_writes=true`,
/// suppresses) the FlagDefinitions counter for `/flags/definitions`. The
/// `/flags` endpoint is covered by tests in `test_flags.rs`; this test is
/// the equivalent for the FlagDefinitions code path so a regression that
/// honored `skip_writes` for one endpoint but not the other can't slip
/// through.
#[rstest::rstest]
#[case(true)]
#[case(false)]
#[tokio::test]
async fn test_flag_definitions_billing_counter(#[case] skip_writes: bool) {
    use feature_flags::config::FlexBool;
    use feature_flags::flags::flag_analytics::{current_bucket, get_team_request_key};
    use feature_flags::flags::flag_request::FlagRequestType;
    use feature_flags::utils::test_utils::{setup_redis_client, TestContext};
    use serde_json::json;

    let mut config = feature_flags::config::Config::default_test_config();
    config.skip_writes = FlexBool(skip_writes);

    let context = TestContext::new(Some(&config)).await;
    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    // Seed the HyperCache with a billable flag (a non-survey, non-product-tour
    // key) so `has_billable_flags` returns true and `record()` actually fires.
    context
        .populate_cache_for_team_with_flags(
            team.id,
            json!({
                "flags": [{"key": "billable-flag", "active": true}],
                "group_type_mapping": {},
                "cohorts": {},
            }),
        )
        .await
        .unwrap();

    // Clean billing key so we can assert on a fresh write.
    let redis = setup_redis_client(Some(config.redis_url.clone())).await;
    let billing_key = get_team_request_key(team.id, FlagRequestType::FlagDefinitions);
    redis.del(billing_key.clone()).await.unwrap();

    let server = common::ServerHandle::for_config(config).await;
    let http = reqwest::Client::new();

    // Bracket the request: the record lands in whichever 2-minute bucket it crosses.
    let bucket_before = current_bucket();

    let response = http
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();
    assert_eq!(
        response.status(),
        200,
        "Response body: {}",
        response.text().await.unwrap()
    );
    let bucket_after = current_bucket();

    if skip_writes {
        // Sleep ~5 flush windows so even a slow CI scheduler couldn't hide
        // an erroneous `record()` behind a delayed first tick.
        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
        for bucket in bucket_before..=bucket_after {
            let counter = redis.hget(billing_key.clone(), bucket.to_string()).await;
            assert!(
                counter.is_err(),
                "FlagDefinitions billing counter should NOT be incremented when skip_writes=true, got {counter:?}"
            );
        }
    } else {
        let counter = common::poll_for_billing_counter_across_buckets(
            &redis,
            &billing_key,
            bucket_before,
            bucket_after,
        )
        .await;
        assert_eq!(
            counter, "1",
            "FlagDefinitions billing counter should be incremented once"
        );
    }
}

/// A 304 lands on the not-modified counter and leaves the full local evaluation
/// counter untouched. It stays unbilled for definitions that hold only survey or
/// product tour flags, and when `skip_writes` is on.
#[rstest::rstest]
#[case::billable_flag("billable-flag", false, true)]
#[case::survey_only("survey-targeting-only", false, false)]
#[case::skip_writes("billable-flag", true, false)]
#[tokio::test]
async fn test_flag_definitions_304_records_not_modified_billing_counter(
    #[case] flag_key: &str,
    #[case] skip_writes: bool,
    #[case] expect_billed: bool,
) {
    use feature_flags::config::FlexBool;
    use feature_flags::flags::flag_analytics::{current_bucket, get_team_request_key};
    use feature_flags::flags::flag_request::FlagRequestType;
    use feature_flags::utils::test_utils::{setup_redis_client, TestContext};
    use serde_json::json;

    let mut config = feature_flags::config::Config::default_test_config();
    config.skip_writes = FlexBool(skip_writes);
    let context = TestContext::new(Some(&config)).await;
    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let etag_value = "billed304etag0001";
    context
        .populate_cache_for_team_with_flags_and_etag(
            team.id,
            json!({
                "flags": [{"key": flag_key, "active": true}],
                "group_type_mapping": {},
                "cohorts": {},
            }),
            etag_value,
        )
        .await
        .unwrap();

    let redis = setup_redis_client(Some(config.redis_url.clone())).await;
    let not_modified_key =
        get_team_request_key(team.id, FlagRequestType::FlagDefinitionsNotModified);
    let full_response_key = get_team_request_key(team.id, FlagRequestType::FlagDefinitions);
    redis.del(not_modified_key.clone()).await.unwrap();
    redis.del(full_response_key.clone()).await.unwrap();

    let server = common::ServerHandle::for_config(config).await;
    let http = reqwest::Client::new();

    let bucket_before = current_bucket();
    let response = http
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .header("If-None-Match", format!("W/\"{etag_value}\""))
        .send()
        .await
        .unwrap();
    assert_eq!(response.status(), 304);
    let bucket_after = current_bucket();

    if expect_billed {
        let counter = common::poll_for_billing_counter_across_buckets(
            &redis,
            &not_modified_key,
            bucket_before,
            bucket_after,
        )
        .await;
        assert_eq!(
            counter, "1",
            "304 should be recorded once on the not-modified billing counter"
        );
    } else {
        tokio::time::sleep(std::time::Duration::from_millis(500)).await;
        for bucket in bucket_before..=bucket_after {
            let counter = redis
                .hget(not_modified_key.clone(), bucket.to_string())
                .await;
            assert!(
                counter.is_err(),
                "304 for non-billable flags must not be billed, got {counter:?}"
            );
        }
    }
    for bucket in bucket_before..=bucket_after {
        let full = redis
            .hget(full_response_key.clone(), bucket.to_string())
            .await;
        assert!(
            full.is_err(),
            "304 must not increment the full local evaluation counter, got {full:?}"
        );
    }
}

/// Only a payload served from Redis decides whether a 304 is billed. With the
/// ETag in Redis but the payload only in S3, neither the full response that
/// precedes the poll nor the poll's own read may memoize S3's answer, so the
/// 304 stays unbilled even though S3 says the definitions are billable.
#[tokio::test]
async fn test_flag_definitions_304_is_not_billed_from_an_s3_payload() {
    use feature_flags::flags::flag_analytics::{current_bucket, get_team_request_key};
    use feature_flags::flags::flag_request::FlagRequestType;
    use feature_flags::utils::test_utils::{setup_redis_client, static_s3_client, TestContext};
    use serde_json::json;

    let config = feature_flags::config::Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;
    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let payload = json!({
        "flags": [{"key": "billable-flag", "active": true}],
        "group_type_mapping": {},
        "cohorts": {},
    });
    let etag_value = "s3only304etag0001";
    context
        .populate_cache_for_team_with_flags_and_etag(team.id, payload.clone(), etag_value)
        .await
        .unwrap();

    let redis = setup_redis_client(Some(config.redis_url.clone())).await;
    // Leave the ETag in Redis and the payload only in S3.
    redis
        .del(format!(
            "posthog:1:cache/teams/{}/feature_flags/flags_with_cohorts.json",
            team.id
        ))
        .await
        .unwrap();
    let not_modified_key =
        get_team_request_key(team.id, FlagRequestType::FlagDefinitionsNotModified);
    redis.del(not_modified_key.clone()).await.unwrap();

    let server = common::ServerHandle::for_config_with_s3(
        config,
        // S3 serves the same billable payload for every key, standing in for a copy
        // that may lag the ETag Redis holds by a version.
        Some(static_s3_client(&payload.to_string())),
    )
    .await;
    let http = reqwest::Client::new();
    let url = format!(
        "http://{}/flags/definitions?token={}",
        server.addr, team.api_token
    );

    let bucket_before = current_bucket();
    let response = http
        .get(&url)
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();
    assert_eq!(response.status(), 200);

    let response = http
        .get(&url)
        .header("Authorization", format!("Bearer {secret_token}"))
        .header("If-None-Match", format!("W/\"{etag_value}\""))
        .send()
        .await
        .unwrap();
    assert_eq!(response.status(), 304);
    let bucket_after = current_bucket();

    tokio::time::sleep(std::time::Duration::from_millis(500)).await;
    for bucket in bucket_before..=bucket_after {
        let counter = redis
            .hget(not_modified_key.clone(), bucket.to_string())
            .await;
        assert!(
            counter.is_err(),
            "304 must not be billed from an S3 payload, got {counter:?}"
        );
    }
}

/// A full response served from Redis memoizes whether its ETag is billable, so
/// a later 304 for that ETag is billed from the memo even after the Redis
/// payload is gone and the 304's own read would find nothing.
#[tokio::test]
async fn test_flag_definitions_304_is_billed_from_the_full_response_memo() {
    use feature_flags::flags::flag_analytics::{current_bucket, get_team_request_key};
    use feature_flags::flags::flag_request::FlagRequestType;
    use feature_flags::utils::test_utils::{setup_redis_client, TestContext};
    use serde_json::json;

    let config = feature_flags::config::Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;
    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();

    let etag_value = "memo304etag00001";
    context
        .populate_cache_for_team_with_flags_and_etag(
            team.id,
            json!({
                "flags": [{"key": "billable-flag", "active": true}],
                "group_type_mapping": {},
                "cohorts": {},
            }),
            etag_value,
        )
        .await
        .unwrap();

    let redis = setup_redis_client(Some(config.redis_url.clone())).await;
    let not_modified_key =
        get_team_request_key(team.id, FlagRequestType::FlagDefinitionsNotModified);
    redis.del(not_modified_key.clone()).await.unwrap();

    let server = common::ServerHandle::for_config(config).await;
    let http = reqwest::Client::new();
    let url = format!(
        "http://{}/flags/definitions?token={}",
        server.addr, team.api_token
    );

    let response = http
        .get(&url)
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();
    assert_eq!(response.status(), 200);

    // Keep the ETag in Redis but drop the payload, so only the memo can bill the 304.
    redis
        .del(format!(
            "posthog:1:cache/teams/{}/feature_flags/flags_with_cohorts.json",
            team.id
        ))
        .await
        .unwrap();

    let bucket_before = current_bucket();
    let response = http
        .get(&url)
        .header("Authorization", format!("Bearer {secret_token}"))
        .header("If-None-Match", format!("W/\"{etag_value}\""))
        .send()
        .await
        .unwrap();
    assert_eq!(response.status(), 304);
    let bucket_after = current_bucket();

    let counter = common::poll_for_billing_counter_across_buckets(
        &redis,
        &not_modified_key,
        bucket_before,
        bucket_after,
    )
    .await;
    assert_eq!(
        counter, "1",
        "304 should be billed from the memo the full response wrote"
    );
}

/// Poll the self-heal rebuild-requests set until it contains `team_id`, or return
/// false after ~2s. The enqueue runs in a background task, so a bounded retry is
/// needed rather than a single read.
async fn poll_for_rebuild_enqueue(redis_url: &str, team_id: i32, s3_hit: bool) -> bool {
    use feature_flags::utils::test_utils::{
        read_flag_definitions_rebuild_requests, read_s3_rebuild_requests,
    };
    use tokio::time::{sleep, Duration};

    for _ in 0..40 {
        let members = if s3_hit {
            read_s3_rebuild_requests(redis_url).await
        } else {
            read_flag_definitions_rebuild_requests(redis_url).await
        };
        if members.contains(&team_id.to_string()) {
            return true;
        }
        sleep(Duration::from_millis(50)).await;
    }
    false
}

#[tokio::test]
async fn test_cache_miss_enqueues_rebuild_when_self_heal_enabled() {
    use feature_flags::{
        config::{Config, FlexBool},
        utils::test_utils::{
            dummy_s3_client, remove_flag_definitions_rebuild_request, TestContext,
        },
    };
    use reqwest;

    let mut config = Config::default_test_config();
    config.flag_definitions_self_heal_enabled = FlexBool(true);
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();
    remove_flag_definitions_rebuild_request(&config.redis_url, team.id).await;
    // Don't populate the cache, and inject a NotFound S3 so the read is a genuine
    // cache_miss (the flags test job has no object store, so real S3 would error).
    let server =
        common::ServerHandle::for_config_with_s3(config.clone(), Some(dummy_s3_client())).await;
    let response = reqwest::Client::new()
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    assert_eq!(response.status(), 503, "expected a cache-miss 503");
    assert!(
        poll_for_rebuild_enqueue(&config.redis_url, team.id, false).await,
        "team {} should be enqueued for rebuild after a cache-miss 503",
        team.id
    );
}

#[tokio::test]
async fn test_cache_miss_enqueues_rebuild_on_dedicated_redis() {
    use feature_flags::{
        config::{Config, FlexBool},
        utils::test_utils::{
            dummy_s3_client, read_flag_definitions_rebuild_requests,
            remove_flag_definitions_rebuild_request, TestContext,
        },
    };
    use reqwest;

    // The Celery drain reads the queue from the dedicated cluster, so the enqueue has to follow
    // the writer regardless of which cluster the reader is on. This case leaves
    // FLAG_DEFINITIONS_DEDICATED_REDIS_ENABLED off, so the reader is still on shared.
    let mut config = Config::default_test_config();
    config.flags_redis_url = DEDICATED_REDIS_URL.to_string();
    config.flag_definitions_self_heal_enabled = FlexBool(true);
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();
    remove_flag_definitions_rebuild_request(&config.flags_redis_url, team.id).await;
    remove_flag_definitions_rebuild_request(&config.redis_url, team.id).await;
    let shared_before = read_flag_definitions_rebuild_requests(&config.redis_url).await;

    // Leave both caches unseeded and inject a NotFound S3 so the read is a genuine
    // cache_miss rather than an s3_error.
    let server =
        common::ServerHandle::for_config_with_s3(config.clone(), Some(dummy_s3_client())).await;
    let response = reqwest::Client::new()
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    assert_eq!(response.status(), 503, "expected a cache-miss 503");
    assert!(
        poll_for_rebuild_enqueue(&config.flags_redis_url, team.id, false).await,
        "team {} should be enqueued on the dedicated redis, where the drain reads",
        team.id
    );

    let shared_after = read_flag_definitions_rebuild_requests(&config.redis_url).await;
    let added: Vec<&String> = shared_after
        .iter()
        .filter(|member| !shared_before.contains(member))
        .collect();
    assert!(
        !added.contains(&&team.id.to_string()),
        "team {} must not be enqueued on the shared redis, where nothing drains",
        team.id
    );
}

#[tokio::test]
async fn test_s3_hit_enqueues_rebuild_and_keeps_first_score() {
    use feature_flags::{
        config::{Config, FlexBool},
        utils::test_utils::{
            remove_s3_rebuild_request, setup_redis_client, static_s3_client, TestContext,
        },
    };
    use reqwest;
    use std::time::{SystemTime, UNIX_EPOCH};

    let mut config = Config::default_test_config();
    config.flag_definitions_self_heal_enabled = FlexBool(true);
    config.flag_definitions_rebuild_on_s3_hit_enabled = FlexBool(true);
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();
    remove_s3_rebuild_request(&config.redis_url, team.id).await;

    let server = common::ServerHandle::for_config_with_s3(
        config.clone(),
        Some(static_s3_client(r#"{"flags": [], "cohorts": {}}"#)),
    )
    .await;
    let response = reqwest::Client::new()
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    assert_eq!(
        response.status(),
        200,
        "expected the S3 payload to be served"
    );
    assert!(
        response.headers().get("etag").is_none(),
        "an S3-served response has no ETag to send, which is what the rebuild repairs"
    );

    assert!(
        poll_for_rebuild_enqueue(&config.redis_url, team.id, true).await,
        "team {} should be enqueued for rebuild after an S3-served response",
        team.id
    );

    let redis = setup_redis_client(Some(config.redis_url.clone())).await;
    let first_score_upper_bound = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .unwrap()
        .as_millis() as i64;
    let queue = "flag_definitions:rebuild_s3_requests".to_string();
    redis
        .zadd_nx(
            queue.clone(),
            team.id.to_string(),
            first_score_upper_bound + 60_000,
        )
        .await
        .unwrap();
    let original_score_members = redis
        .zrangebyscore(
            queue,
            "-inf".to_string(),
            first_score_upper_bound.to_string(),
        )
        .await
        .unwrap();
    assert!(
        original_score_members.contains(&team.id.to_string()),
        "a repeated NX write must keep the first enqueue score"
    );
}

#[tokio::test]
async fn test_cache_miss_does_not_enqueue_rebuild_when_self_heal_disabled() {
    use feature_flags::{
        config::Config,
        utils::test_utils::{
            dummy_s3_client, read_flag_definitions_rebuild_requests,
            remove_flag_definitions_rebuild_request, TestContext,
        },
    };
    use reqwest;
    use tokio::time::{sleep, Duration};

    // default_test_config leaves flag_definitions_self_heal_enabled = false.
    let config = Config::default_test_config();
    let context = TestContext::new(Some(&config)).await;

    let (team, secret_token, _) = context
        .create_team_with_secret_token(None, None, None)
        .await
        .unwrap();
    remove_flag_definitions_rebuild_request(&config.redis_url, team.id).await;

    // Inject a NotFound S3 so this is a genuine cache_miss: the only reason no enqueue
    // happens is the flag being off, not the miss classifying as s3_error.
    let server =
        common::ServerHandle::for_config_with_s3(config.clone(), Some(dummy_s3_client())).await;
    let response = reqwest::Client::new()
        .get(format!(
            "http://{}/flags/definitions?token={}",
            server.addr, team.api_token
        ))
        .header("Authorization", format!("Bearer {secret_token}"))
        .send()
        .await
        .unwrap();

    assert_eq!(response.status(), 503, "expected a cache-miss 503");

    // Give any (erroneous) background enqueue time to land, then assert it did not.
    sleep(Duration::from_millis(500)).await;
    let members = read_flag_definitions_rebuild_requests(&config.redis_url).await;
    assert!(
        !members.contains(&team.id.to_string()),
        "team {} must not be enqueued when self-heal is disabled",
        team.id
    );
}