:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
mod common;
/// Poll until `last_used_at` is set for the given PAK, or panic after ~4s.
async fn poll_for_pak_last_used_at(
context: &feature_flags::utils::test_utils::TestContext,
pak_id: &str,
message: &str,
) {
use tokio::time::{sleep, Duration};
let mut conn = context.get_non_persons_connection().await.unwrap();
for _ in 0..80 {
let count: (i64,) = sqlx::query_as(
"SELECT COUNT(*) FROM posthog_personalapikey WHERE id = $1 AND last_used_at IS NOT NULL",
)
.bind(pak_id)
.fetch_one(&mut *conn)
.await
.unwrap();
if count.0 > 0 {
return;
}
sleep(Duration::from_millis(50)).await;
}
panic!("{message}");
}
#[tokio::test]
async fn test_hypercache_config_generation() {
use common_hypercache::{HyperCacheConfig, KeyType};
use common_types::TeamIdentifier;
// Create a test team that implements TeamIdentifier
#[derive(Debug, Clone)]
struct TestTeam {
id: i32,
token: String,
}
impl TeamIdentifier for TestTeam {
fn team_id(&self) -> i32 {
self.id
}
fn api_token(&self) -> &str {
&self.token
}
}
let test_team = TestTeam {
id: 123,
token: "test_token".to_string(),
};
// Test that HyperCache configurations generate correct cache keys matching Django format
let config_with_cohorts = HyperCacheConfig::new(
"feature_flags".to_string(),
"flags_with_cohorts.json".to_string(),
"us-east-1".to_string(),
"test-bucket".to_string(),
);
let config_without_cohorts = HyperCacheConfig::new(
"feature_flags".to_string(),
"flags_without_cohorts.json".to_string(),
"us-east-1".to_string(),
"test-bucket".to_string(),
);
let team_key = KeyType::team(test_team.clone());
// Test Redis cache key generation (includes posthog:1: prefix)
let redis_key_with_cohorts = config_with_cohorts.get_redis_cache_key(&team_key);
assert_eq!(
redis_key_with_cohorts,
"posthog:1:cache/teams/123/feature_flags/flags_with_cohorts.json"
);
let redis_key_without_cohorts = config_without_cohorts.get_redis_cache_key(&team_key);
assert_eq!(
redis_key_without_cohorts,
"posthog:1:cache/teams/123/feature_flags/flags_without_cohorts.json"
);
// Test S3 cache key generation (no prefix, matches Django object_storage)
let s3_key_with_cohorts = config_with_cohorts.get_s3_cache_key(&team_key);
assert_eq!(
s3_key_with_cohorts,
"cache/teams/123/feature_flags/flags_with_cohorts.json"
);
let s3_key_without_cohorts = config_without_cohorts.get_s3_cache_key(&team_key);
assert_eq!(
s3_key_without_cohorts,
"cache/teams/123/feature_flags/flags_without_cohorts.json"
);
}
#[tokio::test]
async fn test_flag_definitions_endpoint_exists() {
use feature_flags::config::Config;
use reqwest;
let config = Config::default_test_config();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
// Test that the endpoint exists and returns some response (even if unauthorized)
let response = client
.get(format!(
"http://{}/flags/definitions?token=any_token",
server.addr
))
.send()
.await
.unwrap();
// Should return 401 Unauthorized when no authentication is provided
assert_eq!(response.status(), 401);
}
#[tokio::test]
async fn test_personal_api_key_authentication_no_key() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
use serde_json::Value;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create a real team so we pass the token validation
let team = context.insert_new_team(None).await.unwrap();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
// Test that the endpoint returns 401 when no API key is provided
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.send()
.await
.unwrap();
let status = response.status();
let body_text = response.text().await.unwrap();
assert_eq!(status, 401);
// Verify the response body matches Django's format
let body: Value = serde_json::from_str(&body_text).unwrap();
assert_eq!(body["type"], "authentication_error");
assert_eq!(body["code"], "not_authenticated");
assert_eq!(
body["detail"],
"Authentication credentials were not provided."
);
assert_eq!(body["attr"], Value::Null);
}
#[tokio::test]
async fn test_personal_api_key_authentication_invalid_key() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
use serde_json::Value;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create a real team so we pass the token validation
let team = context.insert_new_team(None).await.unwrap();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
// Test that the endpoint returns 401 when an invalid API key is provided
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", "Bearer phx_invalid_key_12345")
.send()
.await
.unwrap();
let status = response.status();
let body_text = response.text().await.unwrap();
assert_eq!(status, 401);
// Verify the response body matches Django's format
let body: Value = serde_json::from_str(&body_text).unwrap();
assert_eq!(body["type"], "authentication_error");
assert_eq!(body["code"], "authentication_failed");
assert_eq!(body["detail"], "Personal API key is invalid.");
assert_eq!(body["attr"], Value::Null);
}
#[tokio::test]
async fn test_personal_api_key_authentication_valid_key_with_scopes() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create team, user, and personal API key using helpers
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let user_email = feature_flags::utils::test_utils::TestContext::generate_test_email("test_pak");
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
context
.add_user_to_organization(user_id, &org_id, 15)
.await
.unwrap();
let (_pak_id, api_key_value) = context
.create_personal_api_key(user_id, "Test PAK", vec!["feature_flag:read"], None, None)
.await
.unwrap();
// Populate cache to avoid 503 errors
context.populate_cache_for_team(team.id).await.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Test that the endpoint returns 200 when a valid API key with proper scopes is provided
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Response body: {}",
response.text().await.unwrap()
);
}
#[tokio::test]
async fn test_personal_api_key_authentication_without_feature_flag_scopes() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create team, user, and personal API key with different scope
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let user_email =
feature_flags::utils::test_utils::TestContext::generate_test_email("test_no_scopes");
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
let (_pak_id, api_key_value) = context
.create_personal_api_key(
user_id,
"Test PAK No Scopes",
vec!["insight:read"],
None,
None,
)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Test that the endpoint returns 403 when API key is valid but lacks feature_flag scopes
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
let status = response.status();
let body_text = response.text().await.unwrap();
assert_eq!(status, 403);
let body: serde_json::Value = serde_json::from_str(&body_text).unwrap();
assert_eq!(body["type"], "authentication_error");
assert_eq!(body["code"], "permission_denied");
assert_eq!(
body["detail"],
"Personal API key lacks required scopes (feature_flag:read or feature_flag:write)."
);
assert_eq!(body["attr"], serde_json::Value::Null);
}
#[tokio::test]
async fn test_personal_api_key_authentication_inactive_user() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create team
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
// Create INACTIVE user using helper (is_active = false)
let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
let user_email = format!("test_inactive_{test_uuid}@posthog.com");
let user_id = context
.create_user_with_options(&user_email, &org_id, Some(team.id), false)
.await
.unwrap();
let (_pak_id, api_key_value) = context
.create_personal_api_key(
user_id,
"Test PAK Inactive",
vec!["feature_flag:read"],
None,
None,
)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Test that the endpoint returns 401 when user is inactive
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 401);
}
#[tokio::test]
async fn test_secret_api_token_authentication_valid_token() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create team with secret API token using helper
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
// Populate cache to avoid 503 errors
context.populate_cache_for_team(team.id).await.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Test that the endpoint returns 200 when a valid secret API token is provided
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Response body: {}",
response.text().await.unwrap()
);
}
#[tokio::test]
async fn test_secret_api_token_authentication_invalid_token() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
use serde_json::Value;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create a real team so we pass the token validation
let team = context.insert_new_team(None).await.unwrap();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
// Test that the endpoint returns 401 when an invalid secret token is provided
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", "Bearer phs_invalid_secret_token_xyz")
.send()
.await
.unwrap();
let status = response.status();
let body_text = response.text().await.unwrap();
assert_eq!(status, 401);
// Verify the response body matches Django's format
let body: Value = serde_json::from_str(&body_text).unwrap();
assert_eq!(body["type"], "authentication_error");
assert_eq!(body["code"], "authentication_failed");
assert_eq!(body["detail"], "Secret API token is invalid.");
assert_eq!(body["attr"], Value::Null);
}
/// ACCEPT_TEAM_SECRET_TOKENS is the kill switch for legacy team secret tokens (#66179).
/// When it is false, the lookup skips posthog_team.
/// A legacy token then authenticates only through a backfilled PSAK mirror row with the same string.
/// `test_secret_api_token_authentication_valid_token` covers the default (true) path.
#[rstest::rstest]
#[case::with_mirror_row(true, 200)]
#[case::without_mirror_row(false, 401)]
#[tokio::test]
async fn test_kill_switch_off_legacy_token_only_matches_mirror_row(
#[case] mirror_row: bool,
#[case] expected_status: u16,
) {
use feature_flags::{
config::{Config, FlexBool},
utils::test_utils::TestContext,
};
use reqwest;
let mut config = Config::default_test_config();
config.accept_team_secret_tokens = FlexBool(false);
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
if mirror_row {
context
.create_project_secret_api_key_with_value(
team.id,
"Migrated legacy secret API key",
Some(vec!["feature_flag:read", "support_ticket:read"]),
&secret_token,
)
.await
.unwrap();
}
context.populate_cache_for_team(team.id).await.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
let status = response.status();
let body_text = response.text().await.unwrap();
assert_eq!(status, expected_status, "Response body: {body_text}");
}
/// Token param absent with a valid team-scoped secret — team is derived from the token.
#[rstest::rstest]
#[case::team_secret_token("secret")]
#[case::project_secret_key("project_secret")]
#[tokio::test]
async fn test_missing_token_param_success(#[case] auth_type: &str) {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let (team, bearer_value) = match auth_type {
"secret" => {
let (team, secret, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
(team, secret)
}
"project_secret" => {
let team = context.insert_new_team(None).await.unwrap();
let (_, key) = context
.create_project_secret_api_key(team.id, "Test Key", Some(vec!["feature_flag:read"]))
.await
.unwrap();
(team, key)
}
_ => unreachable!(),
};
context.populate_cache_for_team(team.id).await.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let response = client
.get(format!("http://{}/flags/definitions", server.addr))
.header("Authorization", format!("Bearer {bearer_value}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Response body: {}",
response.text().await.unwrap()
);
}
/// Token param absent — error cases that don't need DB setup.
#[rstest::rstest]
#[case::no_auth(None, 401)]
#[case::invalid_phs_token(Some("Bearer phs_invalid_token_xyz"), 401)]
#[tokio::test]
async fn test_missing_token_param_error(
#[case] auth_header: Option<&str>,
#[case] expected_status: u16,
) {
use feature_flags::config::Config;
use reqwest;
let config = Config::default_test_config();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
let mut req = client.get(format!("http://{}/flags/definitions", server.addr));
if let Some(header) = auth_header {
req = req.header("Authorization", header);
}
let response = req.send().await.unwrap();
assert_eq!(response.status(), expected_status);
}
/// When token param is missing, the response should contain flags for the team
/// that owns the phs_ token — not a different team's flags.
#[tokio::test]
async fn test_missing_token_param_returns_correct_team_flags() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
use serde_json::{json, Value};
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create two teams with different secret tokens
let (team1, secret1, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let (team2, secret2, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
// Seed each team's cache with distinguishable flags so we can verify isolation
context
.populate_cache_for_team_with_flags(
team1.id,
json!({"flags": [{"key": "team1-flag", "active": true}], "group_type_mapping": {}, "cohorts": {}}),
)
.await
.unwrap();
context
.populate_cache_for_team_with_flags(
team2.id,
json!({"flags": [{"key": "team2-flag", "active": true}], "group_type_mapping": {}, "cohorts": {}}),
)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Request with team1's secret token (no ?token= param)
let resp1 = client
.get(format!("http://{}/flags/definitions", server.addr))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(resp1.status(), 200);
let body1: Value = serde_json::from_str(&resp1.text().await.unwrap()).unwrap();
// Request with team2's secret token (no ?token= param)
let resp2 = client
.get(format!("http://{}/flags/definitions", server.addr))
.header("Authorization", format!("Bearer {secret2}"))
.send()
.await
.unwrap();
assert_eq!(resp2.status(), 200);
let body2: Value = serde_json::from_str(&resp2.text().await.unwrap()).unwrap();
// Each team should get its own flags, not the other's
assert_eq!(body1["flags"][0]["key"], "team1-flag");
assert_eq!(body2["flags"][0]["key"], "team2-flag");
assert_ne!(
body1.get("flags"),
body2.get("flags"),
"Different teams' phs_ tokens should resolve to different flags"
);
}
/// When token param is missing and auth is a personal API key (multi-team),
/// should return 400 since we can't derive the team.
#[tokio::test]
async fn test_missing_token_param_with_personal_api_key_returns_400() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let user_email = format!("test-pak-{}@posthog.com", uuid::Uuid::new_v4());
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
let (_, pak_token) = context
.create_personal_api_key(user_id, "Test PAK", vec!["feature_flag:read"], None, None)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let response = client
.get(format!("http://{}/flags/definitions", server.addr))
.header("Authorization", format!("Bearer {pak_token}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 400);
let body = response.text().await.unwrap();
assert!(
body.contains("token"),
"Error should mention the token parameter: {body}"
);
}
#[tokio::test]
async fn test_token_mismatch_secret_token_for_different_team() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create two teams with secret tokens
let (team1, _secret1, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let (_team2, secret2, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Test: Use team1's public token but team2's secret token - should fail
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret2}"))
.send()
.await
.unwrap();
// Should return 401 because the secret token doesn't match the team
assert_eq!(response.status(), 401);
}
#[tokio::test]
async fn test_personal_api_key_with_scoped_teams_allowed() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create team using helper (creates org, project, and team)
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
let user_email = format!("test_scoped_teams_{test_uuid}@posthog.com");
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
context
.add_user_to_organization(user_id, &org_id, 15)
.await
.unwrap();
// Create personal API key with scoped_teams restriction that includes our team
let (_pak_id, api_key_value) = context
.create_personal_api_key(
user_id,
"Test PAK Scoped Teams",
vec!["feature_flag:read"],
Some(vec![team.id]),
None,
)
.await
.unwrap();
// Populate cache to avoid 503 errors
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
context
.populate_flag_definitions_cache(redis_client, team.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Should succeed because the team is in the scoped_teams list
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap();
assert_eq!(
status, 200,
"Should authenticate when team is in scoped_teams. Response body: {body}"
);
}
#[tokio::test]
async fn test_personal_api_key_with_scoped_teams_denied() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
let user_email = format!("test_denied_{test_uuid}@posthog.com");
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
// Create personal API key with scoped_teams restriction that excludes our team
let (_pak_id, api_key_value) = context
.create_personal_api_key(
user_id,
"Test PAK Denied",
vec!["feature_flag:read"],
Some(vec![99999]), // Different team ID
None,
)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Should fail because the team is not in the scoped_teams list
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
401,
"Should deny access when team is not in scoped_teams"
);
}
#[tokio::test]
async fn test_personal_api_key_with_scoped_organizations_allowed() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
let user_email = format!("test_org_allowed_{test_uuid}@posthog.com");
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
context
.add_user_to_organization(user_id, &org_id, 15)
.await
.unwrap();
// Create personal API key with scoped_organizations restriction that includes our org
let org_id_str = org_id.to_string();
let (_pak_id, api_key_value) = context
.create_personal_api_key(
user_id,
"Test PAK Org Allowed",
vec!["feature_flag:read"],
None,
Some(vec![org_id_str]),
)
.await
.unwrap();
// Populate cache to avoid 503 errors
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
context
.populate_flag_definitions_cache(redis_client, team.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Should succeed because the organization is in the scoped_organizations list
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap();
assert_eq!(
status, 200,
"Should authenticate when organization is in scoped_organizations. Response body: {body}"
);
}
#[tokio::test]
async fn test_personal_api_key_with_scoped_organizations_denied() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create team using helper (creates org, project, and team)
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
let user_email = format!("test_org_denied_{test_uuid}@posthog.com");
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
// Create personal API key with scoped_organizations restriction that excludes our org
let different_org_id = uuid::Uuid::new_v4().to_string();
let (_pak_id, api_key_value) = context
.create_personal_api_key(
user_id,
"Test PAK Org Denied",
vec!["feature_flag:read"],
None,
Some(vec![different_org_id]), // Different organization ID
)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Should fail because the organization is not in the scoped_organizations list
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
401,
"Should deny access when organization is not in scoped_organizations"
);
}
#[tokio::test]
async fn test_personal_api_key_with_scoped_organizations_removed_member() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
let user_email = format!("test_org_removed_{test_uuid}@posthog.com");
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
context
.add_user_to_organization(user_id, &org_id, 15)
.await
.unwrap();
let org_id_str = org_id.to_string();
let (_pak_id, api_key_value) = context
.create_personal_api_key(
user_id,
"Test PAK Org Removed",
vec!["feature_flag:read"],
None,
Some(vec![org_id_str]),
)
.await
.unwrap();
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
context
.populate_flag_definitions_cache(redis_client, team.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Verify access works while user is a member
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Should authenticate while user is an org member"
);
// Remove user from organization
context
.remove_user_from_organization(user_id, &org_id)
.await
.unwrap();
// Simulate Django signal-based cache invalidation (Python handles this in production)
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
feature_flags::utils::test_utils::invalidate_personal_api_key_auth_cache(
redis_client,
&api_key_value,
)
.await
.unwrap();
// Should now fail because the user is no longer an org member
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
401,
"Should deny access after user is removed from organization"
);
}
#[tokio::test]
async fn test_personal_api_key_unscoped_removed_member() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
let user_email = format!("test_unscoped_removed_{test_uuid}@posthog.com");
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
context
.add_user_to_organization(user_id, &org_id, 15)
.await
.unwrap();
// Create PAK without scoped_organizations to confirm the mandatory membership
// check works for all PAK configurations, not just those with explicit org scoping.
let (_pak_id, api_key_value) = context
.create_personal_api_key(
user_id,
"Test PAK Unscoped",
vec!["feature_flag:read"],
None,
None,
)
.await
.unwrap();
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
context
.populate_flag_definitions_cache(redis_client, team.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Verify access works while user is a member
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Should authenticate while user is an org member"
);
// Remove user from organization
context
.remove_user_from_organization(user_id, &org_id)
.await
.unwrap();
// Simulate Django signal-based cache invalidation (Python handles this in production)
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
feature_flags::utils::test_utils::invalidate_personal_api_key_auth_cache(
redis_client,
&api_key_value,
)
.await
.unwrap();
// Should fail because the user is no longer an org member, even without scoped_organizations
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
401,
"Should deny access after user is removed from organization, even without scoped_organizations"
);
}
#[tokio::test]
async fn test_cache_miss_returns_503() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
use serde_json::Value;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create team with secret API token
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
// DO NOT populate cache - we want to test cache miss behavior
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Request should return 503 when cache is empty
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
let status = response.status();
let body_text = response.text().await.unwrap();
assert_eq!(
status, 503,
"Should return 503 on cache miss. Body: {body_text}"
);
// Verify the response body has proper error format (if JSON)
if let Ok(body) = serde_json::from_str::<Value>(&body_text) {
assert_eq!(body["type"], "server_error");
assert_eq!(body["code"], "service_unavailable");
assert!(
body["detail"]
.as_str()
.unwrap()
.contains("A service dependency is temporarily unavailable"),
"Error message should mention cache miss"
);
} else {
// If not JSON, verify the error message mentions cache
assert!(
body_text.contains("A service dependency is temporarily unavailable"),
"Body should mention cache miss. Got: {body_text}"
);
}
}
#[tokio::test]
async fn test_personal_api_key_with_all_access_scopes() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create team, user, and personal API key with {*} (all access) scopes
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
let user_email = format!("test_all_access_{test_uuid}@posthog.com");
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
context
.add_user_to_organization(user_id, &org_id, 15)
.await
.unwrap();
// Create PAK with {*} scope (all access)
let (_pak_id, api_key_value) = context
.create_personal_api_key(user_id, "Test PAK All Access", vec!["*"], None, None)
.await
.unwrap();
// Populate cache to avoid 503 errors
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
context
.populate_flag_definitions_cache(redis_client, team.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Test that the endpoint returns 200 when PAK has all access
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Response body: {}",
response.text().await.unwrap()
);
}
#[tokio::test]
async fn test_personal_api_key_with_feature_flag_write_scope() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create team, user, and personal API key with feature_flag:write scope
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
let user_email = format!("test_write_scope_{test_uuid}@posthog.com");
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
context
.add_user_to_organization(user_id, &org_id, 15)
.await
.unwrap();
// Create PAK with feature_flag:write scope (write includes read access)
let (_pak_id, api_key_value) = context
.create_personal_api_key(
user_id,
"Test PAK Write Scope",
vec!["feature_flag:write"],
None,
None,
)
.await
.unwrap();
// Populate cache to avoid 503 errors
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
context
.populate_flag_definitions_cache(redis_client, team.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Test that the endpoint returns 200 with feature_flag:write scope
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Should succeed with feature_flag:write scope. Response body: {}",
response.text().await.unwrap()
);
}
#[tokio::test]
async fn test_backup_secret_api_token_authentication() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create a team with both secret_api_token and secret_api_token_backup using helper
let backup_token_value = feature_flags::utils::test_utils::random_string("phs_backup_", 12);
let (team, _secret_token, backup_secret_token) = context
.create_team_with_secret_token(None, None, Some(&backup_token_value))
.await
.unwrap();
// Populate cache to avoid 503 errors
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
context
.populate_flag_definitions_cache(redis_client, team.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Test that the backup secret token works
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header(
"Authorization",
format!("Bearer {}", backup_secret_token.unwrap()),
)
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Should authenticate with backup secret token. Response body: {}",
response.text().await.unwrap()
);
}
#[tokio::test]
async fn test_invalid_project_api_key() {
use feature_flags::config::Config;
use reqwest;
use serde_json::Value;
let config = Config::default_test_config();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
// Test with an explicitly invalid token (not just missing, but actually wrong)
let response = client
.get(format!(
"http://{}/flags/definitions?token=phc_definitely_invalid_token_12345",
server.addr
))
.send()
.await
.unwrap();
let status = response.status();
let body_text = response.text().await.unwrap();
assert_eq!(
status, 401,
"Should return 401 for invalid token. Body: {body_text}"
);
// An invalid project token must return the structured DRF-style JSON envelope,
// matching the secret/personal API key paths.
let body: Value = serde_json::from_str(&body_text)
.unwrap_or_else(|_| panic!("Body should be JSON. Got: {body_text}"));
assert_eq!(body["type"], "authentication_error");
assert_eq!(body["code"], "authentication_failed");
assert_eq!(
body["detail"],
"The provided API key is invalid or has expired. Please check your API key and try again."
);
assert_eq!(body["attr"], Value::Null);
}
#[tokio::test]
async fn test_secret_token_takes_priority_over_personal_api_key() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create two teams with different data
let (team1, secret_token1, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let team2 = context.insert_new_team(None).await.unwrap();
let org_id2 = context.get_organization_id_for_team(&team2).await.unwrap();
// Create PAK for team2
let test_uuid = uuid::Uuid::new_v4().to_string()[..8].to_string();
let user_email = format!("test_priority_{test_uuid}@posthog.com");
let user_id = context
.create_user(&user_email, &org_id2, team2.id)
.await
.unwrap();
let (_pak_id, _personal_key) = context
.create_personal_api_key(
user_id,
"Test PAK Priority",
vec!["feature_flag:read"],
None,
None,
)
.await
.unwrap();
// Populate cache for team1 only
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
context
.populate_flag_definitions_cache(redis_client, team1.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Provide BOTH authentication methods:
// - Secret token for team1 (in Authorization header - phs_ prefix)
// - Personal API key for team2 (phx_ prefix)
// Expected: Secret token takes priority, so we get team1's data
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret_token1}"))
.send()
.await
.unwrap();
// Should succeed with team1's secret token (200 status)
// If PAK was tried first, it would fail because PAK belongs to team2
assert_eq!(
response.status(),
200,
"Should authenticate with secret token when both auth methods provided. Response body: {}",
response.text().await.unwrap()
);
}
#[tokio::test]
async fn test_flag_definitions_rate_limit_enforced() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let context = TestContext::new(None).await;
// Create team with secret token
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
// `/minute` (not `/second`) throughout this file: governor's QuantaClock (TSC-backed)
// can replenish sub-second windows too fast on non-Depot runners. Don't revert.
let mut config = Config::default_test_config();
config.flag_definitions_rate_limits =
format!(r#"{{"{}": "1/minute"}}"#, team.id).parse().unwrap();
// Populate cache to avoid 503 errors
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
context
.populate_flag_definitions_cache(redis_client, team.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// First request should succeed (within rate limit)
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"First request should succeed. Response body: {}",
response.text().await.unwrap()
);
// Second request immediately after should be rate limited (429)
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
let status = response.status();
let body = response.text().await.unwrap();
assert_eq!(
status, 429,
"Second request should be rate limited. Response body: {body}"
);
assert!(
body.contains("Rate limit exceeded"),
"Response should mention rate limit. Got: {body}"
);
}
#[tokio::test]
async fn test_flag_definitions_custom_rate_limit_overrides_default() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let context = TestContext::new(None).await;
// Create teams with secret tokens
let (custom_team, custom_secret, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let (default_team, default_secret, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
// Create config with custom rate limit for custom_team (2 requests per minute)
// Default is 600/minute, so custom should be more restrictive
let mut config = Config::default_test_config();
config.flag_definitions_rate_limits = format!(r#"{{"{}": "2/minute"}}"#, custom_team.id)
.parse()
.unwrap();
// Populate cache for both teams
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
context
.populate_flag_definitions_cache(redis_client.clone(), custom_team.id)
.await
.unwrap();
context
.populate_flag_definitions_cache(redis_client, default_team.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Test custom rate limit (2/minute)
// Send all 3 requests concurrently to ensure they hit the rate limiter simultaneously
let requests = (0..3).map(|_| {
client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, custom_team.api_token
))
.header("Authorization", format!("Bearer {custom_secret}"))
.send()
});
let responses = futures::future::join_all(requests).await;
let success_count = responses
.into_iter()
.filter(|r| r.as_ref().unwrap().status() == 200)
.count();
assert!(
success_count <= 2,
"Should allow at most 2 requests per minute for custom team. Got: {success_count}"
);
// Test default rate limit (600/minute)
// Should allow more requests than custom limit
// Send all 5 requests concurrently to ensure they hit the rate limiter simultaneously
let requests = (0..5).map(|_| {
client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, default_team.api_token
))
.header("Authorization", format!("Bearer {default_secret}"))
.send()
});
let responses = futures::future::join_all(requests).await;
let default_success_count = responses
.into_iter()
.filter(|r| r.as_ref().unwrap().status() == 200)
.count();
assert!(
default_success_count > 2,
"Default team should allow more than 2 requests per minute. Got: {default_success_count}"
);
}
#[tokio::test]
async fn test_flag_definitions_rate_limit_metrics_incremented() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let context = TestContext::new(None).await;
// Create team with secret token
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
// Create config with very low rate limit for this specific team
let mut config = Config::default_test_config();
config.flag_definitions_rate_limits =
format!(r#"{{"{}": "1/minute"}}"#, team.id).parse().unwrap();
config.flag_definitions_conditional_rate_per_minute = 1;
config.enable_metrics = true; // Enable metrics collection
// Populate cache
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
context
.populate_flag_definitions_cache(redis_client.clone(), team.id)
.await
.unwrap();
// The rate-limited team's `:etag` value cannot be decoded. Deleting `etag_read_failure_label`
// left `redis_error` with no coverage, and this is the only test that can scrape /metrics.
// No other test writes an undecodable `:etag`, so other tests running in the same process
// cannot add to the `redis_error` count asserted below.
redis_client
.set_bytes(
format!(
"posthog:1:cache/teams/{}/feature_flags/flags_with_cohorts.json:etag",
team.id
),
b"not-a-pickle".to_vec(),
None,
)
.await
.unwrap();
let (other_team, other_secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
context
.populate_flag_definitions_cache(redis_client.clone(), other_team.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let url = format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
);
let response = get_definitions(&client, &url, &secret_token, None).await;
assert_eq!(response.status(), 200);
let response = get_definitions(&client, &url, &secret_token, None).await;
assert_eq!(response.status(), 429);
let response = get_definitions(&client, &url, &secret_token, Some("W/\"stale\"")).await;
assert_eq!(
response.status(),
429,
"A stale ETag should pass the conditional budget and be refused by the spent full-response budget"
);
let response = get_definitions(&client, &url, &secret_token, Some("W/\"stale\"")).await;
assert_eq!(
response.status(),
429,
"The spent conditional budget should refuse a stale ETag before the ETag read"
);
// The second team is not rate limited, so this request reaches the ETag read.
let other_url = format!(
"http://{}/flags/definitions?token={}",
server.addr, other_team.api_token
);
let response = get_definitions(&client, &other_url, &other_secret_token, None).await;
assert_eq!(response.status(), 200);
// Fetch metrics from /metrics endpoint
let metrics_response = client
.get(format!("http://{}/metrics", server.addr))
.send()
.await
.unwrap();
let metrics_text = metrics_response.text().await.unwrap();
let metric_value = |prefix: &str, labels: &[&str]| {
metrics_text
.lines()
.find(|line| {
line.starts_with(prefix) && labels.iter().all(|label| line.contains(label))
})
.and_then(|line| line.rsplit(' ').next())
.and_then(|value| value.parse::<f64>().ok())
};
// Both limiters share metric names and differ only in the `budget` label, so a dashboard
// that sums a counter sees every request. A stale ETag passes both limiters but counts
// once as a request. The full-response limiter still counts the 429 it returns for it.
let team_key = format!(r#"key="{}""#, team.id);
let team_count = |metric: &str, budget: &str| {
metric_value(
&format!("{metric}{{"),
&[&team_key, &format!(r#"budget="{budget}""#)],
)
};
assert_eq!(
team_count("flags_flag_definitions_requests_total", "full"),
Some(2.0),
"A stale ETag should not count as a second request. Metrics: {metrics_text}"
);
assert_eq!(
team_count("flags_flag_definitions_requests_total", "conditional"),
Some(2.0),
"Requests with an ETag in If-None-Match should count under the conditional budget. Metrics: {metrics_text}"
);
assert_eq!(
team_count("flags_flag_definitions_rate_limited_total", "full"),
Some(2.0),
"The full-response limiter should count the 429 it returns for a stale ETag. Metrics: {metrics_text}"
);
assert_eq!(
team_count("flags_flag_definitions_rate_limited_total", "conditional"),
Some(1.0),
"429s from the conditional limiter should count under the conditional budget. Metrics: {metrics_text}"
);
// Runbook step 2 picks which cluster to query from this gauge, so assert the series and
// its value and not just the metric name. This config leaves the toggle off.
let gauge_value = metric_value(
"flags_flag_definitions_reads_dedicated_redis{",
&[r#"reason="disabled""#],
);
assert_eq!(
gauge_value,
Some(0.0),
"A reader left on shared Redis should report the gauge at 0 with reason=\"disabled\". Metrics: {metrics_text}"
);
// An absent ETag writes no log record, so this counter is the only trace that it
// happened. populate_flag_definitions_cache writes no `:etag` key for the second team.
assert!(
metric_value(
"flags_flag_definitions_etag_total{",
&[r#"result="redis_missing""#]
)
.is_some(),
"An absent ETag should count as redis_missing. Metrics: {metrics_text}"
);
// Routing a decode failure back to Ok(None) would revert this to redis_missing, which sends
// the on-call to rebuild a cache tier when the fault is corrupt data. Only the rate-limited
// team's 200 and its first stale ETag read its undecodable ETag. A limiter check that runs
// after the ETag read raises the count, because the refused requests then read Redis too.
let redis_error = metric_value(
"flags_flag_definitions_etag_total{",
&[r#"result="redis_error""#],
);
assert_eq!(
redis_error,
Some(2.0),
"An undecodable ETag should count as redis_error, and only the 200 and the first stale ETag should read it. Metrics: {metrics_text}"
);
}
async fn get_definitions(
client: &reqwest::Client,
url: &str,
secret_token: &str,
if_none_match: Option<&str>,
) -> reqwest::Response {
let request = client
.get(url)
.header("Authorization", format!("Bearer {secret_token}"));
match if_none_match {
Some(etag) => request.header("If-None-Match", etag),
None => request,
}
.send()
.await
.unwrap()
}
#[tokio::test]
async fn test_etag_304_does_not_consume_full_response_rate_limit() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let mut config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
config.flag_definitions_rate_limits =
format!(r#"{{"{}": "1/minute"}}"#, team.id).parse().unwrap();
let etag_value = "a1b2c3d4e5f6g7h8";
context
.populate_cache_for_team_with_etag(team.id, etag_value)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let url = format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
);
let matching_etag = format!("W/\"{etag_value}\"");
let response = get_definitions(&client, &url, &secret_token, None).await;
assert_eq!(response.status(), 200, "First full response should succeed");
// The full-response budget (1/minute) is now spent, but revalidation polls
// with a matching ETag draw from the separate conditional budget.
for _ in 0..3 {
let response = get_definitions(&client, &url, &secret_token, Some(&matching_etag)).await;
assert_eq!(
response.status(),
304,
"Matching ETag should return 304 even when the full-response budget is spent"
);
}
let response = get_definitions(&client, &url, &secret_token, None).await;
assert_eq!(
response.status(),
429,
"Full response should still be rate limited"
);
}
#[tokio::test]
async fn test_etag_304_rate_limit_enforced() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let mut config = Config::default_test_config();
config.flag_definitions_conditional_rate_per_minute = 1;
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let etag_value = "a1b2c3d4e5f6g7h8";
context
.populate_cache_for_team_with_etag(team.id, etag_value)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let url = format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
);
let matching_etag = format!("W/\"{etag_value}\"");
let response = get_definitions(&client, &url, &secret_token, Some(&matching_etag)).await;
assert_eq!(response.status(), 304, "First 304 should succeed");
let response = get_definitions(&client, &url, &secret_token, Some(&matching_etag)).await;
let status = response.status();
let body = response.text().await.unwrap();
assert_eq!(
status, 429,
"Second 304 should be rate limited. Response body: {body}"
);
// The limiter runs before the ETag read. It cannot tell a stale ETag from a matching one.
let response = get_definitions(&client, &url, &secret_token, Some("W/\"stale\"")).await;
assert_eq!(
response.status(),
429,
"A stale ETag should spend the conditional budget"
);
let response = get_definitions(&client, &url, &secret_token, None).await;
assert_eq!(
response.status(),
200,
"Full response should not be limited by the conditional budget"
);
}
#[tokio::test]
async fn test_stale_etag_200_spends_both_budgets() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let mut config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let one_per_minute = format!(r#"{{"{}": "1/minute"}}"#, team.id);
config.flag_definitions_rate_limits = one_per_minute.parse().unwrap();
config.flag_definitions_conditional_rate_limits = one_per_minute.parse().unwrap();
let etag_value = "a1b2c3d4e5f6g7h8";
context
.populate_cache_for_team_with_etag(team.id, etag_value)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let url = format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
);
let matching_etag = format!("W/\"{etag_value}\"");
let response = get_definitions(&client, &url, &secret_token, Some("W/\"stale\"")).await;
assert_eq!(
response.status(),
200,
"A stale ETag should get a full response"
);
let response = get_definitions(&client, &url, &secret_token, Some(&matching_etag)).await;
assert_eq!(
response.status(),
429,
"The stale ETag should have spent the team's 1/minute conditional override"
);
let response = get_definitions(&client, &url, &secret_token, None).await;
assert_eq!(
response.status(),
429,
"The stale ETag should have spent the team's 1/minute full-response override"
);
}
#[tokio::test]
async fn test_etag_returns_304_when_matching() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let etag_value = "a1b2c3d4e5f6g7h8";
context
.populate_cache_for_team_with_etag(team.id, etag_value)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.header("If-None-Match", format!("W/\"{etag_value}\""))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
304,
"Should return 304 when ETag matches. Body: {}",
response.text().await.unwrap_or_default()
);
}
#[tokio::test]
async fn test_etag_304_includes_etag_and_cache_control_headers() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let etag_value = "abcdef1234567890";
context
.populate_cache_for_team_with_etag(team.id, etag_value)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.header("If-None-Match", format!("W/\"{etag_value}\""))
.send()
.await
.unwrap();
assert_eq!(response.status(), 304);
assert_eq!(
response.headers().get("etag").unwrap().to_str().unwrap(),
format!("W/\"{etag_value}\"")
);
assert_eq!(
response
.headers()
.get("cache-control")
.unwrap()
.to_str()
.unwrap(),
"private, must-revalidate"
);
}
#[tokio::test]
async fn test_etag_returns_200_when_not_matching() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
context
.populate_cache_for_team_with_etag(team.id, "current_etag_value")
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.header("If-None-Match", "W/\"stale_etag_value\"")
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Should return 200 when ETag does not match"
);
assert_eq!(
response.headers().get("etag").unwrap().to_str().unwrap(),
"W/\"current_etag_value\""
);
assert_eq!(
response
.headers()
.get("cache-control")
.unwrap()
.to_str()
.unwrap(),
"private, must-revalidate"
);
}
#[tokio::test]
async fn test_etag_200_includes_etag_header_without_if_none_match() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let etag_value = "freshdata12345678";
context
.populate_cache_for_team_with_etag(team.id, etag_value)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Request WITHOUT If-None-Match header
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
assert_eq!(
response.headers().get("etag").unwrap().to_str().unwrap(),
format!("W/\"{etag_value}\""),
"200 response should include ETag header even without If-None-Match"
);
}
#[tokio::test]
async fn test_etag_graceful_degradation_without_stored_etag() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
// Populate cache WITHOUT an ETag
context.populate_cache_for_team(team.id).await.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
// Send If-None-Match even though no ETag is stored — should get 200 (graceful degradation)
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.header("If-None-Match", "W/\"some_stale_etag\"")
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Should return 200 when no ETag is stored (graceful degradation)"
);
assert!(
response.headers().get("etag").is_none(),
"Should not include ETag header when no ETag is stored"
);
}
/// Stands in for the dedicated cluster. The shared cache is database 0, so keys cannot collide.
const DEDICATED_REDIS_URL: &str = "redis://localhost:6379/1";
/// Setup both dedicated-cluster tests need: the reader pointed at the dedicated database with
/// the toggle on, plus a team to request.
async fn dedicated_context_and_team() -> (
feature_flags::config::Config,
feature_flags::utils::test_utils::TestContext,
feature_flags::team::team_models::Team,
String,
) {
use feature_flags::config::{Config, FlexBool};
use feature_flags::utils::test_utils::TestContext;
let config = Config {
flags_redis_url: DEDICATED_REDIS_URL.to_string(),
flag_definitions_dedicated_redis_enabled: FlexBool(true),
..Config::default_test_config()
};
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
(config, context, team, secret_token)
}
/// Seeds only the dedicated database: a reader left on shared 503s, and a payload that moved
/// without its ETag loses the header.
#[tokio::test]
async fn test_dedicated_redis_serves_payload_and_etag() {
use feature_flags::utils::test_utils::setup_redis_client;
let (config, context, team, secret_token) = dedicated_context_and_team().await;
let etag_value = "dedicated_etag_01";
let dedicated = setup_redis_client(Some(DEDICATED_REDIS_URL.to_string())).await;
context
.populate_cache_for_team_with_etag_on(dedicated, team.id, etag_value)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
let status = response.status();
let etag_header = response
.headers()
.get("etag")
.map(|v| v.to_str().unwrap().to_string());
let body_text = response.text().await.unwrap();
assert_eq!(
status, 200,
"Should read the payload from the dedicated Redis. Body: {body_text}"
);
let body: serde_json::Value = serde_json::from_str(&body_text).unwrap();
assert!(
body.get("flags").is_some(),
"Body should carry the dedicated payload. Body: {body_text}"
);
assert_eq!(
etag_header.as_deref(),
Some(format!("W/\"{etag_value}\"").as_str()),
"Should serve the dedicated ETag"
);
}
/// The split-brain 304, which is silent in production: a shared ETag matching while the
/// dedicated cluster holds a newer payload pins the SDK to stale definitions.
#[tokio::test]
async fn test_dedicated_redis_ignores_shared_etag() {
use feature_flags::utils::test_utils::setup_redis_client;
let (config, context, team, secret_token) = dedicated_context_and_team().await;
let shared_etag = "shared_etag_0001";
let dedicated_etag = "dedicated_etag_1";
let shared = setup_redis_client(Some(config.redis_url.clone())).await;
let dedicated = setup_redis_client(Some(DEDICATED_REDIS_URL.to_string())).await;
context
.populate_cache_for_team_with_etag_on(shared, team.id, shared_etag)
.await
.unwrap();
context
.populate_cache_for_team_with_etag_on(dedicated, team.id, dedicated_etag)
.await
.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
assert_eq!(
response.headers().get("etag").unwrap().to_str().unwrap(),
format!("W/\"{dedicated_etag}\""),
"Should serve the dedicated ETag, not the shared one"
);
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.header("If-None-Match", format!("W/\"{shared_etag}\""))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"The shared cluster's ETag must not produce a 304 once reads are dedicated"
);
}
#[tokio::test]
async fn test_flag_definitions_billing_limited_returns_402() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// Create team with secret token in real PG (needed for auth)
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
// Start server with mock Redis where the team's token is billing-limited
let server = common::ServerHandle::for_config_with_mock_redis(
config,
vec![team.api_token.clone()], // billing-limited
vec![(team.api_token.clone(), team.id)], // valid for team lookup
)
.await;
let client = reqwest::Client::new();
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
let status = response.status();
let body_text = response.text().await.unwrap();
assert_eq!(
status, 402,
"Should return 402 when billing quota is exceeded. Body: {body_text}"
);
// Response body matches Django's JSON format for SDK compatibility
let body: serde_json::Value = serde_json::from_str(&body_text).unwrap();
assert_eq!(body["type"], "quota_limited");
assert_eq!(body["code"], "payment_required");
}
#[rstest::rstest]
#[case::read_scope(Some(vec!["feature_flag:read"]), true, 200)]
#[case::write_scope(Some(vec!["feature_flag:write"]), true, 200)]
#[case::null_scopes_full_access(None, true, 200)]
#[case::wildcard_scope(Some(vec!["*"]), true, 200)]
#[case::wrong_scope(Some(vec!["insight:read"]), true, 401)]
#[case::wrong_team(Some(vec!["feature_flag:read"]), false, 401)]
#[tokio::test]
async fn test_flag_definitions_project_secret_api_key(
#[case] scopes: Option<Vec<&str>>,
#[case] same_team: bool,
#[case] expected_status: u16,
) {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let team = context.insert_new_team(None).await.unwrap();
let key_team_id = if same_team {
team.id
} else {
let other_team = context.insert_new_team(None).await.unwrap();
other_team.id
};
let (_, raw_key) = context
.create_project_secret_api_key(key_team_id, "Test Key", scopes)
.await
.unwrap();
if expected_status == 200 {
context.populate_cache_for_team(team.id).await.unwrap();
}
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {raw_key}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
expected_status,
"Response body: {}",
response.text().await.unwrap()
);
}
#[tokio::test]
async fn test_valid_pak_used_to_authenticate_from_cache_updates_last_used_at() {
use feature_flags::{
api::api_key_usage::{debounce_key, ApiKeyKind},
config::Config,
utils::test_utils::TestContext,
};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let team = context.insert_new_team(None).await.unwrap();
let org_id = context.get_organization_id_for_team(&team).await.unwrap();
let user_email = TestContext::generate_test_email("pak_cache_last_used");
let user_id = context
.create_user(&user_email, &org_id, team.id)
.await
.unwrap();
context
.add_user_to_organization(user_id, &org_id, 15)
.await
.unwrap();
let (pak_id, api_key_value) = context
.create_personal_api_key(
user_id,
"Test PAK Cache",
vec!["feature_flag:read"],
None,
None,
)
.await
.unwrap();
context.populate_cache_for_team(team.id).await.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
server.wait_until_ready().await;
let client = reqwest::Client::new();
let url = format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
);
// First request: populates the auth token cache and triggers last_used_at update
let response = client
.get(&url)
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
// Poll for the spawned background task to complete the DB write
poll_for_pak_last_used_at(
&context,
&pak_id,
"Timed out waiting for background task to set last_used_at for PAK",
)
.await;
// Clear the Redis debounce key so the next request can write to DB again
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone())).await;
redis_client
.del(debounce_key(ApiKeyKind::Personal, &pak_id))
.await
.expect("Failed to delete debounce key");
// Reset last_used_at to NULL so we can verify the second request sets it
let mut conn = context.get_non_persons_connection().await.unwrap();
sqlx::query("UPDATE posthog_personalapikey SET last_used_at = NULL WHERE id = $1")
.bind(&pak_id)
.execute(&mut *conn)
.await
.unwrap();
// Second request: auth comes from the token cache (no DB query for auth),
// but should still trigger the last_used_at update via record_api_key_last_used
let response = client
.get(&url)
.header("Authorization", format!("Bearer {api_key_value}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
// Poll for the spawned background task to complete the DB write
poll_for_pak_last_used_at(
&context,
&pak_id,
"last_used_at should be set after authenticating from the auth token cache",
)
.await;
}
#[rstest::rstest]
#[case::with_token_param(true)]
#[case::bearer_only(false)]
#[tokio::test]
async fn test_project_secret_api_key_updates_last_used_at(#[case] with_token_param: bool) {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let team = context.insert_new_team(None).await.unwrap();
let (key_id, key) = context
.create_project_secret_api_key(team.id, "PSAK LastUsed", Some(vec!["feature_flag:read"]))
.await
.unwrap();
context.populate_cache_for_team(team.id).await.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
server.wait_until_ready().await;
let url = if with_token_param {
format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
)
} else {
format!("http://{}/flags/definitions", server.addr)
};
let response = reqwest::Client::new()
.get(url)
.header("Authorization", format!("Bearer {key}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
common::poll_for_psak_last_used_at(
&context,
&key_id,
"Timed out waiting for last_used_at to be set for the project secret API key",
)
.await;
}
#[tokio::test]
async fn test_flag_definitions_with_legacy_secret_token_fallback() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
use reqwest;
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
// create_team_with_secret_token creates a legacy phs_ token on posthog_team
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
// Do NOT insert a project secret API key — the phs_ token should fall back to legacy
context.populate_cache_for_team(team.id).await.unwrap();
let server = common::ServerHandle::for_config(config.clone()).await;
let client = reqwest::Client::new();
let response = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Response body: {}",
response.text().await.unwrap()
);
}
/// Tests the DB-backed rate limit allowlist end-to-end.
/// Combined into a single test to avoid DB-level interference — all scenarios
/// share the same `posthog_instancesetting` row.
///
/// Scenarios tested:
/// 1. Allowlisted team bypasses both rate limits (200 or 304 instead of 429)
/// 2. Non-allowlisted team gets rate limited (429)
/// 3. With two teams, only the listed one bypasses
/// 4. Env var allowlist preserved when DB row is missing
/// 5. null DB value treated as empty allowlist
/// 6. Invalid team IDs skipped, valid ones kept
/// 7. Django-format JSON-encoded string (json.dumps wrapping)
#[tokio::test]
async fn test_db_rate_limit_allowlist() {
use feature_flags::{config::Config, utils::test_utils::TestContext};
let context = TestContext::new(None).await;
// Clean up any leftover rows from a previous failed run
context
.delete_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS")
.await
.unwrap();
let (team1, secret1, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let (team2, secret2, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
// --- Scenario 1: allowlisted team bypasses rate limit ---
{
let mut config = Config::default_test_config();
config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
.parse()
.unwrap();
config.flag_definitions_conditional_rate_per_minute = 1;
context
.set_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS", &team1.id.to_string())
.await
.unwrap();
let etag_value = "a1b2c3d4e5f6g7h8";
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
.await;
context
.populate_cache_for_team_with_etag_on(redis_client, team1.id, etag_value)
.await
.unwrap();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
// First request triggers DB refresh
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200);
// Second request would be rate limited, but allowlist bypasses it
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(
resp.status(),
200,
"Allowlisted team should bypass rate limit"
);
// The conditional budget (1/minute) would reject the second poll, but the
// DB-refreshed allowlist applies to the conditional limiter too
let url = format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
);
let matching_etag = format!("W/\"{etag_value}\"");
for i in 0..2 {
let resp = get_definitions(&client, &url, &secret1, Some(&matching_etag)).await;
assert_eq!(
resp.status(),
304,
"Allowlisted team should bypass the conditional rate limit (poll {i})"
);
}
}
// --- Scenario 2: non-allowlisted team gets rate limited ---
{
let mut config = Config::default_test_config();
config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
.parse()
.unwrap();
context
.delete_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS")
.await
.unwrap();
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
.await;
context
.populate_flag_definitions_cache(redis_client, team1.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200);
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(
resp.status(),
429,
"Non-allowlisted team should be rate limited"
);
}
// --- Scenario 3: only listed team bypasses, other team gets limited ---
{
let mut config = Config::default_test_config();
config.flag_definitions_rate_limits = format!(
r#"{{"{}": "1/minute", "{}": "1/minute"}}"#,
team1.id, team2.id
)
.parse()
.unwrap();
context
.set_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS", &team1.id.to_string())
.await
.unwrap();
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
.await;
context
.populate_flag_definitions_cache(redis_client.clone(), team1.id)
.await
.unwrap();
context
.populate_flag_definitions_cache(redis_client, team2.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
// Team1 (allowlisted) can make many requests
for i in 0..5 {
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(
resp.status(),
200,
"Allowlisted team1 request {i} should succeed"
);
}
// Team2 (not allowlisted) should eventually be rate limited
let mut saw_rate_limit = false;
for _ in 0..10 {
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team2.api_token
))
.header("Authorization", format!("Bearer {secret2}"))
.send()
.await
.unwrap();
if resp.status() == 429 {
saw_rate_limit = true;
break;
}
}
assert!(
saw_rate_limit,
"Non-allowlisted team2 should eventually be rate limited"
);
}
// --- Scenario 4: env var allowlist preserved when DB row is missing ---
{
// Configure team1 as allowlisted via the env var (set at server startup)
let mut config = Config::default_test_config();
config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
.parse()
.unwrap();
config.rate_limiting_allow_list_teams = team1.id.to_string().parse().unwrap();
config.flag_definitions_conditional_rate_per_minute = 1;
// Ensure no DB row exists — env var default should be kept
context
.delete_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS")
.await
.unwrap();
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
.await;
let etag_value = "a1b2c3d4e5f6g7h8";
context
.populate_cache_for_team_with_etag_on(redis_client, team1.id, etag_value)
.await
.unwrap();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
// First request triggers DB refresh — row missing, so env var default is kept
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200);
// Second request: team1 should still bypass via env var allowlist
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(
resp.status(),
200,
"Env var allowlist should be preserved when DB row is missing"
);
let url = format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
);
let matching_etag = format!("W/\"{etag_value}\"");
for _ in 0..2 {
let resp = get_definitions(&client, &url, &secret1, Some(&matching_etag)).await;
assert_eq!(
resp.status(),
304,
"Env var allowlist should bypass the conditional rate limit"
);
}
}
// --- Scenario 5: null DB value treated as empty allowlist ---
{
let mut config = Config::default_test_config();
config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
.parse()
.unwrap();
context
.set_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS", "null")
.await
.unwrap();
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
.await;
context
.populate_flag_definitions_cache(redis_client, team1.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200);
// null means empty allowlist — team should be rate limited
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(
resp.status(),
429,
"null DB value should be treated as empty allowlist"
);
}
// --- Scenario 6: invalid team IDs skipped, valid ones kept ---
{
let mut config = Config::default_test_config();
config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
.parse()
.unwrap();
// Mix of valid and invalid IDs — team1 should still be allowlisted
context
.set_instance_setting(
"RATE_LIMITING_ALLOW_LIST_TEAMS",
&format!("{},abc,xyz", team1.id),
)
.await
.unwrap();
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
.await;
context
.populate_flag_definitions_cache(redis_client, team1.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200);
// team1 is in the valid portion of the allowlist — should bypass
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(
resp.status(),
200,
"Valid team IDs should be kept even when invalid ones are present"
);
}
// --- Scenario 7: Django-format JSON-encoded string ---
{
let mut config = Config::default_test_config();
config.flag_definitions_rate_limits = format!(r#"{{"{}": "1/minute"}}"#, team1.id)
.parse()
.unwrap();
// Django's set_instance_setting calls json.dumps(value), so "23047"
// is stored as "\"23047\"" in raw_value
context
.set_instance_setting(
"RATE_LIMITING_ALLOW_LIST_TEAMS",
&format!("\"{}\"", team1.id),
)
.await
.unwrap();
let redis_client =
feature_flags::utils::test_utils::setup_redis_client(Some(config.redis_url.clone()))
.await;
context
.populate_flag_definitions_cache(redis_client, team1.id)
.await
.unwrap();
let server = common::ServerHandle::for_config(config).await;
let client = reqwest::Client::new();
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200);
// team1 should bypass via JSON-encoded allowlist
let resp = client
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team1.api_token
))
.header("Authorization", format!("Bearer {secret1}"))
.send()
.await
.unwrap();
assert_eq!(
resp.status(),
200,
"Django-format JSON-encoded allowlist should work"
);
}
// Clean up
context
.delete_instance_setting("RATE_LIMITING_ALLOW_LIST_TEAMS")
.await
.unwrap();
}
/// Verifies the billing path writes (or, when `skip_writes=true`,
/// suppresses) the FlagDefinitions counter for `/flags/definitions`. The
/// `/flags` endpoint is covered by tests in `test_flags.rs`; this test is
/// the equivalent for the FlagDefinitions code path so a regression that
/// honored `skip_writes` for one endpoint but not the other can't slip
/// through.
#[rstest::rstest]
#[case(true)]
#[case(false)]
#[tokio::test]
async fn test_flag_definitions_billing_counter(#[case] skip_writes: bool) {
use feature_flags::config::FlexBool;
use feature_flags::flags::flag_analytics::{current_bucket, get_team_request_key};
use feature_flags::flags::flag_request::FlagRequestType;
use feature_flags::utils::test_utils::{setup_redis_client, TestContext};
use serde_json::json;
let mut config = feature_flags::config::Config::default_test_config();
config.skip_writes = FlexBool(skip_writes);
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
// Seed the HyperCache with a billable flag (a non-survey, non-product-tour
// key) so `has_billable_flags` returns true and `record()` actually fires.
context
.populate_cache_for_team_with_flags(
team.id,
json!({
"flags": [{"key": "billable-flag", "active": true}],
"group_type_mapping": {},
"cohorts": {},
}),
)
.await
.unwrap();
// Clean billing key so we can assert on a fresh write.
let redis = setup_redis_client(Some(config.redis_url.clone())).await;
let billing_key = get_team_request_key(team.id, FlagRequestType::FlagDefinitions);
redis.del(billing_key.clone()).await.unwrap();
let server = common::ServerHandle::for_config(config).await;
let http = reqwest::Client::new();
// Bracket the request: the record lands in whichever 2-minute bucket it crosses.
let bucket_before = current_bucket();
let response = http
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"Response body: {}",
response.text().await.unwrap()
);
let bucket_after = current_bucket();
if skip_writes {
// Sleep ~5 flush windows so even a slow CI scheduler couldn't hide
// an erroneous `record()` behind a delayed first tick.
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
for bucket in bucket_before..=bucket_after {
let counter = redis.hget(billing_key.clone(), bucket.to_string()).await;
assert!(
counter.is_err(),
"FlagDefinitions billing counter should NOT be incremented when skip_writes=true, got {counter:?}"
);
}
} else {
let counter = common::poll_for_billing_counter_across_buckets(
&redis,
&billing_key,
bucket_before,
bucket_after,
)
.await;
assert_eq!(
counter, "1",
"FlagDefinitions billing counter should be incremented once"
);
}
}
/// A 304 lands on the not-modified counter and leaves the full local evaluation
/// counter untouched. It stays unbilled for definitions that hold only survey or
/// product tour flags, and when `skip_writes` is on.
#[rstest::rstest]
#[case::billable_flag("billable-flag", false, true)]
#[case::survey_only("survey-targeting-only", false, false)]
#[case::skip_writes("billable-flag", true, false)]
#[tokio::test]
async fn test_flag_definitions_304_records_not_modified_billing_counter(
#[case] flag_key: &str,
#[case] skip_writes: bool,
#[case] expect_billed: bool,
) {
use feature_flags::config::FlexBool;
use feature_flags::flags::flag_analytics::{current_bucket, get_team_request_key};
use feature_flags::flags::flag_request::FlagRequestType;
use feature_flags::utils::test_utils::{setup_redis_client, TestContext};
use serde_json::json;
let mut config = feature_flags::config::Config::default_test_config();
config.skip_writes = FlexBool(skip_writes);
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let etag_value = "billed304etag0001";
context
.populate_cache_for_team_with_flags_and_etag(
team.id,
json!({
"flags": [{"key": flag_key, "active": true}],
"group_type_mapping": {},
"cohorts": {},
}),
etag_value,
)
.await
.unwrap();
let redis = setup_redis_client(Some(config.redis_url.clone())).await;
let not_modified_key =
get_team_request_key(team.id, FlagRequestType::FlagDefinitionsNotModified);
let full_response_key = get_team_request_key(team.id, FlagRequestType::FlagDefinitions);
redis.del(not_modified_key.clone()).await.unwrap();
redis.del(full_response_key.clone()).await.unwrap();
let server = common::ServerHandle::for_config(config).await;
let http = reqwest::Client::new();
let bucket_before = current_bucket();
let response = http
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.header("If-None-Match", format!("W/\"{etag_value}\""))
.send()
.await
.unwrap();
assert_eq!(response.status(), 304);
let bucket_after = current_bucket();
if expect_billed {
let counter = common::poll_for_billing_counter_across_buckets(
&redis,
¬_modified_key,
bucket_before,
bucket_after,
)
.await;
assert_eq!(
counter, "1",
"304 should be recorded once on the not-modified billing counter"
);
} else {
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
for bucket in bucket_before..=bucket_after {
let counter = redis
.hget(not_modified_key.clone(), bucket.to_string())
.await;
assert!(
counter.is_err(),
"304 for non-billable flags must not be billed, got {counter:?}"
);
}
}
for bucket in bucket_before..=bucket_after {
let full = redis
.hget(full_response_key.clone(), bucket.to_string())
.await;
assert!(
full.is_err(),
"304 must not increment the full local evaluation counter, got {full:?}"
);
}
}
/// Only a payload served from Redis decides whether a 304 is billed. With the
/// ETag in Redis but the payload only in S3, neither the full response that
/// precedes the poll nor the poll's own read may memoize S3's answer, so the
/// 304 stays unbilled even though S3 says the definitions are billable.
#[tokio::test]
async fn test_flag_definitions_304_is_not_billed_from_an_s3_payload() {
use feature_flags::flags::flag_analytics::{current_bucket, get_team_request_key};
use feature_flags::flags::flag_request::FlagRequestType;
use feature_flags::utils::test_utils::{setup_redis_client, static_s3_client, TestContext};
use serde_json::json;
let config = feature_flags::config::Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let payload = json!({
"flags": [{"key": "billable-flag", "active": true}],
"group_type_mapping": {},
"cohorts": {},
});
let etag_value = "s3only304etag0001";
context
.populate_cache_for_team_with_flags_and_etag(team.id, payload.clone(), etag_value)
.await
.unwrap();
let redis = setup_redis_client(Some(config.redis_url.clone())).await;
// Leave the ETag in Redis and the payload only in S3.
redis
.del(format!(
"posthog:1:cache/teams/{}/feature_flags/flags_with_cohorts.json",
team.id
))
.await
.unwrap();
let not_modified_key =
get_team_request_key(team.id, FlagRequestType::FlagDefinitionsNotModified);
redis.del(not_modified_key.clone()).await.unwrap();
let server = common::ServerHandle::for_config_with_s3(
config,
// S3 serves the same billable payload for every key, standing in for a copy
// that may lag the ETag Redis holds by a version.
Some(static_s3_client(&payload.to_string())),
)
.await;
let http = reqwest::Client::new();
let url = format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
);
let bucket_before = current_bucket();
let response = http
.get(&url)
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
let response = http
.get(&url)
.header("Authorization", format!("Bearer {secret_token}"))
.header("If-None-Match", format!("W/\"{etag_value}\""))
.send()
.await
.unwrap();
assert_eq!(response.status(), 304);
let bucket_after = current_bucket();
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
for bucket in bucket_before..=bucket_after {
let counter = redis
.hget(not_modified_key.clone(), bucket.to_string())
.await;
assert!(
counter.is_err(),
"304 must not be billed from an S3 payload, got {counter:?}"
);
}
}
/// A full response served from Redis memoizes whether its ETag is billable, so
/// a later 304 for that ETag is billed from the memo even after the Redis
/// payload is gone and the 304's own read would find nothing.
#[tokio::test]
async fn test_flag_definitions_304_is_billed_from_the_full_response_memo() {
use feature_flags::flags::flag_analytics::{current_bucket, get_team_request_key};
use feature_flags::flags::flag_request::FlagRequestType;
use feature_flags::utils::test_utils::{setup_redis_client, TestContext};
use serde_json::json;
let config = feature_flags::config::Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
let etag_value = "memo304etag00001";
context
.populate_cache_for_team_with_flags_and_etag(
team.id,
json!({
"flags": [{"key": "billable-flag", "active": true}],
"group_type_mapping": {},
"cohorts": {},
}),
etag_value,
)
.await
.unwrap();
let redis = setup_redis_client(Some(config.redis_url.clone())).await;
let not_modified_key =
get_team_request_key(team.id, FlagRequestType::FlagDefinitionsNotModified);
redis.del(not_modified_key.clone()).await.unwrap();
let server = common::ServerHandle::for_config(config).await;
let http = reqwest::Client::new();
let url = format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
);
let response = http
.get(&url)
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 200);
// Keep the ETag in Redis but drop the payload, so only the memo can bill the 304.
redis
.del(format!(
"posthog:1:cache/teams/{}/feature_flags/flags_with_cohorts.json",
team.id
))
.await
.unwrap();
let bucket_before = current_bucket();
let response = http
.get(&url)
.header("Authorization", format!("Bearer {secret_token}"))
.header("If-None-Match", format!("W/\"{etag_value}\""))
.send()
.await
.unwrap();
assert_eq!(response.status(), 304);
let bucket_after = current_bucket();
let counter = common::poll_for_billing_counter_across_buckets(
&redis,
¬_modified_key,
bucket_before,
bucket_after,
)
.await;
assert_eq!(
counter, "1",
"304 should be billed from the memo the full response wrote"
);
}
/// Poll the self-heal rebuild-requests set until it contains `team_id`, or return
/// false after ~2s. The enqueue runs in a background task, so a bounded retry is
/// needed rather than a single read.
async fn poll_for_rebuild_enqueue(redis_url: &str, team_id: i32, s3_hit: bool) -> bool {
use feature_flags::utils::test_utils::{
read_flag_definitions_rebuild_requests, read_s3_rebuild_requests,
};
use tokio::time::{sleep, Duration};
for _ in 0..40 {
let members = if s3_hit {
read_s3_rebuild_requests(redis_url).await
} else {
read_flag_definitions_rebuild_requests(redis_url).await
};
if members.contains(&team_id.to_string()) {
return true;
}
sleep(Duration::from_millis(50)).await;
}
false
}
#[tokio::test]
async fn test_cache_miss_enqueues_rebuild_when_self_heal_enabled() {
use feature_flags::{
config::{Config, FlexBool},
utils::test_utils::{
dummy_s3_client, remove_flag_definitions_rebuild_request, TestContext,
},
};
use reqwest;
let mut config = Config::default_test_config();
config.flag_definitions_self_heal_enabled = FlexBool(true);
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
remove_flag_definitions_rebuild_request(&config.redis_url, team.id).await;
// Don't populate the cache, and inject a NotFound S3 so the read is a genuine
// cache_miss (the flags test job has no object store, so real S3 would error).
let server =
common::ServerHandle::for_config_with_s3(config.clone(), Some(dummy_s3_client())).await;
let response = reqwest::Client::new()
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 503, "expected a cache-miss 503");
assert!(
poll_for_rebuild_enqueue(&config.redis_url, team.id, false).await,
"team {} should be enqueued for rebuild after a cache-miss 503",
team.id
);
}
#[tokio::test]
async fn test_cache_miss_enqueues_rebuild_on_dedicated_redis() {
use feature_flags::{
config::{Config, FlexBool},
utils::test_utils::{
dummy_s3_client, read_flag_definitions_rebuild_requests,
remove_flag_definitions_rebuild_request, TestContext,
},
};
use reqwest;
// The Celery drain reads the queue from the dedicated cluster, so the enqueue has to follow
// the writer regardless of which cluster the reader is on. This case leaves
// FLAG_DEFINITIONS_DEDICATED_REDIS_ENABLED off, so the reader is still on shared.
let mut config = Config::default_test_config();
config.flags_redis_url = DEDICATED_REDIS_URL.to_string();
config.flag_definitions_self_heal_enabled = FlexBool(true);
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
remove_flag_definitions_rebuild_request(&config.flags_redis_url, team.id).await;
remove_flag_definitions_rebuild_request(&config.redis_url, team.id).await;
let shared_before = read_flag_definitions_rebuild_requests(&config.redis_url).await;
// Leave both caches unseeded and inject a NotFound S3 so the read is a genuine
// cache_miss rather than an s3_error.
let server =
common::ServerHandle::for_config_with_s3(config.clone(), Some(dummy_s3_client())).await;
let response = reqwest::Client::new()
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 503, "expected a cache-miss 503");
assert!(
poll_for_rebuild_enqueue(&config.flags_redis_url, team.id, false).await,
"team {} should be enqueued on the dedicated redis, where the drain reads",
team.id
);
let shared_after = read_flag_definitions_rebuild_requests(&config.redis_url).await;
let added: Vec<&String> = shared_after
.iter()
.filter(|member| !shared_before.contains(member))
.collect();
assert!(
!added.contains(&&team.id.to_string()),
"team {} must not be enqueued on the shared redis, where nothing drains",
team.id
);
}
#[tokio::test]
async fn test_s3_hit_enqueues_rebuild_and_keeps_first_score() {
use feature_flags::{
config::{Config, FlexBool},
utils::test_utils::{
remove_s3_rebuild_request, setup_redis_client, static_s3_client, TestContext,
},
};
use reqwest;
use std::time::{SystemTime, UNIX_EPOCH};
let mut config = Config::default_test_config();
config.flag_definitions_self_heal_enabled = FlexBool(true);
config.flag_definitions_rebuild_on_s3_hit_enabled = FlexBool(true);
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
remove_s3_rebuild_request(&config.redis_url, team.id).await;
let server = common::ServerHandle::for_config_with_s3(
config.clone(),
Some(static_s3_client(r#"{"flags": [], "cohorts": {}}"#)),
)
.await;
let response = reqwest::Client::new()
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(
response.status(),
200,
"expected the S3 payload to be served"
);
assert!(
response.headers().get("etag").is_none(),
"an S3-served response has no ETag to send, which is what the rebuild repairs"
);
assert!(
poll_for_rebuild_enqueue(&config.redis_url, team.id, true).await,
"team {} should be enqueued for rebuild after an S3-served response",
team.id
);
let redis = setup_redis_client(Some(config.redis_url.clone())).await;
let first_score_upper_bound = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_millis() as i64;
let queue = "flag_definitions:rebuild_s3_requests".to_string();
redis
.zadd_nx(
queue.clone(),
team.id.to_string(),
first_score_upper_bound + 60_000,
)
.await
.unwrap();
let original_score_members = redis
.zrangebyscore(
queue,
"-inf".to_string(),
first_score_upper_bound.to_string(),
)
.await
.unwrap();
assert!(
original_score_members.contains(&team.id.to_string()),
"a repeated NX write must keep the first enqueue score"
);
}
#[tokio::test]
async fn test_cache_miss_does_not_enqueue_rebuild_when_self_heal_disabled() {
use feature_flags::{
config::Config,
utils::test_utils::{
dummy_s3_client, read_flag_definitions_rebuild_requests,
remove_flag_definitions_rebuild_request, TestContext,
},
};
use reqwest;
use tokio::time::{sleep, Duration};
// default_test_config leaves flag_definitions_self_heal_enabled = false.
let config = Config::default_test_config();
let context = TestContext::new(Some(&config)).await;
let (team, secret_token, _) = context
.create_team_with_secret_token(None, None, None)
.await
.unwrap();
remove_flag_definitions_rebuild_request(&config.redis_url, team.id).await;
// Inject a NotFound S3 so this is a genuine cache_miss: the only reason no enqueue
// happens is the flag being off, not the miss classifying as s3_error.
let server =
common::ServerHandle::for_config_with_s3(config.clone(), Some(dummy_s3_client())).await;
let response = reqwest::Client::new()
.get(format!(
"http://{}/flags/definitions?token={}",
server.addr, team.api_token
))
.header("Authorization", format!("Bearer {secret_token}"))
.send()
.await
.unwrap();
assert_eq!(response.status(), 503, "expected a cache-miss 503");
// Give any (erroneous) background enqueue time to land, then assert it did not.
sleep(Duration::from_millis(500)).await;
let members = read_flag_definitions_rebuild_requests(&config.redis_url).await;
assert!(
!members.contains(&team.id.to_string()),
"team {} must not be enqueued when self-heal is disabled",
team.id
);
}