:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use crate::api::api_key_usage::ApiKeyKind;
use crate::{
api::{
auth,
errors::{ClientFacingError, FlagError},
instance_setting::{constance_key, fetch_instance_setting_raw_value},
},
flags::{flag_analytics::is_billable_flag_key, flag_request::FlagRequestType},
handler::types::Library,
metrics::consts::{
FLAG_DEFINITIONS_AUTH_COUNTER, FLAG_DEFINITIONS_CACHE_HIT_COUNTER,
FLAG_DEFINITIONS_CACHE_MISS_COUNTER, FLAG_DEFINITIONS_ETAG_COUNTER,
FLAG_DEFINITIONS_NOT_MODIFIED_BILLING_COUNTER, FLAG_DEFINITIONS_REBUILD_REQUESTED_COUNTER,
},
router::State as AppState,
team::team_models::Team,
};
use axum::{
debug_handler,
extract::{Query, State},
http::{HeaderMap, Method, StatusCode},
response::{IntoResponse, Json, Response},
};
use common_hypercache::{CacheSource, HyperCacheError, KeyType};
use common_metrics::inc;
use common_redis::Client as RedisClient;
use common_types::TeamId;
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sqlx::PgPool;
use std::collections::HashSet;
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tracing::{info, warn};
const ALLOWLIST_TTL_SECS: u64 = 60;
/// Redis sorted set holding team IDs whose flag-definitions Redis entry is gone and
/// needs a rebuild. A Celery worker drains it (member = team_id, score = the first
/// request's time in epoch millis, kept by the NX write below). Must stay in sync with
/// `REBUILD_REQUESTS_ZSET` in `products/feature_flags/backend/rebuild_queue.py` (pinned by
/// the Python test `test_request_zset_key_matches_rust_contract`).
pub(crate) const FLAG_DEFINITIONS_REBUILD_REQUESTS_ZSET: &str = "flag_definitions:rebuild_requests";
pub(crate) const FLAG_DEFINITIONS_S3_REBUILD_REQUESTS_ZSET: &str =
"flag_definitions:rebuild_s3_requests";
/// `trigger` label values for `FLAG_DEFINITIONS_REBUILD_REQUESTED_COUNTER`. The two triggers
/// ramp independently, so the dashboard has to separate them.
const REBUILD_TRIGGER_CACHE_MISS: &str = "cache_miss";
const REBUILD_TRIGGER_S3_HIT: &str = "s3_hit";
static CONSTANCE_KEY: Lazy<String> = Lazy::new(|| constance_key("RATE_LIMITING_ALLOW_LIST_TEAMS"));
/// Refresh the rate limit allowlist from the database if stale, then update the limiter.
/// Matches Django's `get_team_allow_list(round(time.time() / 60))` pattern.
/// The cache is per-instance (stored on the rate limiter), not global.
///
/// To avoid stampeding the database at the TTL boundary (10k+ req/s),
/// `claim_allowlist_refresh` atomically checks staleness and marks as refreshed,
/// so only one request triggers the DB query. If the query fails, we serve stale
/// data for another TTL cycle.
async fn refresh_rate_limit_allowlist_if_stale(state: &AppState) {
if !state
.flag_definitions_full_limiter
.claim_allowlist_refresh(ALLOWLIST_TTL_SECS)
{
return;
}
match fetch_allowlist_from_db(&state.database_pools.non_persons_reader).await {
Ok(Some(new_allowlist)) => {
state
.flag_definitions_conditional_limiter
.update_allowlist(new_allowlist.clone());
state
.flag_definitions_full_limiter
.update_allowlist(new_allowlist);
}
Ok(None) => {
// Row not in DB — keep the env var default
}
Err(e) => {
warn!(error = %e, "Failed to refresh rate limit allowlist from database, using cached value");
}
}
}
/// Query the posthog_instancesetting table for the rate limit allowlist.
/// Returns None if the row doesn't exist (env var default should be kept),
/// Some(set) if the row exists (overrides the env var).
pub async fn fetch_allowlist_from_db(pool: &PgPool) -> Result<Option<HashSet<TeamId>>, String> {
let raw = match fetch_instance_setting_raw_value(pool, &CONSTANCE_KEY).await? {
Some(v) => v,
None => return Ok(None),
};
let value = raw.trim();
if value.is_empty() {
return Ok(Some(HashSet::new()));
}
let mut team_ids = HashSet::new();
for part in value.split(',').map(|p| p.trim()) {
if part.is_empty() {
continue;
}
match part.parse::<TeamId>() {
Ok(id) => {
team_ids.insert(id);
}
Err(e) => {
warn!(
part = part,
error = %e,
"Skipping invalid team ID in RATE_LIMITING_ALLOW_LIST_TEAMS"
);
}
}
}
Ok(Some(team_ids))
}
/// Response for flag definitions endpoint
/// This is returned as raw JSON from cache to avoid deserialization overhead
pub type FlagDefinitionsResponse = Value;
/// Bound on the payload read a 304 makes when the memo has no entry for the
/// current ETag. Matches the hypercache Redis timeout. A slow read delays the
/// 304 by at most this long, and then the poll goes unbilled and the next one
/// retries.
const BILLABLE_LOOKUP_TIMEOUT: Duration = Duration::from_millis(500);
/// How long a failed, slow, or non-Redis payload read is remembered. While
/// the payload tier is degraded this caps the retries at one per pod per
/// team per interval instead of one per poll, and every poll in between
/// stays unbilled.
const BILLABLE_MEMO_FAILURE_TTL: Duration = Duration::from_secs(30);
type BillableMemoKey = (i32, String);
/// A memoized answer outlives a mismatch between the ETag and the payload it
/// was computed from for at most `answer_ttl`: the writer can publish a new
/// version between the ETag read and the payload read of one request. A
/// failure to answer is kept only briefly so the next poll retries soon.
struct BillableMemoExpiry {
answer_ttl: Duration,
failure_ttl: Duration,
}
impl moka::Expiry<BillableMemoKey, Option<bool>> for BillableMemoExpiry {
fn expire_after_create(
&self,
_key: &BillableMemoKey,
value: &Option<bool>,
_created_at: std::time::Instant,
) -> Option<Duration> {
Some(match value {
Some(_) => self.answer_ttl,
None => self.failure_ttl,
})
}
}
/// Memo of whether the definitions behind a `(team, etag)` contain a billable
/// flag, so a 304 for definitions that hold only survey or product tour flags
/// is not billed, without reading the payload the 304 deliberately skips.
#[derive(Clone)]
pub struct DefinitionsBillableCache {
entries: moka::future::Cache<BillableMemoKey, Option<bool>>,
}
impl DefinitionsBillableCache {
pub fn new(max_capacity: u64, answer_ttl: Duration) -> Self {
Self::with_failure_ttl(max_capacity, answer_ttl, BILLABLE_MEMO_FAILURE_TTL)
}
fn with_failure_ttl(max_capacity: u64, answer_ttl: Duration, failure_ttl: Duration) -> Self {
Self {
entries: moka::future::Cache::builder()
.max_capacity(max_capacity)
.expire_after(BillableMemoExpiry {
answer_ttl,
failure_ttl,
})
.build(),
}
}
pub async fn insert(&self, team_id: i32, etag: &str, billable: bool) {
self.entries
.insert((team_id, etag.to_string()), Some(billable))
.await;
}
/// Returns the memoized status, or runs `load` once for all concurrent
/// callers with the same key and stores the result. `None` when the load
/// returns `None`, fails, or exceeds the bound; that `None` is kept for
/// `BILLABLE_MEMO_FAILURE_TTL` before a later call retries.
pub async fn get_or_load<F>(&self, team_id: i32, etag: &str, load: F) -> Option<bool>
where
F: std::future::Future<Output = Option<bool>>,
{
self.entries
.get_with((team_id, etag.to_string()), async {
tokio::time::timeout(BILLABLE_LOOKUP_TIMEOUT, load)
.await
.ok()
.flatten()
})
.await
}
}
/// Query parameters for the flag definitions endpoint
#[derive(Debug, Deserialize, Serialize)]
pub struct FlagDefinitionsQueryParams {
/// Team API token. Required when authenticating with a personal API key (which
/// can access multiple teams). Optional when authenticating with a `phs_` secret
/// token or project secret API key, since those are team-scoped and the team can
/// be derived from the token itself.
pub token: Option<String>,
}
/// Flag definitions endpoint handler
///
/// This endpoint provides flag definitions for client-side evaluation.
///
/// **HTTP Method:** GET only (HEAD and OPTIONS also supported)
///
/// **Required Query Parameter:**
/// - `token`: Team API token (e.g., `phc_...`) - specifies which team's flags to return
///
/// **Authentication Methods (one required via Authorization header):**
/// 1. Team secret API tokens (secret_api_token, secret_api_token_backup)
/// 2. Personal API keys with feature_flag:read or feature_flag:write scopes
///
/// The authentication must have access to the team specified by the token parameter.
///
/// **Response:**
/// The response is retrieved directly from Redis cache using Django's cache keys.
/// No database fallback is provided - if the cache is empty, an error is returned.
/// The response always includes cohort definitions.
///
/// **ETag support:**
/// Supports `If-None-Match` header for conditional requests. Returns 304 Not Modified
/// when the client's ETag matches the current cache state, avoiding redundant data transfer.
/// ETags are stored by Django in Redis alongside the cached data.
#[debug_handler]
pub async fn flags_definitions(
State(state): State<AppState>,
Query(params): Query<FlagDefinitionsQueryParams>,
headers: HeaderMap,
method: Method,
) -> Result<Response, FlagError> {
info!(
method = %method,
token = ?params.token,
"Processing flag definitions request"
);
// Only GET is supported for this read-only endpoint
// HEAD and OPTIONS are handled for HTTP compliance
if method != Method::GET {
return Ok(handle_non_get_method(&method));
}
// Resolve the team. Two flows:
// 1. Token provided: look up team by token, then authenticate against it (standard SDK flow)
// 2. Token absent: authenticate with phs_ secret token first, derive team_id from it
// (supports direct API callers who authenticate with Bearer phs_<key> only)
let team = if let Some(ref token) = params.token {
let team = fetch_team_by_token(&state, token).await?;
authenticate_flag_definitions(&state, &team, &headers).await?;
team
} else {
resolve_team_from_auth(&state, &headers).await?
};
// Refresh the rate limit allowlist from the database if stale (every ~60s)
refresh_rate_limit_allowlist_if_stale(&state).await;
// This check runs before the ETag read so that a request it refuses costs no Redis
// call. The handler cannot tell a 304 from a full response until Redis answers.
// Whether If-None-Match carries an ETag therefore picks the budget.
let client_etag = extract_etag_from_header(headers.get("if-none-match"));
if client_etag.is_some() {
state
.flag_definitions_conditional_limiter
.check_rate_limit(team.id)?;
} else {
state
.flag_definitions_full_limiter
.check_rate_limit(team.id)?;
}
// Check billing quota — matches Django's DECIDE_FEATURE_FLAG_QUOTA_CHECK behavior.
if state
.feature_flags_billing_limiter
.is_limited(&team.api_token)
.await
{
return Err(FlagError::ClientFacing(ClientFacingError::BillingLimit));
}
let team_key = KeyType::team(team.clone());
let current_etag = match state
.flags_with_cohorts_hypercache_reader
.get_etag(&team_key)
.await
{
Ok(Some(etag)) => Some(etag),
Ok(None) => {
// Redis answered and held no ETag key for this team. Counted apart from a
// cluster fault because the two need opposite responses: rebuild the cache
// tier, or treat Redis as the fault.
inc(
FLAG_DEFINITIONS_ETAG_COUNTER,
&[("result".to_string(), "redis_missing".to_string())],
1,
);
None
}
Err(e) => {
warn!(
team_id = team.id,
error = %e,
"Failed to read flag definitions ETag"
);
inc(
FLAG_DEFINITIONS_ETAG_COUNTER,
&[("result".to_string(), "redis_error".to_string())],
1,
);
None
}
};
// If client sent a matching ETag, short-circuit with 304 (skip full data fetch)
if let (Some(ref client_val), Some(ref current_val)) = (&client_etag, ¤t_etag) {
if client_val == current_val {
inc(
FLAG_DEFINITIONS_ETAG_COUNTER,
&[("result".to_string(), "hit".to_string())],
1,
);
if !*state.config.skip_writes
&& is_billable_for_etag(&state, &team_key, team.id, current_val).await
{
state.billing_aggregator.record(
team.id,
FlagRequestType::FlagDefinitionsNotModified,
Some(Library::from_headers(&headers)),
);
}
return Ok(not_modified_response(current_val));
}
}
// A stale ETag gets a full response, so it spends the full-response budget too. The
// conditional limiter already counted this request.
if client_etag.is_some() {
state
.flag_definitions_full_limiter
.check_rate_limit_without_request_count(team.id)?;
}
let etag_result = if client_etag.is_some() {
"miss"
} else {
"none"
};
inc(
FLAG_DEFINITIONS_ETAG_COUNTER,
&[("result".to_string(), etag_result.to_string())],
1,
);
// Retrieve cached response from HyperCache (always with cohorts)
let (cached_response, cache_source) = get_from_cache(&state, &team_key, team.id).await?;
// Record usage for billing, filtering out non-billable flags (surveys, product tours).
let billable = has_billable_flags(&cached_response);
if let (Some(etag), Some(billable)) = (
¤t_etag,
billable_for_etag_from(&cached_response, cache_source),
) {
state
.definitions_billable_cache
.insert(team.id, etag, billable)
.await;
}
if !*state.config.skip_writes && billable {
let library = Library::from_headers(&headers);
state
.billing_aggregator
.record(team.id, FlagRequestType::FlagDefinitions, Some(library));
}
Ok(ok_response_with_etag(
cached_response,
current_etag.as_deref(),
))
}
pub(crate) fn format_weak_etag(raw: &str) -> String {
format!("W/\"{}\"", raw)
}
/// Build a 304 Not Modified response with ETag and Cache-Control headers.
pub(crate) fn not_modified_response(etag: &str) -> Response {
(
StatusCode::NOT_MODIFIED,
[
("etag", format_weak_etag(etag)),
("cache-control", "private, must-revalidate".to_string()),
],
)
.into_response()
}
/// Build a 200 OK JSON response, attaching ETag and Cache-Control if available.
fn ok_response_with_etag(data: Value, etag: Option<&str>) -> Response {
match etag {
Some(etag_val) => (
StatusCode::OK,
[
("content-type", "application/json".to_string()),
("etag", format_weak_etag(etag_val)),
("cache-control", "private, must-revalidate".to_string()),
],
Json(data),
)
.into_response(),
None => Json(data).into_response(),
}
}
/// Extract the raw ETag value from an `If-None-Match` header.
///
/// Handles both strong ETags (`"abc123"`) and weak ETags (`W/"abc123"`) per RFC 7232.
/// Returns `None` if the header is absent or empty.
pub(crate) fn extract_etag_from_header(header: Option<&axum::http::HeaderValue>) -> Option<String> {
let value = header?.to_str().ok()?;
let trimmed = value.trim();
if trimmed.is_empty() {
return None;
}
let etag = if trimmed.starts_with("W/\"") {
// Weak ETag: W/"abc123" → strip W/ prefix, then strip quotes
&trimmed[2..]
} else {
trimmed
};
let etag = etag.trim_matches('"');
if etag.is_empty() {
None
} else {
Some(etag.to_string())
}
}
/// Handles non-GET HTTP methods (HEAD, OPTIONS, and unsupported methods)
pub(crate) fn handle_non_get_method(method: &Method) -> Response {
match *method {
Method::HEAD => (
StatusCode::OK,
[("content-type", "application/json")],
axum::body::Body::empty(),
)
.into_response(),
Method::OPTIONS => {
(StatusCode::NO_CONTENT, [("allow", "GET, OPTIONS, HEAD")]).into_response()
}
_ => (
StatusCode::METHOD_NOT_ALLOWED,
[("allow", "GET, OPTIONS, HEAD")],
)
.into_response(),
}
}
/// Fetches a team by its API token, delegating to FlagService for consistent
/// negative caching, metrics, and error handling across all endpoints.
async fn fetch_team_by_token(state: &AppState, token: &str) -> Result<Team, FlagError> {
state.flag_service().verify_token_and_get_team(token).await
}
/// Resolves a team from the Authorization header when no `?token=` param is provided.
///
/// Only works with `phs_` secret tokens and project secret API keys, which are
/// team-scoped. Personal API keys can access multiple teams and require an explicit
/// `?token=` param to disambiguate — returns an error in that case.
async fn resolve_team_from_auth(state: &AppState, headers: &HeaderMap) -> Result<Team, FlagError> {
if let Some(token) = auth::extract_team_secret_token(headers) {
let secret = auth::validate_secret_api_token(state, &token).await?;
inc(
FLAG_DEFINITIONS_AUTH_COUNTER,
&[("method".to_string(), secret.method_label().to_string())],
1,
);
// The team comes from the token itself, so the key is fully authenticated here. Stamp
// before the lookup so an infrastructure failure there does not leave it "never used".
state
.record_project_secret_key_usage(secret.project_secret_key_id)
.await;
// Prefer HyperCache via api_token (new cache entries include it).
// Fall back to PG for old cache entries that predate the field.
let svc = state.flag_service();
return match secret.api_token {
Some(t) => svc.verify_token_and_get_team(&t).await,
None => svc.get_team_by_id(secret.team_id).await,
};
}
// Non-phs_ auth (e.g. personal API key) can't derive team without a token param
if auth::extract_personal_api_key(headers)?.is_some() {
return Err(FlagError::ClientFacing(ClientFacingError::BadRequest(
"The 'token' query parameter is required unless authenticating with a \
phs_-prefixed token."
.to_string(),
)));
}
Err(FlagError::NoAuthenticationProvided)
}
fn cache_hit_metric_source_label(source: &CacheSource) -> &'static str {
match source {
CacheSource::S3AfterRedisError => "s3_after_redis_error",
_ => source.as_log_str(),
}
}
/// Only Redis holds the payload the ETag was published with. An S3 fallback
/// may be another version, so it decides nothing.
fn billable_for_etag_from(response: &FlagDefinitionsResponse, source: CacheSource) -> Option<bool> {
matches!(source, CacheSource::Redis).then(|| has_billable_flags(response))
}
/// Reads the payload at most once per pod per ETag per memo TTL. An
/// unreadable, slow, or non-Redis payload is not memoized as an answer and
/// counts as not billable, because over-billing is the worse failure.
async fn is_billable_for_etag(
state: &AppState,
team_key: &KeyType,
team_id: i32,
etag: &str,
) -> bool {
let billable = state
.definitions_billable_cache
.get_or_load(team_id, etag, async {
// Reads the hypercache directly: this lookup serves a 304, so it must not count as
// a definitions cache hit or queue a rebuild the way a full response does. It reads
// Redis only, because an S3 copy cannot decide (see `billable_for_etag_from`), so a
// fallback read would only add latency and S3 load. A Redis miss is not an answer.
let response = state
.flags_with_cohorts_hypercache_reader
.get_typed_from_redis::<FlagDefinitionsResponse>(team_key)
.await
.ok()?;
Some(response.is_some_and(|response| has_billable_flags(&response)))
})
.await;
let outcome = match billable {
Some(true) => "billable",
Some(false) => "not_billable",
None => "unknown",
};
inc(
FLAG_DEFINITIONS_NOT_MODIFIED_BILLING_COUNTER,
&[("outcome".to_string(), outcome.to_string())],
1,
);
billable.unwrap_or(false)
}
/// Retrieves the cached response using the pre-initialized HyperCacheReader
///
/// Always uses the cache with cohorts included to match Django's behavior and ensure
/// consistency across all clients accessing the same team's data. The cohorts are required
/// for proper local evaluation of flags that depend on cohort membership.
///
/// Emits metrics on both cache hit (with source label) and cache miss (with reason label)
/// to support dashboards and alerting during the migration from Django.
async fn get_from_cache(
state: &AppState,
team_key: &KeyType,
team_id: i32,
) -> Result<(FlagDefinitionsResponse, CacheSource), FlagError> {
let result = state
.flags_with_cohorts_hypercache_reader
.get_with_source(team_key)
.await;
match result {
Ok((data, source)) => {
let source_name = source.as_log_str();
inc(
FLAG_DEFINITIONS_CACHE_HIT_COUNTER,
&[(
"source".to_string(),
cache_hit_metric_source_label(&source).to_string(),
)],
1,
);
info!(
team_id = team_id,
source = source_name,
"Cache hit for flag definitions"
);
if should_rebuild_after_hit(
&source,
*state.config.flag_definitions_rebuild_on_s3_hit_enabled,
) {
enqueue_flag_definitions_rebuild(state, team_id, REBUILD_TRIGGER_S3_HIT);
}
Ok((data, source))
}
Err(e) => {
let reason = match &e {
HyperCacheError::CacheMiss => "cache_miss",
HyperCacheError::S3(_) => "s3_error",
HyperCacheError::Redis(_) => "redis_error",
HyperCacheError::Json(_) => "json_parse_error",
HyperCacheError::Pickle(_) => "pickle_parse_error",
HyperCacheError::Timeout(_) => "timeout",
};
inc(
FLAG_DEFINITIONS_CACHE_MISS_COUNTER,
&[("reason".to_string(), reason.to_string())],
1,
);
warn!(
team_id = team_id,
reason = reason,
error = %e,
"Flag definitions cache miss"
);
// Self-heal: a genuinely empty cache (not a transient redis/s3/parse
// error) has no DB fallback here, so it would 503 until something
// rewrites it. Enqueue a debounced rebuild request for a Celery worker.
if reason == "cache_miss" {
enqueue_flag_definitions_rebuild(state, team_id, REBUILD_TRIGGER_CACHE_MISS);
}
Err(FlagError::from(e))
}
}
}
/// Only `CacheSource::S3` qualifies, because only that source proves Redis answered and did
/// not hold the key. `S3AfterRedisError` reaches S3 because the Redis read failed, timed out,
/// or would not decode, which says nothing about whether the entry exists. Queueing on that
/// would enqueue every team served during a Redis incident and point a rebuild storm at the
/// cluster that is already failing.
fn should_rebuild_after_hit(source: &CacheSource, enabled: bool) -> bool {
enabled && matches!(source, CacheSource::S3)
}
/// Fire-and-forget enqueue of a flag-definitions rebuild request.
///
/// Writes to a Redis sorted set on the flags-namespace client, because that is where the
/// Django writer lives. The Celery drain derives its Redis from
/// `flag_definitions_hypercache.redis_url` (`rebuild_queue.py`), which resolves from the same
/// `FLAGS_REDIS_URL`. A request written to any other cluster is never drained and the team
/// never gets rebuilt.
///
/// The two ends agree on configuration, not on connection state. A process that cannot reach
/// the dedicated cluster at startup falls back to the shared one and enqueues there for its
/// whole life, while Celery keeps draining the dedicated one. Those teams wait for the hourly
/// verifier instead. The same startup failure already sends the flags.json, team-metadata, and
/// remote-config readers to the shared cluster, where Django writes nothing, so it degrades
/// more than this queue.
///
/// This is deliberately not the client the payload and the ETag are read from. The reader has its
/// own cluster switch (`FLAG_DEFINITIONS_DEDICATED_REDIS_ENABLED`, resolved in `server.rs`), so
/// during the cutover it can sit on either cluster while the queue must stay on the one Celery
/// drains. Unifying the queue with the reader severs the queue from the drain whenever the two
/// clusters differ.
///
/// Spawned so it never adds response latency. `write_rebuild_request` holds the write.
///
/// `FLAG_DEFINITIONS_SELF_HEAL_ENABLED` is checked here rather than at each call site, so it
/// stops every trigger. A trigger that also needs its own ramp adds that switch at its call
/// site.
fn enqueue_flag_definitions_rebuild(state: &AppState, team_id: i32, trigger: &'static str) {
if !*state.config.flag_definitions_self_heal_enabled {
return;
}
tokio::spawn(write_rebuild_request(
state.flags_namespace_redis_client(),
team_id,
trigger,
));
}
async fn write_rebuild_request(
redis: Arc<dyn RedisClient + Send + Sync>,
team_id: i32,
trigger: &'static str,
) {
let score = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0);
// NX, so a team that has been waiting keeps the score it was queued with. An
// overwriting write would move it behind requests made later, and the drain takes the
// lowest scores first.
let result = redis
.zadd_nx(
if trigger == REBUILD_TRIGGER_S3_HIT {
FLAG_DEFINITIONS_S3_REBUILD_REQUESTS_ZSET
} else {
FLAG_DEFINITIONS_REBUILD_REQUESTS_ZSET
}
.to_string(),
team_id.to_string(),
score,
)
.await;
inc(
FLAG_DEFINITIONS_REBUILD_REQUESTED_COUNTER,
&[
(
"result".to_string(),
if result.is_ok() { "ok" } else { "error" }.to_string(),
),
("trigger".to_string(), trigger.to_string()),
],
1,
);
}
/// Authenticates flag definitions requests using team secret API tokens or personal API keys
///
/// Validates that the authentication credential has access to the specified team.
///
/// Supports two authentication methods:
/// 1. Team secret API tokens (secret_api_token, secret_api_token_backup) from Authorization header
/// 2. Personal API keys with feature_flag:read or feature_flag:write scopes
///
/// Priority: Secret API tokens take precedence over personal API keys when both are provided.
///
/// Returns Ok(()) if authentication succeeds, Err otherwise
async fn authenticate_flag_definitions(
state: &AppState,
team: &Team,
headers: &HeaderMap,
) -> Result<(), FlagError> {
// Try team secret token or project secret API key (from Authorization header only)
// Both use phs_ prefix and share the same cache; the unified loader handles both.
if let Some(token) = auth::extract_team_secret_token(headers) {
let secret = auth::validate_secret_api_token_for_team(state, &token, team.id).await?;
inc(
FLAG_DEFINITIONS_AUTH_COUNTER,
&[("method".to_string(), secret.method_label().to_string())],
1,
);
state
.record_project_secret_key_usage(secret.project_secret_key_id)
.await;
return Ok(());
}
// Try personal API key (with scope validation)
if let Some(key) = auth::extract_personal_api_key(headers)? {
let pak_id =
auth::validate_personal_api_key_with_scopes_for_team(state, &key, team).await?;
inc(
FLAG_DEFINITIONS_AUTH_COUNTER,
&[("method".to_string(), "personal_api_key".to_string())],
1,
);
// PAK last_used_at tracking is advisory; shared with remote_config via the State helper.
state
.record_api_key_last_used(ApiKeyKind::Personal, pak_id)
.await;
return Ok(());
}
Err(FlagError::NoAuthenticationProvided)
}
/// Checks whether the cached flag definitions contain any billable flags.
///
/// Returns false if all flags are survey or product tour targeting flags,
/// matching Django's `local_evaluation` billing filter. The cached response
/// has a `"flags"` array where each entry has a `"key"` field.
fn has_billable_flags(response: &Value) -> bool {
let Some(flags) = response.get("flags").and_then(|f| f.as_array()) else {
return false;
};
flags.iter().any(|flag| {
let key = flag.get("key").and_then(|k| k.as_str()).unwrap_or("");
is_billable_flag_key(key)
})
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
use serde_json::json;
#[rstest]
#[case::regular_flags(json!({"flags": [{"key": "my-feature"}, {"key": "another-flag"}]}), true)]
#[case::only_survey_flags(json!({"flags": [{"key": "survey-targeting-abc"}, {"key": "survey-targeting-xyz"}]}), false)]
#[case::only_product_tour_flags(json!({"flags": [{"key": "product-tour-targeting-abc"}]}), false)]
#[case::mixed_survey_and_regular(json!({"flags": [{"key": "survey-targeting-abc"}, {"key": "my-feature"}]}), true)]
#[case::empty_flags_array(json!({"flags": []}), false)]
#[case::no_flags_key(json!({"cohorts": {}}), false)]
#[case::only_survey_and_tour_flags(json!({"flags": [{"key": "survey-targeting-abc"}, {"key": "product-tour-targeting-xyz"}]}), false)]
fn test_has_billable_flags(#[case] response: Value, #[case] expected: bool) {
assert_eq!(has_billable_flags(&response), expected);
}
#[rstest]
#[case::confirmed_redis_miss(CacheSource::S3, true, true)]
#[case::redis_read_failed(CacheSource::S3AfterRedisError, true, false)]
#[case::served_by_redis(CacheSource::Redis, true, false)]
#[case::served_by_fallback(CacheSource::Fallback, true, false)]
#[case::switch_off(CacheSource::S3, false, false)]
fn test_should_rebuild_after_hit(
#[case] source: CacheSource,
#[case] enabled: bool,
#[case] expected: bool,
) {
assert_eq!(should_rebuild_after_hit(&source, enabled), expected);
}
#[rstest]
#[case::confirmed_redis_miss(CacheSource::S3, "s3")]
#[case::redis_read_failed(CacheSource::S3AfterRedisError, "s3_after_redis_error")]
fn test_cache_hit_metric_source_label(#[case] source: CacheSource, #[case] expected: &str) {
assert_eq!(cache_hit_metric_source_label(&source), expected);
}
#[tokio::test]
async fn test_rebuild_request_does_not_overwrite_an_earlier_score() {
let mock = common_redis::MockRedisClient::new();
let redis: Arc<dyn RedisClient + Send + Sync> = Arc::new(mock.clone());
write_rebuild_request(redis.clone(), 42, REBUILD_TRIGGER_S3_HIT).await;
write_rebuild_request(redis, 42, REBUILD_TRIGGER_S3_HIT).await;
let calls = mock.get_calls();
assert_eq!(calls.len(), 2, "one queue write per request");
for call in calls {
assert_eq!(
call.op, "zadd_nx",
"repeated requests must keep the first score"
);
assert_eq!(call.key, FLAG_DEFINITIONS_S3_REBUILD_REQUESTS_ZSET);
assert!(
matches!(&call.value, common_redis::MockRedisValue::MemberScore(member, _) if member == "42"),
"the member is the team id the drain parses, got {:?}",
call.value
);
}
}
#[tokio::test]
async fn test_billable_cache_memoizes_a_successful_load() {
let cache = DefinitionsBillableCache::new(10, Duration::from_secs(3600));
assert_eq!(
cache.get_or_load(1, "etag", async { Some(true) }).await,
Some(true)
);
// A later load is never run once the key is memoized.
assert_eq!(
cache.get_or_load(1, "etag", async { None }).await,
Some(true)
);
// A new etag for the same team is a fresh key.
assert_eq!(
cache.get_or_load(1, "etag2", async { Some(false) }).await,
Some(false)
);
}
#[tokio::test]
async fn test_billable_cache_memoizes_a_failed_load_until_the_failure_ttl() {
let cache = DefinitionsBillableCache::with_failure_ttl(
10,
Duration::from_secs(3600),
Duration::from_millis(500),
);
assert_eq!(cache.get_or_load(1, "etag", async { None }).await, None);
assert_eq!(
cache.get_or_load(1, "etag", async { Some(true) }).await,
None
);
// moka expires entries on its own clock, so a paused tokio clock would not age this one.
tokio::time::sleep(Duration::from_secs(1)).await;
assert_eq!(
cache.get_or_load(1, "etag", async { Some(true) }).await,
Some(true)
);
}
#[tokio::test(start_paused = true)]
async fn test_billable_cache_bounds_a_slow_load() {
let cache = DefinitionsBillableCache::new(10, Duration::from_secs(3600));
assert_eq!(
cache.get_or_load(1, "etag", std::future::pending()).await,
None
);
}
#[rstest]
#[case::redis(CacheSource::Redis, Some(true))]
#[case::s3(CacheSource::S3, None)]
#[case::s3_after_redis_error(CacheSource::S3AfterRedisError, None)]
#[case::fallback(CacheSource::Fallback, None)]
fn test_billable_for_etag_from_only_trusts_redis(
#[case] source: CacheSource,
#[case] expected: Option<bool>,
) {
let response = json!({"flags": [{"key": "billable-flag"}]});
assert_eq!(billable_for_etag_from(&response, source), expected);
}
#[test]
fn test_extract_etag_from_header_weak() {
let val = axum::http::HeaderValue::from_static("W/\"a1b2c3d4e5f6g7h8\"");
assert_eq!(
extract_etag_from_header(Some(&val)),
Some("a1b2c3d4e5f6g7h8".to_string())
);
}
#[test]
fn test_extract_etag_from_header_strong() {
let val = axum::http::HeaderValue::from_static("\"a1b2c3d4e5f6g7h8\"");
assert_eq!(
extract_etag_from_header(Some(&val)),
Some("a1b2c3d4e5f6g7h8".to_string())
);
}
#[test]
fn test_extract_etag_from_header_none() {
assert_eq!(extract_etag_from_header(None), None);
}
#[test]
fn test_extract_etag_from_header_empty() {
let val = axum::http::HeaderValue::from_static("");
assert_eq!(extract_etag_from_header(Some(&val)), None);
}
#[test]
fn test_extract_etag_from_header_bare_value() {
let val = axum::http::HeaderValue::from_static("a1b2c3d4e5f6g7h8");
assert_eq!(
extract_etag_from_header(Some(&val)),
Some("a1b2c3d4e5f6g7h8".to_string())
);
}
#[test]
fn test_extract_etag_from_header_empty_weak() {
let val = axum::http::HeaderValue::from_static("W/\"\"");
assert_eq!(extract_etag_from_header(Some(&val)), None);
}
#[test]
fn test_extract_etag_from_header_wildcard_treated_as_literal() {
// RFC 7232 allows `*` to match any ETag, but we treat it as a literal
// value. This means it will never match a stored ETag, which is safe —
// the client just gets a 200 with full data instead of a 304.
let val = axum::http::HeaderValue::from_static("*");
assert_eq!(extract_etag_from_header(Some(&val)), Some("*".to_string()));
}
#[test]
fn test_extract_etag_from_header_multiple_etags_no_special_handling() {
// RFC 7232 allows comma-separated ETags, but we don't parse them
// individually. The whole value is treated as a single string, so it
// won't match any stored ETag — the client gets a 200 with full data.
let val = axum::http::HeaderValue::from_static("\"etag1\", \"etag2\"");
assert_eq!(
extract_etag_from_header(Some(&val)),
Some("etag1\", \"etag2".to_string())
);
}
}