:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Integration tests for the internal batch flag evaluation endpoint
//! (`POST /internal/batch_flag_evaluation`), which backs flag-driven static cohort
//! generation.
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use anyhow::Result;
use reqwest::StatusCode;
use serde_json::{json, Value};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf};
use tokio::net::{TcpListener, TcpStream};
use url::Url;
use uuid::Uuid;
pub mod common;
use crate::common::ServerHandle;
use feature_flags::cohorts::cohort_models::CohortType;
use feature_flags::config::{Config, DEFAULT_TEST_CONFIG};
use feature_flags::flags::flag_matching_utils::calculate_hash;
use feature_flags::flags::flag_models::FeatureFlagRow;
use feature_flags::utils::test_utils::{random_string, TestContext};
const INTERNAL_TOKEN: &str = "test-internal-request-token";
const ENDPOINT: &str = "/internal/batch_flag_evaluation";
fn batch_eval_config() -> Config {
let mut config = DEFAULT_TEST_CONFIG.clone();
config.internal_request_token = Some(INTERNAL_TOKEN.to_string());
config
}
async fn send_batch_request(
server: &ServerHandle,
bearer_token: Option<&str>,
body: &Value,
) -> reqwest::Response {
let client = reqwest::Client::new();
let mut request = client
.post(format!("http://{}{}", server.addr, ENDPOINT))
.header("content-type", "application/json")
.body(body.to_string());
if let Some(token) = bearer_token {
request = request.header("authorization", format!("Bearer {token}"));
}
request.send().await.expect("Failed to send batch request")
}
fn batch_body(team_id: i32, flag_key: &str, expected_version: i32) -> Value {
json!({
"team_id": team_id,
"project_id": team_id,
"flag_key": flag_key,
"expected_version": expected_version,
})
}
fn flag_row(team_id: i32, key: &str, filters: Value) -> FeatureFlagRow {
FeatureFlagRow {
id: 0,
team_id,
name: Some(format!("{key} description")),
key: key.to_string(),
filters,
deleted: false,
active: true,
ensure_experience_continuity: Some(false),
version: None,
evaluation_runtime: None,
evaluation_tags: None,
bucketing_identifier: None,
has_experiment: false,
}
}
/// Matched UUIDs as a sorted Vec for set comparison.
fn matched_uuids(response_json: &Value) -> Vec<Uuid> {
let mut uuids: Vec<Uuid> = response_json["matched_person_uuids"]
.as_array()
.expect("matched_person_uuids should be an array")
.iter()
.map(|v| v.as_str().unwrap().parse().unwrap())
.collect();
uuids.sort();
uuids
}
#[tokio::test]
async fn test_rejects_unauthenticated_requests() -> Result<()> {
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
let body = batch_body(team.id, "any-flag", 0);
// No Authorization header
let res = send_batch_request(&server, None, &body).await;
assert_eq!(res.status(), StatusCode::UNAUTHORIZED);
// Wrong bearer token
let res = send_batch_request(&server, Some("wrong-token"), &body).await;
assert_eq!(res.status(), StatusCode::UNAUTHORIZED);
Ok(())
}
#[tokio::test]
async fn test_rejects_all_requests_when_no_internal_token_configured() -> Result<()> {
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let mut config = DEFAULT_TEST_CONFIG.clone();
config.internal_request_token = None;
let server = ServerHandle::for_config(config).await;
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "any-flag", 0),
)
.await;
assert_eq!(res.status(), StatusCode::UNAUTHORIZED);
Ok(())
}
#[tokio::test]
async fn test_rejects_invalid_limits() -> Result<()> {
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
for bad_limit in [0, -5, 10_001] {
let mut body = batch_body(team.id, "any-flag", 0);
body["limit"] = json!(bad_limit);
let res = send_batch_request(&server, Some(INTERNAL_TOKEN), &body).await;
assert_eq!(
res.status(),
StatusCode::BAD_REQUEST,
"limit {bad_limit} should be rejected"
);
let json_response = res.json::<Value>().await?;
assert_eq!(json_response["error"], "invalid_request");
}
Ok(())
}
#[tokio::test]
async fn test_missing_and_deleted_flags_return_not_found() -> Result<()> {
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
// Non-existent flag
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "does-not-exist", 0),
)
.await;
assert_eq!(res.status(), StatusCode::NOT_FOUND);
// Deleted flag — the flag fetch excludes deleted flags, so this is
// indistinguishable from a missing flag.
let mut deleted = flag_row(
team.id,
"deleted-flag",
json!({"groups": [{"properties": [{"key": "key", "value": "value", "type": "person"}]}]}),
);
deleted.deleted = true;
context.insert_flag(team.id, Some(deleted)).await?;
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "deleted-flag", 0),
)
.await;
assert_eq!(res.status(), StatusCode::NOT_FOUND);
Ok(())
}
#[tokio::test]
async fn test_inactive_flag_rejected() -> Result<()> {
// The Django caller returns [] for inactive flags without calling us; this guard
// is defensive.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
let mut inactive = flag_row(
team.id,
"inactive-flag",
json!({"groups": [{"rollout_percentage": 100}]}),
);
inactive.active = false;
context.insert_flag(team.id, Some(inactive)).await?;
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "inactive-flag", 0),
)
.await;
assert_eq!(res.status(), StatusCode::BAD_REQUEST);
let json_response = res.json::<Value>().await?;
assert_eq!(json_response["error"], "flag_inactive");
Ok(())
}
#[tokio::test]
async fn test_unsupported_config_format_rejected() -> Result<()> {
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
for version in [json!(2), json!(3), json!("1")] {
for active in [true, false] {
let key = format!("unsupported-{version}-{active}");
let mut row = flag_row(
team.id,
&key,
json!({"version": version, "aggregation_group_type_index": 1}),
);
row.active = active;
context.insert_flag(team.id, Some(row)).await?;
let res =
send_batch_request(&server, Some(INTERNAL_TOKEN), &batch_body(team.id, &key, 0))
.await;
assert_eq!(res.status(), StatusCode::BAD_REQUEST);
let json_response = res.json::<Value>().await?;
assert_eq!(json_response["error"], "unsupported_config_format");
}
}
Ok(())
}
#[tokio::test]
async fn test_group_aggregated_flag_rejected() -> Result<()> {
// The Django caller returns [] for group-aggregated flags without calling us;
// this guard is defensive.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"group-flag",
json!({
"groups": [{"properties": [{"key": "key", "value": "value", "type": "group", "group_type_index": 1}]}],
"aggregation_group_type_index": 1,
}),
)),
)
.await?;
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "group-flag", 0),
)
.await;
assert_eq!(res.status(), StatusCode::BAD_REQUEST);
let json_response = res.json::<Value>().await?;
assert_eq!(json_response["error"], "group_aggregated_flag");
Ok(())
}
#[tokio::test]
async fn test_version_mismatch_returns_conflict_with_actual_version() -> Result<()> {
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
let flag = context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"versioned-flag",
json!({"groups": [{"rollout_percentage": 100}]}),
)),
)
.await?;
let mut conn = context.get_non_persons_connection().await?;
sqlx::query("UPDATE posthog_featureflag SET version = 3 WHERE id = $1")
.bind(flag.id)
.execute(&mut *conn)
.await?;
// Stale expected_version → 409 with the actual version, and no evaluation happened.
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "versioned-flag", 1),
)
.await;
assert_eq!(res.status(), StatusCode::CONFLICT);
let json_response = res.json::<Value>().await?;
assert_eq!(json_response["error"], "version_conflict");
assert_eq!(json_response["actual_version"], json!(3));
// Matching expected_version → 200.
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "versioned-flag", 3),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
Ok(())
}
#[tokio::test]
async fn test_property_matching_version_mismatch_returns_conflict() -> Result<()> {
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"matching-version-flag",
json!({"groups": [{"rollout_percentage": 100}]}),
)),
)
.await?;
let mut body = batch_body(team.id, "matching-version-flag", 0);
body["expected_property_matching_version"] = json!(2);
let res = send_batch_request(&server, Some(INTERNAL_TOKEN), &body).await;
assert_eq!(res.status(), StatusCode::CONFLICT);
let json_response = res.json::<Value>().await?;
assert_eq!(json_response["error"], "property_matching_version_conflict");
assert_eq!(json_response["actual_property_matching_version"], json!(1));
Ok(())
}
#[tokio::test]
async fn test_team_property_matching_version_selects_live_and_batch_semantics() -> Result<()> {
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"boolean-property-flag",
json!({
"groups": [{
"properties": [{
"key": "key",
"value": false,
"type": "person",
"operator": "exact"
}],
"rollout_percentage": 100
}]
}),
)),
)
.await?;
let distinct_id = "non-boolean-value";
context
.insert_person(
team.id,
distinct_id.to_string(),
Some(json!({"key": "banana"})),
)
.await?;
let person_uuid = context
.get_person_uuid_by_distinct_id(team.id, distinct_id)
.await?;
let legacy_response = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "boolean-property-flag", 0),
)
.await
.json::<Value>()
.await?;
assert_eq!(legacy_response["property_matching_version"], json!(1));
assert_eq!(matched_uuids(&legacy_response), vec![person_uuid]);
let mut conn = context.get_non_persons_connection().await?;
sqlx::query(
"INSERT INTO feature_flags_teamfeatureflagsconfig \
(team_id, minimal_flag_called_events, property_matching_version) \
VALUES ($1, false, 2)",
)
.bind(team.id)
.execute(&mut *conn)
.await?;
let live_response = server
.send_flags_request(
json!({"token": team.api_token, "distinct_id": distinct_id}).to_string(),
Some("2"),
None,
)
.await;
assert_eq!(live_response.status(), StatusCode::OK);
let live_response = live_response.json::<Value>().await?;
assert_eq!(
live_response["flags"]["boolean-property-flag"]["enabled"],
json!(false)
);
let mut explicit_body = batch_body(team.id, "boolean-property-flag", 0);
explicit_body["expected_property_matching_version"] = json!(2);
let explicit_response = send_batch_request(&server, Some(INTERNAL_TOKEN), &explicit_body)
.await
.json::<Value>()
.await?;
assert_eq!(explicit_response["property_matching_version"], json!(2));
assert!(matched_uuids(&explicit_response).is_empty());
Ok(())
}
#[tokio::test]
async fn test_null_expected_version_coerces_to_zero() -> Result<()> {
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"null-version-flag",
json!({"groups": [{"rollout_percentage": 100}]}),
)),
)
.await?;
// JSON null and an omitted field both mean "expect version 0", matching a flag
// whose version column is NULL.
let null_body = json!({
"team_id": team.id,
"flag_key": "null-version-flag",
"expected_version": null,
});
let res = send_batch_request(&server, Some(INTERNAL_TOKEN), &null_body).await;
assert_eq!(res.status(), StatusCode::OK);
let omitted_body = json!({
"team_id": team.id,
"flag_key": "null-version-flag",
});
let res = send_batch_request(&server, Some(INTERNAL_TOKEN), &omitted_body).await;
assert_eq!(res.status(), StatusCode::OK);
Ok(())
}
#[tokio::test]
async fn test_property_filter_selects_matching_persons() -> Result<()> {
// Port of the core matching behavior in test_creating_static_cohort_iterator: persons
// matching the property filter are included, others are not.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"property-flag",
json!({
"groups": [{
"properties": [{"key": "key", "value": "value", "type": "person"}],
"rollout_percentage": 100,
}],
}),
)),
)
.await?;
for (distinct_id, properties) in [
("prop_person1", json!({"key": "value"})),
("prop_person2", json!({"key": "value"})),
("prop_person3", json!({"key": "other"})),
("prop_person4", json!({})),
] {
context
.insert_person(team.id, distinct_id.to_string(), Some(properties))
.await?;
}
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "property-flag", 0),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
let json_response = res.json::<Value>().await?;
let mut expected = vec![
context
.get_person_uuid_by_distinct_id(team.id, "prop_person1")
.await?,
context
.get_person_uuid_by_distinct_id(team.id, "prop_person2")
.await?,
];
expected.sort();
assert_eq!(matched_uuids(&json_response), expected);
assert_eq!(json_response["errors_count"], json!(0));
assert_eq!(json_response["next_cursor"], Value::Null);
Ok(())
}
#[tokio::test]
async fn test_datetime_filter_uses_team_timezone_not_utc() -> Result<()> {
// Batch eval must interpret naive datetime person values in the team's timezone, the
// same as live `/flags` evaluation and HogQL/ClickHouse cohort membership. The handler
// reads `team.timezone` from Postgres server-side; the Django caller sends no timezone.
//
// Phoenix is UTC-7 year-round (no DST), so the arithmetic is unambiguous. The filter
// pins an explicit-UTC instant (12:00Z), while the person value is a naive wall-clock
// time interpreted in the team timezone:
// - In America/Phoenix, "2024-06-01 08:00:00" is 2024-06-01 15:00 UTC → after 12:00Z → matches.
// - Under a wrong UTC fallback it would be 08:00 UTC → not after 12:00Z → would NOT match.
// So this person is matched only if the team timezone is applied.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
context
.update_team_timezone(team.id, "America/Phoenix")
.await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"date-flag",
json!({
"groups": [{
"properties": [{
"key": "signup_date",
"type": "person",
"value": "2024-06-01T12:00:00Z",
"operator": "is_date_after",
}],
"rollout_percentage": 100,
}],
}),
)),
)
.await?;
for (distinct_id, signup_date) in [
// 15:00 UTC in Phoenix → after the filter only because the team tz is applied.
("tz_dependent_match", "2024-06-01 08:00:00"),
// Well before the filter under any timezone → never matches (selection control).
("clearly_before", "2020-01-01 00:00:00"),
] {
context
.insert_person(
team.id,
distinct_id.to_string(),
Some(json!({ "signup_date": signup_date })),
)
.await?;
}
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "date-flag", 0),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
let json_response = res.json::<Value>().await?;
let matched = context
.get_person_uuid_by_distinct_id(team.id, "tz_dependent_match")
.await?;
assert_eq!(matched_uuids(&json_response), vec![matched]);
assert_eq!(json_response["errors_count"], json!(0));
assert_eq!(json_response["next_cursor"], Value::Null);
Ok(())
}
#[tokio::test]
async fn test_rollout_percentage_bucketing_matches_live_hash() -> Result<()> {
// Rollout bucketing must use the same `_hash(key, identifier)` as live evaluation.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
let flag_key = "rollout-flag";
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
flag_key,
json!({"groups": [{"rollout_percentage": 50}]}),
)),
)
.await?;
let mut expected: Vec<Uuid> = Vec::new();
let mut expected_out = 0;
for i in 0..20 {
let distinct_id = format!("rollout_person_{i}");
context
.insert_person(team.id, distinct_id.clone(), Some(json!({})))
.await?;
let hash = calculate_hash(&format!("{flag_key}."), &distinct_id, "")?;
if hash <= 0.5 {
expected.push(
context
.get_person_uuid_by_distinct_id(team.id, &distinct_id)
.await?,
);
} else {
expected_out += 1;
}
}
expected.sort();
assert!(
!expected.is_empty() && expected_out > 0,
"test should exercise both sides of the rollout boundary"
);
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, flag_key, 0),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
let json_response = res.json::<Value>().await?;
assert_eq!(matched_uuids(&json_response), expected);
assert_eq!(json_response["errors_count"], json!(0));
Ok(())
}
#[tokio::test]
async fn test_cursor_pagination_covers_all_persons() -> Result<()> {
// Port of test_creating_static_cohort_iterator's batching behavior, now cursor-paged.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"paging-flag",
json!({"groups": [{"rollout_percentage": 100}]}),
)),
)
.await?;
let mut expected: Vec<Uuid> = Vec::new();
for i in 0..5 {
let distinct_id = format!("paging_person_{i}");
context
.insert_person(team.id, distinct_id.clone(), Some(json!({})))
.await?;
expected.push(
context
.get_person_uuid_by_distinct_id(team.id, &distinct_id)
.await?,
);
}
expected.sort();
let mut collected: Vec<Uuid> = Vec::new();
let mut cursor: i64 = 0;
let mut pages = 0;
loop {
let mut body = batch_body(team.id, "paging-flag", 0);
body["cursor"] = json!(cursor);
body["limit"] = json!(2);
let res = send_batch_request(&server, Some(INTERNAL_TOKEN), &body).await;
assert_eq!(res.status(), StatusCode::OK);
let json_response = res.json::<Value>().await?;
collected.extend(matched_uuids(&json_response));
pages += 1;
assert!(pages <= 4, "pagination should terminate");
match json_response["next_cursor"].as_i64() {
Some(next) => {
assert!(next > cursor, "cursor must advance");
cursor = next;
}
None => break,
}
}
// 5 persons with limit 2 → 3 pages (2 + 2 + 1, last page short-circuits next_cursor).
assert_eq!(pages, 3);
collected.sort();
assert_eq!(collected, expected);
Ok(())
}
#[tokio::test]
async fn test_person_with_no_distinct_ids_is_skipped() -> Result<()> {
// Almost-deleted persons (no distinct_ids) are skipped without counting as errors.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"skip-flag",
json!({"groups": [{"rollout_percentage": 100}]}),
)),
)
.await?;
// A person with no posthog_persondistinctid rows at all.
let orphan_uuid = Uuid::now_v7();
let mut conn = context.get_persons_connection().await?;
sqlx::query(
r#"INSERT INTO posthog_person
(created_at, properties, properties_last_updated_at, properties_last_operation,
team_id, is_user_id, is_identified, uuid, version)
VALUES ('2023-04-05', '{}', '{}', '{}', $1, NULL, true, $2, 0)"#,
)
.bind(team.id)
.bind(orphan_uuid)
.execute(&mut *conn)
.await?;
drop(conn);
context
.insert_person(team.id, "normal_person".to_string(), Some(json!({})))
.await?;
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "skip-flag", 0),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
let json_response = res.json::<Value>().await?;
let expected = vec![
context
.get_person_uuid_by_distinct_id(team.id, "normal_person")
.await?,
];
assert_eq!(matched_uuids(&json_response), expected);
assert_eq!(
json_response["errors_count"],
json!(0),
"skipped persons are not errors"
);
Ok(())
}
#[tokio::test]
async fn test_distinct_id_choice_is_deterministic_min_id() -> Result<()> {
// Evaluation uses the distinct_id with the lowest posthog_persondistinctid.id.
// For a 50% flag, the outcome must follow the FIRST distinct_id's hash, not the
// second's.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
let flag_key = "deterministic-flag";
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
flag_key,
json!({"groups": [{"rollout_percentage": 50}]}),
)),
)
.await?;
// Find a (first, second) distinct_id pair whose hashes land on opposite sides of the
// rollout boundary, so the test can detect which one was used.
let salt_prefix = format!("{flag_key}.");
let (first, second) = (0..10_000)
.find_map(|i| {
let first = format!("det_first_{i}");
let second = format!("det_second_{i}");
let first_in = calculate_hash(&salt_prefix, &first, "").unwrap() <= 0.5;
let second_in = calculate_hash(&salt_prefix, &second, "").unwrap() <= 0.5;
(first_in && !second_in).then_some((first, second))
})
.expect("should find a hash pair on opposite sides of the boundary");
let person_id = context
.insert_person(team.id, first.clone(), Some(json!({})))
.await?;
// Second distinct_id gets a strictly higher posthog_persondistinctid.id.
let mut conn = context.get_persons_connection().await?;
sqlx::query(
"INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id, version)
VALUES ($1, $2, $3, 0)",
)
.bind(&second)
.bind(person_id)
.bind(team.id)
.execute(&mut *conn)
.await?;
drop(conn);
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, flag_key, 0),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
let json_response = res.json::<Value>().await?;
let person_uuid = context
.get_person_uuid_by_distinct_id(team.id, &first)
.await?;
assert_eq!(
matched_uuids(&json_response),
vec![person_uuid],
"the first (min pd.id) distinct_id hashes inside the rollout, so the person must match"
);
Ok(())
}
#[tokio::test]
async fn test_experience_continuity_overrides_read_but_never_written() -> Result<()> {
// An existing hash key override must be honored (read), and the batch endpoint
// must never write new overrides (the matcher runs with skip_writes).
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
let flag_key = "continuity-flag";
let mut flag = flag_row(
team.id,
flag_key,
json!({"groups": [{"rollout_percentage": 50}]}),
);
flag.ensure_experience_continuity = Some(true);
context.insert_flag(team.id, Some(flag)).await?;
let salt_prefix = format!("{flag_key}.");
// A person whose own distinct_id hashes OUT of the rollout, with an override hash key
// that hashes IN — the override must flip them to matched.
let overridden = (0..10_000)
.map(|i| format!("continuity_overridden_{i}"))
.find(|d| calculate_hash(&salt_prefix, d, "").unwrap() > 0.5)
.unwrap();
let override_hash_key = (0..10_000)
.map(|i| format!("override_key_{i}"))
.find(|d| calculate_hash(&salt_prefix, d, "").unwrap() <= 0.5)
.unwrap();
// A person who'd be out of the rollout with no override — must stay out.
let plain = (0..10_000)
.map(|i| format!("continuity_plain_{i}"))
.find(|d| calculate_hash(&salt_prefix, d, "").unwrap() > 0.5)
.unwrap();
let overridden_person_id = context
.insert_person(team.id, overridden.clone(), Some(json!({})))
.await?;
context
.insert_person(team.id, plain.clone(), Some(json!({})))
.await?;
let mut conn = context.get_persons_connection().await?;
sqlx::query(
"INSERT INTO posthog_featureflaghashkeyoverride (team_id, person_id, feature_flag_key, hash_key)
VALUES ($1, $2, $3, $4)",
)
.bind(team.id)
.bind(overridden_person_id)
.bind(flag_key)
.bind(&override_hash_key)
.execute(&mut *conn)
.await?;
drop(conn);
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, flag_key, 0),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
let json_response = res.json::<Value>().await?;
let overridden_uuid = context
.get_person_uuid_by_distinct_id(team.id, &overridden)
.await?;
assert_eq!(
matched_uuids(&json_response),
vec![overridden_uuid],
"the override hash key must flip the overridden person into the rollout"
);
// skip_writes: the batch call must not have written any new override rows.
let mut conn = context.get_persons_connection().await?;
let override_count: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM posthog_featureflaghashkeyoverride WHERE team_id = $1",
)
.bind(team.id)
.fetch_one(&mut *conn)
.await?;
assert_eq!(
override_count, 1,
"batch evaluation must never write hash key overrides"
);
Ok(())
}
/// Returns a persons database URL that goes through a proxy. The proxy forwards traffic until a
/// client sends bytes that contain `needle`. From then on it sends nothing more to Postgres on any
/// connection and keeps every socket open. Postgres never receives the frozen query. Its statement
/// timeout therefore cannot end the query. A database that has stopped answering behaves the same
/// way.
async fn persons_db_url_frozen_by_query_containing(needle: &'static [u8]) -> Result<String> {
let mut url = Url::parse(&DEFAULT_TEST_CONFIG.persons_read_database_url)?;
let upstream = format!(
"{}:{}",
url.host_str().expect("the persons database URL has a host"),
url.port().unwrap_or(5432)
);
let listener = TcpListener::bind("127.0.0.1:0").await?;
let proxy_port = listener.local_addr()?.port();
let frozen = Arc::new(AtomicBool::new(false));
tokio::spawn(async move {
while let Ok((client, _)) = listener.accept().await {
let upstream = upstream.clone();
let frozen = frozen.clone();
tokio::spawn(async move {
let Ok(server) = TcpStream::connect(&upstream).await else {
return;
};
let (client_reader, mut client_writer) = client.into_split();
let (mut server_reader, server_writer) = server.into_split();
tokio::spawn(async move {
tokio::io::copy(&mut server_reader, &mut client_writer).await
});
forward_to_postgres_until_frozen(client_reader, server_writer, &frozen, needle)
.await;
});
}
});
url.set_host(Some("127.0.0.1"))?;
url.set_port(Some(proxy_port))
.expect("a postgres URL accepts a port");
// The proxy matches `needle` in plaintext, so the connection must not use TLS.
url.query_pairs_mut().append_pair("sslmode", "disable");
Ok(url.to_string())
}
async fn forward_to_postgres_until_frozen(
mut from: OwnedReadHalf,
mut to: OwnedWriteHalf,
frozen: &AtomicBool,
needle: &[u8],
) {
let mut buf = vec![0; 16 * 1024];
// Keeps the end of the previous read so that a needle split across two reads still matches.
let mut window = Vec::new();
loop {
let n = match from.read(&mut buf).await {
Ok(0) | Err(_) => return,
Ok(n) => n,
};
window.extend_from_slice(&buf[..n]);
if window.windows(needle.len()).any(|w| w == needle) {
frozen.store(true, Ordering::SeqCst);
}
window.drain(..window.len().saturating_sub(needle.len() - 1));
if frozen.load(Ordering::SeqCst) {
std::future::pending::<()>().await;
}
if to.write_all(&buf[..n]).await.is_err() {
return;
}
}
}
#[tokio::test]
async fn test_unanswered_persons_query_fails_the_person_at_the_deadline() -> Result<()> {
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
// A continuity flag at 100% rollout skips the hash key override lookup. This flag uses 50% so
// that the lookup runs.
let flag_key = "continuity-flag";
let mut flag = flag_row(
team.id,
flag_key,
json!({"groups": [{"rollout_percentage": 50}]}),
);
flag.ensure_experience_continuity = Some(true);
context.insert_flag(team.id, Some(flag)).await?;
context
.insert_person(team.id, "unanswered_user".to_string(), Some(json!({})))
.await?;
let mut config = batch_eval_config();
// Only the per-person evaluation reads the override table. The page scan runs before it
// through the same proxy and gets its answer.
config.persons_read_database_url =
persons_db_url_frozen_by_query_containing(b"posthog_featureflaghashkeyoverride").await?;
config.persons_db_deadline_ms = 200;
// Without the deadline, the page does not finish before this timeout. The endpoint then
// returns 503.
config.batch_flag_eval_timeout_ms = 30_000;
let server = ServerHandle::for_config(config).await;
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, flag_key, 0),
)
.await;
let status = res.status();
let body = res.text().await?;
assert_eq!(status, StatusCode::OK, "{body}");
let json_response: Value = serde_json::from_str(&body)?;
assert_eq!(json_response["errors_count"], json!(1));
assert_eq!(json_response["matched_person_uuids"], json!([]));
Ok(())
}
#[tokio::test]
async fn test_static_and_dynamic_cohort_conditions() -> Result<()> {
// Port of test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too,
// with live-evaluation semantics: static cohorts resolve via posthog_cohortpeople,
// dynamic cohorts evaluate in-memory against person properties.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
let static_cohort = context
.insert_cohort(team.id, None, json!({}), true)
.await?;
let dynamic_cohort = context
.insert_cohort(
team.id,
None,
json!({"properties": {"type": "OR", "values": [{
"type": "OR",
"values": [{"key": "plan", "value": "premium", "type": "person"}],
}]}}),
false,
)
.await?;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"cohort-flag",
json!({
"groups": [
{
"properties": [{"key": "id", "value": static_cohort.id, "type": "cohort"}],
"rollout_percentage": 100,
},
{
"properties": [{"key": "id", "value": dynamic_cohort.id, "type": "cohort"}],
"rollout_percentage": 100,
},
],
}),
)),
)
.await?;
// in_static: a member of the static cohort.
let in_static_id = context
.insert_person(team.id, "cohort_in_static".to_string(), Some(json!({})))
.await?;
context
.add_person_to_cohort(static_cohort.id, in_static_id)
.await?;
// in_dynamic: matches the dynamic cohort's property filter.
context
.insert_person(
team.id,
"cohort_in_dynamic".to_string(),
Some(json!({"plan": "premium"})),
)
.await?;
// outsider: in neither cohort.
context
.insert_person(
team.id,
"cohort_outsider".to_string(),
Some(json!({"plan": "free"})),
)
.await?;
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "cohort-flag", 0),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
let json_response = res.json::<Value>().await?;
let mut expected = vec![
context
.get_person_uuid_by_distinct_id(team.id, "cohort_in_static")
.await?,
context
.get_person_uuid_by_distinct_id(team.id, "cohort_in_dynamic")
.await?,
];
expected.sort();
assert_eq!(matched_uuids(&json_response), expected);
assert_eq!(json_response["errors_count"], json!(0));
Ok(())
}
#[tokio::test]
async fn test_realtime_cohort_condition_reads_membership_for_a_serving_team() -> Result<()> {
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
let cohort = context
.insert_cohort_with_type_and_condition_type(
team.id,
None,
json!({"properties": {"type": "OR", "values": [{
"type": "OR",
"values": [{"key": "plan", "value": "enterprise", "type": "person"}],
}]}}),
false,
Some(CohortType::Realtime),
None,
Some(chrono::Utc::now()),
Some(json!({"behavioral": true})),
None,
)
.await?;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"realtime-cohort-flag",
json!({
"groups": [{
"properties": [{"key": "id", "value": cohort.id, "type": "cohort"}],
"rollout_percentage": 100,
}],
}),
)),
)
.await?;
// The person fails the cohort's person filter, so only their `cohort_membership` row can match them.
context
.insert_person(
team.id,
"realtime_member".to_string(),
Some(json!({"plan": "free"})),
)
.await?;
let member = context
.get_person_uuid_by_distinct_id(team.id, "realtime_member")
.await?;
context
.insert_cohort_membership(team.id, cohort.id, member, true)
.await?;
let mut conn = context.get_non_persons_connection().await?;
sqlx::query(
"INSERT INTO cohort_realtime_team_access \
(team_id, level, purpose, save_triggers_backfill, processing_since, changed_at) \
VALUES ($1, 'serving', 'pilot', false, now(), now())",
)
.bind(team.id)
.execute(&mut *conn)
.await?;
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "realtime-cohort-flag", 0),
)
.await;
// Delete before asserting, so a failed assertion leaves no row behind.
sqlx::query("DELETE FROM cohort_realtime_team_access WHERE team_id = $1")
.bind(team.id)
.execute(&mut *conn)
.await?;
assert_eq!(res.status(), StatusCode::OK);
assert_eq!(matched_uuids(&res.json::<Value>().await?), vec![member]);
Ok(())
}
#[tokio::test]
async fn test_missing_property_negative_operator_uses_live_semantics() -> Result<()> {
// Live Rust evaluation explicitly treats a missing property as matching `is_not`
// (see the IsNot arm of `match_property`). The batch endpoint must follow live:
// a person missing the property still matches `is_not`.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"negation-flag",
json!({
"groups": [{
"properties": [{"key": "key", "value": "value", "type": "person", "operator": "is_not"}],
"rollout_percentage": 100,
}],
}),
)),
)
.await?;
context
.insert_person(
team.id,
"neg_has_other_value".to_string(),
Some(json!({"key": "other"})),
)
.await?;
context
.insert_person(team.id, "neg_missing_property".to_string(), Some(json!({})))
.await?;
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "negation-flag", 0),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
let json_response = res.json::<Value>().await?;
let mut expected = vec![
context
.get_person_uuid_by_distinct_id(team.id, "neg_has_other_value")
.await?,
context
.get_person_uuid_by_distinct_id(team.id, "neg_missing_property")
.await?,
];
expected.sort();
assert_eq!(
matched_uuids(&json_response),
expected,
"a person missing the property must match is_not, like live evaluation does"
);
Ok(())
}
#[tokio::test]
async fn test_multivariate_flag_variants_count_as_matched() -> Result<()> {
// Cohort membership is "does the live flag match", regardless of which variant.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"multivariate-flag",
json!({
"groups": [{"rollout_percentage": 100}],
"multivariate": {
"variants": [
{"key": "control", "rollout_percentage": 50},
{"key": "test", "rollout_percentage": 50},
],
},
}),
)),
)
.await?;
let mut expected = Vec::new();
for i in 0..4 {
let distinct_id = format!("variant_person_{i}");
context
.insert_person(team.id, distinct_id.clone(), Some(json!({})))
.await?;
expected.push(
context
.get_person_uuid_by_distinct_id(team.id, &distinct_id)
.await?,
);
}
expected.sort();
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "multivariate-flag", 0),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
let json_response = res.json::<Value>().await?;
assert_eq!(matched_uuids(&json_response), expected);
Ok(())
}
#[tokio::test]
async fn test_no_flag_analytics_emitted_for_batch_requests() -> Result<()> {
// The batch handler bypasses the /flags pipeline by construction (no billing, no
// flag analytics); this test pins that so a refactor can't silently reintroduce it.
use feature_flags::flags::flag_analytics::{current_bucket, get_team_request_key};
use feature_flags::flags::flag_request::FlagRequestType;
use feature_flags::utils::test_utils::setup_redis_client;
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
let server = ServerHandle::for_config(batch_eval_config()).await;
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
"analytics-flag",
json!({"groups": [{"rollout_percentage": 100}]}),
)),
)
.await?;
context
.insert_person(
team.id,
random_string("analytics_person", 8),
Some(json!({})),
)
.await?;
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, "analytics-flag", 0),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
let redis_client = setup_redis_client(None).await;
for request_type in [FlagRequestType::Decide, FlagRequestType::FlagDefinitions] {
let key = get_team_request_key(team.id, request_type);
let bucket = current_bucket();
// Billing aggregator keys are `<team_key>:<bucket>`; analytics counters use the
// same keyspace. Neither must exist after a batch-only workload.
for candidate in [key.clone(), format!("{key}:{bucket}")] {
let value = redis_client.get(candidate.clone()).await;
assert!(
value.is_err(),
"no analytics/billing key should exist for batch requests, found {candidate}"
);
}
}
Ok(())
}
#[tokio::test]
async fn test_batch_results_match_live_flags_evaluation() -> Result<()> {
// The contract for static cohort generation is "cohort membership = who the live
// flag matches". Evaluate the same flag for the same persons via the live /flags
// endpoint and via the batch endpoint, and require identical outcomes.
let context = TestContext::new(None).await;
let team = context.insert_new_team(None).await?;
// Mock redis validates the team token for the live path; flags are only in PG, so
// both the live path (PG fallback) and the batch path read the same definitions.
let server = ServerHandle::for_config_with_mock_redis(
batch_eval_config(),
vec![],
vec![(team.api_token.clone(), team.id)],
)
.await;
let flag_key = "parity-flag";
context
.insert_flag(
team.id,
Some(flag_row(
team.id,
flag_key,
json!({
"groups": [{
"properties": [{"key": "plan", "value": "premium", "type": "person"}],
"rollout_percentage": 50,
}],
}),
)),
)
.await?;
// Pick distinct_ids whose rollout hashes are known to land on both sides of the 50%
// boundary, so the test deterministically exercises matched and unmatched persons.
let salt_prefix = format!("{flag_key}.");
let mut in_rollout_ids = (0..10_000)
.map(|i| format!("parity_in_{i}"))
.filter(|d| calculate_hash(&salt_prefix, d, "").unwrap() <= 0.5);
let mut out_of_rollout_ids = (0..10_000)
.map(|i| format!("parity_out_{i}"))
.filter(|d| calculate_hash(&salt_prefix, d, "").unwrap() > 0.5);
let persons = [
(in_rollout_ids.next().unwrap(), json!({"plan": "premium"})),
(in_rollout_ids.next().unwrap(), json!({"plan": "premium"})),
(
out_of_rollout_ids.next().unwrap(),
json!({"plan": "premium"}),
),
(in_rollout_ids.next().unwrap(), json!({"plan": "free"})),
(out_of_rollout_ids.next().unwrap(), json!({"plan": "free"})),
];
let mut live_matched: Vec<Uuid> = Vec::new();
for (distinct_id, properties) in persons {
context
.insert_person(team.id, distinct_id.clone(), Some(properties))
.await?;
let live_response = server
.send_flags_request(
json!({"token": team.api_token, "distinct_id": distinct_id}).to_string(),
Some("2"),
None,
)
.await;
assert_eq!(live_response.status(), StatusCode::OK);
let live_json = live_response.json::<Value>().await?;
if live_json["flags"][flag_key]["enabled"] == json!(true) {
live_matched.push(
context
.get_person_uuid_by_distinct_id(team.id, &distinct_id)
.await?,
);
}
}
live_matched.sort();
assert_eq!(
live_matched.len(),
2,
"exactly the two premium persons inside the rollout should match live"
);
let res = send_batch_request(
&server,
Some(INTERNAL_TOKEN),
&batch_body(team.id, flag_key, 0),
)
.await;
assert_eq!(res.status(), StatusCode::OK);
let json_response = res.json::<Value>().await?;
assert_eq!(
matched_uuids(&json_response),
live_matched,
"batch evaluation must match live /flags evaluation exactly"
);
assert_eq!(json_response["errors_count"], json!(0));
Ok(())
}