:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use feature_flags::flags::config_v2::{Outcome, ParseError};
use feature_flags::flags::evaluate_v2::{
Evaluation, EvaluationError, Evaluator, HashUse, PersonProperties,
};
use feature_flags::flags::feature_flag_list::PreparedFlags;
use feature_flags::flags::flag_matching_utils::calculate_hash;
use feature_flags::flags::flag_request::MAX_DISTINCT_ID_LEN;
use serde_json::{json, Value};
use uuid::Uuid;
pub mod common;
#[path = "test_rules_v2_evaluation/corpus.rs"]
mod corpus;
#[path = "test_rules_v2_evaluation/wire.rs"]
mod wire;
/// `wire::validate_v4` accepts every wire fixture case marked `valid` and fails every other
/// case at the layer its `expected_failure.layer` declares.
#[test]
fn vendored_response_fixtures_agree_with_the_validator() {
let fixtures = corpus::load("fixtures/wire/responses.json");
let (mut valid, mut invalid) = (0, 0);
for case in fixtures["cases"].as_array().unwrap() {
let id = case["id"].as_str().unwrap();
let verdict = wire::validate_v4(&wire::fixture_case(&fixtures, case));
if case["expected"] == "valid" {
assert_eq!(verdict, Ok(()), "{id}");
valid += 1;
} else {
let (layer, detail) = verdict.expect_err(id);
assert_eq!(layer, case["expected_failure"]["layer"], "{id}: {detail}");
invalid += 1;
}
}
assert_eq!((valid, invalid), (26, 44));
}
#[test]
fn evaluation_corpora_have_unique_ids_and_the_pinned_families() {
let mut counts = BTreeMap::new();
for (component, cases) in [
("v2_boolean_evaluation", corpus::cases()),
("v2_value_evaluation", corpus::value_cases()),
("v2_experiment_evaluation", corpus::experiment_cases()),
] {
let ids: Vec<_> = cases.iter().map(|case| case["id"].clone()).collect();
assert_eq!(
ids.iter()
.map(Value::to_string)
.collect::<BTreeSet<_>>()
.len(),
ids.len()
);
for case in &cases {
*counts
.entry((component, case["family"].as_str().unwrap().to_owned()))
.or_insert(0) += 1;
}
}
let family = |component, family: &str, count| ((component, family.to_owned()), count);
assert_eq!(
counts,
BTreeMap::from([
family("v2_boolean_evaluation", "ordering", 18),
family("v2_boolean_evaluation", "properties", 67),
family("v2_boolean_evaluation", "context", 8),
family("v2_boolean_evaluation", "errors", 6),
family("v2_boolean_evaluation", "hashing", 21),
family("v2_boolean_evaluation", "white_box", 10),
family("v2_boolean_evaluation", "eligibility", 3),
family("v2_boolean_evaluation", "parser", 2),
family("v2_value_evaluation", "ordering", 27),
family("v2_value_evaluation", "values", 11),
family("v2_value_evaluation", "parser", 11),
family("v2_experiment_evaluation", "ordering", 20),
family("v2_experiment_evaluation", "assignment", 21),
family("v2_experiment_evaluation", "values", 9),
family("v2_experiment_evaluation", "white_box", 10),
family("v2_experiment_evaluation", "parser", 13),
])
);
}
#[test]
fn parsed_configs_match_every_core_case_without_mutating_cached_inputs() {
let mut executed = 0;
let mut rejected = 0;
for case in corpus::cases()
.into_iter()
.chain(corpus::value_cases())
.chain(corpus::experiment_cases())
{
let id = case["id"].as_str().unwrap();
match case["family"].as_str().unwrap() {
"white_box" | "eligibility" => continue,
"ordering" | "properties" | "context" | "errors" | "hashing" | "values"
| "assignment" | "parser" => {}
other => panic!("unhandled family: {other}"),
}
let flag = corpus::read(&case);
let before = serde_json::to_value(&flag).unwrap();
let cached = PreparedFlags::seal(vec![flag]);
let retained = cached[0].filters.non_v1.as_ref().unwrap();
if case["family"] == "parser" {
// A schema rejection: a too-long list is reported as a limit.
let kind = match retained.parsed_v2.as_ref() {
Some(Err(ParseError::Malformed(_) | ParseError::LimitExceeded(_))) => "malformed",
Some(Err(ParseError::Unsupported(_))) | None => "unsupported",
other => panic!("{id}: unexpected parse result {other:?}"),
};
assert_eq!(
json!({"status":"parse_error","kind":kind}),
case["expected"],
"{id}"
);
rejected += 1;
continue;
}
let config = corpus::config(&cached[0]);
let evaluator = Evaluator::new(config);
let properties = corpus::properties(&case);
let properties_before = properties.clone();
let context = corpus::context(&case, &properties);
for _ in 0..2 {
assert_eq!(
corpus::result_json(evaluator.evaluate(&context)),
case["expected"],
"{id}"
);
}
let cloned = cached[0].clone();
assert!(Arc::ptr_eq(
retained,
cloned.filters.non_v1.as_ref().unwrap()
));
assert_eq!(serde_json::to_value(&cached[0]).unwrap(), before, "{id}");
assert_eq!(properties, properties_before);
if let Some(Value::Array(draws)) = case.get("hash_evidence") {
assert_experiment_hash_evidence(id, &context, draws);
} else if let Some(evidence) = case.get("hash_evidence") {
let (seed, rollout_percentage) = config
.rules
.iter()
.find_map(|rule| match &rule.outcome {
Outcome::PercentageRollout {
seed,
rollout_percentage,
..
} => Some((seed, rollout_percentage)),
_ => None,
})
.unwrap();
let subject: String = context
.person_identifier
.chars()
.take(MAX_DISTINCT_ID_LEN)
.collect();
assert_eq!(subject, evidence["identifier"]);
let hash = calculate_hash(&format!("{seed}."), &subject, "").unwrap();
assert_eq!(
format!("{:016x}", hash.to_bits()),
evidence["hash01_binary64_hex"],
"{id}"
);
assert_eq!(
format!("{:016x}", (rollout_percentage / 100.0).to_bits()),
evidence["threshold_binary64_hex"],
"{id}"
);
}
executed += 1;
}
assert_eq!((executed, rejected), (120 + 38 + 50, 2 + 11 + 13));
}
fn assert_experiment_hash_evidence(
id: &str,
context: &feature_flags::flags::evaluate_v2::EvaluationContext<'_>,
draws: &[Value],
) {
let hex = |value: f64| format!("{:016x}", value.to_bits());
let subject: String = context
.person_identifier
.chars()
.take(MAX_DISTINCT_ID_LEN)
.collect();
for draw in draws {
assert_eq!(subject, draw["identifier"], "{id}");
let hash = calculate_hash(
draw["prefix"].as_str().unwrap(),
&subject,
draw["salt"].as_str().unwrap(),
)
.unwrap();
assert_eq!(hex(hash), draw["hash01_binary64_hex"], "{id}");
}
}
#[tokio::test]
async fn corpus_eligibility_uses_the_request_boundary_and_valid_v2_is_evaluated() {
use feature_flags::config::DEFAULT_TEST_CONFIG;
use feature_flags::utils::test_utils::{
insert_flags_for_team_in_redis, insert_new_team_in_redis, setup_redis_client,
};
let config = DEFAULT_TEST_CONFIG.clone();
let redis = setup_redis_client(Some(config.redis_url.clone())).await;
let team = insert_new_team_in_redis(redis.clone()).await.unwrap();
let cases: Vec<_> = corpus::cases()
.into_iter()
.filter(|case| case["family"] == "eligibility")
.collect();
let mut flags = vec![
json!({"id":1,"team_id":team.id,"key":"healthy","active":true,"filters":{"groups":[{"rollout_percentage":100}]}}),
];
let mut requested = vec!["healthy".to_string(), "eligible-v2".to_string()];
for (index, case) in cases.iter().enumerate() {
let mut flag = corpus::read(case);
flag.id = (index + 2) as i32;
flag.team_id = team.id;
flag.key = case["id"].as_str().unwrap().to_string();
if case["flag"]["requested"] == true {
requested.push(flag.key.clone());
}
flags.push(serde_json::to_value(flag).unwrap());
}
let mut eligible = corpus::read(&cases[0]);
eligible.id = 20;
eligible.team_id = team.id;
eligible.key = "eligible-v2".to_string();
eligible.active = true;
let evaluator = Evaluator::new(corpus::config(&eligible));
assert_eq!(
corpus::result_json(
evaluator.evaluate(&corpus::context(&cases[0], &corpus::properties(&cases[0])))
)["value"],
true
);
flags.push(serde_json::to_value(eligible).unwrap());
insert_flags_for_team_in_redis(redis, team.id, Some(json!(flags).to_string()))
.await
.unwrap();
let server = common::ServerHandle::for_config(config).await;
let body: Value = reqwest::Client::new()
.post(format!("http://{}/flags?v=2&config=false", server.addr))
.json(&json!({"token":team.api_token,"distinct_id":"example-person","flag_keys":requested}))
.send()
.await
.unwrap()
.error_for_status()
.unwrap()
.json()
.await
.unwrap();
assert_eq!(body["flags"]["healthy"]["enabled"], true);
assert_eq!(body["flags"]["eligible-v2"]["enabled"], true);
assert!(body["flags"]["eligible-v2"].get("failed").is_none());
assert_eq!(body["errorsWhileComputingFlags"], false);
for case in &cases {
assert!(body["flags"].get(case["id"].as_str().unwrap()).is_none());
assert_eq!(case["expected"], json!({"status":"omitted"}));
}
assert_eq!(cases.len(), 3);
}
#[test]
fn canonical_white_box_cases_exercise_the_production_ordered_evaluator() {
let mut count = 0;
for case in corpus::cases()
.into_iter()
.chain(corpus::experiment_cases())
.filter(|case| case["family"] == "white_box")
{
let flag = corpus::read(&case);
let config = corpus::config(&flag);
let evaluator = Evaluator::new(config);
let properties = corpus::properties(&case);
let context = corpus::context(&case, &properties);
let hash = f64::from_bits(
u64::from_str_radix(
case["white_box"]["hash01_binary64_hex"].as_str().unwrap(),
16,
)
.unwrap(),
);
let result = evaluator.evaluate_with_hash(&context, |_, _, _| Ok(hash));
assert_eq!(
corpus::result_json(result),
case["expected"],
"{}",
case["id"]
);
count += 1;
}
assert_eq!(count, 10 + 10);
}
#[test]
fn hashing_is_lazy_subject_is_resolved_once_and_repeated_seeds_reuse_the_hash() {
let case = corpus::cases()
.into_iter()
.find(|case| case["id"] == "v2_boolean.hash.full")
.unwrap();
let mut flag = corpus::read(&case);
let config = Arc::make_mut(flag.filters.non_v1.as_mut().unwrap())
.parsed_v2
.as_mut()
.unwrap()
.as_mut()
.unwrap();
let properties = corpus::properties(&case);
let mut context = corpus::context(&case, &properties);
assert!(Evaluator::new(config)
.evaluate_with_hash(&context, |_, _, _| panic!("100% must not hash"))
.is_ok());
context.person_identifier = "";
assert!(matches!(
Evaluator::new(config)
.evaluate_with_hash(&context, |_, _, _| panic!("empty subject must not hash")),
Ok(Evaluation::NoRuleMatch { .. })
));
if let Outcome::PercentageRollout {
rollout_percentage, ..
} = &mut config.rules[0].outcome
{
*rollout_percentage = 0.0;
}
let mut second = config.rules[0].clone();
second.id = Uuid::new_v4();
config.rules.push(second);
let subject = "😀".repeat(201);
context.person_identifier = &subject;
let mut calls = 0;
assert!(matches!(
Evaluator::new(config).evaluate_with_hash(&context, |hash_use, seed, subject| {
calls += 1;
assert_eq!(hash_use, HashUse::Rollout);
assert_eq!(seed, "example-allocation");
assert_eq!(subject, "😀".repeat(200));
Ok(0.5)
}),
Ok(Evaluation::NoRuleMatch { .. })
));
assert_eq!(calls, 1);
assert_eq!(
Evaluator::new(config).evaluate_with_hash(&context, |_, _, _| Err(EvaluationError::Hash)),
Err(EvaluationError::Hash)
);
}
#[test]
fn experiment_draws_follow_pause_holdout_rollout_variant_order() {
for (id, expected) in [
("v2_experiment.ordering.paused_before_holdout", vec![]),
(
"v2_experiment.assignment.holdout_out",
vec![
(HashUse::Holdout, "example-holdout"),
(HashUse::Variant, "example-split"),
],
),
(
"v2_experiment.assignment.rollout_hit",
vec![
(HashUse::Rollout, "example-split"),
(HashUse::Variant, "example-split"),
],
),
(
"v2_experiment.assignment.shared_rollout_seed",
vec![
(HashUse::Rollout, "example-shared"),
(HashUse::Variant, "example-shared"),
],
),
] {
let case = corpus::experiment_cases()
.into_iter()
.find(|case| case["id"] == id)
.unwrap();
let flag = corpus::read(&case);
let properties = corpus::properties(&case);
let context = corpus::context(&case, &properties);
let mut draws = Vec::new();
let result = Evaluator::new(corpus::config(&flag)).evaluate_with_hash(
&context,
|hash_use, seed, subject| {
draws.push((hash_use, seed.to_owned()));
hash_use.hash(seed, subject)
},
);
assert_eq!(corpus::result_json(result), case["expected"], "{id}");
let draws: Vec<_> = draws.iter().map(|(u, s)| (*u, s.as_str())).collect();
assert_eq!(draws, expected, "{id}");
}
}
#[test]
fn regex_execution_errors_are_not_inverted_by_negation() {
let mut case = corpus::cases()
.into_iter()
.find(|case| case["id"] == "v2_boolean.property.regex")
.unwrap();
case["config"]["rules"][0]["targeting"]["properties"][0]["value"] = json!(r"^(a+)+\1$");
case["context"]["properties"]["color"] = json!("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!");
for negation in [false, true] {
case["config"]["rules"][0]["targeting"]["properties"][0]["negation"] = json!(negation);
let flag = corpus::read(&case);
let config = corpus::config(&flag);
let properties = corpus::properties(&case);
let context = corpus::context(&case, &properties);
assert_eq!(
Evaluator::new(config).evaluate(&context),
Err(EvaluationError::InvalidRegex),
"negation={negation}"
);
}
}
#[test]
fn evaluation_is_repeatable_and_diagnostics_do_not_retain_inputs() {
let mut case = corpus::cases()
.into_iter()
.find(|case| case["id"] == "v2_boolean.property.regex")
.unwrap();
case["config"]["rules"][0]["targeting"]["properties"][0]["value"] = json!("sensitive-pattern");
let flag = corpus::read(&case);
let config = corpus::config(&flag);
let evaluator = Evaluator::new(config);
let properties = corpus::properties(&case);
let context = corpus::context(&case, &properties);
let result = evaluator.evaluate(&context);
assert_eq!(result, evaluator.evaluate(&context));
assert!(config.estimated_heap_bytes() >= 2048);
let debug = format!("{evaluator:?} {context:?} {result:?}");
for sensitive in ["sensitive-pattern", "person-example", "color"] {
assert!(!debug.contains(sensitive));
}
let mut case = corpus::cases()
.into_iter()
.find(|case| case["id"] == "v2_boolean.hash.full")
.unwrap();
case["config"]["rules"][0]["seed"] = json!("sensitive-seed");
let flag = corpus::read(&case);
let evaluator = Evaluator::new(corpus::config(&flag));
let properties = corpus::properties(&case);
let context = corpus::context(&case, &properties);
let result = evaluator.evaluate(&context);
assert!(matches!(result, Ok(Evaluation::TargetingMatch { .. })));
assert!(!format!("{evaluator:?} {result:?}").contains("sensitive-seed"));
}
/// Complete properties come from a stored person and run through the request path. The
/// service reads the person for a missing predicate key and treats an overrides-only
/// request as complete, so partial and unavailable context exist only below that path:
/// those cases reach `get_match` directly, with no preparation.
#[tokio::test]
async fn corpus_cases_project_through_the_matcher_and_the_legacy_formats() {
use feature_flags::api::types::{
DecideV1Response, DecideV2Response, FlagValue, FlagsResponse, FlagsResponseV4,
LegacyFlagsResponse,
};
use feature_flags::cohorts::cohort_cache_manager::CohortCacheManager;
use feature_flags::flags::flag_matching::FeatureFlagMatcher;
use feature_flags::utils::test_utils::{
flag_list_with_metadata, mock_group_type_cache, TestContext,
};
use std::collections::HashMap;
let db = TestContext::new(None).await;
let cohort_cache = Arc::new(CohortCacheManager::new(
db.non_persons_reader.clone(),
None,
None,
));
let (mut projected, mut direct, mut skipped) = (0, 0, 0);
for case in corpus::cases()
.into_iter()
.chain(corpus::value_cases())
.chain(corpus::experiment_cases())
{
let id = case["id"].as_str().unwrap();
let properties = corpus::properties(&case);
let context = corpus::context(&case, &properties);
// White-box needs the hash seam, eligibility the request boundary.
if matches!(
case["family"].as_str().unwrap(),
"white_box" | "eligibility"
) {
skipped += 1;
continue;
}
let team = db.insert_new_team(None).await.unwrap();
let mut flag = corpus::read(&case);
flag.team_id = team.id;
let mut matcher = FeatureFlagMatcher::new(
context.person_identifier.to_string(),
None,
team.id,
db.create_postgres_router(),
cohort_cache.clone(),
mock_group_type_cache(HashMap::new()),
None,
)
.with_timezone(context.timezone)
.with_explicit_exact_matching(context.use_explicit_exact_matching)
.with_now(context.now);
let expected = &case["expected"];
let failed = expected["status"] != "success";
let legacy = case
.get("legacy")
.cloned()
.unwrap_or_else(|| corpus::legacy(&expected["value"]));
let enabled = legacy["enabled"].as_bool().unwrap();
let variant = legacy["variant"].as_str().map(str::to_owned);
let payload = Some(&legacy["payload"]).filter(|payload| !payload.is_null());
let decoded = |payload: &Option<Value>| {
payload
.as_ref()
.map(|p| serde_json::from_str::<Value>(p.as_str().unwrap()).unwrap())
};
let reason = match expected["reason"].as_str() {
Some("targeting_match" | "experiment_split") => "condition_match",
Some("rollout_miss") => "out_of_rollout_bound",
Some("holdout") => "holdout_condition_value",
_ => "no_condition_match",
};
let rule_index = expected["rule"]["index"].as_i64();
let PersonProperties::Complete(_) = context.properties else {
let overrides =
matches!(context.properties, PersonProperties::Partial(_)).then_some(&properties);
match matcher.get_match(&flag, overrides, None, None, &None) {
Ok(outcome) => {
assert!(!failed, "{id}");
assert_eq!(
(
outcome.matches,
outcome.reason.to_string(),
outcome.condition_index
),
(enabled, reason.to_string(), rule_index.map(|i| i as usize)),
"{id}"
);
assert_eq!(
(outcome.variant, decoded(&outcome.payload).as_ref()),
(variant, payload),
"{id}"
);
}
Err(error) => {
assert!(failed, "{id}");
assert_eq!(
error.evaluation_error_code(),
"flag_evaluation_error",
"{id}"
);
}
}
direct += 1;
continue;
};
if !properties.is_empty() {
db.insert_person(
team.id,
context.person_identifier.to_string(),
Some(json!(properties)),
)
.await
.unwrap();
}
let key = flag.key.clone();
let response = matcher
.evaluate_all_feature_flags(
flag_list_with_metadata(vec![flag]),
None,
None,
None,
Uuid::new_v4(),
None,
false,
)
.await
.unwrap();
let details = &response.flags[&key];
assert_eq!(
(
details.failed,
details.enabled,
&details.variant,
decoded(&details.metadata.payload).as_ref()
),
(failed, enabled, &variant, payload),
"{id}"
);
assert_eq!(response.errors_while_computing_flags, failed, "{id}");
if failed {
let expected_code = if case["family"] == "parser" {
"flag_data_parsing_error"
} else {
"flag_evaluation_error"
};
assert_eq!(details.reason.code, expected_code, "{id}");
} else {
assert_eq!(details.reason.code, reason, "{id}");
assert_eq!(
details.reason.condition_index.map(i64::from),
rule_index,
"{id}"
);
}
let wire = serde_json::to_value(&response).unwrap();
let reparse = || serde_json::from_value::<FlagsResponse>(wire.clone()).unwrap();
let map_value: FlagValue = serde_json::from_value(legacy["value"].clone()).unwrap();
let flags_v1 = LegacyFlagsResponse::from_response(reparse());
assert_eq!(flags_v1.feature_flags[&key], map_value, "{id}");
assert_eq!(
decoded(&flags_v1.feature_flag_payloads.get(&key).cloned()).as_ref(),
payload,
"{id}"
);
assert_eq!(
DecideV1Response::from_response(reparse())
.feature_flags
.contains(&key),
enabled,
"{id}"
);
assert_eq!(
DecideV2Response::from_response(reparse())
.feature_flags
.get(&key),
enabled.then_some(map_value).as_ref(),
"{id}"
);
let v4 = serde_json::to_value(FlagsResponseV4::from_response(response)).unwrap();
wire::validate_v4(&v4).unwrap_or_else(|error| panic!("{id}: {error:?}"));
let record = &v4["flags"][&key];
assert_eq!(record["metadata"]["config_version"], 2, "{id}");
assert_eq!(record.get("enabled"), None, "{id}");
if failed {
assert_eq!(
(
&record["value"],
&record["reason"]["code"],
&record["failed"]
),
(&Value::Null, &json!("error"), &json!(true)),
"{id}"
);
} else {
assert_eq!(record["value"], expected["value"], "{id}");
assert_eq!(record["reason"]["code"], expected["reason"], "{id}");
assert_eq!(
record["reason"]["condition_index"], expected["rule"]["index"],
"{id}"
);
assert_eq!(
record["metadata"].get("rule_id"),
expected["rule"].get("id"),
"{id}"
);
assert_eq!(
record["metadata"].get("rule_type"),
expected["rule"].get("rule_type"),
"{id}"
);
assert_eq!(
record["metadata"].get("variant_key"),
expected.get("variant"),
"{id}"
);
for field in ["experiment_id", "holdout_id"] {
assert_eq!(record["metadata"].get(field), None, "{id}");
}
}
projected += 1;
}
assert_eq!((projected, direct, skipped), (114 + 49 + 63, 8, 13 + 10));
}