Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

25.6 KB
Raw Blame
//! Runs the vendored Feature Flag Rules v2 contract corpus against production code:
//! the `sha1_60_v1` hash vectors against `calculate_hash` and the evaluator's rollout,
//! holdout, and variant paths; the version 1 lock-down cases against
//! `FeatureFlagMatcher::evaluate_all_feature_flags`; and the legacy projection matrix
//! against the `/flags` and `/decide` response conversions.
//!
//! The corpus lives under `tests/fixtures/rules_v2_contract/`; CI never fetches the harness.

use std::collections::{BTreeSet, HashMap, HashSet};
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::Arc;

use feature_flags::api::types::{
    DecideV1Response, DecideV2Response, FlagDetails, FlagDetailsMetadata, FlagEvaluationReason,
    FlagsResponse, LegacyFlagsResponse,
};
use feature_flags::cohorts::cohort_cache_manager::CohortCacheManager;
use feature_flags::flags::cache_builder::compute_flag_dependencies;
use feature_flags::flags::feature_flag_list::PreparedFlags;
use feature_flags::flags::flag_matching::FeatureFlagMatcher;
use feature_flags::flags::flag_matching_utils::calculate_hash;
use feature_flags::flags::flag_models::{FeatureFlag, FeatureFlagList, MultivariateFlagVariant};
use feature_flags::flags::flag_request::FlagRequest;
use feature_flags::flags::v1_bucketing::{is_in_rollout, select_variant};
use feature_flags::utils::test_utils::{mock_group_type_cache, TestContext};
use regex::Regex;
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
use uuid::Uuid;

const CORPUS_DIR: &str = "tests/fixtures/rules_v2_contract";

/// A dependency target absent from the fixture definitions maps to an id no fixture flag
/// uses, so the target is missing from the flag list entirely. That is the only way the
/// evaluator reports `missing_dependency`: a deleted target that another flag references
/// stays in the cached list and pre-seeds as false.
const MISSING_DEPENDENCY_ID: i64 = 0;

#[test]
fn vendored_contract_is_intact() {
    let source = load_json("SOURCE.json");

    let recorded = source["files"].as_object().expect("SOURCE.json files");
    let mut on_disk = BTreeSet::new();
    collect_files(&corpus_dir(), &corpus_dir(), &mut on_disk);
    on_disk.remove("SOURCE.json");
    assert_eq!(
        on_disk,
        recorded.keys().cloned().collect::<BTreeSet<_>>(),
        "vendored files must match the files listed in SOURCE.json"
    );
    for (rel, digest) in recorded {
        let bytes = fs::read(corpus_dir().join(rel)).unwrap();
        assert_eq!(
            hex::encode(Sha256::digest(&bytes)),
            digest.as_str().unwrap(),
            "{rel}: bytes differ from the digest in SOURCE.json; re-vendor from the harness instead of editing"
        );
    }
    assert_eq!(
        source["contract_version"],
        load_json("manifest.json")["contract"]["version"]
    );
}

#[tokio::test]
async fn corpus_matches_production_evaluator_and_projections() {
    let db = TestContext::new(None).await;
    let team_id = db.insert_new_team(None).await.expect("insert team").id;

    let mut covered = HashSet::new();
    covered.extend(run_hash_corpus(&db, team_id).await);
    covered.extend(run_v1_corpus(&db, team_id).await);
    covered.extend(run_projection_corpus());

    assert_eq!(covered, corpus_case_ids(), "every corpus case must execute");
}

fn corpus_case_ids() -> HashSet<String> {
    let manifest = load_json("manifest.json");
    let mut ids = HashSet::new();
    let mut counts = Vec::new();
    for artifact in corpus_artifacts(&manifest) {
        let path = str_field(artifact, "path");
        let in_file = file_case_ids(&load_json(path));
        counts.push((path.to_string(), in_file.len()));
        for id in in_file {
            assert!(ids.insert(id.clone()), "{path}: duplicate case id {id}");
        }
    }
    assert_eq!(
        counts,
        [
            ("corpus/hash_sha1_60_v1.json".to_string(), 32),
            ("corpus/v1_evaluation.json".to_string(), 31),
            ("corpus/legacy_projection.json".to_string(), 11),
        ]
    );
    ids
}

async fn run_hash_corpus(db: &TestContext, team_id: i32) -> HashSet<String> {
    let corpus = load_json("corpus/hash_sha1_60_v1.json");
    let mut covered = HashSet::new();

    for vector in corpus["vectors"].as_array().unwrap() {
        let id = str_field(vector, "id");
        let prefix = str_field(vector, "prefix");
        let identifier = str_field(vector, "identifier");
        let salt = str_field(vector, "salt");

        let input = format!("{prefix}{identifier}{salt}");
        assert_eq!(
            input,
            str_field(vector, "input"),
            "{id}: input concatenation"
        );
        assert_eq!(
            hex::encode(input.as_bytes()),
            str_field(vector, "input_utf8_hex"),
            "{id}: input bytes"
        );
        let hash = calculate_hash(prefix, identifier, salt).unwrap();
        assert_hash01(id, hash, &vector["hash01"]);
        assert_eq!(
            format!("{:016x}", hash.to_bits()),
            str_field(vector, "hash01_binary64_hex"),
            "{id}: hash01 binary64 bits"
        );

        if let Some(before) = vector["identifier_before_truncation"].as_str() {
            let request = FlagRequest {
                distinct_id: Some(before.to_string()),
                ..Default::default()
            };
            assert_eq!(
                request.extract_distinct_id().unwrap(),
                identifier,
                "{id}: request-level identifier truncation"
            );
        }

        // Rollout, holdout, and variant checks run through the evaluator so the comparison
        // and the boundary accumulation are the production ones, not a copy kept here.
        // Production hashes `<flag key>.`, so the flag key is the prefix minus its dot. The
        // holdout vector's `holdout-` prefix has no dot: `get_holdout_hash` always hashes
        // that literal, so the flag key never reaches it.
        let key = prefix.strip_suffix('.').unwrap_or(prefix);
        let (context, group_type_mapping, aggregation) = match vector["identifier_source"]
            .get("json_group_key")
        {
            Some(group_key) => (
                json!({"distinct_id": "person", "person_properties": {}, "groups": {"organization": group_key}}),
                json!({"0": "organization"}),
                Some(0),
            ),
            None => (
                json!({"distinct_id": identifier, "person_properties": {}}),
                Value::Null,
                None,
            ),
        };

        for check in vector["rollout_checks"].as_array().into_iter().flatten() {
            let percentage = check["rollout_percentage"].as_f64().unwrap();
            assert_eq!(
                percentage / 100.0,
                check["threshold"].as_f64().unwrap(),
                "{id}: threshold for {percentage}"
            );
            let flag = hash_check_flag(
                team_id,
                key,
                aggregation,
                json!({"groups": [{"properties": [], "rollout_percentage": percentage}]}),
            );
            let details = evaluate_single(db, team_id, flag, &group_type_mapping, &context).await;
            assert_eq!(
                json!(details.enabled),
                check["included"],
                "{id}: rollout at {percentage} percent"
            );
        }
        for check in vector["holdout_checks"].as_array().into_iter().flatten() {
            let percentage = check["exclusion_percentage"].as_f64().unwrap();
            let flag = hash_check_flag(
                team_id,
                key,
                None,
                json!({"groups": [{"properties": [], "rollout_percentage": 100}], "holdout": {"id": 7, "exclusion_percentage": percentage}}),
            );
            let details = evaluate_single(db, team_id, flag, &group_type_mapping, &context).await;
            assert_eq!(
                json!(details.variant.as_deref() == Some("holdout-7")),
                check["member"],
                "{id}: holdout membership at {percentage} percent"
            );
        }
        if let Some(check) = vector.get("variant_check") {
            let flag = hash_check_flag(
                team_id,
                key,
                aggregation,
                json!({"groups": [{"properties": [], "rollout_percentage": 100}], "multivariate": {"variants": check["variants"]}}),
            );
            let details = evaluate_single(db, team_id, flag, &group_type_mapping, &context).await;
            assert_eq!(
                details.variant.as_deref(),
                check["selected"].as_str(),
                "{id}: variant selection"
            );
        }
        covered.insert(id.to_string());
    }

    for vector in corpus["threshold_vectors"].as_array().unwrap() {
        let id = str_field(vector, "id");
        assert_eq!(
            json!(is_in_rollout::<std::convert::Infallible>(
                vector["rollout_percentage"].as_f64().unwrap(),
                || Ok(prescribed_hash(vector)),
            )
            .unwrap()),
            vector["included"],
            "{id}: rollout inclusion"
        );
        covered.insert(id.to_string());
    }
    for vector in corpus["variant_vectors"].as_array().unwrap() {
        let id = str_field(vector, "id");
        let variants: Vec<MultivariateFlagVariant> =
            serde_json::from_value(vector["variants"].clone()).unwrap();
        assert_eq!(
            json!(select_variant(prescribed_hash(vector), &variants)),
            vector["selected"],
            "{id}: v1 variant selection"
        );
        covered.insert(id.to_string());
    }

    // Parity rows pin that the v1 input and the converted v2 input hash identically.
    // `identical_outcome` and `divergence` describe version 2 semantics, which nothing in
    // this crate implements yet, so they are not asserted here.
    for parity in corpus["seed_parity"].as_array().unwrap() {
        let id = str_field(parity, "id");
        let Some(v2) = parity.get("v2") else {
            // The corpus links this white-box row to its v1 vector only through `note`.
            assert_eq!(id, "parity.final_variant_fallback_divergent");
            let linked = corpus["variant_vectors"]
                .as_array()
                .unwrap()
                .iter()
                .find(|vector| str_field(vector, "id") == "variant.hash_one_off_the_end")
                .expect("linked v1 variant vector");
            let variants: Vec<MultivariateFlagVariant> =
                serde_json::from_value(linked["variants"].clone()).unwrap();
            let hash = prescribed_hash(parity);
            assert_eq!(hash, prescribed_hash(linked), "{id}: linked v1 hash");
            assert_eq!(select_variant(hash, &variants), None, "{id}: v1 selection");
            covered.insert(id.to_string());
            continue;
        };
        let v1_input = str_field(parity, "v1_input");
        let v1_hash = hash_of_input(v1_input);
        let v2_hash = calculate_hash(
            str_field(v2, "prefix"),
            str_field(v2, "identifier"),
            str_field(v2, "salt"),
        )
        .unwrap();
        assert_hash01(id, v2_hash, &parity["hash01"]);
        assert_eq!(
            v1_hash.to_bits(),
            v2_hash.to_bits(),
            "{id}: v1 and converted v2 inputs must hash identically"
        );
        assert_eq!(
            json!(v1_input == str_field(v2, "input")),
            parity["identical_bytes"],
            "{id}: identical_bytes"
        );
        covered.insert(id.to_string());
    }
    covered
}

async fn run_v1_corpus(db: &TestContext, team_id: i32) -> HashSet<String> {
    let corpus = load_json("corpus/v1_evaluation.json");
    let mut covered = HashSet::new();

    for case in corpus["cases"].as_array().unwrap() {
        let id = str_field(case, "id");
        for evidence in case["hash_evidence"].as_array().into_iter().flatten() {
            assert_hash01(
                id,
                hash_of_input(str_field(evidence, "input")),
                &evidence["hash01"],
            );
        }

        let (flags, filtered_out) = corpus_flags(team_id, &case["definitions"]["flags"]);
        let response = evaluate(
            db,
            team_id,
            flags,
            filtered_out,
            &case["definitions"]["group_type_mapping"],
            &case["context"],
        )
        .await;
        assert!(
            !response.errors_while_computing_flags,
            "{id}: errorsWhileComputingFlags"
        );

        let mut unexpected: BTreeSet<&String> = response.flags.keys().collect();
        for (key, expected) in case["expected"]["flags"].as_object().unwrap() {
            let actual = response
                .flags
                .get(key)
                .unwrap_or_else(|| panic!("{id}: flag {key} missing from the response"));
            unexpected.remove(key);
            assert_eq!(
                json!(actual.to_value()),
                expected["value"],
                "{id}: {key} value"
            );
            assert_eq!(
                json!(actual.enabled),
                expected["enabled"],
                "{id}: {key} enabled"
            );
            assert_eq!(
                json!(actual.variant),
                expected["variant"],
                "{id}: {key} variant"
            );
            assert_eq!(
                json!(actual.metadata.payload),
                expected["payload"],
                "{id}: {key} payload"
            );
            assert_eq!(
                json!(actual.reason.code),
                expected["reason"]["code"],
                "{id}: {key} reason code"
            );
            assert_eq!(
                json!(actual.reason.condition_index),
                expected["reason"]["condition_index"],
                "{id}: {key} condition index"
            );
        }
        for omitted in case["expected"]["omitted_flags"].as_array().unwrap() {
            let omitted = omitted.as_str().unwrap();
            assert!(
                !response.flags.contains_key(omitted),
                "{id}: {omitted} must be omitted from the response"
            );
        }
        assert!(
            unexpected.is_empty(),
            "{id}: response carries flags the corpus does not expect: {unexpected:?}"
        );
        covered.insert(id.to_string());
    }
    covered
}

fn run_projection_corpus() -> HashSet<String> {
    let corpus = load_json("corpus/legacy_projection.json");
    let mut covered = HashSet::new();

    for case in corpus["cases"].as_array().unwrap() {
        let id = str_field(case, "id");
        let outcome = &case["outcome"];
        let build = || response_from_outcome(outcome);
        let actuals = [
            ("flags_v2", serde_json::to_value(build()).unwrap()),
            (
                "flags_v1",
                serde_json::to_value(LegacyFlagsResponse::from_response(build())).unwrap(),
            ),
            (
                "decide_v2",
                serde_json::to_value(DecideV2Response::from_response(build())).unwrap(),
            ),
            (
                "decide_v1",
                serde_json::to_value(DecideV1Response::from_response(build())).unwrap(),
            ),
        ];
        for (protocol, actual) in actuals {
            assert_projection(id, protocol, &case["projections"][protocol], &actual);
        }
        covered.insert(id.to_string());
    }
    covered
}

/// Builds the detailed response the evaluator would have produced for a corpus outcome.
fn response_from_outcome(outcome: &Value) -> FlagsResponse {
    let mut flags = HashMap::new();
    if !outcome["omitted"].as_bool().unwrap() {
        let key = str_field(outcome, "key").to_string();
        let detail = FlagDetails {
            key: key.clone(),
            enabled: outcome["enabled"].as_bool().unwrap(),
            variant: outcome["variant"].as_str().map(String::from),
            failed: outcome["failed"].as_bool().unwrap(),
            reason: FlagEvaluationReason {
                code: str_field(&outcome["reason"], "code").to_string(),
                condition_index: outcome["reason"]["condition_index"]
                    .as_i64()
                    .map(|i| i as i32),
                description: None,
            },
            metadata: FlagDetailsMetadata {
                id: outcome["flag_id"].as_i64().unwrap() as i32,
                version: outcome["flag_version"].as_i64().unwrap() as i32,
                description: None,
                payload: Some(outcome["payload"].clone()).filter(|p| !p.is_null()),
                has_experiment: false,
            },
            conditions: None,
            config_outcome: Default::default(),
        };
        flags.insert(key, detail);
    }
    FlagsResponse::new(
        outcome["failed"].as_bool().unwrap(),
        flags,
        None,
        Uuid::nil(),
    )
}

/// The corpus pins a subset of each response: every listed field must match, the flag maps
/// must contain exactly the listed keys, and `failed` must be absent unless listed.
fn assert_projection(id: &str, protocol: &str, expected: &Value, actual: &Value) {
    for (field, expected_value) in expected.as_object().unwrap() {
        if field == "response_version" {
            continue;
        }
        let actual_value = actual
            .get(field)
            .unwrap_or_else(|| panic!("{id}: {protocol} response lacks {field}"));
        assert_subset(id, protocol, field, expected_value, actual_value);
        if let (Some(e), Some(a)) = (expected_value.as_object(), actual_value.as_object()) {
            assert_eq!(
                e.keys().collect::<BTreeSet<_>>(),
                a.keys().collect::<BTreeSet<_>>(),
                "{id}: {protocol} {field} keys"
            );
        }
    }
    for (key, detail) in expected["flags"].as_object().into_iter().flatten() {
        if detail.get("failed").is_none() {
            assert!(
                actual["flags"][key].get("failed").is_none(),
                "{id}: {protocol} {key} must not carry failed"
            );
        }
    }
}

fn assert_subset(id: &str, protocol: &str, path: &str, expected: &Value, actual: &Value) {
    match (expected.as_object(), actual.as_object()) {
        (Some(e), Some(a)) => {
            for (field, expected_value) in e {
                let actual_value = a
                    .get(field)
                    .unwrap_or_else(|| panic!("{id}: {protocol} {path}.{field} missing"));
                assert_subset(
                    id,
                    protocol,
                    &format!("{path}.{field}"),
                    expected_value,
                    actual_value,
                );
            }
        }
        _ => assert_eq!(expected, actual, "{id}: {protocol} {path}"),
    }
}

/// Maps corpus flag definitions onto the evaluator's input shape: the team id is attached,
/// dependency filters name their target by id instead of key, and inactive flags join the
/// filtered-out set exactly as the request handler builds it.
fn corpus_flags(team_id: i32, definitions: &Value) -> (Vec<FeatureFlag>, HashSet<i32>) {
    let definitions = definitions.as_array().unwrap();
    let key_to_id: HashMap<&str, i64> = definitions
        .iter()
        .map(|flag| (str_field(flag, "key"), flag["id"].as_i64().unwrap()))
        .collect();

    let mut flags = Vec::new();
    let mut filtered_out = HashSet::new();
    for definition in definitions {
        let mut definition = definition.clone();
        definition["team_id"] = json!(team_id);
        for group in definition["filters"]["groups"]
            .as_array_mut()
            .into_iter()
            .flatten()
        {
            for property in group["properties"].as_array_mut().into_iter().flatten() {
                if property["type"] == "flag" {
                    let target = key_to_id
                        .get(str_field(property, "key"))
                        .copied()
                        .unwrap_or(MISSING_DEPENDENCY_ID);
                    property["key"] = json!(target.to_string());
                }
            }
        }
        let flag: FeatureFlag =
            serde_json::from_value(definition).expect("corpus flag must deserialize");
        if !flag.active {
            filtered_out.insert(flag.id);
        }
        flags.push(flag);
    }
    (flags, filtered_out)
}

fn hash_check_flag(
    team_id: i32,
    key: &str,
    aggregation: Option<i32>,
    mut filters: Value,
) -> FeatureFlag {
    if let Some(index) = aggregation {
        filters["aggregation_group_type_index"] = json!(index);
    }
    serde_json::from_value(
        json!({"id": 1, "team_id": team_id, "key": key, "active": true, "filters": filters}),
    )
    .expect("hash check flag must deserialize")
}

async fn evaluate_single(
    db: &TestContext,
    team_id: i32,
    flag: FeatureFlag,
    group_type_mapping: &Value,
    context: &Value,
) -> FlagDetails {
    let key = flag.key.clone();
    let mut response = evaluate(
        db,
        team_id,
        vec![flag],
        HashSet::new(),
        group_type_mapping,
        context,
    )
    .await;
    response
        .flags
        .remove(&key)
        .unwrap_or_else(|| panic!("flag {key} missing from the response"))
}

async fn evaluate(
    db: &TestContext,
    team_id: i32,
    flags: Vec<FeatureFlag>,
    filtered_out_flag_ids: HashSet<i32>,
    group_type_mapping: &Value,
    context: &Value,
) -> FlagsResponse {
    let types_to_indexes: HashMap<String, i32> = group_type_mapping
        .as_object()
        .into_iter()
        .flatten()
        .map(|(index, name)| (name.as_str().unwrap().to_string(), index.parse().unwrap()))
        .collect();
    let groups: Option<HashMap<String, Value>> = context
        .get("groups")
        .map(|g| serde_json::from_value(g.clone()).unwrap());
    let person_properties: HashMap<String, Value> =
        serde_json::from_value(context["person_properties"].clone()).unwrap();
    let group_properties: Option<HashMap<String, HashMap<String, Value>>> = context
        .get("group_properties")
        .map(|g| serde_json::from_value(g.clone()).unwrap());

    let mut matcher = FeatureFlagMatcher::new(
        str_field(context, "distinct_id").to_string(),
        None,
        team_id,
        db.create_postgres_router(),
        Arc::new(CohortCacheManager::new(
            db.non_persons_reader.clone(),
            None,
            None,
        )),
        mock_group_type_cache(types_to_indexes),
        groups,
    );
    // Dependency stages and missing targets come from the production builder, the same
    // way the batch evaluation handler assembles its flag list.
    let evaluation_metadata = compute_flag_dependencies(&flags).expect("dependency metadata");
    let flag_list = FeatureFlagList {
        flags: PreparedFlags::seal(flags),
        filtered_out_flag_ids,
        evaluation_metadata: Arc::new(evaluation_metadata),
        cohorts: None,
    };
    matcher
        .evaluate_all_feature_flags(
            flag_list,
            Some(person_properties),
            group_properties,
            None,
            Uuid::new_v4(),
            None,
            false,
        )
        .await
        .expect("evaluation must not fail")
}

/// Hashes a corpus row that supplies the whole concatenated input instead of its parts.
fn hash_of_input(input: &str) -> f64 {
    calculate_hash(input, "", "").unwrap()
}

fn prescribed_hash(case: &Value) -> f64 {
    str_field(case, "hash01").parse::<f64>().unwrap()
}

fn assert_hash01(id: &str, actual: f64, expected: &Value) {
    let expected: f64 = expected
        .as_str()
        .and_then(|literal| literal.parse().ok())
        .unwrap_or_else(|| panic!("{id}: hash01 must be a decimal literal"));
    assert_eq!(
        actual.to_bits(),
        expected.to_bits(),
        "{id}: hash01 is {actual:?}, corpus expects {expected:?}"
    );
}

fn corpus_dir() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).join(CORPUS_DIR)
}

/// serde_json parses long decimals with best-effort precision unless its `float_roundtrip`
/// feature is on, and 17-digit `hash01` literals can land one ulp off. Those literals are
/// quoted here so `assert_hash01` can parse them exactly with `str::parse::<f64>`.
fn load_json(rel: &str) -> Value {
    let path = corpus_dir().join(rel);
    let text = fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()));
    let text = Regex::new(r#"("hash01"\s*:\s*)([-+.0-9eE]+)"#)
        .unwrap()
        .replace_all(&text, "$1\"$2\"");
    serde_json::from_str(&text).unwrap_or_else(|e| panic!("parse {rel}: {e}"))
}

fn corpus_artifacts(manifest: &Value) -> impl Iterator<Item = &Value> {
    manifest["artifacts"]
        .as_array()
        .unwrap()
        .iter()
        .filter(|artifact| artifact["kind"] == "corpus")
}

/// Every `id` of an object inside a top-level array, which is how all three corpus files
/// lay out their cases and vectors.
fn file_case_ids(file: &Value) -> Vec<String> {
    file.as_object()
        .unwrap()
        .values()
        .filter_map(Value::as_array)
        .flatten()
        .filter_map(|row| row["id"].as_str())
        .map(String::from)
        .collect()
}

fn collect_files(root: &Path, dir: &Path, out: &mut BTreeSet<String>) {
    for entry in fs::read_dir(dir).unwrap() {
        let path = entry.unwrap().path();
        if path.is_dir() {
            collect_files(root, &path, out);
        } else {
            let rel = path.strip_prefix(root).unwrap();
            out.insert(
                rel.iter()
                    .map(|s| s.to_string_lossy())
                    .collect::<Vec<_>>()
                    .join("/"),
            );
        }
    }
}

fn str_field<'a>(value: &'a Value, field: &str) -> &'a str {
    value[field]
        .as_str()
        .unwrap_or_else(|| panic!("corpus field {field} must be a string in {value}"))
}