Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

20.8 KB
Raw Blame
use anyhow::Result;
use serde_json::{json, Value};

use feature_flags::config::DEFAULT_TEST_CONFIG;
use feature_flags::utils::test_utils::{
    insert_flags_for_team_in_redis, insert_new_team_in_redis, setup_redis_client, TestContext,
};

use crate::common::*;

pub mod common;
#[path = "test_rules_v2_evaluation/corpus.rs"]
#[allow(dead_code)]
mod corpus;
#[path = "test_rules_v2_evaluation/wire.rs"]
#[allow(dead_code)]
mod wire;

const RULE_A: &str = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
const RULE_B: &str = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb";
const INTERNAL_TOKEN: &str = "internal-test-token";
const DECIDE: &[(&str, &str)] = &[("X-Original-Endpoint", "decide")];
const INTERNAL: &[(&str, &str)] = &[("Authorization", "Bearer internal-test-token")];

fn v2(key: &str, id: i32, default_value: Value, rules: Value) -> Value {
    json!({
        "id": id, "team_id": 0, "key": key, "name": key, "active": true, "deleted": false,
        "version": 3, "has_experiment": false,
        "filters": {"version": 2, "return_type": "boolean", "default_value": default_value, "rules": rules},
    })
}

fn targeted(id: &str, value: bool, properties: Value) -> Value {
    json!({"id": id, "rule_type": "targeted_release", "targeting": {"properties": properties}, "value": value})
}

fn rollout(id: &str, percentage: u8, on_rollout_miss: &str) -> Value {
    json!({
        "id": id, "rule_type": "percentage_rollout", "targeting": {"properties": []}, "value": true,
        "rollout_percentage": percentage, "on_rollout_miss": on_rollout_miss,
        "assignment_algorithm": "sha1_60_v1", "seed": "seed-for-tests",
    })
}

fn email(pattern: &str) -> Value {
    json!([{"key": "email", "type": "person", "operator": "icontains", "value": pattern}])
}

fn corpus_flag(case: &Value, id: i32) -> Value {
    json!({
        "id": id, "team_id": 0, "key": case["id"], "name": case["id"], "active": true,
        "deleted": false, "version": 1, "has_experiment": false, "filters": case["config"],
    })
}

async fn serve(
    v4_enabled: bool,
    distinct_id: &str,
    properties: Value,
    mut flags: Vec<Value>,
) -> Result<(ServerHandle, String)> {
    let mut config = DEFAULT_TEST_CONFIG.clone();
    config.internal_request_token = Some(INTERNAL_TOKEN.to_string());
    config.flags_v4_response_enabled = v4_enabled;
    let redis = setup_redis_client(Some(config.redis_url.clone())).await;
    let team = insert_new_team_in_redis(redis.clone()).await?;
    let db = TestContext::new(None).await;
    db.insert_new_team(Some(team.id)).await?;
    db.insert_person(team.id, distinct_id.to_string(), Some(properties))
        .await?;
    for flag in &mut flags {
        flag["team_id"] = json!(team.id);
    }
    insert_flags_for_team_in_redis(redis, team.id, Some(json!(flags).to_string())).await?;
    Ok((ServerHandle::for_config(config).await, team.api_token))
}

/// A team with one person, three v1 flags and six v2 flags covering every boolean outcome.
async fn server(v4_enabled: bool) -> Result<(ServerHandle, String)> {
    let flags = boolean_flags();
    serve(
        v4_enabled,
        "ann",
        json!({"email": "[email protected]"}),
        flags,
    )
    .await
}

fn boolean_flags() -> Vec<Value> {
    vec![
        json!({
            "id": 1, "team_id": 0, "key": "v1-on", "name": "v1-on", "active": true,
            "deleted": false, "version": 5, "has_experiment": false,
            "filters": {"groups": [{"properties": [], "rollout_percentage": 100}]},
        }),
        json!({
            "id": 2, "team_id": 0, "key": "v1-variant", "name": "v1-variant", "active": true,
            "deleted": false, "version": 6, "has_experiment": false,
            "filters": {
                "groups": [{"properties": [], "rollout_percentage": 100, "variant": "compact"}],
                "multivariate": {"variants": [{"key": "compact", "name": "Compact", "rollout_percentage": 100}]},
                "payloads": {"compact": "{\"columns\":2}"},
            },
        }),
        json!({
            "id": 9, "team_id": 0, "key": "v1-free-key", "name": "v1-free-key", "active": true,
            "deleted": false, "version": 7, "has_experiment": false,
            "filters": {
                "groups": [{"properties": [], "rollout_percentage": 100, "variant": "provider/model-1.2 beta"}],
                "multivariate": {"variants": [{"key": "provider/model-1.2 beta", "name": "Free key", "rollout_percentage": 100}]},
            },
        }),
        v2(
            "v2-true",
            3,
            json!(false),
            json!([targeted(RULE_A, true, email("@example.com"))]),
        ),
        v2(
            "v2-false",
            4,
            json!(true),
            json!([targeted(RULE_A, false, json!([]))]),
        ),
        v2(
            "v2-null-default",
            5,
            Value::Null,
            json!([targeted(RULE_A, true, email("@nowhere"))]),
        ),
        v2(
            "v2-rollout-return-default",
            6,
            json!(false),
            json!([rollout(RULE_B, 0, "return_default")]),
        ),
        v2(
            "v2-rollout-continue",
            7,
            json!(true),
            json!([rollout(RULE_B, 0, "continue")]),
        ),
        v2(
            "v2-error",
            8,
            json!(false),
            json!([targeted(
                RULE_A,
                true,
                json!([
                    {"key": "email", "type": "person", "operator": "regex", "value": "[", "negation": false}
                ])
            )]),
        ),
    ]
}

async fn post(
    server: &ServerHandle,
    query: &str,
    headers: &[(&str, &str)],
    body: Value,
) -> Result<Value> {
    let mut request = reqwest::Client::new()
        .post(format!("http://{}/flags?{query}", server.addr))
        .header("Content-Type", "application/json")
        .body(body.to_string());
    for (name, value) in headers {
        request = request.header(*name, *value);
    }
    let response = request.send().await?;
    assert_eq!(response.status(), 200);
    Ok(response.json().await?)
}

fn keys(value: &Value) -> Vec<&str> {
    let mut keys: Vec<_> = value
        .as_object()
        .unwrap()
        .keys()
        .map(String::as_str)
        .collect();
    keys.sort_unstable();
    keys
}

#[tokio::test]
async fn v4_returns_the_typed_record_for_v1_and_v2_flags() -> Result<()> {
    let (server, token) = server(true).await?;
    let body = json!({"token": token, "distinct_id": "ann"});

    let v4 = post(&server, "v=4&config=false", &[], body.clone()).await?;
    wire::validate_v4(&v4).unwrap_or_else(|error| panic!("{error:?}"));
    assert_eq!(v4["errorsWhileComputingFlags"], true);
    let flags = &v4["flags"];
    assert_eq!(flags.as_object().unwrap().len(), 9);

    let expected = [
        (
            "v1-on",
            json!(true),
            "condition_match",
            json!(0),
            1,
            None,
            None,
        ),
        (
            "v1-variant",
            json!("compact"),
            "condition_match",
            json!(0),
            1,
            None,
            None,
        ),
        (
            "v1-free-key",
            json!("provider/model-1.2 beta"),
            "condition_match",
            json!(0),
            1,
            None,
            None,
        ),
        (
            "v2-true",
            json!(true),
            "targeting_match",
            json!(0),
            2,
            Some("targeted_release"),
            Some(RULE_A),
        ),
        (
            "v2-false",
            json!(false),
            "targeting_match",
            json!(0),
            2,
            Some("targeted_release"),
            Some(RULE_A),
        ),
        (
            "v2-null-default",
            Value::Null,
            "no_rule_match",
            Value::Null,
            2,
            None,
            None,
        ),
        (
            "v2-rollout-return-default",
            json!(false),
            "rollout_miss",
            json!(0),
            2,
            Some("percentage_rollout"),
            Some(RULE_B),
        ),
        (
            "v2-rollout-continue",
            json!(true),
            "no_rule_match",
            Value::Null,
            2,
            None,
            None,
        ),
        ("v2-error", Value::Null, "error", Value::Null, 2, None, None),
    ];
    for (key, value, code, condition_index, config_version, rule_type, rule_id) in expected {
        let record = &flags[key];
        assert_eq!(record["key"], key, "{key}");
        assert_eq!(record["value"], value, "{key}");
        assert_eq!(record["reason"]["code"], code, "{key}");
        assert_eq!(
            record["reason"]["condition_index"], condition_index,
            "{key}"
        );
        assert_eq!(
            record["metadata"]["config_version"], config_version,
            "{key}"
        );
        assert_eq!(
            record["metadata"].get("rule_type"),
            rule_type.map(Value::from).as_ref(),
            "{key}"
        );
        assert_eq!(
            record["metadata"].get("rule_id"),
            rule_id.map(Value::from).as_ref(),
            "{key}"
        );
        assert_eq!(
            record.get("failed"),
            (key == "v2-error").then_some(&json!(true)),
            "{key}"
        );
        assert!(
            record.get("enabled").is_none() && record.get("variant").is_none(),
            "{key}"
        );
        if config_version == 2 {
            assert_eq!(record["metadata"]["payload"], Value::Null, "{key}");
        }
    }
    for (key, description) in [
        ("v2-true", "Matched rule 1"),
        ("v2-rollout-return-default", "Rule 1 rollout miss"),
        ("v2-null-default", "No rule matched"),
    ] {
        assert_eq!(flags[key]["reason"]["description"], description, "{key}");
    }
    assert_eq!(flags["v1-variant"]["metadata"]["variant_key"], "compact");
    assert_eq!(
        flags["v1-free-key"]["metadata"]["variant_key"],
        "provider/model-1.2 beta"
    );
    assert_eq!(
        flags["v1-variant"]["metadata"]["payload"],
        "{\"columns\":2}"
    );
    assert_eq!(flags["v1-on"]["metadata"].get("variant_key"), None);
    assert_eq!(flags["v1-on"]["metadata"]["version"], 5);
    assert_eq!(flags["v2-true"]["metadata"]["version"], 3);

    let subset = post(
        &server,
        "v=4&config=false",
        &[],
        json!({"token": token, "distinct_id": "ann", "flag_keys": ["v2-true", "missing"]}),
    )
    .await?;
    assert_eq!(keys(&subset["flags"]), ["v2-true"]);
    wire::validate_v4(&subset).unwrap();

    let detailed = post(
        &server,
        "v=4&config=false&detailed_analysis=true",
        INTERNAL,
        body,
    )
    .await?;
    wire::validate_v4(&detailed).unwrap();
    assert_eq!(
        detailed["flags"]["v1-on"]["conditions"]
            .as_array()
            .unwrap()
            .len(),
        1
    );
    assert_eq!(detailed["flags"]["v2-true"]["conditions"], json!([]));
    Ok(())
}

#[tokio::test]
async fn v2_legacy_and_decide_shapes_are_unchanged() -> Result<()> {
    let (server, token) = server(true).await?;
    let body = json!({"token": token, "distinct_id": "ann"});

    let v2 = post(&server, "v=2&config=false", &[], body.clone()).await?;
    for (key, record) in v2["flags"].as_object().unwrap() {
        let mut expected = vec!["enabled", "key", "metadata", "reason", "variant"];
        if key == "v2-error" {
            expected.insert(1, "failed");
        }
        assert_eq!(keys(record), expected, "{key}");
        assert_eq!(
            keys(&record["metadata"]),
            ["description", "has_experiment", "id", "payload", "version"],
            "{key}"
        );
    }
    assert_eq!(v2["flags"]["v2-true"]["enabled"], true);
    assert_eq!(v2["flags"]["v2-true"]["reason"]["code"], "condition_match");
    assert_eq!(v2["flags"]["v2-null-default"]["enabled"], false);
    assert_eq!(v2["flags"]["v1-variant"]["variant"], "compact");

    for query in ["v=1&config=false", "v=abc&config=false"] {
        let legacy = post(&server, query, &[], body.clone()).await?;
        assert_eq!(legacy.get("flags"), None, "{query}");
        assert_eq!(legacy["featureFlags"]["v2-true"], true, "{query}");
        assert_eq!(legacy["featureFlags"]["v1-variant"], "compact", "{query}");
    }

    let decide_v3 = post(&server, "v=3&config=false", DECIDE, body.clone()).await?;
    assert_eq!(decide_v3.get("flags"), None);
    assert_eq!(decide_v3["featureFlags"]["v2-true"], true);
    let decide_v4 = post(&server, "v=4&config=false", DECIDE, body).await?;
    assert_eq!(decide_v4["flags"]["v2-true"]["enabled"], true);
    assert_eq!(decide_v4["flags"]["v2-true"].get("value"), None);

    let minimal: Value = reqwest::get(format!("http://{}/flags?v=4", server.addr))
        .await?
        .json()
        .await?;
    assert_eq!(minimal["flags"], json!({}));
    assert_eq!(minimal["supportedCompression"], json!(["gzip", "gzip-js"]));
    wire::validate_v4(&minimal).unwrap();
    Ok(())
}

#[tokio::test]
async fn v4_serves_the_v2_record_while_the_setting_is_off() -> Result<()> {
    let string_case = corpus::value_cases()
        .into_iter()
        .find(|case| case["id"] == "v2_value.string.target")
        .unwrap();
    let mut flags = boolean_flags();
    flags.push(corpus_flag(&string_case, 10));
    let (server, token) = serve(false, "ann", json!({"email": "[email protected]"}), flags).await?;
    let body = json!({"token": token, "distinct_id": "ann"});
    let request_id = [("X-REQUEST-ID", "00000000-0000-4000-8000-000000000003")];
    let without_time = |mut response: Value| {
        response.as_object_mut().unwrap().remove("evaluatedAt");
        response
    };

    for config in ["config=false", "config=true"] {
        let v2 = post(&server, &format!("v=2&{config}"), &request_id, body.clone()).await?;
        assert_eq!(v2["flags"]["v2_value.string.target"]["variant"], "compact");
        assert_eq!(v2["flags"]["v2-true"]["enabled"], true);
        assert_eq!(v2["flags"]["v1-variant"]["variant"], "compact");
        for version in ["v=3", "v=4", "v=99"] {
            let response = post(
                &server,
                &format!("{version}&{config}"),
                &request_id,
                body.clone(),
            )
            .await?;
            assert_eq!(
                without_time(response),
                without_time(v2.clone()),
                "{version}&{config}"
            );
        }
    }

    let get = |version: &str| {
        reqwest::Client::new()
            .get(format!("http://{}/flags?{version}", server.addr))
            .header(request_id[0].0, request_id[0].1)
            .send()
    };
    let minimal_v2: Value = get("v=2").await?.json().await?;
    let minimal_v4: Value = get("v=4").await?.json().await?;
    assert_eq!(without_time(minimal_v4), without_time(minimal_v2));
    Ok(())
}

#[tokio::test]
async fn only_an_exact_v4_selects_the_v4_record() -> Result<()> {
    let (server, token) = server(true).await?;
    let body = json!({"token": token, "distinct_id": "ann"});
    let request_id = [("X-REQUEST-ID", "00000000-0000-4000-8000-000000000004")];
    let without_time = |mut response: Value| {
        response.as_object_mut().unwrap().remove("evaluatedAt");
        response
    };

    let v4 = post(&server, "v=4&config=false", &request_id, body.clone()).await?;
    assert_eq!(v4["flags"]["v2-true"]["value"], true);
    assert_eq!(v4["flags"]["v2-true"].get("enabled"), None);

    let v2 = without_time(post(&server, "v=2&config=false", &request_id, body.clone()).await?);
    for version in ["v=3", "v=5", "v=99", "v=9992", "v=17380067"] {
        let response = post(
            &server,
            &format!("{version}&config=false"),
            &request_id,
            body.clone(),
        )
        .await?;
        assert_eq!(without_time(response), v2, "{version}");
    }

    let v1 = without_time(post(&server, "v=1&config=false", &request_id, body.clone()).await?);
    for version in [
        "v=4.0",
        "v=4.5",
        "v=12345678901234567890123456789012345678901234",
        "v=abc",
    ] {
        let response = post(
            &server,
            &format!("{version}&config=false"),
            &request_id,
            body.clone(),
        )
        .await?;
        assert_eq!(without_time(response), v1, "{version}");
    }

    let decide_v4 = post(&server, "v=4&config=false", DECIDE, body).await?;
    assert_eq!(decide_v4["flags"]["v2-true"]["enabled"], true);
    assert_eq!(decide_v4["flags"]["v2-true"].get("value"), None);

    let get = |version: &str| {
        reqwest::Client::new()
            .get(format!("http://{}/flags?{version}", server.addr))
            .header(request_id[0].0, request_id[0].1)
            .send()
    };
    let minimal_v2: Value = get("v=2").await?.json().await?;
    for version in ["v=3", "v=5"] {
        let minimal: Value = get(version).await?.json().await?;
        assert_eq!(
            without_time(minimal),
            without_time(minimal_v2.clone()),
            "{version}"
        );
    }
    let minimal_v4: Value = get("v=4").await?.json().await?;
    wire::validate_v4(&minimal_v4).unwrap();
    Ok(())
}

/// Serves the success cases and checks their v4 and v2 records. Each identifier gets its own
/// server, because the experiment corpus picks identifiers per outcome. Returns the cases checked.
async fn check_on_the_wire(cases: Vec<Value>) -> Result<usize> {
    let cases: Vec<_> = cases
        .into_iter()
        .filter(|case| case["expected"]["status"] == "success")
        .collect();
    let mut identifiers: Vec<&str> = cases
        .iter()
        .map(|case| case["context"]["identifier"].as_str().unwrap())
        .collect();
    identifiers.sort_unstable();
    identifiers.dedup();
    let decoded = |payload: &Value| {
        payload
            .as_str()
            .map_or(Value::Null, |p| serde_json::from_str(p).unwrap())
    };
    let mut checked = 0;
    // Requests require a distinct ID; the empty-identifier rows run below the request path.
    for distinct_id in identifiers.into_iter().filter(|id| !id.is_empty()) {
        let group: Vec<_> = cases
            .iter()
            .filter(|case| case["context"]["identifier"] == distinct_id)
            .collect();
        let flags = group
            .iter()
            .zip(100..)
            .map(|(case, id)| corpus_flag(case, id));
        let (server, token) = serve(
            true,
            distinct_id,
            group[0]["context"]["properties"].clone(),
            flags.collect(),
        )
        .await?;
        let body = json!({"token": token, "distinct_id": distinct_id});
        let v4 = post(&server, "v=4&config=false", &[], body.clone()).await?;
        wire::validate_v4(&v4).unwrap_or_else(|error| panic!("{error:?}"));
        let v2 = post(&server, "v=2&config=false", &[], body).await?;
        for case in group {
            let key = case["id"].as_str().unwrap();
            let (expected, legacy) = (&case["expected"], &case["legacy"]);
            let record = &v4["flags"][key];
            assert_eq!(record["value"], expected["value"], "{key}");
            assert_eq!(record["reason"]["code"], expected["reason"], "{key}");
            assert_eq!(
                record["reason"]["condition_index"], expected["rule"]["index"],
                "{key}"
            );
            assert_eq!(
                record["metadata"].get("rule_type"),
                expected["rule"].get("rule_type"),
                "{key}"
            );
            assert_eq!(
                record["metadata"].get("rule_id"),
                expected["rule"].get("id"),
                "{key}"
            );
            assert_eq!(
                record["metadata"].get("variant_key"),
                expected.get("variant"),
                "{key}"
            );
            assert_eq!(record["metadata"]["payload"], Value::Null, "{key}");
            assert_eq!(record["metadata"]["has_experiment"], false, "{key}");
            for field in ["experiment_id", "holdout_id"] {
                assert_eq!(record["metadata"].get(field), None, "{key}");
            }
            let v2_record = &v2["flags"][key];
            assert_eq!(v2_record["enabled"], legacy["enabled"], "{key}");
            assert_eq!(v2_record["variant"], legacy["variant"], "{key}");
            assert_eq!(
                decoded(&v2_record["metadata"]["payload"]),
                legacy["payload"],
                "{key}"
            );
            checked += 1;
        }
    }
    Ok(checked)
}

#[tokio::test]
async fn v4_carries_every_typed_corpus_value_through_the_request_path() -> Result<()> {
    assert_eq!(check_on_the_wire(corpus::value_cases()).await?, 38);
    Ok(())
}

#[tokio::test]
async fn experiment_rules_carry_the_assigned_variant_and_no_experiment_identity_on_the_wire(
) -> Result<()> {
    let cases = corpus::experiment_cases()
        .into_iter()
        .filter(|case| case["family"] != "white_box")
        .collect();
    assert_eq!(check_on_the_wire(cases).await?, 47);
    Ok(())
}