Skip to content

Code graph and memory for your coding agents, in one Rust program. For Claude Code, Codex, Cursor and any MCP client.

93.7 KB
Raw Blame
use graphicle_memory::{
    ErrorKind, archived, community, dispatch, entities, lifecycle, packing, process, retrieval,
    temporal, text,
};
use serde_json::{Value, json};

#[test]
fn reciprocal_rank_fusion_has_explicit_scores_and_deterministic_ties() {
    let rankings = vec![vec!["b".into(), "a".into()], vec!["a".into(), "b".into()]];
    let result = retrieval::rrf(&rankings, 1.0, 0.0).unwrap();
    assert_eq!(result, vec![("a".into(), 1.5), ("b".into(), 1.5)]);
    assert!(retrieval::rrf(&rankings, 1.0, 1.6).unwrap().is_empty());
    for constant in [0.0, -1.0, f64::NAN, f64::INFINITY] {
        assert_eq!(
            retrieval::rrf(&rankings, constant, 0.0).unwrap_err().kind(),
            ErrorKind::InvalidInput
        );
    }
}

#[test]
fn lexical_retrieval_rejects_duplicate_identities_and_unknown_modes() {
    let row = json!({"identity":"one","title":"Authentication","text":"The token expires after ten minutes"});
    let mut request = json!({"scope":"native-contract-ranking","question":"token expires","rows":[row.clone(),row]});
    assert!(
        retrieval::rank(&request)
            .unwrap_err()
            .to_string()
            .contains("unique")
    );
    request["rows"] = json!([]);
    request["mode"] = json!("unsupported");
    assert!(retrieval::rank(&request).is_err());
    // The overview mode is not supported (unit `community-reports`): auto
    // mode ranks a
    // question that asks for an overview with the other modes' rules.
    let rows = json!([passage("one", "d1", "user: The service has three layers.")]);
    let overview = retrieval::rank(&json!({"scope":"native-contract-ranking",
        "question":"Give an overview of the architecture","rows":rows}))
    .unwrap();
    assert_eq!(overview["mode"], "balanced", "{overview}");
}

#[test]
fn diversity_selection_preserves_origin_caps_duplicate_suppression_and_need_coverage() {
    let rows: Vec<_> = [
        (
            "a",
            "source",
            "token rotation",
            "token rotation rollback policy common",
            "one",
        ),
        (
            "duplicate",
            "source",
            "token rotation",
            "token rotation rollback policy common",
            "two",
        ),
        ("b", "source", "token", "token expires common", "one"),
        (
            "c",
            "source",
            "rollback",
            "rollback recovery safety unique",
            "three",
        ),
        ("d", "source", "token", "token cache common", "one"),
        (
            "m",
            "memory",
            "rotation lesson",
            "rotation safely recovery rules",
            "memory",
        ),
    ]
    .into_iter()
    .map(|(identity, kind, title, body, origin)| {
        json!({
            "identity":identity,"kind":kind,"title":title,"text":body,"origin":origin,
            "source":{"path":format!("{identity}.py")},"memory":{"kind":"lesson"}
        })
    })
    .collect();
    let request = json!({"question":"token rotation rollback","scope":"diversity-contract",
        "rows":rows,"limit":4,"needs":["rotation rollback"],
        "rankings":{"vector":["a","duplicate","b","c","d","m"]}});
    let result = retrieval::rank(&request).unwrap();
    assert_eq!(result["sources"], json!(["a", "c", "b"]));
    assert_eq!(result["memories"], json!(["m"]));
    assert_eq!(result["decisions"], 4);
    assert_eq!(result["unresolved_needs"], json!([]));
    assert_eq!(result["complete_evidence_guaranteed"], false);
}

#[test]
fn contradictions_close_only_overlapping_older_edges_and_preserve_input() {
    let request = json!({"scope":"repo","recorded_at":"2026-09-23T10:00:00Z",
        "resolved_edge":{"scope":"repo","valid_at":"2026-09-20T00:00:00Z"},
        "candidates":[{"id":"old","scope":"repo","valid_at":"2026-09-01T00:00:00Z"},
        {"id":"expired","scope":"repo","valid_at":"2026-08-01T00:00:00Z","invalid_at":"2026-09-19T00:00:00Z"},
        {"id":"unknown","scope":"repo"}]});
    let original = request.clone();
    let result = temporal::contradictions(&request).unwrap();
    assert_eq!(request, original);
    assert_eq!(result["invalidated_edges"].as_array().unwrap().len(), 1);
    assert_eq!(
        result["invalidated_edges"][0]["invalid_at"],
        "2026-09-20T00:00:00Z"
    );
    assert_eq!(
        result["invalidated_edges"][0]["expired_at"],
        "2026-09-23T10:00:00Z"
    );
    assert_eq!(result["unresolved"][0]["id"], "unknown");
    assert_eq!(result["semantic_contradiction_checked"], false);
    assert_eq!(result["automatic_commit"], false);
}

#[test]
fn temporal_scope_mismatch_and_inverted_intervals_are_rejected() {
    let mut request = json!({"scope":"one","recorded_at":"2026-09-23T00:00:00Z","resolved_edge":{"scope":"two"},"candidates":[]});
    assert!(temporal::contradictions(&request).is_err());
    request["resolved_edge"] = json!({"scope":"one","valid_at":"2026-09-23T00:00:00Z","invalid_at":"2026-09-22T00:00:00Z"});
    assert!(temporal::contradictions(&request).is_err());
    assert!(temporal::instant("2026-09-23T10:00:00").is_err());
}

#[test]
fn history_distinguishes_knowledge_time_event_time_and_unknown_dates() {
    let request = json!({"known_at":"2026-09-22T23:00:00Z","event_at":"2026-09-20T00:00:00Z","records":[
        {"id":"fact","revision":1,"recorded_at":"2026-09-21T00:00:00Z","valid_at":"2026-09-01T00:00:00Z","state":"reviewed","value":"old"},
        {"id":"fact","revision":2,"recorded_at":"2026-09-23T00:00:00Z","valid_at":"2026-09-01T00:00:00Z","state":"reviewed","value":"future knowledge"},
        {"id":"undated","revision":1,"recorded_at":"2026-09-21T00:00:00Z","state":"reviewed"},
        {"id":"draft","revision":1,"recorded_at":"2026-09-21T00:00:00Z","state":"draft"}]});
    let result = temporal::history(&request).unwrap();
    assert_eq!(result["records"].as_array().unwrap().len(), 1);
    assert_eq!(result["records"][0]["value"], "old");
    assert_eq!(result["unknown_event_time"][0]["id"], "undated");
    assert_eq!(result["current_source_validity_not_implied"], true);
}

#[test]
fn dependency_cycles_missing_revisions_and_depth_limits_withhold_reuse() {
    let cyclic = json!({"root":"a","nodes":[{"id":"a","revision":1,"state":"reviewed","parents":{"b":1}},{"id":"b","revision":1,"state":"reviewed","parents":{"a":1}}]});
    let result = lifecycle::dependencies(&cyclic).unwrap();
    assert_eq!(result["stale"], true);
    assert!(
        result["reasons"]
            .as_array()
            .unwrap()
            .contains(&json!("dependency_cycle:a"))
    );
    assert_eq!(result["automatic_deletion"], false);
    let missing =
        lifecycle::dependencies(&json!({"root":"a","nodes":[{"id":"a","parents":{"b":2}}]}))
            .unwrap();
    assert_eq!(missing["reasons"], json!(["decision_changed:b"]));
    let nodes:Vec<_> = (0..103).map(|i|json!({"id":i.to_string(),"revision":1,"state":"reviewed","parents":{(i+1).to_string():1}})).collect();
    let result = lifecycle::dependencies(&json!({"root":"0","nodes":nodes})).unwrap();
    assert_eq!(result["complete"], false);
    assert!(
        result["reasons"]
            .as_array()
            .unwrap()
            .contains(&json!("dependency_depth_exceeded"))
    );
}

/// What a listing asks per record: a root carrying the record's parents and
/// reasons, walked alongside every record.
fn per_root(nodes: &[Value], node: &Value) -> Value {
    let mut all = vec![json!({"id":"root","revision":0,"state":"reviewed",
        "parents":node.get("parents").cloned().unwrap_or(json!({})),
        "reasons":node.get("reasons").cloned().unwrap_or(json!([]))})];
    all.extend(nodes.iter().cloned());
    lifecycle::dependencies(&json!({"root":"root","nodes":all})).unwrap()["reasons"].clone()
}

#[test]
fn batched_dependencies_equal_one_walk_per_record() {
    // Deterministic pseudo-random graphs: shared ancestry, cycles, changed
    // revisions, unreviewed parents, missing parents and the reserved id.
    let mut seed: u64 = 0x5eed;
    let mut next = |bound: u64| {
        seed = seed
            .wrapping_mul(6364136223846793005)
            .wrapping_add(1442695040888963407);
        (seed >> 33) % bound
    };
    for round in 0..60 {
        let count = 1 + next(40) as usize;
        let nodes: Vec<Value> = (0..count)
            .map(|i| {
                let mut parents = serde_json::Map::new();
                for _ in 0..next(4) {
                    let target = match next(20) {
                        0 => "missing".to_string(),
                        1 if round % 7 == 0 => "root".to_string(),
                        // Mostly older records, sometimes newer ones (cycles).
                        _ if next(5) == 0 => format!("m{}", next(count as u64)),
                        _ => format!("m{}", next(i.max(1) as u64)),
                    };
                    let revision = if next(8) == 0 { 2 } else { 1 };
                    parents.insert(target, json!(revision));
                }
                let reasons: Vec<String> = (0..next(3))
                    .map(|r| format!("source_changed_or_unavailable:s{}", next(5) + r))
                    .collect();
                json!({"id":format!("m{i}"),"revision":1,
                    "state":if next(6) == 0 {"draft"} else {"reviewed"},
                    "parents":parents,"reasons":reasons})
            })
            .collect();
        let batched = lifecycle::dependencies_all(&json!({"nodes":nodes})).unwrap();
        for node in &nodes {
            let id = node["id"].as_str().unwrap();
            assert_eq!(
                batched["reasons"][id],
                per_root(&nodes, node),
                "round {round}, {id}: {nodes:#?}"
            );
        }
    }
    // A chain deeper than the walk limit reports the same early stop.
    let chain: Vec<Value> = (0..130)
        .map(|i| {
            json!({"id":format!("c{i}"),"revision":1,"state":"reviewed",
            "parents":if i + 1 < 130 {json!({format!("c{}", i + 1):1})} else {json!({})},
            "reasons":if i == 129 {json!(["source_changed_or_unavailable:deep"])} else {json!([])}})
        })
        .collect();
    let batched = lifecycle::dependencies_all(&json!({"nodes":chain})).unwrap();
    for node in &chain {
        let id = node["id"].as_str().unwrap();
        assert_eq!(batched["reasons"][id], per_root(&chain, node), "{id}");
    }
    assert_eq!(
        batched["reasons"]["c0"],
        json!(["dependency_depth_exceeded"])
    );
    assert_eq!(
        batched["reasons"]["c100"],
        json!(["source_changed_or_unavailable:deep"])
    );
    assert!(
        lifecycle::dependencies_all(&json!({"nodes":[{"id":"root"}]})).is_err(),
        "the root id is reserved"
    );
}

#[test]
fn retractions_prune_grounding_without_mutating_history() {
    let request = json!({"live_ids":["live"],"based_on":{"world":[{"id":"live"},{"id":"gone"}],"observation":[{"id":"gone"}]}});
    let original = request.clone();
    let result = lifecycle::retractions(&request).unwrap();
    assert_eq!(request, original);
    assert_eq!(result["pruned"]["world"], json!([{"id":"live"}]));
    assert_eq!(result["unresolvable"], false);
    assert_eq!(result["requires_review"], true);
    assert_eq!(result["original_preserved"], true);
}

#[test]
fn readiness_uses_current_snapshot_and_cost_without_rescheduling_running_jobs() {
    let result = lifecycle::readiness(&json!({"snapshot":"new","budget":3,"jobs":[
        {"id":"ready","status":"ready","snapshot":"new","cost":1},
        {"id":"stale","status":"ready","snapshot":"old","priority":9,"cost":2},
        {"id":"busy","status":"running","priority":10,"cost":1},
        {"id":"cheap","status":"queued","priority":1,"cost":1}]}))
    .unwrap();
    assert_eq!(result["ready"], json!(["ready"]));
    assert_eq!(result["suggested"], json!(["stale", "cheap"]));
    assert_eq!(result["cost"], 3);
    assert_eq!(result["automatic_execution"], false);
}

#[test]
fn community_weight_counts_distinct_sources_instead_of_mentions() {
    let result = community::weights(
        &json!({"reports":[{"community_id":"a"},{"community_id":"b"}],"entities":[
        {"community_ids":["a"],"text_unit_ids":["one","one","two"]},
        {"community_ids":["a","b"],"text_unit_ids":["one"]}]}),
    )
    .unwrap();
    assert_eq!(result["reports"][0]["attributes"]["occurrence"], 1.0);
    assert_eq!(result["reports"][1]["attributes"]["occurrence"], 0.5);
}

#[test]
fn community_context_requires_sources_and_respects_final_token_budget() {
    let result = community::context(&json!({"entities":[],"max_tokens":128,"reports":[
        {"id":"missing","community_id":"a","title":"No provenance","summary":"Must not be included","source_ids":[]},
        {"id":"large","community_id":"b","title":"Too long","summary":"long report ".repeat(500),"source_ids":["one"]},
        {"id":"small","community_id":"c","title":"Small","summary":"Supported statement","source_ids":["two"]}]})).unwrap();
    assert_eq!(result["reports"].as_array().unwrap().len(), 1);
    assert_eq!(result["reports"][0]["id"], "small");
    assert!(result["tokens"].as_u64().unwrap() <= 128);
    assert_eq!(result["model_calls"], 0);
}

#[test]
fn packed_utf8_excerpts_have_exact_byte_offsets_and_never_cut_procedures() {
    let body = "世界 🌍 authentication procedure\n".repeat(1200);
    let result = packing::pack(&json!({"repo":{"id":"r","name":"Repo","snapshot":"one"},"question":"authentication","hits":[
        {"identity":"source","text":body,"source":{"source_kind":"repository_file","path":"readme.md","id":"s","sha256":"hash","byte_start":9,"line":1}}],
        "memories":[{"memory":{"id":"p","kind":"procedure","title":"All steps","body":"required step ".repeat(3000)}}],"max_tokens":768,"max_chars":4000})).unwrap();
    let markdown = result["markdown"].as_str().unwrap();
    assert_eq!(
        result["tokens"].as_u64().unwrap() as usize,
        text::count(markdown)
    );
    assert_eq!(
        result["characters"].as_u64().unwrap() as usize,
        markdown.chars().count()
    );
    assert!(text::count(markdown) <= 768);
    assert!(markdown.chars().count() <= 4000);
    assert!(result["memories"].as_array().unwrap().is_empty());
    assert_eq!(result["omitted"]["memories"], 1);
    let source = &result["sources"][0];
    let end = source["byte_end"].as_u64().unwrap() as usize - 9;
    assert!(body.is_char_boundary(end));
    assert!(markdown.contains(&body[..end]));
    assert_eq!(source["next_byte"], source["byte_end"]);
    assert_eq!(source["complete"], false);
}

#[test]
fn dated_excerpts_cite_their_change_and_age_marks_and_keep_them_when_focused() {
    let mut body = "unrelated words\n\n".repeat(400);
    body.push_str("The build host is ci-runner-07.\n");
    let request = |focus: bool| {
        let mut request = json!({"repo":{"id":"r","name":"Repo","snapshot":"one"},"question":"which build host","hits":[
            {"identity":"a","text":body,"source":{"source_kind":"repository_file","path":"notes/node.md","id":"file:notes/node.md",
                "sha256":"hash","byte_start":0,"line":1,"changed":"2026-10-15 06:52 UTC",
                "age_marks":["may be stale: src/sync.rs (defines `sync_all`) changed 2026-10-15 07:00 UTC"]}},
            {"identity":"b","text":"plain text\n","source":{"source_kind":"repository_file","path":"plain.md","id":"file:plain.md",
                "sha256":"hash2","byte_start":0,"line":1}}],
            "memories":[],"max_tokens":3000,"max_chars":6000});
        if focus {
            request["pack_focus"] = json!(true);
        }
        request
    };
    for focus in [false, true] {
        let result = packing::pack(&request(focus)).unwrap();
        let markdown = result["markdown"].as_str().unwrap();
        let line = result["sources"][0]["line"].as_u64().unwrap();
        assert!(markdown.contains(&format!(
            "\nSource [repository_file]: notes/node.md:{line} (changed 2026-10-15 06:52 UTC)\nNote: may be stale: src/sync.rs (defines `sync_all`) changed 2026-10-15 07:00 UTC\n"
        )));
        // An undated source is cited as before.
        assert!(markdown.contains("\nSource [repository_file]: plain.md:1\nID: file:plain.md\n"));
        assert_eq!(result["sources"][0]["changed"], "2026-10-15 06:52 UTC");
        assert_eq!(focus, line > 1);
    }
}

#[test]
fn abi_and_in_process_boundaries_reject_malformed_input_without_panics() {
    for bytes in [
        b"not json".as_slice(),
        b"{}",
        b"{\"abi\":2,\"operation\":\"info\"}",
    ] {
        let response: Value = serde_json::from_slice(&process(bytes)).unwrap();
        assert_eq!(response["ok"], false);
    }
    let response: Value =
        serde_json::from_slice(&process(b"{\"abi\":1,\"operation\":\"info\"}")).unwrap();
    assert_eq!(response["ok"], true);
    for value in [Value::Null, json!([]), json!({"operation":"absent"})] {
        let result = std::panic::catch_unwind(|| dispatch(value)).unwrap();
        assert!(result.is_err());
        assert_ne!(result.unwrap_err().kind(), ErrorKind::Internal);
    }
    let result =
        dispatch(json!({"operation":"graph.info","unused":"x".repeat(8*1024*1024)})).unwrap();
    assert_eq!(result["implementation"], "rust");
}

fn named(question: &str) -> Vec<(&'static str, String)> {
    entities::terms(question)
        .into_iter()
        .map(|t| (t.kind, t.text))
        .collect()
}

#[test]
fn question_terms_are_names_quotes_identifiers_and_quantities() {
    assert_eq!(
        named("How many fish are in my 30-gallon tank?"),
        [("quantity", "30-gallon".to_string())]
    );
    // mem0 drops candidates of two characters or fewer ("Q3").
    assert_eq!(
        named("What did my manager at NovaTech say about the Q3 launch?"),
        [("proper", "NovaTech".to_string())]
    );
    assert_eq!(
        named("Which cafe in Shinjuku did I like, and what did Sarah's sister recommend?"),
        [
            ("proper", "Shinjuku".to_string()),
            ("proper", "Sarah".to_string())
        ]
    );
    // Connectors join names; calendar words and sentence openers are not names.
    assert_eq!(
        named("Remind me what I bought at the Museum of Modern Art last Saturday in March."),
        [("proper", "Museum of Modern Art".to_string())]
    );
    // As upstream, a quoted phrase keeps the names inside it.
    assert_eq!(
        named(
            r#"What did I say about "The Night Circus" and config.yaml in 2023, and the $250 fee?"#
        ),
        [
            ("proper", "Night Circus".to_string()),
            ("identifier", "config.yaml".to_string()),
            ("quantity", "2023".to_string()),
            ("quantity", "$250".to_string()),
            ("quoted", "The Night Circus".to_string()),
        ]
    );
    assert!(named("What is my favourite colour?").is_empty());
    assert!(named("How long have I been running on weekends?").is_empty());
}

#[test]
fn absent_terms_occur_in_no_source_under_any_spacing_case_or_plural() {
    let terms = entities::terms(
        "How many fish are in my 30-gallon tank from Aqua World and \"Blue Lagoon\"?",
    );
    let sources = [
        "user: I set up a 20 gallon tank from AQUA world's shop.",
        "user: We went to the blue lagoons.",
    ];
    let missing: Vec<_> = entities::absent(&terms, sources)
        .into_iter()
        .map(|t| t.text.clone())
        .collect();
    assert_eq!(missing, ["30-gallon"]);
    let later = ["user: Upgraded to a 30 gallons setup."];
    assert!(entities::absent(&terms[..1], later).is_empty());
    let result = dispatch(json!({"operation":"entities.terms","text":"Did I visit Harajuku with Priya?","sources":["user: Priya and I went to Shinjuku."]})).unwrap();
    assert_eq!(
        result["terms"],
        json!([{"kind":"proper","text":"Harajuku"},{"kind":"proper","text":"Priya"}])
    );
    assert_eq!(
        result["absent"],
        json!([{"kind":"proper","text":"Harajuku"}])
    );
}

#[test]
fn question_periods_resolve_against_the_date_asked() {
    let asked = chrono::NaiveDate::from_ymd_opt(2023, 5, 30).unwrap(); // a Tuesday
    let period = |question: &str| {
        temporal::window(question, asked).map(|(a, b, rule)| (a.to_string(), b.to_string(), rule))
    };
    let expect = |a: &str, b: &str, rule: &'static str| Some((a.to_string(), b.to_string(), rule));
    assert_eq!(
        period("How many times did I bake in the past two weeks?"),
        expect("2023-05-16", "2023-05-30", "past_count")
    );
    assert_eq!(
        period("What did I buy 10 days ago?"),
        expect("2023-05-19", "2023-05-21", "count_ago")
    );
    assert_eq!(
        period("Which event was two weeks ago?"),
        expect("2023-05-13", "2023-05-19", "count_ago")
    );
    assert_eq!(
        period("What certification did I complete last month?"),
        expect("2023-04-01", "2023-04-30", "last_unit")
    );
    assert_eq!(
        period("Where was I last week?"),
        expect("2023-05-22", "2023-05-28", "last_unit")
    );
    assert_eq!(
        period("What game did I beat last weekend?"),
        expect("2023-05-27", "2023-05-28", "last_weekend")
    );
    assert_eq!(
        period("Who did I meet last Tuesday?"),
        expect("2023-05-23", "2023-05-23", "last_weekday")
    );
    assert_eq!(
        period("What did I cook a couple of days ago?"),
        expect("2023-05-27", "2023-05-29", "couple_ago")
    );
    assert_eq!(
        period("How much did I earn this month?"),
        expect("2023-05-01", "2023-05-30", "this_unit")
    );
    assert_eq!(
        period("Which museums did I visit in December?"),
        expect("2022-12-01", "2022-12-31", "in_month")
    );
    assert_eq!(
        period("What did I do in March 2023?"),
        expect("2023-03-01", "2023-03-31", "month_year")
    );
    assert_eq!(
        period("Where did I travel in 2021?"),
        expect("2021-01-01", "2021-12-31", "in_year")
    );
    // No period named, a day-month date, or a port number: no window.
    assert_eq!(period("How many days ago did I buy the lamp?"), None);
    assert_eq!(period("What happened on 13 July 2023?"), None);
    assert_eq!(period("Is the server listening in 8080?"), None);
    assert_eq!(
        period("Which did I start first, the fence or the barn?"),
        None
    );
    // "Before today" bounds a span instead of naming the day asked.
    assert_eq!(
        period("What is the order of airlines I flew with, before today?"),
        None
    );
    assert_eq!(period("How many books had I read as of today?"), None);
    assert_eq!(
        period("What did I order for dinner today?"),
        expect("2023-05-30", "2023-05-30", "today")
    );
    // Written dates of sources: a recorded clock first, else the header line.
    assert_eq!(
        temporal::source_date(
            None,
            "Conversation date: 2023/05/20 (Sat) 02:21\n\nuser: hi"
        )
        .map(|d| d.to_string()),
        Some("2023-05-20".into())
    );
    assert_eq!(
        temporal::source_date(
            Some("2024-01-02T09:00:00Z"),
            "Conversation date: 2023/05/20"
        )
        .map(|d| d.to_string()),
        Some("2024-01-02".into())
    );
    assert_eq!(temporal::source_date(None, "# Notes\n\nNo date here"), None);
    let result = dispatch(json!({"operation":"temporal.window","question":"What did I buy last month?","asked":"2023/05/30 (Tue) 23:40"})).unwrap();
    assert_eq!(
        result,
        json!({"start":"2023-04-01","end":"2023-04-30","rule":"last_unit"})
    );
}

/// The previous `text::prefix`: every probe tokenizes the whole prefix.
fn prefix_by_full_counts(value: &str, tokens: usize, characters: usize) -> String {
    if tokens == 0 || characters == 0 {
        return String::new();
    }
    let boundaries: Vec<usize> = value
        .char_indices()
        .map(|(i, _)| i)
        .chain([value.len()])
        .take(characters.saturating_add(1))
        .collect();
    let last = *boundaries.last().unwrap_or(&0);
    if text::count(&value[..last]) <= tokens {
        return value[..last].to_owned();
    }
    let (mut low, mut high) = (0usize, boundaries.len() - 1);
    while low < high {
        let mid = low + (high - low).div_ceil(2);
        if text::count(&value[..boundaries[mid]]) <= tokens {
            low = mid;
        } else {
            high = mid - 1;
        }
    }
    while low > 0 && text::count(&value[..boundaries[low]]) > tokens {
        low -= 1;
    }
    value[..boundaries[low]].to_owned()
}

#[test]
fn line_reusing_prefix_counts_equal_whole_prefix_counts() {
    // Fragments chosen to stress cl100k's pre-tokenizer at line breaks:
    // indentation, CRLF, trailing whitespace, punctuation runs that absorb
    // breaks, contractions, digit runs, non-ASCII letters and spaces, emoji.
    const PARTS: &[&str] = &[
        "\n",
        "\r\n",
        "\n\n",
        " ",
        "  ",
        "\t",
        " \t ",
        "a",
        "Hello",
        "world",
        "don't",
        "'s",
        "'LL",
        "12",
        "34567",
        ".",
        "...",
        "!?",
        ",",
        "-",
        "_",
        "#",
        "(",
        ")",
        "\"",
        "é",
        "日本語",
        "🙂",
        "\u{a0}",
        "\u{2003}",
        "x1",
        "  \n",
        ".\n",
        "\n  ",
        "\n\t- item",
        "\n   \n",
        ":\r\n",
        "user: ",
        "assistant: ",
        "\u{0b}",
        "\u{0c}",
        "'",
    ];
    let mut seed: u64 = 0x005e_ed0f_11ce;
    let mut next = |bound: usize| {
        seed = seed
            .wrapping_mul(6364136223846793005)
            .wrapping_add(1442695040888963407);
        ((seed >> 33) % bound as u64) as usize
    };
    let mut texts: Vec<String> = vec![
        String::new(),
        "\n".into(),
        "a\n b".into(),
        "a\n\n  b".into(),
        "x  \n  \n  y".into(),
        "end with spaces   ".into(),
        "line\r\n\r\n  next".into(),
        "p.\n\n.q".into(),
    ];
    for _ in 0..90 {
        let length = 1 + next(90);
        texts.push((0..length).map(|_| PARTS[next(PARTS.len())]).collect());
    }
    for value in &texts {
        let ends: Vec<usize> = value
            .char_indices()
            .map(|(i, _)| i)
            .chain([value.len()])
            .collect();
        let direct: Vec<usize> = ends.iter().map(|&end| text::count(&value[..end])).collect();
        // Every candidate split (spacing 1), a middle spacing and the default.
        for segment in [1, 7, 64] {
            assert_eq!(
                text::prefix_counts(value, &ends, segment),
                direct,
                "{value:?} segment={segment}"
            );
        }
        for tokens in [1, 2, 3, 5, 8, 13, 40, 400] {
            for characters in [1, 7, 30, 10_000] {
                assert_eq!(
                    text::prefix(value, tokens, characters),
                    prefix_by_full_counts(value, tokens, characters),
                    "{value:?} tokens={tokens} characters={characters}"
                );
            }
        }
    }
}

#[test]
fn prefix_counts_split_long_runs_after_letters_and_digits() {
    // Runs with no space or line break: base64 and markup from a drawio SVG,
    // lock-file hashes and URLs, identifiers with digits and contractions.
    let texts = [
        "content=\"7Vttk6I4EP41Vu19kILwIn6cmR13r2r3yjqnau8+ZiAit0C4EGf0fv0lEBBCUFR0nLvRmlGaJgndTzpPd3BkPsSbLwSmq+uY/2j==\"",
        "&lt;mxfile&gt;&lt;diagram id=&quot;jyERGzDynktFHFRGN0ph&quot;&gt;",
        "sha256:6f1d6f2f4c2b9a0e1e5f0a6b7c8d9e0f1a2b3c4d5e6f708192a3b4c5d6e7f809",
        "https://files.pythonhosted.org/packages/a1/b2/abc123def-4.5.6-py3-none-any.whl#x=1",
        "don'tIt'sWe'LLx1y22z333w4444'd'S'Ve'RE9a",
        "ab12cd34ef56gh78ij90kl!?mn--op__qr..st,,uv((wx))yz",
    ];
    for value in texts {
        let value = value.repeat(3);
        let ends: Vec<usize> = value
            .char_indices()
            .map(|(i, _)| i)
            .chain([value.len()])
            .collect();
        let direct: Vec<usize> = ends.iter().map(|&end| text::count(&value[..end])).collect();
        for segment in [1, 7, 64] {
            assert_eq!(
                text::prefix_counts(&value, &ends, segment),
                direct,
                "{value:?} segment={segment}"
            );
        }
        for tokens in [1, 2, 3, 5, 8, 13, 40, 400] {
            for characters in [1, 7, 30, 10_000] {
                assert_eq!(
                    text::prefix(&value, tokens, characters),
                    prefix_by_full_counts(&value, tokens, characters),
                    "{value:?} tokens={tokens} characters={characters}"
                );
            }
        }
    }
}

#[test]
fn compact_word_features_answer_like_a_word_count_map() {
    use graphicle_memory::cache;
    let texts = [
        "The tank holds 30 fish; the fish are tetras. Tank water is 24C.",
        "Zürich trip: museum, museum café, and the Kunsthaus museum shop",
        "",
        "a b c a b a",
    ];
    for (i, value) in texts.iter().enumerate() {
        let features = cache::features("compact-test", value).unwrap();
        let mut expected: std::collections::BTreeMap<String, usize> = Default::default();
        for word in text::words(value) {
            *expected.entry(word).or_default() += 1;
        }
        assert_eq!(features.length, expected.values().sum::<usize>(), "{value}");
        assert_eq!(features.words.len(), expected.len(), "{value}");
        assert_eq!(
            features.words.keys().collect::<Vec<_>>(),
            expected.keys().map(String::as_str).collect::<Vec<_>>()
        );
        for (word, n) in &expected {
            assert!(features.words.contains_key(word));
            assert_eq!(features.words.get(word), *n, "{word}");
        }
        for absent in ["", "zzz", "tan", "tanks", "museums"] {
            assert_eq!(
                features.words.contains_key(absent),
                expected.contains_key(absent),
                "{absent}"
            );
        }
        for other in &texts[i..] {
            let b = cache::features("compact-test", other).unwrap();
            let manual = expected.keys().filter(|w| b.words.contains_key(w)).count();
            assert_eq!(features.words.common(&b.words), manual, "{value} / {other}");
        }
    }
    // A repeated lookup is a hit on the same shared value.
    let first = cache::features("compact-test", texts[0]).unwrap();
    let again = cache::features("compact-test", texts[0]).unwrap();
    assert!(std::sync::Arc::ptr_eq(&first, &again));
}

#[test]
fn chunks_keep_the_newest_documents_at_the_passage_bound_in_input_order() {
    let document = |id: &str, observed: Option<&str>, recorded: Option<&str>| {
        let content = format!("user: note {id} about the tank\n");
        json!({"id":id,"title":id,"sha256":text::sha256(content.as_bytes()),"content":content,
            "observed_at":observed,"recorded_at":recorded})
    };
    let documents = json!([
        document("b", Some("2026-01-02T00:00:00Z"), None),
        document("a", Some("2026-01-01T00:00:00Z"), None),
        // The source's own time wins over when it was observed.
        document(
            "c",
            Some("2026-01-01T00:00:00Z"),
            Some("2026-03-01T00:00:00+02:00")
        ),
        document("d", None, None),
    ]);
    let chunks = |prefer: Value, max: usize| {
        dispatch(
            json!({"operation":"chunks","scope":"prefer-test","documents":documents,
            "max_passages":max,"characters":2600,"tokens":640,"prefer":prefer}),
        )
    };
    let origins = |result: &Value| -> Vec<String> {
        result["rows"]
            .as_array()
            .unwrap()
            .iter()
            .map(|r| r["origin"].as_str().unwrap().to_owned())
            .collect()
    };
    // When everything fits, the preference changes nothing.
    let all = chunks(json!("newest"), 10).unwrap();
    assert_eq!(all["rows"], chunks(Value::Null, 10).unwrap()["rows"]);
    assert_eq!(origins(&all), ["b", "a", "c", "d"]);
    assert_eq!(all["passage_limit_reached"], false);
    assert_eq!(all["omitted_documents_are"], "none");
    let newest = chunks(json!("newest"), 2).unwrap();
    assert_eq!(origins(&newest), ["b", "c"]);
    assert_eq!(newest["passage_limit_reached"], true);
    assert_eq!(newest["omitted_documents"], 2);
    // a (January 1) and the undated d are older than both kept documents.
    assert_eq!(newest["omitted_documents_are"], "oldest");
    assert_eq!(newest["preference"], "newest");
    let input = chunks(json!("input"), 2).unwrap();
    assert_eq!(origins(&input), ["b", "a"]);
    assert_eq!(input["omitted_documents_are"], "last_listed");
    assert_eq!(
        chunks(json!("newest"), 0).unwrap()["omitted_documents_are"],
        "all"
    );
    assert!(chunks(json!("oldest"), 2).is_err());
    assert!(chunks(json!(1), 2).is_err());
}

/// P-CAP (EVAL_PLAN R5): files synced from one folder share one time and are
/// listed by path, so at the passage bound the documents of equal time are
/// taken from the end of the listing, the latest session files, and the
/// report says which ones were left out.
#[test]
fn equal_times_at_the_passage_bound_keep_the_latest_listed_documents() {
    let document = |id: &str, observed: Option<&str>| {
        let content = format!("user: session {id} about the tank\n");
        json!({"id":id,"title":id,"sha256":text::sha256(content.as_bytes()),"content":content,
            "observed_at":observed,"recorded_at":null})
    };
    let chunks = |documents: &Value, max: usize| {
        dispatch(
            json!({"operation":"chunks","scope":"tie-test","documents":documents,
            "max_passages":max,"characters":2600,"tokens":640,"prefer":"newest"}),
        )
        .unwrap()
    };
    let origins = |result: &Value| -> Vec<String> {
        result["rows"]
            .as_array()
            .unwrap()
            .iter()
            .map(|r| r["origin"].as_str().unwrap().to_owned())
            .collect()
    };
    let synced = Some("2026-02-01T00:00:00Z");
    let folder = json!([
        document("session-00001.txt", synced),
        document("session-00002.txt", synced),
        document("session-00003.txt", synced),
        document("session-00004.txt", synced),
    ]);
    let kept = chunks(&folder, 2);
    assert_eq!(origins(&kept), ["session-00003.txt", "session-00004.txt"]);
    assert_eq!(kept["omitted_documents"], 2);
    assert_eq!(kept["omitted_documents_are"], "earliest_listed");
    // Undated documents tie with each other the same way.
    let undated = json!([
        document("a", None),
        document("b", None),
        document("c", None)
    ]);
    let kept = chunks(&undated, 1);
    assert_eq!(origins(&kept), ["c"]);
    assert_eq!(kept["omitted_documents_are"], "earliest_listed");
    // A newer document is still taken before every equally old one, and the
    // ones left out are then older than everything kept.
    let mixed = json!([
        document("old-1", synced),
        document("old-2", synced),
        document("new", Some("2026-03-01T00:00:00Z")),
    ]);
    let kept = chunks(&mixed, 1);
    assert_eq!(origins(&kept), ["new"]);
    assert_eq!(kept["omitted_documents_are"], "oldest");
    let kept = chunks(&mixed, 2);
    assert_eq!(origins(&kept), ["old-2", "new"]);
    assert_eq!(kept["omitted_documents_are"], "earliest_listed");
}

/// P-CAP: one request may ask for 40,000 passages (it was 20,000), and the
/// passage-bound consumers follow the constant.
#[test]
fn a_request_may_ask_for_forty_thousand_passages() {
    use graphicle_memory::ingest;
    const { assert!(ingest::MAX_PASSAGES >= 40_000) };
    assert_eq!(
        retrieval::MAX_ROWS,
        ingest::MAX_PASSAGES + retrieval::MAX_RECALL_MEMORIES
    );
    let documents: Vec<Value> = (0..ingest::MAX_PASSAGES + 2)
        .map(|i| {
            let content = format!("note {i}");
            json!({"id":format!("n{i:05}"),"title":"t","sha256":text::sha256(content.as_bytes()),
                "content":content,"observed_at":"2026-01-01T00:00:00Z"})
        })
        .collect();
    let request = |max: usize| {
        json!({"operation":"chunks","scope":"cap-test","documents":documents,
        "max_passages":max,"characters":2600,"tokens":640,"prefer":"newest"})
    };
    let result = dispatch(request(ingest::MAX_PASSAGES)).unwrap();
    let rows = result["rows"].as_array().unwrap();
    assert_eq!(rows.len(), ingest::MAX_PASSAGES);
    // The two earliest listed of the equally dated notes are the ones left out.
    assert_eq!(rows[0]["origin"], "n00002");
    assert_eq!(result["omitted_documents"], 2);
    assert_eq!(result["omitted_documents_are"], "earliest_listed");
    assert!(
        dispatch(request(ingest::MAX_PASSAGES + 1))
            .unwrap_err()
            .to_string()
            .contains("supported range")
    );
}

#[test]
fn chunks_and_dependency_passes_accept_past_the_former_bounds() {
    use graphicle_memory::ingest;
    let documents: Vec<Value> = (0..6000)
        .map(|i| {
            let content = format!("note {i}");
            json!({"id":format!("n{i}"),"title":"t","sha256":text::sha256(content.as_bytes()),"content":content})
        })
        .collect();
    let result = dispatch(
        json!({"operation":"chunks","scope":"bounds-test","documents":documents,
        "max_passages":ingest::MAX_PASSAGES,"characters":2600,"tokens":640}),
    )
    .unwrap();
    assert_eq!(result["rows"].as_array().unwrap().len(), 6000);
    assert!(
        result.get("document_limit_reached").is_none(),
        "{}",
        result["omitted_documents"]
    );
    // Past the document bound the request degrades instead of
    // failing. The documents taken last are left out unread (a malformed one
    // there is never read) and reported; the rest are chunked as before.
    let mut too_many: Vec<Value> = (0..ingest::MAX_DOCUMENTS)
        .map(|i| {
            let content = format!("note {i}");
            json!({"id":format!("n{i}"),"title":"t","sha256":text::sha256(content.as_bytes()),"content":content})
        })
        .collect();
    too_many.push(json!({}));
    let degraded = dispatch(
        json!({"operation":"chunks","scope":"bounds-test","documents":too_many,
        "max_passages":10,"characters":2600,"tokens":640}),
    )
    .unwrap();
    let rows = degraded["rows"].as_array().unwrap();
    assert_eq!(rows.len(), 10);
    assert_eq!(rows[0]["origin"], "n0");
    assert_eq!(degraded["document_limit_reached"], true);
    assert_eq!(degraded["omitted_by_document_limit"], 1);
    assert_eq!(
        degraded["omitted_documents"],
        ingest::MAX_DOCUMENTS + 1 - 10
    );
    assert_eq!(degraded["omitted_documents_are"], "last_listed");
    assert_eq!(degraded["passage_limit_reached"], true);
    // Every memory of a repository (past the former 5,000 and 10,000) in one pass.
    let nodes: Vec<Value> = (0..12_000)
        .map(|i| json!({"id":format!("m{i}"),"revision":1,"state":"reviewed","parents":{},"reasons":[]}))
        .collect();
    let all = lifecycle::dependencies_all(&json!({"nodes":nodes})).unwrap();
    assert_eq!(all["reasons"].as_object().unwrap().len(), 12_000);
    let mut rooted = nodes.clone();
    rooted.push(json!({"id":"root","revision":0,"state":"reviewed","parents":{},"reasons":[]}));
    assert!(lifecycle::dependencies(&json!({"root":"root","nodes":rooted})).is_ok());
    let over = vec![json!({"id":"x"}); lifecycle::MAX_MEMORIES + 1];
    assert!(lifecycle::dependencies_all(&json!({"nodes":over})).is_err());
}

#[test]
fn section_anchors_resolve_batch_headings_slugs_and_headings_and_blocks_report_their_section() {
    let split =
        dispatch(json!({"operation":"document.split","markdown":"# A\n\nText.\n"})).unwrap();
    let delta = |document: &Value, operations: Value| {
        dispatch(json!({"operation":"document.delta","document":document,"operations":operations}))
            .unwrap()
    };
    let ids = |result: &Value| -> Vec<String> {
        result["document"]["sections"]
            .as_array()
            .unwrap()
            .iter()
            .map(|s| s["id"].as_str().unwrap().to_owned())
            .collect()
    };
    let chained = delta(
        &split["document"],
        json!([
            {"op":"add_section","heading":"A","blocks":["Second A."]},
            // "A" is no section id; it is the heading of the section just added.
            {"op":"add_section","heading":"B","blocks":[],"after_section_id":"A"},
            {"op":"add_section","heading":"C","blocks":[],"new_id":"custom","after_section_id":"a"},
            {"op":"add_section","heading":"D","blocks":[],"after_section_id":"custom"},
            {"op":"add_section","heading":"E","blocks":[],"after_section_id":"Missing Heading"}
        ]),
    );
    assert_eq!(ids(&chained), ["a", "custom", "d", "a-2", "b"], "{chained}");
    assert_eq!(chained["skipped"].as_array().unwrap().len(), 1);
    assert_eq!(chained["skipped"][0]["index"], 4);
    assert_eq!(
        chained["skipped"][0]["reason"],
        "unknown after_section_id: Missing Heading"
    );
    // Slug and plain-heading fallbacks for existing sections; exact ids win.
    let document = json!({"version":2,"sections":[
        {"id":"team-plans","heading":"Team plans","level":2,"blocks":[{"id":"x","text":"One."}]},
        {"id":"s-1","heading":"Budget Notes","level":2,"blocks":[{"id":"y","text":"Two."}]},
        {"id":"budget-notes","heading":"Other","level":2,"blocks":[]}]});
    let fallbacks = delta(
        &document,
        json!([
            {"op":"add_section","heading":"After plans","blocks":[],"after_section_id":"Team Plans!"},
            {"op":"add_section","heading":"After s-1","blocks":[],"after_section_id":"s-1"},
            // The slug `budget-notes` is an exact id and wins over the heading.
            {"op":"add_section","heading":"After budget","blocks":[],"after_section_id":"Budget Notes"}
        ]),
    );
    assert_eq!(
        ids(&fallbacks),
        [
            "team-plans",
            "after-plans",
            "s-1",
            "after-s-1",
            "budget-notes",
            "after-budget"
        ],
        "{fallbacks}"
    );
    let heading_only = delta(
        &json!({"version":2,"sections":[{"id":"s-9","heading":"Budget Notes","level":2,"blocks":[]}]}),
        json!([{"op":"add_section","heading":"Next","blocks":[],"after_section_id":"Budget Notes"}]),
    );
    assert_eq!(ids(&heading_only), ["s-9", "next"]);
    let blocks = delta(
        &document,
        json!([
            {"op":"replace_block","section_id":"team-plans","block_id":"y","text":"Moved?"},
            {"op":"remove_block","section_id":"team-plans","block_id":"zzz"},
            {"op":"append_block","section_id":"nope","text":"x"}
        ]),
    );
    let reasons: Vec<&str> = blocks["skipped"]
        .as_array()
        .unwrap()
        .iter()
        .map(|s| s["reason"].as_str().unwrap())
        .collect();
    assert_eq!(
        reasons,
        [
            "block_id y belongs to section s-1, not team-plans",
            "unknown block_id: zzz",
            "unknown section_id: nope"
        ]
    );
    assert_eq!(blocks["changed"], false);
}

fn passage(identity: &str, origin: &str, text: &str) -> Value {
    json!({"identity":identity,"kind":"source","origin":origin,"title":"","text":text,
        "source":{"id":origin,"byte_start":0,"byte_end":text.len()}})
}

/// Unsupported `rank` options (src/archived.rs): turned on, each is refused;
/// at an inert value it changes nothing. Unsupported values of options that
/// stay are refused the same way.
#[test]
fn archived_rank_options_are_refused_and_their_inert_values_change_nothing() {
    let mut rows = vec![passage(
        "ship",
        "d1",
        "user: The release ships on Friday after the canary.",
    )];
    for i in 0..6 {
        rows.push(passage(
            &format!("x{i}"),
            &format!("n{i}"),
            &format!("user: Note {i} about compost bins in the garden."),
        ));
    }
    let base = json!({"scope":"archived-options","question":"When does the release ship?",
        "rows":rows,"limit":3,"debug_fused":3});
    let plain = retrieval::rank(&base).unwrap();
    assert_eq!(plain["sources"][0], "ship", "{plain}");
    assert!(!archived::OPTIONS.is_empty());
    for entry in archived::OPTIONS {
        assert_eq!(
            entry.message(),
            format!("`{}` is not supported by Graphicle", entry.option)
        );
        let mut request = base.clone();
        request[entry.option] = json!(true);
        let error = retrieval::rank(&request).unwrap_err();
        assert_eq!(error.kind(), ErrorKind::InvalidInput, "{}", entry.option);
        assert_eq!(error.to_string(), entry.message());
        for value in [Value::Null, json!(false), json!(0), json!({})] {
            if !(entry.inert)(&value) {
                continue;
            }
            request[entry.option] = value.clone();
            let output = retrieval::rank(&request).unwrap();
            assert_eq!(output["fused"], plain["fused"], "{}: {value}", entry.option);
            assert_eq!(
                output["sources"], plain["sources"],
                "{}: {value}",
                entry.option
            );
        }
    }
    for entry in archived::VALUES {
        let mut request = base.clone();
        request[entry.option] = json!(entry.value);
        let error = retrieval::rank(&request).unwrap_err();
        assert_eq!(error.kind(), ErrorKind::InvalidInput, "{}", entry.value);
        assert_eq!(error.to_string(), entry.message());
    }
}

#[test]
fn weighted_fusion_equals_plain_fusion_at_unit_weights_and_reorders_otherwise() {
    let rankings = vec![
        vec!["a".to_string(), "b".into()],
        vec!["b".to_string(), "a".into()],
    ];
    assert_eq!(
        retrieval::rrf(&rankings, 61.0, 0.0).unwrap(),
        retrieval::rrf_weighted(&rankings, &[1.0, 1.0], 61.0, 0.0).unwrap()
    );
    let weighted = retrieval::rrf_weighted(&rankings, &[1.0, 2.0], 61.0, 0.0).unwrap();
    assert_eq!(weighted[0].0, "b");
    assert!(retrieval::rrf_weighted(&rankings, &[1.0], 61.0, 0.0).is_err());
    assert!(retrieval::rrf_weighted(&rankings, &[1.0, -1.0], 61.0, 0.0).is_err());
}

#[test]
fn stemming_matches_inflected_forms_only_when_requested() {
    let mut rows = vec![
        passage(
            "s1",
            "d1",
            "user: We planned camping trips with the kids twice last summer.",
        ),
        passage("s2", "d2", "user: My plan for the business trip is ready."),
    ];
    for i in 0..10 {
        rows.push(passage(
            &format!("x{i}"),
            &format!("n{i}"),
            &format!("user: Note {i} about compost bins in the garden."),
        ));
    }
    let base = json!({"scope":"stem-test","question":"When did we plan the camping trip?",
        "rows":rows,"limit":2,"debug_fused":2});
    // Exact words: "plan" and "trip" outweigh "camping" alone.
    let plain = retrieval::rank(&base).unwrap();
    assert!(plain.get("stemmer").is_none());
    assert_eq!(plain["fused"][0], "s2", "{plain}");
    // Stems: "planned"/"plan", "camping"/"camp" and "trips"/"trip" all match.
    let mut request = base.clone();
    request["stem"] = json!(true);
    let stemmed = retrieval::rank(&request).unwrap();
    assert_eq!(stemmed["stemmer"], "porter");
    assert_eq!(stemmed["fused"][0], "s1", "{stemmed}");
    request["stem"] = json!("yes");
    assert!(retrieval::rank(&request).is_err());
    request["stem"] = json!(false);
    assert_eq!(retrieval::rank(&request).unwrap()["fused"], plain["fused"]);
}

#[test]
fn query_variants_fuse_rephrasings_keywords_and_hypothetical_passages_as_weighted_arms() {
    let mut rows = vec![
        passage(
            "s1",
            "d1",
            "user: I adopted a rescue greyhound named Pixel in March.",
        ),
        passage(
            "s2",
            "d2",
            "user: The pet store sells dog food and leashes.",
        ),
    ];
    for i in 0..10 {
        rows.push(passage(
            &format!("x{i}"),
            &format!("n{i}"),
            &format!("user: Note {i} about compost bins in the garden."),
        ));
    }
    // The question shares no word with s1.
    let base = json!({"scope":"variants-test","question":"What is the name of my dog?",
        "rows":rows,"limit":2,"debug_fused":3});
    let plain = retrieval::rank(&base).unwrap();
    assert!(plain.get("query_variants").is_none());
    assert_eq!(plain["fused"], json!(["s2"]), "{plain}");
    let mut request = base.clone();
    request["query_variants"] = json!([
        {"text":"What did I call the greyhound I adopted?","weight":1.3},
        {"text":"greyhound name","kind":"keyword"},
        {"text":"  GREYHOUND name ","kind":"keyword","weight":0.5},
        {"text":"The user adopted a greyhound named Pixel.","kind":"passage","weight":0.7}
    ]);
    let fused = retrieval::rank(&request).unwrap();
    // Case-insensitive duplicates of one kind merge and add their weights.
    assert_eq!(
        fused["query_variants"],
        json!([
            {"arm":"lexical_q1","kind":"query","weight":1.3,"terms":5,"ranked":1},
            {"arm":"lexical_q2","kind":"keyword","weight":1.5,"terms":2,"ranked":1},
            {"arm":"lexical_q3","kind":"passage","weight":0.7,"terms":5,"ranked":12}
        ]),
        "{fused}"
    );
    assert_eq!(fused["fused"][0], "s1", "{fused}");
    let reasons = fused["reasons"]["s1"].as_array().unwrap();
    for arm in ["lexical_q1", "lexical_q2", "lexical_q3"] {
        assert!(reasons.contains(&json!(arm)), "{fused}");
    }
    // `arm_weights` overrides a variant's own weight.
    request["arm_weights"] = json!({"lexical_q1":0,"lexical_q2":0,"lexical_q3":0});
    assert_eq!(retrieval::rank(&request).unwrap()["fused"][0], "s2");
    // A caller's dense ranking for variant i counts the variant's weight times
    // the `semantic` arm's, unless `arm_weights` names it.
    let mut dense = base.clone();
    dense["query_variants"] = json!([{"text":"greyhound","weight":1.5}]);
    dense["rankings"] = json!({"semantic_q1":["x5"]});
    dense["arm_weights"] = json!({"semantic":0.5});
    let low = retrieval::rank(&dense).unwrap();
    assert_eq!(low["query_variants"][0]["dense"], "semantic_q1");
    assert_eq!(low["fused"], json!(["s1", "s2", "x5"]), "{low}");
    dense["arm_weights"] = json!({"semantic":2});
    assert_eq!(retrieval::rank(&dense).unwrap()["fused"][0], "x5");
    dense["arm_weights"] = json!({"semantic":2,"semantic_q1":0.5});
    assert_eq!(retrieval::rank(&dense).unwrap()["fused"][0], "s1");
    // Variant terms are stemmed with the question's.
    let mut stemmed = base.clone();
    stemmed["query_variants"] = json!([{"text":"adopting greyhounds"}]);
    assert_eq!(
        retrieval::rank(&stemmed).unwrap()["query_variants"][0]["ranked"],
        0
    );
    stemmed["stem"] = json!(true);
    assert_eq!(
        retrieval::rank(&stemmed).unwrap()["query_variants"][0]["ranked"],
        1
    );
    // Invalid variants are rejected; unset or empty variants change nothing.
    for invalid in [
        json!("greyhound"),
        json!([{"text":""}]),
        json!([{"text":"   "}]),
        json!([{"text":"x".repeat(4001)}]),
        json!([{"text":"greyhound","kind":"hyde"}]),
        json!([{"text":"greyhound","weight":4.5}]),
        json!([{"text":"greyhound","weight":-1}]),
        json!([{"text":"greyhound","note":1}]),
        json!(["greyhound"]),
        json!(
            (0..9)
                .map(|i| json!({"text":format!("q{i}")}))
                .collect::<Vec<_>>()
        ),
    ] {
        let mut request = base.clone();
        request["query_variants"] = invalid.clone();
        assert!(retrieval::rank(&request).is_err(), "{invalid}");
    }
    let mut empty = base.clone();
    empty["query_variants"] = json!([]);
    let empty = retrieval::rank(&empty).unwrap();
    assert!(empty.get("query_variants").is_none());
    assert_eq!(empty["fused"], plain["fused"]);
    assert_eq!(retrieval::rank(&base).unwrap()["fused"], plain["fused"]);
}

#[test]
fn the_speaker_arm_ranks_passages_by_what_the_user_said() {
    let mut rows = vec![
        passage(
            "told",
            "d1",
            "user: I finally bought running shoes from Brooks.\n\nassistant: Enjoy them!",
        ),
        passage(
            "advised",
            "d2",
            "user: Any tips for a beginner?\n\nassistant: Good running shoes matter. \
             Try running shoes from Brooks or Hoka; buy running shoes after a gait test.",
        ),
    ];
    for i in 0..10 {
        rows.push(passage(
            &format!("x{i}"),
            &format!("n{i}"),
            &format!("user: Note {i} about compost bins in the garden."),
        ));
    }
    let base = json!({"scope":"speaker-test","question":"Which brand of running shoes did I buy?",
        "rows":rows,"limit":2,"debug_fused":2});
    let plain = retrieval::rank(&base).unwrap();
    assert!(plain.get("speaker").is_none());
    assert_eq!(plain["fused"][0], "advised", "{plain}");
    let mut request = base.clone();
    request["speaker"] = json!(true);
    let keyed = retrieval::rank(&request).unwrap();
    assert_eq!(keyed["fused"][0], "told", "{keyed}");
    assert_eq!(keyed["speaker"]["keep"], json!(["user"]));
    assert_eq!(keyed["speaker"]["rows"], 12);
    assert_eq!(keyed["speaker"]["ranked"], 1);
    assert!(
        keyed["reasons"]["told"]
            .as_array()
            .unwrap()
            .contains(&json!("speaker"))
    );
    // `auto` keys the arm on the side the question asks about.
    let mut auto = base.clone();
    auto["speaker"] = json!("auto");
    let mine = retrieval::rank(&auto).unwrap();
    assert_eq!(mine["speaker"]["auto"], "user");
    assert_eq!(mine["speaker"]["keep"], json!(["user"]));
    assert_eq!(mine["fused"], keyed["fused"]);
    auto["question"] = json!("Which brand of running shoes did you recommend?");
    let theirs = retrieval::rank(&auto).unwrap();
    assert_eq!(theirs["speaker"]["keep"], json!(["assistant"]));
    assert_eq!(theirs["speaker"]["rows"], 2);
    assert_eq!(theirs["fused"][0], "advised", "{theirs}");
    auto["question"] = json!("In our previous chat, which running shoes came up?");
    assert_eq!(
        retrieval::rank(&auto).unwrap()["speaker"],
        json!({"auto":"none"})
    );
    for (question, side) in [
        ("What did you say about compost?", "assistant"),
        ("You told me to stretch first; why?", "assistant"),
        ("Remind me of your last answer about tents.", "assistant"),
        ("Can you recommend running shoes?", "user"),
        ("What did I tell you about my shoes?", "user"),
        ("We talked about shoes, right?", "none"),
    ] {
        assert_eq!(retrieval::auto_speaker(question), side, "{question}");
    }
    request["arm_weights"] = json!({"speaker":0.0});
    assert_eq!(retrieval::rank(&request).unwrap()["fused"], plain["fused"]);
    request["speaker"] = json!(false);
    assert_eq!(retrieval::rank(&request).unwrap()["fused"], plain["fused"]);
    for invalid in [
        json!("user"),
        json!({"keep":[]}),
        json!({"keep":["a:b"]}),
        json!({"keep":["user"],"other":1}),
    ] {
        request["speaker"] = invalid;
        assert!(retrieval::rank(&request).is_err());
    }
    let user = std::collections::BTreeSet::from(["user".to_owned()]);
    let transcript = "tail of an earlier turn\nUser: first line\nsecond line\nNote: still mine\n\
                      assistant: reply\nuser: again";
    assert_eq!(
        retrieval::speaker_text(transcript, &user),
        " first line\nsecond line\nNote: still mine\n again\n"
    );
    // A kept name ends at any other capitalized speaker name.
    let caroline = std::collections::BTreeSet::from(["caroline".to_owned()]);
    let chat =
        "Caroline: I went to a LGBTQ support group.\nMelanie: That's brave!\nCaroline: Thanks";
    assert_eq!(
        retrieval::speaker_text(chat, &caroline),
        " I went to a LGBTQ support group.\n Thanks\n"
    );
}

#[test]
fn key_expansion_merges_a_rows_abstractions_into_its_lexical_key() {
    let mut rows = vec![
        passage(
            "kept",
            "a",
            "user: Brought him home today. He chewed a slipper.",
        ),
        passage(
            "said",
            "b",
            "user: Which shelter has beagle puppies? I want to adopt one.",
        ),
    ];
    for i in 0..6 {
        rows.push(passage(
            &format!("x{i}"),
            &format!("n{i}"),
            &format!("user: Note {i} about compost bins in the garden."),
        ));
    }
    rows[0]["abstractions"] = json!(["The user adopted a beagle puppy named Max."]);
    let mut request = json!({"scope":"key-expansion-test",
        "question":"What name did I give the beagle puppy I adopted?",
        "rows":rows,"limit":2,"debug_fused":3});
    let plain = retrieval::rank(&request).unwrap();
    assert!(plain.get("key_expansion").is_none());
    assert!(
        !plain["fused"].as_array().unwrap().contains(&json!("kept")),
        "{plain}"
    );
    request["key_expansion"] = json!("merge");
    let merged = retrieval::rank(&request).unwrap();
    assert_eq!(merged["key_expansion"], json!({"mode":"merge","rows":1}));
    assert_eq!(merged["fused"][0], "kept", "{merged}");
    assert_eq!(merged["sources"][0], "kept");
    request["key_expansion"] = json!(false);
    assert_eq!(retrieval::rank(&request).unwrap()["fused"], plain["fused"]);
    for invalid in [json!(true), json!("replace")] {
        request["key_expansion"] = invalid;
        assert!(retrieval::rank(&request).is_err());
    }
    request["key_expansion"] = json!("merge");
    request["rows"][0]["abstractions"] = json!(["x".repeat(1001)]);
    assert!(retrieval::rank(&request).is_err());
}

#[test]
fn origin_quotas_let_the_best_origin_give_more_and_counting_keeps_the_limit() {
    let mut rows = Vec::new();
    for i in 0..5 {
        rows.push(passage(
            &format!("a{i}"),
            "A",
            &format!("user: The aquarium tank {i} holds neon tetras and guppies."),
        ));
    }
    for i in 0..3 {
        rows.push(passage(
            &format!("b{i}"),
            &format!("B{i}"),
            &format!("user: A tank {i} note about fish."),
        ));
    }
    let base = json!({"scope":"quota-test","question":"Which fish does the aquarium tank hold?","rows":rows,"limit":6});
    let count = |ranked: &Value| {
        ranked["sources"]
            .as_array()
            .unwrap()
            .iter()
            .filter(|s| s.as_str().unwrap().starts_with('a'))
            .count()
    };
    assert_eq!(count(&retrieval::rank(&base).unwrap()), 2);
    let mut quota = base.clone();
    quota["origin_quota"] = json!({"max": 3, "min": 1});
    let ranked = retrieval::rank(&quota).unwrap();
    assert!(count(&ranked) >= 3, "{ranked}");
    // A counting question is still reported as one; the selection keeps the
    // requested limit (the deeper read, `adaptive_depth`, is not supported).
    let mut counting = base.clone();
    counting["question"] = json!("How many fish does the aquarium tank hold?");
    counting["limit"] = json!(2);
    counting["debug_fused"] = json!(5);
    let counted = retrieval::rank(&counting).unwrap();
    assert_eq!(counted["evidence_strength"]["aggregate"], true);
    assert_eq!(counted["evidence_strength"]["limit"], 2);
    assert!(counted["evidence_strength"].get("adaptive_depth").is_none());
    assert_eq!(counted["sources"].as_array().unwrap().len(), 2, "{counted}");
    assert_eq!(counted["fused"].as_array().unwrap().len(), 5);
}

#[test]
fn an_origin_prior_scales_its_rows_before_the_order_is_taken() {
    let rows = vec![
        passage(
            "doc",
            "file:docs/parser.md",
            "The parser reads the header and the parser checks the header length.",
        ),
        passage(
            "code",
            "file:src/parser.rs",
            "fn read_header() { let length = 4; }",
        ),
    ];
    let base = json!({"scope":"prior-test","question":"parser header length","rows":rows,
        "limit":2,"debug_fused":2});
    let order = |ranked: &Value| ranked["fused"].clone();
    let plain = retrieval::rank(&base).unwrap();
    assert_eq!(order(&plain), json!(["doc", "code"]));
    assert!(plain.get("origin_prior").is_none());
    let mut prior = base.clone();
    prior["origin_prior"] = json!({"file:docs/parser.md": 0.1});
    let ranked = retrieval::rank(&prior).unwrap();
    assert_eq!(order(&ranked), json!(["code", "doc"]), "{ranked}");
    assert_eq!(ranked["sources"], json!(["code", "doc"]));
    assert_eq!(ranked["origin_prior"]["origins"], 1);
    // A weight of 1 changes nothing; weights outside 0 to 1 are refused.
    prior["origin_prior"] = json!({"file:docs/parser.md": 1.0});
    assert_eq!(order(&retrieval::rank(&prior).unwrap()), order(&plain));
    for bad in [
        json!({"file:docs/parser.md": 1.5}),
        json!({"x": "a"}),
        json!([0.5]),
    ] {
        prior["origin_prior"] = bad;
        assert!(retrieval::rank(&prior).is_err());
    }
}

#[test]
fn a_packet_lists_the_next_files_its_excerpts_do_not_quote() {
    let body = "fn parse() {}\n".repeat(400);
    let hit = json!({"identity":"a","text":body,"source":{"id":"file:src/a.rs","path":"src/a.rs",
        "source_kind":"repository_file","byte_start":0,"byte_end":body.len(),"line":1,"sha256":"x"}});
    let request = |next: Value| {
        json!({"repo":{"name":"r","snapshot":"s"},"question":"parse","hits":[hit.clone()],
            "memories":[],"max_tokens":3000,"max_chars":8400,"next_files":next})
    };
    let packed = packing::pack(&request(
        json!({"paths":["src/a.rs","src/b.rs","src/c.rs","src/d.rs"],"limit":2}),
    ))
    .unwrap();
    let markdown = packed["markdown"].as_str().unwrap();
    assert_eq!(packed["next_files"], json!(["src/b.rs", "src/c.rs"]));
    assert!(markdown.contains(packing::NEXT_FILES_HEADING), "{markdown}");
    assert!(markdown.contains("- src/b.rs\n- src/c.rs\n"), "{markdown}");
    assert!(!markdown.contains("- src/a.rs") && !markdown.contains("- src/d.rs"));
    // The list is after the excerpt and before the budget line, within the
    // budget, which the excerpt otherwise fills.
    let list = markdown.find("- src/b.rs").unwrap();
    assert!(markdown.find("fn parse").unwrap() < list);
    assert!(list < markdown.find("\nBudget: ").unwrap());
    assert!(packed["characters"].as_u64().unwrap() <= 8400);
    // Without the option, or with a limit of 0, no list.
    let plain = packing::pack(&request(Value::Null)).unwrap();
    assert!(plain.get("next_files").is_none());
    assert!(
        !plain["markdown"]
            .as_str()
            .unwrap()
            .contains(packing::NEXT_FILES_HEADING)
    );
    let none = packing::pack(&request(json!({"paths":["src/b.rs"],"limit":0}))).unwrap();
    assert!(none.get("next_files").is_none());
    for bad in [
        json!({"paths":["src/b.rs"],"limit":41}),
        json!({"paths":[1],"limit":2}),
        json!({"limit":2}),
    ] {
        assert!(packing::pack(&request(bad)).is_err());
    }
}

#[test]
fn an_overflowing_window_keeps_its_best_matching_paragraphs() {
    let filler = "assistant: ".to_string()
        + &"Here is a long general answer about many unrelated topics. ".repeat(40);
    let body = format!(
        "{filler}\n\n{filler}\n\nuser: My new aquarium holds six neon tetras.\n\nassistant: Nice.\n"
    );
    let hit = json!({"identity":"h","text":body,"source":{"id":"file:s.txt","path":"s.txt","source_kind":"repository_file",
        "byte_start":100,"byte_end":100 + body.len(),"line":7,"sha256":"x"}});
    let request = |focus: bool| {
        json!({"repo":{"name":"r","snapshot":"s"},"question":"How many neon tetras are in my aquarium?","hits":[hit.clone()],
            "memories":[],"max_tokens":600,"max_chars":4000,"pack_focus":focus})
    };
    let plain = packing::pack(&request(false)).unwrap();
    assert!(
        !plain["markdown"].as_str().unwrap().contains("neon tetras"),
        "the prefix cut drops the answer"
    );
    let focused = packing::pack(&request(true)).unwrap();
    assert!(
        focused["markdown"]
            .as_str()
            .unwrap()
            .contains("six neon tetras"),
        "{}",
        focused["markdown"]
    );
    let source = &focused["sources"][0];
    assert_eq!(source["focused"], true);
    let start = source["byte_start"].as_u64().unwrap() as usize;
    let end = source["byte_end"].as_u64().unwrap() as usize;
    assert!(start > 100 && end <= 100 + body.len());
    assert!(body[start - 100..end - 100].contains("six neon tetras"));
    assert_eq!(source["window_byte_start"], 100);
    assert_eq!(
        source["line"].as_u64().unwrap(),
        7 + body[..start - 100].matches('\n').count() as u64
    );
    assert!(focused["markdown"].as_str().unwrap().contains(&format!(
        "Source [repository_file]: s.txt:{}\n",
        source["line"]
    )));
    assert!(
        !focused["markdown"]
            .as_str()
            .unwrap()
            .contains("Original source header"),
        "a window after the document start has no opening header to keep"
    );
}

#[test]
fn a_focused_excerpt_keeps_the_document_header_it_skips() {
    let filler = "Caroline: ".to_string()
        + &"I talked about many unrelated weekend plans and errands. ".repeat(40);
    let header = "Conversation date: 1:56 pm on 8 May, 2023";
    let body = format!(
        "{header}\n\n{filler}\n\n{filler}\n\nMelanie: I adopted six neon tetras yesterday.\n\nCaroline: Nice.\n"
    );
    let hit = json!({"identity":"h","text":body,"source":{"id":"file:s.txt","path":"s.txt","source_kind":"repository_file",
        "byte_start":0,"byte_end":body.len(),"line":1,"sha256":"x"}});
    let request = json!({"repo":{"name":"r","snapshot":"s"},"question":"When did Melanie adopt the neon tetras?",
        "hits":[hit],"memories":[],"max_tokens":600,"max_chars":4000,"pack_focus":true});
    let packed = packing::pack(&request).unwrap();
    let markdown = packed["markdown"].as_str().unwrap();
    assert!(markdown.contains("six neon tetras"), "{markdown}");
    assert!(
        markdown.contains(&format!(
            "Original source header (quoted data): {}\n",
            serde_json::to_string(header).unwrap()
        )),
        "{markdown}"
    );
    let source = &packed["sources"][0];
    assert_eq!(source["focused"], true);
    assert_eq!(source["source_context"]["text"], header);
    assert_eq!(source["source_context"]["byte_end"], header.len());
    let start = source["byte_start"].as_u64().unwrap() as usize;
    let end = source["byte_end"].as_u64().unwrap() as usize;
    assert!(start > header.len() && body[start..end].contains("six neon tetras"));
    assert!(packed["tokens"].as_u64().unwrap() <= 600);
}

#[test]
fn focus_options_match_stems_and_bound_the_run() {
    let filler = |i: usize| {
        let speaker = if i.is_multiple_of(2) {
            "Caroline"
        } else {
            "Ana"
        };
        format!("{speaker}: Turn {i} is about errands and weekend plans of every sort.\n\n")
    };
    let answer = "Ana: I painted the harbour at dawn.\n\nCaroline: Lovely.\n\n";
    let body = (0..30).map(filler).collect::<String>()
        + answer
        + &(30..60).map(filler).collect::<String>();
    let other: String = (0..60)
        .map(|i| format!("Bo: Line {i} of a different talk about bread and buses.\n\n"))
        .collect();
    let hit = |n: usize, text: &str| {
        json!({"identity":format!("h{n}"),"text":text,"source":{"id":format!("file:s{n}.txt"),"path":format!("s{n}.txt"),
            "source_kind":"repository_file","byte_start":100,"byte_end":100 + text.len(),"line":7,"sha256":"x"}})
    };
    let ask = |question: &str, focus: Value| {
        json!({"repo":{"name":"r","snapshot":"s"},"question":question,"hits":[hit(1, &body), hit(2, &other)],
            "memories":[],"max_tokens":900,"max_chars":8000,"pack_focus":focus})
    };
    let request = |focus: Value| ask("Who paints harbours?", focus);
    let excerpt = |packed: &Value, i: usize, text: &str| {
        let source = &packed["sources"][i];
        let start = source["byte_start"].as_u64().unwrap() as usize - 100;
        let end = source["byte_end"].as_u64().unwrap() as usize - 100;
        text[start..end].to_owned()
    };
    let exact = packing::pack(&request(json!(true))).unwrap();
    assert!(
        !excerpt(&exact, 0, &body).contains("painted the harbour"),
        "no exact word matches, so the prefix stays"
    );
    let stemmed = packing::pack(&request(json!({"stem":true}))).unwrap();
    assert!(excerpt(&stemmed, 0, &body).contains("painted the harbour"));
    assert_eq!(stemmed["sources"][0]["focused"], true);
    let bounded = packing::pack(&request(json!({"stem":true,"context":1}))).unwrap();
    assert_eq!(excerpt(&bounded, 0, &body), filler(29) + answer);
    // "Ana" tags half the turns, so her other turns do not extend the run.
    let named = packing::pack(&ask(
        "When did Ana paint harbours?",
        json!({"stem":true,"context":1}),
    ))
    .unwrap();
    assert_eq!(excerpt(&named, 0, &body), filler(29) + answer);
    assert!(
        excerpt(&bounded, 1, &other).len() > excerpt(&stemmed, 1, &other).len(),
        "the share the bounded run leaves goes to the next source"
    );
    for packed in [&exact, &stemmed, &bounded] {
        assert!(packed["tokens"].as_u64().unwrap() <= 900);
    }
}

#[test]
fn compact_citations_spend_the_citation_lines_on_evidence() {
    let turns: String = (0..60)
        .map(|i| format!("user: Turn {i} mentions a garden visit and the weather.\n\n"))
        .collect();
    let hit = |n: usize, id: &str| {
        json!({"identity":format!("h{n}"),"text":turns,"source":{"id":id,"path":format!("s{n}.txt"),
            "source_kind":"repository_file","byte_start":0,"byte_end":turns.len(),"line":1,
            "sha256":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"},
            "next_byte":turns.len() + 10})
    };
    let request = |compact: bool| {
        json!({"repo":{"name":"r","snapshot":"s"},"question":"garden visit",
            "hits":[hit(1, "file:s1.txt"), hit(2, "memory:m2")],"memories":[],"max_tokens":500,"max_chars":4000,
            "compact_citations":compact})
    };
    let full = packing::pack(&request(false)).unwrap();
    let compact = packing::pack(&request(true)).unwrap();
    let full_md = full["markdown"].as_str().unwrap();
    let compact_md = compact["markdown"].as_str().unwrap();
    assert!(full_md.contains("SHA-256: 0123") && full_md.contains("ID: file:s1.txt"));
    assert!(full_md.contains("Continue at byte"), "{full_md}");
    assert!(!compact_md.contains("SHA-256"), "{compact_md}");
    assert!(!compact_md.contains("ID: file:s1.txt"), "{compact_md}");
    assert!(
        compact_md.contains("ID: memory:m2"),
        "an ID the path cannot rebuild is kept"
    );
    assert!(!compact_md.contains("Continue at byte"), "{compact_md}");
    assert!(compact_md.contains("Source [repository_file]: s1.txt:1\n"));
    assert_eq!(compact["compact_citations"], true);
    assert_eq!(full["compact_citations"], false);
    for (a, b) in full["sources"]
        .as_array()
        .unwrap()
        .iter()
        .zip(compact["sources"].as_array().unwrap())
    {
        assert_eq!(a["id"], b["id"]);
        assert_eq!(a["sha256"], b["sha256"]);
        assert_eq!(a["byte_start"], b["byte_start"]);
        assert!(b["byte_end"].as_u64() >= a["byte_end"].as_u64());
    }
    let quoted = |packet: &serde_json::Value| -> u64 {
        packet["sources"]
            .as_array()
            .unwrap()
            .iter()
            .map(|s| s["byte_end"].as_u64().unwrap() - s["byte_start"].as_u64().unwrap())
            .sum()
    };
    assert!(
        quoted(&compact) > quoted(&full),
        "the saved citation tokens buy more quoted evidence"
    );
    assert!(compact["tokens"].as_u64().unwrap() <= 500);
}

#[test]
fn packing_rejects_wrapping_source_offsets() {
    let mut request = json!({"repo":{"name":"r","snapshot":"s"},
        "hits":[{"identity":"h","text":"evidence","source":{"id":"file:s.txt","path":"s.txt",
            "source_kind":"repository_file","byte_start":u64::MAX,"line":1,"sha256":"x"}}],
        "memories":[],"max_tokens":600,"max_chars":4000});
    assert_eq!(
        packing::pack(&request).unwrap_err().kind(),
        ErrorKind::InvalidInput
    );
    request["hits"][0]["source"]["byte_start"] = json!(0);
    request["hits"][0]["source"]["line"] = json!(u64::MAX);
    request["hits"][0]["text"] = json!("unrelated information\n\n".repeat(200) + "neon tetras\n");
    request["pack_focus"] = json!(true);
    request["question"] = json!("neon tetras");
    assert_eq!(
        packing::pack(&request).unwrap_err().kind(),
        ErrorKind::InvalidInput
    );
}

fn conversation_hit(identity: &str, id: &str, date: &str, start: u64, text: &str) -> Value {
    json!({"identity":identity,"text":text,"source":{"id":id,"path":format!("{id}.md"),
        "source_kind":"conversation","byte_start":start,"byte_end":start + text.len() as u64,
        "line":1,"sha256":format!("{id}-hash"),"recorded_at":date}})
}

/// Hits marked `last` (the runtime's test files) are presented after the
/// others, each group in rank order, while the budget still goes by rank:
/// at a budget that leaves windows out, the packet holds exactly the
/// excerpts of the unmarked packet.
#[test]
fn hits_marked_last_follow_the_others_with_the_same_excerpts() {
    let hits: Vec<Value> = (0..12)
        .map(|n| {
            let path = if n % 3 == 0 {
                format!("tests/test_m{n}.py")
            } else {
                format!("src/m{n}.py")
            };
            let text = format!("def f{n}(value):\n    return value + {n}\n\n").repeat(60);
            json!({"identity":format!("h{n}"),"text":text,"source":{"id":format!("file:{path}"),
                "path":path,"source_kind":"repository_file","byte_start":0,"line":1,
                "sha256":format!("h{n}-hash")}})
        })
        .collect();
    let request = |marked: bool| {
        let mut hits = hits.clone();
        for hit in hits
            .iter_mut()
            .filter(|h| marked && s(h, "path").contains("tests/"))
        {
            hit["last"] = json!(true);
        }
        json!({"repo":{"id":"r","name":"Repo","snapshot":"one"},"question":"what does f1 return",
            "hits":hits,"memories":[],"max_tokens":1500,"max_chars":6000})
    };
    fn s<'a>(hit: &'a Value, key: &str) -> &'a str {
        hit["source"][key].as_str().unwrap()
    }
    let ranked = packing::pack(&request(false)).unwrap();
    let marked = packing::pack(&request(true)).unwrap();
    assert!(ranked.get("moved_last").is_none());
    assert!(
        ranked["omitted"]["sources"].as_u64().unwrap() > 0,
        "the budget leaves windows out: {}",
        ranked["omitted"]
    );
    let excerpts = |packet: &Value| {
        packet["sources"]
            .as_array()
            .unwrap()
            .iter()
            .map(|s| {
                format!(
                    "{}@{}-{}",
                    s["id"].as_str().unwrap(),
                    s["byte_start"],
                    s["byte_end"]
                )
            })
            .collect::<Vec<_>>()
    };
    let (ranked_excerpts, marked_excerpts) = (excerpts(&ranked), excerpts(&marked));
    assert!(ranked_excerpts.iter().any(|e| e.contains("tests/")));
    // The same excerpts, the test files after the others, each group in
    // rank order.
    let (mut others, tests): (Vec<String>, Vec<String>) = ranked_excerpts
        .iter()
        .cloned()
        .partition(|e| !e.contains("tests/"));
    let moved = ranked_excerpts
        .iter()
        .enumerate()
        .filter(|(i, e)| {
            e.contains("tests/")
                && ranked_excerpts[i + 1..]
                    .iter()
                    .any(|x| !x.contains("tests/"))
        })
        .count();
    others.extend(tests);
    assert_eq!(marked_excerpts, others);
    assert!(moved > 0);
    assert_eq!(marked["moved_last"], json!(moved));
    assert_eq!(marked["omitted"], ranked["omitted"]);
    assert_eq!(marked["characters"], ranked["characters"]);
    assert_eq!(marked["tokens"], ranked["tokens"]);
    // The Markdown follows the same order.
    let markdown = marked["markdown"].as_str().unwrap();
    let at = |e: &String| {
        let path = e.split('@').next().unwrap().trim_start_matches("file:");
        markdown.find(&format!(": {path}:")).unwrap()
    };
    let positions: Vec<usize> = marked_excerpts.iter().map(at).collect();
    assert!(positions.windows(2).all(|w| w[0] < w[1]), "{positions:?}");
    // Nothing to move: the order stands and nothing moved.
    let mut tail = request(true);
    tail["hits"] = json!(
        hits.iter()
            .filter(|h| !s(h, "path").contains("tests/"))
            .cloned()
            .collect::<Vec<_>>()
    );
    tail["hits"][0]["last"] = json!(false);
    let unmoved = packing::pack(&tail).unwrap();
    assert!(unmoved.get("moved_last").is_none());
    tail["hits"][0]["last"] = json!("yes");
    assert!(
        packing::pack(&tail)
            .unwrap_err()
            .to_string()
            .contains("last must be true or false")
    );
}

#[test]
fn chronological_packing_spends_by_rank_and_presents_by_date_with_gap_markers() {
    let hits = json!([
        conversation_hit("may", "s2", "2023-05-08T09:00:00Z", 0, "We adopted a beagle named Pip."),
        conversation_hit("late", "s1", "2023-01-02T10:00:00Z", 900, "Later that day we chose the name."),
        {"identity":"code","text":"fn pet() {}","source":{"id":"file:src/pet.rs","path":"src/pet.rs",
            "source_kind":"repository_file","byte_start":0,"line":1,"sha256":"code-hash"}},
        conversation_hit("early", "s1", "2023-01-02T10:00:00Z", 0, "We visited the shelter."),
    ]);
    let request = |order: Value| {
        json!({"repo":{"id":"r","name":"Repo","snapshot":"one"},"question":"When did we adopt the dog?",
            "hits":hits,"memories":[],"max_tokens":1200,"max_chars":8000,"pack_order":order})
    };
    let ranked = packing::pack(&request(Value::Null)).unwrap();
    let dated = packing::pack(&request(json!("chronological"))).unwrap();
    assert!(ranked.get("pack_order").is_none());
    assert_eq!(dated["pack_order"], "chronological");
    let order = |packet: &Value| {
        packet["sources"]
            .as_array()
            .unwrap()
            .iter()
            .map(|s| format!("{}@{}", s["id"].as_str().unwrap(), s["byte_start"]))
            .collect::<Vec<_>>()
    };
    assert_eq!(
        order(&ranked),
        ["s2@0", "s1@900", "file:src/pet.rs@0", "s1@0"]
    );
    // The same excerpts, earliest first, a source's excerpts in reading
    // order and the undated after the dated.
    assert_eq!(
        order(&dated),
        ["s1@0", "s1@900", "s2@0", "file:src/pet.rs@0"]
    );
    let ranks: Vec<_> = dated["sources"]
        .as_array()
        .unwrap()
        .iter()
        .map(|s| s["rank"].as_u64().unwrap())
        .collect();
    assert_eq!(ranks, [4, 2, 1, 3]);
    let markdown = dated["markdown"].as_str().unwrap();
    assert_eq!(dated["gap_markers"], 1);
    let marker = markdown
        .find("[Not shown: bytes 23–900 of this source lie between these excerpts; read them with read_project_source.]")
        .unwrap();
    assert!(markdown.find("We visited the shelter.").unwrap() < marker);
    assert!(marker < markdown.find("Later that day").unwrap());
    assert!(markdown.find("Later that day").unwrap() < markdown.find("beagle").unwrap());
    assert!(markdown.find("beagle").unwrap() < markdown.find("fn pet()").unwrap());
    assert_eq!(
        dated["tokens"].as_u64().unwrap() as usize,
        text::count(markdown)
    );
    assert!(text::count(markdown) <= 1200);
    assert!(
        packing::pack(&request(json!("date")))
            .unwrap_err()
            .to_string()
            .contains("pack_order")
    );
}

#[test]
fn excerpts_the_caller_holds_become_references_and_their_share_goes_on() {
    let long = "The deploy runbook says to drain the queue first. ".repeat(120);
    let hits = json!([
        conversation_hit(
            "held",
            "s1",
            "2023-01-02",
            0,
            "Deploys happen on Tuesdays after review."
        ),
        conversation_hit("fresh", "s2", "2023-01-03", 0, &long),
    ]);
    let request = |shown: Value| {
        json!({"repo":{"id":"r","name":"Repo","snapshot":"one"},"question":"How do deploys work?",
            "hits":hits,"memories":[],"max_tokens":700,"max_chars":6000,"shown":shown})
    };
    let plain = packing::pack(&request(Value::Null)).unwrap();
    assert!(plain.get("already_shown").is_none());
    let held = packing::pack(&request(
        json!([{"id":"s1","sha256":"s1-hash","byte_start":0,"byte_end":20},
        {"id":"s1","byte_start":15,"byte_end":200}]),
    ))
    .unwrap();
    let markdown = held["markdown"].as_str().unwrap();
    assert!(!markdown.contains("Deploys happen on Tuesdays"));
    assert!(markdown.contains(
        "Already in context: bytes 0–40 were quoted to you earlier and are not repeated."
    ));
    assert_eq!(held["already_shown"][0]["id"], "s1");
    assert_eq!(held["already_shown"][0]["byte_end"], 40);
    let ids: Vec<_> = held["sources"]
        .as_array()
        .unwrap()
        .iter()
        .map(|s| s["id"].as_str().unwrap())
        .collect();
    assert_eq!(ids, ["s2"]);
    let quoted = |packet: &Value, id: &str| {
        packet["sources"]
            .as_array()
            .unwrap()
            .iter()
            .filter(|s| s["id"] == id)
            .map(|s| s["byte_end"].as_u64().unwrap() - s["byte_start"].as_u64().unwrap())
            .sum::<u64>()
    };
    assert!(
        quoted(&held, "s2") > quoted(&plain, "s2"),
        "the reference's share goes to the next hit"
    );
    assert!(text::count(markdown) <= 700);
    // A span of another version, or one that does not cover the excerpt,
    // leaves the text in place.
    for shown in [
        json!([{"id":"s1","sha256":"older","byte_start":0,"byte_end":200}]),
        json!([{"id":"s1","byte_start":0,"byte_end":20},{"id":"s1","byte_start":21,"byte_end":200}]),
    ] {
        let packet = packing::pack(&request(shown)).unwrap();
        assert!(
            packet["markdown"]
                .as_str()
                .unwrap()
                .contains("Deploys happen on Tuesdays")
        );
        assert_eq!(packet["already_shown"], json!([]));
    }
    for bad in [
        json!([{"id":"s1","byte_start":5,"byte_end":5}]),
        json!([{"byte_start":0,"byte_end":5}]),
        json!("s1"),
        json!(vec![json!({"id":"s1","byte_start":0,"byte_end":5}); 201]),
    ] {
        assert!(packing::pack(&request(bad)).is_err());
    }
}

#[test]
fn reading_guidance_is_an_opt_in_header_line_within_budget() {
    let request = |guidance: Value| {
        json!({"repo":{"id":"r","name":"Repo","snapshot":"one"},"question":"When is the trip?",
            "hits":[conversation_hit("trip", "s1", "2023-01-02", 0, "Assistant: You could go on May 3.")],
            "memories":[],"max_tokens":600,"max_chars":4000,"reading_guidance":guidance})
    };
    let plain = packing::pack(&request(Value::Null)).unwrap();
    let guided = packing::pack(&request(json!(true))).unwrap();
    assert!(
        !plain["markdown"]
            .as_str()
            .unwrap()
            .contains("Reading conversations")
    );
    assert!(plain.get("reading_guidance").is_none());
    let markdown = guided["markdown"].as_str().unwrap();
    let guidance = markdown
        .find("Reading conversations: what a user says")
        .unwrap();
    assert!(guidance < markdown.find("Source [conversation]").unwrap());
    assert!(markdown.contains("an assistant's proposals, examples and suggested dates"));
    assert_eq!(guided["reading_guidance"], true);
    assert_eq!(
        guided["tokens"].as_u64().unwrap() as usize,
        text::count(markdown)
    );
    assert!(packing::pack(&request(json!("yes"))).is_err());
}

/// Past `MAX_STATIC_EDGES`, `rank` folds the static
/// relationship edges per (source file, target file, kind) instead of failing,
/// with the same relationship lift, and cuts them at the bound only when the
/// distinct pairs alone exceed it. Either way the result says so; within the
/// bound nothing is reported.
#[test]
fn rank_folds_or_caps_static_edges_past_the_bound_instead_of_failing() {
    let row = |path: &str, text: &str| {
        json!({"identity":format!("file:{path}:0"),"kind":"source","origin":format!("file:{path}"),
            "title":path,"text":text,"source":{"id":format!("file:{path}"),"path":path}})
    };
    let rows = vec![
        row(
            "src/auth.py",
            "def refresh_token(): rotate the session token",
        ),
        row("src/session.py", "class Session: stores cookies"),
        row("src/unrelated.py", "def render(): draws widgets"),
    ];
    let request = |edges: Vec<Value>| json!({"scope":"static-edge-bound","question":"refresh token","rows":rows,"limit":3,"edges":edges});
    let call = |line: usize| json!({"source":"src/auth.py","target":"src/session.py","kind":"calls","line":line});
    // Within the bound: one edge per call site, no report.
    let within = retrieval::rank(&request((1..=10).map(call).collect())).unwrap();
    assert!(within.get("static_edges").is_none(), "{within}");
    assert!(
        within["reasons"]["file:src/session.py:0"]
            .as_array()
            .unwrap()
            .contains(&json!("static_relationship:1")),
        "{within}"
    );
    // 60,000 call sites between two files fold to one pair: same lift.
    let folded = retrieval::rank(&request((1..=60_000).map(call).collect())).unwrap();
    assert_eq!(folded["static_edges"]["received"], 60_000);
    assert_eq!(folded["static_edges"]["distinct"], 1);
    assert_eq!(folded["static_edges"]["aggregated"], true);
    assert_eq!(folded["static_edges"]["capped"], false);
    assert_eq!(folded["sources"], within["sources"]);
    assert_eq!(folded["reasons"], within["reasons"]);
    // 60,000 distinct file pairs: cut at the bound and reported as capped.
    let unrelated = || {
        (0..60_000)
            .map(|i| json!({"source":format!("a/{}.py", i / 300),"target":format!("b/{}.py", i % 300),"kind":"calls","line":1}))
            .collect::<Vec<Value>>()
    };
    let capped = retrieval::rank(&request(unrelated())).unwrap();
    assert_eq!(capped["static_edges"]["distinct"], 60_000);
    assert_eq!(capped["static_edges"]["used"], retrieval::MAX_STATIC_EDGES);
    assert_eq!(capped["static_edges"]["capped"], true);
    assert_eq!(capped["static_edges"]["lift_complete"], true);
    // The pair the lift reads comes last in input order and still survives
    // the cut: pairs touching the top passages' files are kept first.
    let mut late = unrelated();
    late.push(call(1));
    let kept = retrieval::rank(&request(late)).unwrap();
    assert_eq!(
        kept["static_edges"]["capped"], true,
        "{}",
        kept["static_edges"]
    );
    assert_eq!(kept["static_edges"]["lift_complete"], true);
    assert_eq!(kept["sources"], within["sources"]);
    assert_eq!(kept["reasons"], within["reasons"]);
    // Two hops (relationships mode): the second hop's pair is kept as well,
    // so the lift is exactly the one the two pairs alone give.
    let two_hops = |edges: Vec<Value>| {
        let mut rows = rows.clone();
        rows.push(row("src/store.py", "def save(): writes rows"));
        json!({"scope":"static-edge-bound","question":"refresh token","rows":rows,"limit":4,
            "mode":"relationships","edges":edges})
    };
    let chain = || {
        vec![
            call(1),
            json!({"source":"src/session.py","target":"src/store.py","kind":"calls","line":1}),
        ]
    };
    let small = retrieval::rank(&two_hops(chain())).unwrap();
    assert!(
        small["reasons"]["file:src/store.py:0"]
            .as_array()
            .unwrap()
            .contains(&json!("static_relationship:2")),
        "{small}"
    );
    let mut long = unrelated();
    long.extend(chain());
    let large = retrieval::rank(&two_hops(long)).unwrap();
    assert_eq!(large["static_edges"]["lift_complete"], true);
    assert_eq!(large["sources"], small["sources"]);
    assert_eq!(large["reasons"], small["reasons"]);
    // More pairs touch the top passages' file than the bound holds: the
    // report says the lift could not read them all.
    let fan: Vec<Value> = (0..=retrieval::MAX_STATIC_EDGES)
        .map(|i| json!({"source":"src/auth.py","target":format!("t/{i}.py"),"kind":"calls","line":1}))
        .collect();
    let partial = retrieval::rank(&request(fan)).unwrap();
    assert_eq!(partial["static_edges"]["capped"], true);
    assert_eq!(partial["static_edges"]["lift_complete"], false);
}

/// A packet over several projects of a workspace names each
/// excerpt's and memory's project (`project` and `repository_id` on the
/// source or memory), and a packet over one project, whose sources carry
/// neither, is unchanged.
#[test]
fn excerpts_and_memories_that_name_their_project_say_so() {
    let request = |project: bool, compact: bool| {
        let mut source = json!({"source_kind":"repository_file","path":"app/invoice.py","id":"file:app/invoice.py",
            "sha256":"hash","byte_start":0,"line":1});
        let mut memory = json!({"id":"m1","kind":"decision","authority":"reviewed","title":"Rounding","body":"Round half to even."});
        if project {
            for item in [&mut source, &mut memory] {
                item["project"] = json!("Shop");
                item["repository_id"] = json!("p-shop");
            }
        }
        json!({"repo":{"id":null,"name":"2 projects of this workspace: Shop, Money","snapshot":"s"},
            "question":"invoice total","hits":[{"identity":"p-shop:file:app/invoice.py:0","text":"def settle_invoice(): pass\n","source":source}],
            "memories":[{"memory":memory}],"max_tokens":800,"max_chars":4000,"compact_citations":compact})
    };
    let plain = packing::pack(&request(false, false)).unwrap();
    let named = packing::pack(&request(true, false)).unwrap();
    let markdown = named["markdown"].as_str().unwrap();
    assert!(
        markdown.contains("Source [repository_file]: app/invoice.py:1\nProject: Shop (p-shop)\nID: file:app/invoice.py\n"),
        "{markdown}"
    );
    assert!(
        markdown.contains("Reference: m1\nProject: Shop (p-shop)\n"),
        "{markdown}"
    );
    assert!(!plain["markdown"].as_str().unwrap().contains("Project: "));
    assert_eq!(named["sources"][0]["repository_id"], "p-shop");
    assert_eq!(named["memories"][0]["project"], "Shop");
    let compact = packing::pack(&request(true, true)).unwrap();
    assert!(
        compact["markdown"]
            .as_str()
            .unwrap()
            .contains("app/invoice.py:1\nProject: Shop\n"),
        "{}",
        compact["markdown"]
    );
}

/// Source IDs repeat across the projects of
/// a workspace (`file:README.md` in each). A `shown` span stands for one
/// project's source, its own `repository_id`'s or else the packet's
/// (`repo.id`, none in a packet that names no project), and the reference
/// it produces names that project. Chronological presentation keeps each
/// project's excerpts of a source together and marks gaps within them only.
#[test]
fn shown_spans_and_gap_markers_keep_the_projects_of_a_workspace_apart() {
    let readme = |project: &str, start: u64, text: &str| {
        json!({"identity":format!("{project}:file:README.md:{start}"),"text":text,
            "source":{"id":"file:README.md","path":"README.md","source_kind":"repository_file",
                "byte_start":start,"byte_end":start + text.len() as u64,"line":1,"sha256":"same-bytes",
                "recorded_at":"2023-01-02T10:00:00Z","project":project.to_uppercase(),"repository_id":project}})
    };
    let hits = json!([
        readme("a", 0, "Alpha installs with make."),
        readme("b", 0, "Beta installs with cargo."),
        readme("a", 500, "Alpha runs on Linux."),
        readme("b", 900, "Beta runs on macOS."),
    ]);
    let request = |named: Option<&str>, shown: Value, order: Value| {
        json!({"repo":{"id":named,"name":"2 projects of this workspace: A, B","snapshot":"s"},
            "question":"How do they install?","hits":hits,"memories":[],"max_tokens":1200,
            "max_chars":8000,"shown":shown,"pack_order":order})
    };
    // A span of project a's README holds a's excerpt, not b's same bytes.
    let held = packing::pack(&request(
        None,
        json!([{"id":"file:README.md","repository_id":"a","byte_start":0,"byte_end":30}]),
        Value::Null,
    ))
    .unwrap();
    let markdown = held["markdown"].as_str().unwrap();
    assert!(!markdown.contains("Alpha installs"), "{markdown}");
    assert!(markdown.contains("Beta installs"), "{markdown}");
    assert_eq!(
        held["already_shown"],
        json!([{"id":"file:README.md","sha256":"same-bytes","path":"README.md","line":1,
            "byte_start":0,"byte_end":25,"repository_id":"a"}])
    );
    // A span that names no project is the packet's project's: in a packet
    // that names none it holds nothing; in one that names b, b's excerpt.
    let unnamed = json!([{"id":"file:README.md","byte_start":0,"byte_end":30}]);
    let nobody = packing::pack(&request(None, unnamed.clone(), Value::Null)).unwrap();
    assert_eq!(nobody["already_shown"], json!([]));
    let anchored = packing::pack(&request(Some("b"), unnamed, Value::Null)).unwrap();
    assert_eq!(anchored["already_shown"][0]["repository_id"], "b");
    let markdown = anchored["markdown"].as_str().unwrap();
    assert!(markdown.contains("Alpha installs") && !markdown.contains("Beta installs"));
    // By date: each project's excerpts of README.md together, in reading
    // order, with a marker inside each project's README and none between
    // the two projects.
    let dated = packing::pack(&request(None, Value::Null, json!("chronological"))).unwrap();
    let order: Vec<String> = dated["sources"]
        .as_array()
        .unwrap()
        .iter()
        .map(|s| {
            format!(
                "{}@{}",
                s["repository_id"].as_str().unwrap(),
                s["byte_start"]
            )
        })
        .collect();
    assert_eq!(order, ["a@0", "a@500", "b@0", "b@900"]);
    assert_eq!(dated["gap_markers"], 2);
    let markdown = dated["markdown"].as_str().unwrap();
    let alpha = markdown.find("Alpha installs").unwrap();
    let alpha_gap = markdown
        .find("[Not shown: bytes 25–500 of this source")
        .unwrap();
    let beta = markdown.find("Beta installs").unwrap();
    let beta_gap = markdown
        .find("[Not shown: bytes 25–900 of this source")
        .unwrap();
    assert!(alpha < alpha_gap && alpha_gap < markdown.find("Alpha runs").unwrap());
    assert!(markdown.find("Alpha runs").unwrap() < beta);
    assert!(beta < beta_gap && beta_gap < markdown.find("Beta runs").unwrap());
}