Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust
7.0 KB
Raw Blame
//! On-demand end-to-end smoke for the retrieval harness (mirrors the
//! `writer_throughput` pattern: `#[ignore]`d, run deliberately).
//!
//! Requirements, both intentionally not provisioned by the test:
//! - `target/release/ai-memory` (`cargo build --release -p ai-memory-cli`)
//! - `evals/datasets/longmemeval_s.json` (run once with `--fetch`)
//!
//! ```bash
//! cargo test -p ai-memory-eval --test smoke -- --ignored
//! ```

use std::path::PathBuf;
use std::process::Command;

#[test]
#[ignore = "needs the release server binary and the downloaded dataset"]
fn the_harness_scores_a_tiny_sample_end_to_end() {
    let repo_root: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
    let server_bin = repo_root.join("target/release/ai-memory");
    assert!(
        server_bin.exists(),
        "build the server first: cargo build --release -p ai-memory-cli"
    );
    let dataset = repo_root.join("evals/datasets/longmemeval_s.json");
    assert!(
        dataset.exists(),
        "fetch the dataset first: cargo run -p ai-memory-eval -- retrieval --fetch --sample 1"
    );

    let out = tempfile::tempdir().unwrap();
    let status = Command::new(env!("CARGO_BIN_EXE_ai-memory-eval"))
        .current_dir(&repo_root)
        .args(["retrieval", "--sample", "2", "--concurrency", "2"])
        .arg("--server-bin")
        .arg(&server_bin)
        .arg("--out")
        .arg(out.path())
        .status()
        .unwrap();
    assert!(status.success(), "harness exited non-zero");

    // Exactly one run dir with a structurally complete report.
    let run_dir = std::fs::read_dir(out.path())
        .unwrap()
        .map(|e| e.unwrap().path())
        .find(|p| p.is_dir())
        .expect("run dir created");
    let report: serde_json::Value =
        serde_json::from_slice(&std::fs::read(run_dir.join("report.json")).unwrap()).unwrap();
    assert_eq!(report["questions_scored"], 2);
    assert_eq!(report["mode"], "zero-llm");
    assert!(report["slices"]["overall"]["hit_at"]["5"].is_number());
    assert!(report["slices"]["overall"]["recall_all_at"]["5"].is_number());
    assert!(report["per_question"][0]["recall_all_at"]["5"].is_number());
    // The R2 triple lands next to accuracy.
    assert!(report["slices"]["overall"]["latency_p50_ms"].is_number());
    assert!(report["slices"]["overall"]["context_tokens_mean"].is_number());
    // Provenance that makes a published number auditable.
    assert!(report["commit"].as_str().unwrap().len() >= 7);
    assert!(
        report["dataset_sha256"]
            .as_str()
            .unwrap()
            .starts_with("08d8dad4")
    );
    assert!(run_dir.join("report.md").exists());
}

/// R2 A/B path: a baseline-vs-baseline run (`--candidate`, no differing
/// knob) must produce a delta report whose accuracy and context-token
/// deltas are exactly zero — the determinism guard.
#[test]
#[ignore = "needs the release server binary and the downloaded dataset"]
fn the_ab_path_reports_a_zero_delta_for_identical_configs() {
    let repo_root: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
    let server_bin = repo_root.join("target/release/ai-memory");
    assert!(
        server_bin.exists(),
        "build the server first: cargo build --release -p ai-memory-cli"
    );
    let dataset = repo_root.join("evals/datasets/longmemeval_s.json");
    assert!(
        dataset.exists(),
        "fetch the dataset first: cargo run -p ai-memory-eval -- retrieval --fetch --sample 1"
    );

    let out = tempfile::tempdir().unwrap();
    let status = Command::new(env!("CARGO_BIN_EXE_ai-memory-eval"))
        .current_dir(&repo_root)
        .args([
            "retrieval",
            "--candidate",
            "--sample",
            "2",
            "--concurrency",
            "2",
        ])
        .arg("--server-bin")
        .arg(&server_bin)
        .arg("--out")
        .arg(out.path())
        .status()
        .unwrap();
    assert!(status.success(), "A/B harness exited non-zero");

    let run_dir = std::fs::read_dir(out.path())
        .unwrap()
        .map(|e| e.unwrap().path())
        .find(|p| p.is_dir())
        .expect("run dir created");
    let report: serde_json::Value =
        serde_json::from_slice(&std::fs::read(run_dir.join("report.json")).unwrap()).unwrap();
    // A/B report shape: baseline + candidate + precomputed delta.
    assert_eq!(report["baseline"]["questions_scored"], 2);
    assert_eq!(report["candidate"]["questions_scored"], 2);
    // Identical configs ⇒ deterministic accuracy and context; only latency
    // (wall-clock) may wobble, so it is not asserted zero here.
    let delta = &report["delta"];
    for k in ["1", "3", "5", "10"] {
        assert_eq!(delta["hit_at"][k], 0.0, "hit@{k} delta");
        assert_eq!(delta["recall_at"][k], 0.0, "recall@{k} delta");
        assert_eq!(delta["recall_all_at"][k], 0.0, "recall-all@{k} delta");
    }
    assert_eq!(delta["context_tokens_mean"], 0.0);
    assert_eq!(delta["context_tokens_median"], 0.0);
    assert!(run_dir.join("report.md").exists());
}

/// The bundled corpus exercises the same server and scorer without downloading
/// LongMemEval. Identical configurations must agree on evidence and context.
#[test]
#[ignore = "needs the release server binary; uses the bundled offline corpus"]
fn coding_corpus_replays_offline_with_identical_ab_results() {
    let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
    let server_bin = repo_root.join("target/release/ai-memory");
    assert!(server_bin.exists(), "build the release server first");
    let out = tempfile::tempdir().unwrap();
    let status = Command::new(env!("CARGO_BIN_EXE_ai-memory-eval"))
        .current_dir(&repo_root)
        .args([
            "retrieval",
            "--dataset-variant",
            "coding-v1",
            "--candidate",
            "--concurrency",
            "1",
        ])
        .arg("--server-bin")
        .arg(&server_bin)
        .arg("--out")
        .arg(out.path())
        .status()
        .unwrap();
    assert!(status.success());
    let run_dir = std::fs::read_dir(out.path())
        .unwrap()
        .map(|entry| entry.unwrap().path())
        .find(|path| path.is_dir())
        .unwrap();
    let report: serde_json::Value =
        serde_json::from_slice(&std::fs::read(run_dir.join("report.json")).unwrap()).unwrap();
    assert_eq!(report["baseline"]["dataset"], "coding_v1");
    for question in report["baseline"]["per_question"].as_array().unwrap() {
        assert_eq!(
            question["recall_at"]["10"], 1.0,
            "missing expected evidence: {}",
            question["question_id"]
        );
    }
    assert_eq!(report["baseline"]["questions_scored"], 4);
    assert_eq!(report["candidate"]["questions_scored"], 4);
    for metric in ["hit_at", "recall_at", "recall_all_at"] {
        for k in ["1", "3", "5", "10"] {
            assert_eq!(report["delta"][metric][k], 0.0, "{metric}@{k}");
        }
    }
    assert_eq!(report["delta"]["context_tokens_mean"], 0.0);
    assert_eq!(report["delta"]["context_tokens_median"], 0.0);
}