Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

test(cli): end-to-end use-case test for ai-memory doctor

AkitaOnRails5bb7113fa113
diff --git a/crates/ai-memory-cli/tests/suite/doctor_e2e.rs b/crates/ai-memory-cli/tests/suite/doctor_e2e.rs
new file mode 100644
index 00000000..884f99f1
--- /dev/null
+++ b/crates/ai-memory-cli/tests/suite/doctor_e2e.rs
@@ -0,0 +1,407 @@
+//! Use-case end-to-end test for `ai-memory doctor` (capture-coverage check).
+//!
+//! Doctor exists to make one silent failure visible: a harness that ran in this
+//! project but has no ai-memory hook installed still writes its own local
+//! session transcripts, yet nothing reaches the server. The operator believes
+//! every harness feeds one memory and is quietly wrong (audit finding F1). For
+//! the current project doctor enumerates each known harness's local native
+//! sessions, asks the server how many it actually captured per agent
+//! (`GET /admin/sessions/by-agent`), and warns — with the exact
+//! `install-hooks --agent <agent> --apply` remediation — about any harness that
+//! ran here recently but captured zero.
+//!
+//! The pure verdict logic (`build_rows`) and the local detector (`scan_local`)
+//! are unit-tested in the command module. The UNTESTED half is the whole live
+//! flow against a real server: does the shipped `doctor` subcommand, driven
+//! exactly as an operator would, actually reconcile the local side against the
+//! server's captured counts and print the right verdict? This drives that flow:
+//!
+//! 1. **Gap detected.** A planted Claude transcript for this cwd with an empty
+//! store (server captured zero) → doctor flags `claude-code` as uncaptured
+//! and prints its `install-hooks --agent claude-code --apply` fix.
+//! 2. **Healthy control.** After `backfill` imports that same transcript through
+//! `/hook`, the server has captured sessions for `claude-code` → doctor no
+//! longer flags it, and reports every harness as captured. This control is
+//! what stops a blanket "everything is uncaptured" warning from passing #1
+//! for the wrong reason.
+//!
+//! Unix-gated: the fixture uses the POSIX-encoded `~/.claude/projects/<enc-cwd>/`
+//! native layout (as the sibling `backfill_e2e` test and the `scan_local` unit
+//! test do); cross-platform native-store discovery is owned by
+//! `ai-memory-workstream`.
+
+#![cfg(unix)]
+
+/// Spawns a server and several subprocesses: seconds, not milliseconds, so this
+/// lives in the slow tier (`cargo tf` / CI), not the everyday loop.
+mod slow {
+ use std::fs;
+ use std::net::TcpListener;
+ use std::path::Path;
+ use std::process::{Child, Command, Stdio};
+ use std::time::{Duration, Instant};
+
+ use serde_json::{Value, json};
+
+ const BIN: &str = env!("CARGO_BIN_EXE_ai-memory");
+ const WORKSPACE: &str = "doctor-e2e-ws";
+ const PROJECT: &str = "doctor-e2e-proj";
+
+ /// Start from a clean, hermetic environment: drop every ambient
+ /// `AI_MEMORY_*` var (a developer box or this project's own MCP config may
+ /// export `AI_MEMORY_AUTH_TOKEN`, `AI_MEMORY_SERVER_URL`, scope names, …)
+ /// so the child sees only what this test sets. Without this the spawned
+ /// server would inherit an auth token and reject the test's own requests.
+ fn hermetic(program: &str) -> Command {
+ let mut cmd = Command::new(program);
+ for (key, _) in std::env::vars_os() {
+ if key.to_string_lossy().starts_with("AI_MEMORY_") {
+ cmd.env_remove(key);
+ }
+ }
+ cmd
+ }
+
+ /// Kill the spawned server when the test ends, pass or fail.
+ struct ServerGuard(Child);
+ impl Drop for ServerGuard {
+ fn drop(&mut self) {
+ let _ = self.0.kill();
+ let _ = self.0.wait();
+ }
+ }
+
+ /// A free loopback port. The brief unbind→rebind race is acceptable for a
+ /// slow-tier test and is the same approach the shell smoke test uses.
+ fn free_port() -> u16 {
+ TcpListener::bind("127.0.0.1:0")
+ .expect("bind ephemeral port")
+ .local_addr()
+ .expect("local addr")
+ .port()
+ }
+
+ /// Write the JSONL lines (each already a `Value`) as one transcript file.
+ fn write_jsonl(path: &Path, lines: &[Value]) {
+ let mut body = String::new();
+ for line in lines {
+ body.push_str(&line.to_string());
+ body.push('\n');
+ }
+ fs::write(path, body).expect("write transcript");
+ }
+
+ /// Sum the server's per-agent session counts for the scope. A 404 (the
+ /// no-create scope lookup for a project that has never been written to)
+ /// counts as zero — the pre-import state.
+ async fn session_count(client: &reqwest::Client, base: &str) -> u64 {
+ let resp = client
+ .get(format!("{base}/admin/sessions/by-agent"))
+ .query(&[("workspace", WORKSPACE), ("project", PROJECT)])
+ .send()
+ .await
+ .expect("by-agent request");
+ if !resp.status().is_success() {
+ return 0;
+ }
+ let body: Value = resp.json().await.expect("by-agent json");
+ body["by_agent"]
+ .as_array()
+ .map(|agents| {
+ agents
+ .iter()
+ .filter_map(|a| a["sessions"].as_u64())
+ .sum::<u64>()
+ })
+ .unwrap_or(0)
+ }
+
+ /// Run a subcommand of the built binary to completion, returning its stdout.
+ /// The scope/server/home environment is shared with the spawned server so
+ /// the client talks to the same store the way a real install does.
+ fn run_cli(args: &[&str], data_dir: &Path, home: &Path, cwd: &Path, base: &str) -> String {
+ let out = hermetic(BIN)
+ .args(args)
+ .current_dir(cwd)
+ .env("AI_MEMORY_DATA_DIR", data_dir)
+ .env("AI_MEMORY_HOME", home)
+ .env("AI_MEMORY_SERVER_URL", base)
+ .env("AI_MEMORY_EMBEDDING_PROVIDER", "none")
+ .env("RUST_LOG", "off")
+ .output()
+ .unwrap_or_else(|e| panic!("spawn `{}`: {e}", args.join(" ")));
+ assert!(
+ out.status.success(),
+ "`{}` failed: {}\nstdout: {}\nstderr: {}",
+ args.join(" "),
+ out.status,
+ String::from_utf8_lossy(&out.stdout),
+ String::from_utf8_lossy(&out.stderr),
+ );
+ String::from_utf8(out.stdout).expect("stdout utf8")
+ }
+
+ /// The row for one agent kind in a `doctor --json` report, or `None`.
+ fn find_row<'a>(report: &'a Value, agent: &str) -> Option<&'a Value> {
+ report["rows"]
+ .as_array()?
+ .iter()
+ .find(|r| r["agent"] == agent)
+ }
+
+ #[tokio::test]
+ async fn doctor_flags_an_uncaptured_harness_then_clears_it_after_backfill() {
+ let data_dir = tempfile::tempdir().expect("data dir");
+ let home = tempfile::tempdir().expect("home");
+ let project = tempfile::tempdir().expect("project cwd");
+ // The child's `std::env::current_dir()` returns the canonical path, and
+ // the native-session discovery matches the transcript's `cwd` header
+ // against it — so plant and encode with the same canonical path.
+ let cwd = fs::canonicalize(project.path()).expect("canonicalize project cwd");
+
+ // Plant a Claude transcript for this cwd under the native layout. This
+ // makes `claude-code` a harness that "ran here recently" for doctor,
+ // while the server starts with zero captured — the F1 gap.
+ let session_id = "11111111-2222-3333-4444-555555555555";
+ let encoded = cwd.to_string_lossy().replace('/', "-");
+ let session_dir = home.path().join(".claude").join("projects").join(encoded);
+ fs::create_dir_all(&session_dir).expect("session dir");
+ write_jsonl(
+ &session_dir.join(format!("{session_id}.jsonl")),
+ &[
+ json!({ "sessionId": session_id, "cwd": cwd.to_string_lossy() }),
+ json!({
+ "type": "user",
+ "message": {
+ "role": "user",
+ "content": [{ "type": "text", "text": "Record the doctor-e2e decision." }],
+ },
+ }),
+ json!({
+ "type": "assistant",
+ "message": {
+ "role": "assistant",
+ "content": [{ "type": "text", "text": "Acknowledged the doctor-e2e plan." }],
+ },
+ }),
+ ],
+ );
+
+ // Start the real server (loopback, no auth, hermetic: no embedder, no
+ // wiki watcher — a machine-global inotify instance concurrent server
+ // children can exhaust, #745).
+ let port = free_port();
+ let base = format!("http://127.0.0.1:{port}");
+ let server = ServerGuard(
+ hermetic(BIN)
+ .args([
+ "serve",
+ "--transport",
+ "http",
+ "--bind",
+ &format!("127.0.0.1:{port}"),
+ "--workspace",
+ WORKSPACE,
+ "--project",
+ PROJECT,
+ "--no-watcher",
+ ])
+ .current_dir(&cwd)
+ .env("AI_MEMORY_DATA_DIR", data_dir.path())
+ .env("AI_MEMORY_HOME", home.path())
+ .env("AI_MEMORY_EMBEDDING_PROVIDER", "none")
+ .env("RUST_LOG", "off")
+ .stdin(Stdio::null())
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .spawn()
+ .expect("spawn serve"),
+ );
+
+ let client = reqwest::Client::new();
+
+ // Wait for the server to accept connections.
+ let deadline = Instant::now() + Duration::from_secs(30);
+ loop {
+ if client
+ .get(format!("{base}/mcp"))
+ .timeout(Duration::from_secs(2))
+ .send()
+ .await
+ .is_ok()
+ {
+ break;
+ }
+ assert!(Instant::now() < deadline, "server never became reachable");
+ tokio::time::sleep(Duration::from_millis(150)).await;
+ }
+
+ // Precondition: nothing captured yet for this scope.
+ assert_eq!(
+ session_count(&client, &base).await,
+ 0,
+ "a brand-new project store must start with no captured sessions",
+ );
+
+ // Phase 1 — gap detected. `--since-days 0` makes every on-disk session
+ // count as recent, so the planted transcript is unambiguously "recent".
+ let report: Value = serde_json::from_str(&run_cli(
+ &[
+ "doctor",
+ "--workspace",
+ WORKSPACE,
+ "--project",
+ PROJECT,
+ "--since-days",
+ "0",
+ "--json",
+ ],
+ data_dir.path(),
+ home.path(),
+ &cwd,
+ &base,
+ ))
+ .expect("doctor --json report");
+
+ let claude = find_row(&report, "claude-code")
+ .unwrap_or_else(|| panic!("expected a claude-code row: {report}"));
+ assert_eq!(
+ claude["uncaptured"], true,
+ "a harness that ran here with zero captured must be flagged: {report}"
+ );
+ assert!(
+ claude["local_recent"].as_u64().unwrap_or(0) >= 1,
+ "the planted session must count as recent: {report}"
+ );
+ assert_eq!(
+ claude["captured"], 0,
+ "the server captured nothing for it yet: {report}"
+ );
+ assert_eq!(
+ report["uncaptured"]
+ .as_array()
+ .map(|a| a.iter().any(|v| v == "claude-code")),
+ Some(true),
+ "claude-code must appear in the top-level uncaptured list: {report}"
+ );
+
+ // Phase 1 — the human-readable output prints the exact remediation the
+ // operator is meant to run, naming the specific agent.
+ let human = run_cli(
+ &[
+ "doctor",
+ "--workspace",
+ WORKSPACE,
+ "--project",
+ PROJECT,
+ "--since-days",
+ "0",
+ ],
+ data_dir.path(),
+ home.path(),
+ &cwd,
+ &base,
+ );
+ assert!(
+ human.contains("ai-memory install-hooks --agent claude-code --apply"),
+ "doctor must print the exact install-hooks remediation: {human}"
+ );
+ assert!(
+ human.contains("ran here but nothing was captured"),
+ "doctor must explain the gap in prose: {human}"
+ );
+
+ // Import the local history through the same `/hook` ingress live capture
+ // uses, so the server now has captured sessions for claude-code.
+ let backfill: Value = serde_json::from_str(&run_cli(
+ &[
+ "backfill",
+ "--workspace",
+ WORKSPACE,
+ "--project",
+ PROJECT,
+ "--json",
+ ],
+ data_dir.path(),
+ home.path(),
+ &cwd,
+ &base,
+ ))
+ .expect("backfill --json report");
+ assert_eq!(
+ backfill["imported_sessions"], 1,
+ "backfill must import the one planted session: {backfill}"
+ );
+
+ // The captured count is now nonzero (poll briefly for commit lag).
+ let deadline = Instant::now() + Duration::from_secs(10);
+ loop {
+ if session_count(&client, &base).await >= 1 {
+ break;
+ }
+ assert!(Instant::now() < deadline, "captured session never appeared");
+ tokio::time::sleep(Duration::from_millis(150)).await;
+ }
+
+ // Phase 2 — healthy control. The same harness with the same local
+ // transcript is now captured, so doctor must NOT flag it.
+ let report2: Value = serde_json::from_str(&run_cli(
+ &[
+ "doctor",
+ "--workspace",
+ WORKSPACE,
+ "--project",
+ PROJECT,
+ "--since-days",
+ "0",
+ "--json",
+ ],
+ data_dir.path(),
+ home.path(),
+ &cwd,
+ &base,
+ ))
+ .expect("second doctor --json report");
+
+ let claude2 = find_row(&report2, "claude-code")
+ .unwrap_or_else(|| panic!("expected a claude-code row after backfill: {report2}"));
+ assert_eq!(
+ claude2["uncaptured"], false,
+ "one captured session must clear the flag: {report2}"
+ );
+ assert!(
+ claude2["captured"].as_u64().unwrap_or(0) >= 1,
+ "the backfilled session must be counted as captured: {report2}"
+ );
+ assert_eq!(
+ report2["uncaptured"]
+ .as_array()
+ .map(|a| a.iter().any(|v| v == "claude-code")),
+ Some(false),
+ "claude-code must no longer be in the uncaptured list: {report2}"
+ );
+
+ // Phase 2 — the human output no longer nags for claude-code's hook.
+ let human2 = run_cli(
+ &[
+ "doctor",
+ "--workspace",
+ WORKSPACE,
+ "--project",
+ PROJECT,
+ "--since-days",
+ "0",
+ ],
+ data_dir.path(),
+ home.path(),
+ &cwd,
+ &base,
+ );
+ assert!(
+ !human2.contains("install-hooks --agent claude-code --apply"),
+ "a captured harness must not get an install-hooks nag: {human2}"
+ );
+
+ drop(server);
+ }
+}
diff --git a/crates/ai-memory-cli/tests/suite/main.rs b/crates/ai-memory-cli/tests/suite/main.rs
index bec1be52..4d2c4c05 100644
--- a/crates/ai-memory-cli/tests/suite/main.rs
+++ b/crates/ai-memory-cli/tests/suite/main.rs
@@ -8,6 +8,7 @@
mod autoscope_env;
mod backfill_e2e;
mod completions;
+mod doctor_e2e;
mod hook_drain;
mod hook_payload;
mod marker_scope;