Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
perf(benchmarks): skip wiki checkpoints in retrieval runs
AkitaOnRails4939499990b2
diff --git a/crates/ai-memory-cli/src/commands/backfill.rs b/crates/ai-memory-cli/src/commands/backfill.rsindex 6bdd00d6..96d45e86 100644--- a/crates/ai-memory-cli/src/commands/backfill.rs+++ b/crates/ai-memory-cli/src/commands/backfill.rs@@ -1848,6 +1848,7 @@ mod tests {mid_session_routing: ai_memory_core::MidSessionRouting::default(),profile: ai_memory_core::profile::ProfileSettings::default(),ingest_drop_list: ai_memory_hooks::IngestDropList::default(),+ skip_wiki_checkpoints: false,};let endpoint = ServerEndpoint::for_hook_target("http://localhost".into(), None);let sid = "native-session-with-a-very-long-identifier-that-would-break-the-old-key";@@ -2049,6 +2050,7 @@ mod tests {mid_session_routing: ai_memory_core::MidSessionRouting::default(),profile: ai_memory_core::profile::ProfileSettings::default(),ingest_drop_list: ai_memory_hooks::IngestDropList::default(),+ skip_wiki_checkpoints: false,};let endpoint = ServerEndpoint::for_hook_target("http://localhost".into(), None);let sid = "native-session-resumed-across-two-offline-intervals";diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rsindex 975910d1..0e30237f 100644--- a/crates/ai-memory-cli/src/commands/serve.rs+++ b/crates/ai-memory-cli/src/commands/serve.rs@@ -1681,6 +1681,9 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {mid_session_routing: config.routing.mid_session,profile: config.profile.clone(),ingest_drop_list: (*config.ingest.drop_list).clone(),+ // Benchmark-only escape (docs/benchmarks/README.md), read once+ // here so the hooks crate keeps making no environment reads.+ skip_wiki_checkpoints: benchmark_skip_wiki_checkpoints(),});let workstreams = workstream_router(WorkstreamState {writer: store.writer.clone(),@@ -2957,6 +2960,20 @@ fn auto_improve_review_config_from_settings(}}+/// The retrieval-benchmark escape hatch (docs/benchmarks/README.md): the+/// LongMemEval harness replays ~24k session ends through one server, and+/// each SessionEnd auto-commit walks a wiki tree and object database that+/// keep growing, which dominates ingest wall clock. `retrieval` never+/// reads git history, so the harness runs its servers with this variable+/// set and the commits skipped. Read once at startup; nothing else may+/// read it, and no product path sets it.+fn benchmark_skip_wiki_checkpoints() -> bool {+ matches!(+ std::env::var("AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS").as_deref(),+ Ok("1") | Ok("true")+ )+}+async fn configure_embedder(config: &Config,store: &Store,diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rsindex dbdc3639..d9cd9181 100644--- a/crates/ai-memory-hooks/src/router.rs+++ b/crates/ai-memory-hooks/src/router.rs@@ -642,6 +642,17 @@ pub struct HookState {/// subagent exclusion so a marked event still seeds its tail tracking./// Empty by default.pub ingest_drop_list: crate::drop_list::IngestDropList,+ /// Benchmark escape hatch (`AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS=1`, set+ /// only by the retrieval-benchmark harness in `evals/`): skip the+ /// SessionEnd auto-commits of the wiki git tree. The LongMemEval replay+ /// ends ~24k sessions through one server, and each checkpoint walks a+ /// tree and object database that keep growing, so commits dominate+ /// ingest wall clock there (see `docs/benchmarks/README.md`). Retrieval+ /// never reads git history, so skipping the commits changes no measured+ /// number — the markdown and the index stay the source of truth. Every+ /// other consumer keeps committing: default false, and the flag is read+ /// once at server startup, never per event.+ pub skip_wiki_checkpoints: bool,}/// The owner to stamp on the session and handoff rows this event creates@@ -4276,7 +4287,11 @@ async fn process_authorized(HookSessionAdmission::AlreadyEnded { session } => {let commit_msg = format!("repair session {}", short_id(&session_id.to_string()));let mut recovered = false;- match state.wiki.commit_all(&commit_msg) {+ match if state.skip_wiki_checkpoints {+ Ok(None)+ } else {+ state.wiki.commit_all(&commit_msg)+ } {Ok(Some(oid)) => {recovered = true;state.ingest_metrics.record_persisted(now_unix_ms());@@ -4518,7 +4533,11 @@ async fn process_authorized("lifecycle-only session {}",short_id(&session_id.to_string()),);- match state.wiki.commit_all(&commit_msg) {+ match if state.skip_wiki_checkpoints {+ Ok(None)+ } else {+ state.wiki.commit_all(&commit_msg)+ } {Ok(Some(oid)) => debug!(commit = %oid, "lifecycle-only log auto-commit"),Ok(None) => debug!("wiki clean after lifecycle-only SessionEnd"),Err(e) => warn!(error = %e, "lifecycle-only log auto-commit failed"),@@ -4728,10 +4747,16 @@ async fn process_authorized(short_id(&session_id.to_string()),new_page.title.chars().take(60).collect::<String>(),);- match state.wiki.commit_all(&commit_msg) {- Ok(Some(oid)) => debug!(commit = %oid, "wiki auto-commit"),- Ok(None) => debug!("wiki clean; no auto-commit"),- Err(e) => warn!(error = %e, "auto-commit failed"),+ // The benchmark escape skips the commit; everything the index and the+ // page reader use has already landed (markdown + store transaction).+ if !state.skip_wiki_checkpoints {+ match state.wiki.commit_all(&commit_msg) {+ Ok(Some(oid)) => debug!(commit = %oid, "wiki auto-commit"),+ Ok(None) => debug!("wiki clean; no auto-commit"),+ Err(e) => warn!(error = %e, "auto-commit failed"),+ }+ } else {+ debug!(session = %session_id, "wiki auto-commit skipped (benchmark mode)");}// Persist the optional LLM work before acknowledging the hook, after// deterministic wiki writes are committed so the worker cannot race@@ -6986,6 +7011,7 @@ pub(crate) mod tests {mid_session_routing: MidSessionRouting::default(),profile: ai_memory_core::profile::ProfileSettings::default(),ingest_drop_list: crate::drop_list::IngestDropList::default(),+ skip_wiki_checkpoints: false,}}@@ -15756,6 +15782,103 @@ pub(crate) mod tests {);}+ #[tokio::test]+ async fn benchmark_skip_wiki_checkpoints_leaves_the_page_but_no_commit() {+ // The control: a default session end auto-commits the wiki tree.+ // The benchmark escape must skip only that commit — the session+ // page, the ended-session row and the store index all still land,+ // because retrieval reads markdown + SQLite, never git history.+ async fn replay_session(state: &Arc<HookState>, tmp: &TempDir) -> SessionId {+ let session = SessionId::new();+ for (event, body) in [+ ("session-start", serde_json::json!({})),+ (+ "user-prompt",+ serde_json::json!({ "prompt": "teach me the parser fix" }),+ ),+ ("session-end", serde_json::json!({ "reason": "exit" })),+ ] {+ let envelope = HookEnvelope::from_query_and_body(+ HookQuery {+ event: event.into(),+ agent: Some("claude-code".into()),+ cwd: Some(tmp.path().to_string_lossy().into_owned()),+ workspace: Some("default".into()),+ project: Some("scratch".into()),+ ..Default::default()+ },+ serde_json::json!({+ "session_id": session.to_string(),+ "cwd": tmp.path(),+ "body": body,+ }),+ );+ process(state, envelope, None, Vec::new()).await.unwrap();+ }+ session+ }++ let control_tmp = TempDir::new().unwrap();+ let control_state = Arc::new(make_state(&control_tmp).await);+ let control_session = replay_session(&control_state, &control_tmp).await;+ let checkpoints = control_state.wiki.recent_checkpoints(10).unwrap();+ assert!(+ checkpoints.iter().any(|c| c.summary.starts_with(&format!(+ "session {}",+ short_id(&control_session.to_string())+ ))),+ "default SessionEnd must auto-commit; got {:?}",+ checkpoints+ .iter()+ .map(|c| c.summary.as_str())+ .collect::<Vec<_>>()+ );++ let skip_tmp = TempDir::new().unwrap();+ let mut skip_state = make_state(&skip_tmp).await;+ skip_state.skip_wiki_checkpoints = true;+ let skip_state = Arc::new(skip_state);+ let skip_session = replay_session(&skip_state, &skip_tmp).await;++ // Everything retrieval reads still landed.+ let pages = skip_state+ .reader+ .recent_pages_for_project(skip_state.workspace_id, skip_state.project_id, 20)+ .await+ .unwrap();+ assert!(+ pages+ .iter()+ .any(|p| p.path.as_str().starts_with("sessions/")),+ "the session summary page must still be written with checkpoints skipped"+ );+ let disposition = skip_state+ .reader+ .session_end_disposition(+ skip_session,+ skip_state.workspace_id,+ skip_state.project_id,+ AgentKind::ClaudeCode,+ )+ .await+ .unwrap();+ assert_eq!(+ disposition,+ ai_memory_store::SessionEndDisposition::AlreadyEnded,+ "the session row must still be ended with checkpoints skipped"+ );+ // And the git tree was not committed.+ let checkpoints = skip_state.wiki.recent_checkpoints(10).unwrap();+ assert!(+ checkpoints.is_empty(),+ "the benchmark escape must leave no session checkpoint; got {:?}",+ checkpoints+ .iter()+ .map(|c| c.summary.as_str())+ .collect::<Vec<_>>()+ );+ }+#[tokio::test]async fn session_end_with_create_on_session_end_disabled_writes_summary_page_without_handoff() {let tmp = TempDir::new().unwrap();diff --git a/crates/ai-memory-hooks/tests/suite/idle_end.rs b/crates/ai-memory-hooks/tests/suite/idle_end.rsindex de3c5317..3b62c4e1 100644--- a/crates/ai-memory-hooks/tests/suite/idle_end.rs+++ b/crates/ai-memory-hooks/tests/suite/idle_end.rs@@ -85,6 +85,7 @@ async fn harness() -> (TempDir, Store, Wiki, HookState) {mid_session_routing: ai_memory_core::MidSessionRouting::default(),profile: ai_memory_core::profile::ProfileSettings::default(),ingest_drop_list: crate::drop_list::IngestDropList::default(),+ skip_wiki_checkpoints: false,};(tmp, store, wiki, state)}diff --git a/crates/ai-memory-mcp/tests/suite/admin_metrics.rs b/crates/ai-memory-mcp/tests/suite/admin_metrics.rsindex c11650eb..3de346d0 100644--- a/crates/ai-memory-mcp/tests/suite/admin_metrics.rs+++ b/crates/ai-memory-mcp/tests/suite/admin_metrics.rs@@ -92,6 +92,7 @@ async fn harness() -> Harness {mid_session_routing: ai_memory_core::MidSessionRouting::default(),profile: ai_memory_core::profile::ProfileSettings::default(),ingest_drop_list: ai_memory_hooks::IngestDropList::default(),+ skip_wiki_checkpoints: false,subagent_sessions: Arc::new(tokio::sync::Mutex::new(SubagentSessionSet::default())),ingest_rate: Arc::new(tokio::sync::Mutex::new(ai_memory_hooks::IngestRateLimiter::disabled(),diff --git a/crates/ai-memory-mcp/tests/suite/autoscope_multiuser.rs b/crates/ai-memory-mcp/tests/suite/autoscope_multiuser.rsindex 8d97e9d0..b93854c3 100644--- a/crates/ai-memory-mcp/tests/suite/autoscope_multiuser.rs+++ b/crates/ai-memory-mcp/tests/suite/autoscope_multiuser.rs@@ -184,6 +184,7 @@ impl MultiUserHarness {mid_session_routing: ai_memory_core::MidSessionRouting::default(),profile: ai_memory_core::profile::ProfileSettings::default(),ingest_drop_list: ai_memory_hooks::IngestDropList::default(),+ skip_wiki_checkpoints: false,subagent_sessions: Arc::new(tokio::sync::Mutex::new(SubagentSessionSet::default())),ingest_rate: Arc::new(tokio::sync::Mutex::new(ai_memory_hooks::IngestRateLimiter::disabled(),diff --git a/crates/ai-memory-mcp/tests/suite/recap_identity.rs b/crates/ai-memory-mcp/tests/suite/recap_identity.rsindex 711f982b..f5be1009 100644--- a/crates/ai-memory-mcp/tests/suite/recap_identity.rs+++ b/crates/ai-memory-mcp/tests/suite/recap_identity.rs@@ -69,6 +69,7 @@ async fn harness() -> Harness {ingest_gates: ai_memory_hooks::IngestGates::default(),cwd_probes: ai_memory_hooks::CwdProbes::default(),ingest_drop_list: ai_memory_hooks::IngestDropList::default(),+ skip_wiki_checkpoints: false,consolidate_on_session_end: false,session_consolidation_notify: None,profile_notify: None,diff --git a/crates/ai-memory-mcp/tests/suite/stress_autoscope.rs b/crates/ai-memory-mcp/tests/suite/stress_autoscope.rsindex aad23642..ef039446 100644--- a/crates/ai-memory-mcp/tests/suite/stress_autoscope.rs+++ b/crates/ai-memory-mcp/tests/suite/stress_autoscope.rs@@ -172,6 +172,7 @@ impl Harness {mid_session_routing: ai_memory_core::MidSessionRouting::default(),profile: ai_memory_core::profile::ProfileSettings::default(),ingest_drop_list: ai_memory_hooks::IngestDropList::default(),+ skip_wiki_checkpoints: false,subagent_sessions: Arc::new(tokio::sync::Mutex::new(SubagentSessionSet::default())),ingest_rate: Arc::new(tokio::sync::Mutex::new(ai_memory_hooks::IngestRateLimiter::disabled(),diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.mdindex 35d17045..d5787366 100644--- a/docs/benchmarks/README.md+++ b/docs/benchmarks/README.md@@ -156,3 +156,75 @@ baseline-vs-baseline determinism check is clean (all accuracy/token deltas `0.00expectation first keeps a run from being read in whichever way suits it;a result that misses it still follows the release policy. The idea camefrom #1255.++## Running it fast++The full release benchmark is two runs (zero-LLM FTS and local embeddings)+over all 470 scored questions. Profiling the ingest path in October 2026+found most wall clock went to wiki git checkpointing, which retrieval+never reads: every SessionEnd runs `commit_all`, each commit walks a wiki+tree and object database that keep growing across the run, and the+per-event `log.md` append queues behind the same commit mutex. On a+100-question prefix those two phases were 55% and 38% of server-side+ingest time. The two changes below follow from that, and neither touches+a published number.++### Skip wiki checkpoints (release runs)++`AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS=1`, passed to the eval server via+the harness's general `--server-env` escape, makes the server skip the+SessionEnd wiki auto-commits. Retrieval never reads git history (the+markdown files and the SQLite index are the source of truth), so+accuracy, query latency and context tokens are unchanged. Verified on the+16- and 100-question prefixes: every hit@k and recall@k matched a default+run to the last digit, and the 100-question FTS wall clock fell from+155.9 s to 12.1 s. The variable appears in the report's config line+(`env:AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS=1`), so a run that used it+says so. Accuracy, latency and context-token numbers stay comparable with+earlier releases; only ingest wall clock differs, because runs before+October 2026 paid for the commits.++```bash+ai-memory-eval retrieval \+ --server-bin target/release/ai-memory \+ --concurrency 6 --http-timeout-secs 900 \+ --server-env AI_MEMORY_RETRIEVAL__OBSERVATION_STREAM=true \+ --server-env AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS=1++ai-memory-eval retrieval \+ --server-bin target/release/ai-memory \+ --concurrency 6 --http-timeout-secs 900 \+ --embeddings local \+ --server-env AI_MEMORY_RETRIEVAL__OBSERVATION_STREAM=true \+ --server-env AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS=1+```++### Run the two modes back to back, not at the same time++Two harness processes are fully independent (separate server, temp data+dir and run dir), and concurrent runs score identically to serial ones:+a concurrent and a serial 100-question pair showed zero ranking+differences. But with checkpoints skipped the FTS run takes minutes,+while the local run saturates the CPU with embedding work. Sharing the+machine slowed the local leg by 20–45% in repeated trials on the+32-thread benchmark box, so run FTS first, then local. Concurrent+execution only pays on hardware with real core headroom.++### Iteration runs (never the release comparison)++`--sample N` (first N questions, one slice), `--stratify N` (first N per+question type, every slice) and `--question-id a,b,c` (chosen questions)+exist for iteration and smoke runs; `evals/README.md` documents each. A+working loop is `--stratify 5` (30 scored questions, every slice) for+direction, then the failing questions by id. Release comparisons stay+full-470 per the release policy above; a subset number never lands in+`docs/benchmarks/`.++### What to expect++On the benchmark hardware (Ryzen 9 7950X3D, 32 threads), with checkpoints+skipped: the FTS run finishes in a couple of minutes (~5,400 items/s+sustained on a 282-question prefix); the local-embeddings run is bounded+by the embedder at roughly 19 pages/s and lands near half an hour. Expect+about ±15% of run-to-run spread on wall clock when the machine has other+work on it; accuracy noise is the ±0.005 band above, not wall clock.