Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

perf(benchmarks): skip wiki checkpoints in retrieval runs

AkitaOnRails4939499990b2
diff --git a/crates/ai-memory-cli/src/commands/backfill.rs b/crates/ai-memory-cli/src/commands/backfill.rs
index 6bdd00d6..96d45e86 100644
--- a/crates/ai-memory-cli/src/commands/backfill.rs
+++ b/crates/ai-memory-cli/src/commands/backfill.rs
@@ -1848,6 +1848,7 @@ mod tests {
mid_session_routing: ai_memory_core::MidSessionRouting::default(),
profile: ai_memory_core::profile::ProfileSettings::default(),
ingest_drop_list: ai_memory_hooks::IngestDropList::default(),
+ skip_wiki_checkpoints: false,
};
let endpoint = ServerEndpoint::for_hook_target("http://localhost".into(), None);
let sid = "native-session-with-a-very-long-identifier-that-would-break-the-old-key";
@@ -2049,6 +2050,7 @@ mod tests {
mid_session_routing: ai_memory_core::MidSessionRouting::default(),
profile: ai_memory_core::profile::ProfileSettings::default(),
ingest_drop_list: ai_memory_hooks::IngestDropList::default(),
+ skip_wiki_checkpoints: false,
};
let endpoint = ServerEndpoint::for_hook_target("http://localhost".into(), None);
let sid = "native-session-resumed-across-two-offline-intervals";
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rs
index 975910d1..0e30237f 100644
--- a/crates/ai-memory-cli/src/commands/serve.rs
+++ b/crates/ai-memory-cli/src/commands/serve.rs
@@ -1681,6 +1681,9 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {
mid_session_routing: config.routing.mid_session,
profile: config.profile.clone(),
ingest_drop_list: (*config.ingest.drop_list).clone(),
+ // Benchmark-only escape (docs/benchmarks/README.md), read once
+ // here so the hooks crate keeps making no environment reads.
+ skip_wiki_checkpoints: benchmark_skip_wiki_checkpoints(),
});
let workstreams = workstream_router(WorkstreamState {
writer: store.writer.clone(),
@@ -2957,6 +2960,20 @@ fn auto_improve_review_config_from_settings(
}
}
+/// The retrieval-benchmark escape hatch (docs/benchmarks/README.md): the
+/// LongMemEval harness replays ~24k session ends through one server, and
+/// each SessionEnd auto-commit walks a wiki tree and object database that
+/// keep growing, which dominates ingest wall clock. `retrieval` never
+/// reads git history, so the harness runs its servers with this variable
+/// set and the commits skipped. Read once at startup; nothing else may
+/// read it, and no product path sets it.
+fn benchmark_skip_wiki_checkpoints() -> bool {
+ matches!(
+ std::env::var("AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS").as_deref(),
+ Ok("1") | Ok("true")
+ )
+}
+
async fn configure_embedder(
config: &Config,
store: &Store,
diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rs
index dbdc3639..d9cd9181 100644
--- a/crates/ai-memory-hooks/src/router.rs
+++ b/crates/ai-memory-hooks/src/router.rs
@@ -642,6 +642,17 @@ pub struct HookState {
/// subagent exclusion so a marked event still seeds its tail tracking.
/// Empty by default.
pub ingest_drop_list: crate::drop_list::IngestDropList,
+ /// Benchmark escape hatch (`AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS=1`, set
+ /// only by the retrieval-benchmark harness in `evals/`): skip the
+ /// SessionEnd auto-commits of the wiki git tree. The LongMemEval replay
+ /// ends ~24k sessions through one server, and each checkpoint walks a
+ /// tree and object database that keep growing, so commits dominate
+ /// ingest wall clock there (see `docs/benchmarks/README.md`). Retrieval
+ /// never reads git history, so skipping the commits changes no measured
+ /// number — the markdown and the index stay the source of truth. Every
+ /// other consumer keeps committing: default false, and the flag is read
+ /// once at server startup, never per event.
+ pub skip_wiki_checkpoints: bool,
}
/// The owner to stamp on the session and handoff rows this event creates
@@ -4276,7 +4287,11 @@ async fn process_authorized(
HookSessionAdmission::AlreadyEnded { session } => {
let commit_msg = format!("repair session {}", short_id(&session_id.to_string()));
let mut recovered = false;
- match state.wiki.commit_all(&commit_msg) {
+ match if state.skip_wiki_checkpoints {
+ Ok(None)
+ } else {
+ state.wiki.commit_all(&commit_msg)
+ } {
Ok(Some(oid)) => {
recovered = true;
state.ingest_metrics.record_persisted(now_unix_ms());
@@ -4518,7 +4533,11 @@ async fn process_authorized(
"lifecycle-only session {}",
short_id(&session_id.to_string()),
);
- match state.wiki.commit_all(&commit_msg) {
+ match if state.skip_wiki_checkpoints {
+ Ok(None)
+ } else {
+ state.wiki.commit_all(&commit_msg)
+ } {
Ok(Some(oid)) => debug!(commit = %oid, "lifecycle-only log auto-commit"),
Ok(None) => debug!("wiki clean after lifecycle-only SessionEnd"),
Err(e) => warn!(error = %e, "lifecycle-only log auto-commit failed"),
@@ -4728,10 +4747,16 @@ async fn process_authorized(
short_id(&session_id.to_string()),
new_page.title.chars().take(60).collect::<String>(),
);
- match state.wiki.commit_all(&commit_msg) {
- Ok(Some(oid)) => debug!(commit = %oid, "wiki auto-commit"),
- Ok(None) => debug!("wiki clean; no auto-commit"),
- Err(e) => warn!(error = %e, "auto-commit failed"),
+ // The benchmark escape skips the commit; everything the index and the
+ // page reader use has already landed (markdown + store transaction).
+ if !state.skip_wiki_checkpoints {
+ match state.wiki.commit_all(&commit_msg) {
+ Ok(Some(oid)) => debug!(commit = %oid, "wiki auto-commit"),
+ Ok(None) => debug!("wiki clean; no auto-commit"),
+ Err(e) => warn!(error = %e, "auto-commit failed"),
+ }
+ } else {
+ debug!(session = %session_id, "wiki auto-commit skipped (benchmark mode)");
}
// Persist the optional LLM work before acknowledging the hook, after
// deterministic wiki writes are committed so the worker cannot race
@@ -6986,6 +7011,7 @@ pub(crate) mod tests {
mid_session_routing: MidSessionRouting::default(),
profile: ai_memory_core::profile::ProfileSettings::default(),
ingest_drop_list: crate::drop_list::IngestDropList::default(),
+ skip_wiki_checkpoints: false,
}
}
@@ -15756,6 +15782,103 @@ pub(crate) mod tests {
);
}
+ #[tokio::test]
+ async fn benchmark_skip_wiki_checkpoints_leaves_the_page_but_no_commit() {
+ // The control: a default session end auto-commits the wiki tree.
+ // The benchmark escape must skip only that commit — the session
+ // page, the ended-session row and the store index all still land,
+ // because retrieval reads markdown + SQLite, never git history.
+ async fn replay_session(state: &Arc<HookState>, tmp: &TempDir) -> SessionId {
+ let session = SessionId::new();
+ for (event, body) in [
+ ("session-start", serde_json::json!({})),
+ (
+ "user-prompt",
+ serde_json::json!({ "prompt": "teach me the parser fix" }),
+ ),
+ ("session-end", serde_json::json!({ "reason": "exit" })),
+ ] {
+ let envelope = HookEnvelope::from_query_and_body(
+ HookQuery {
+ event: event.into(),
+ agent: Some("claude-code".into()),
+ cwd: Some(tmp.path().to_string_lossy().into_owned()),
+ workspace: Some("default".into()),
+ project: Some("scratch".into()),
+ ..Default::default()
+ },
+ serde_json::json!({
+ "session_id": session.to_string(),
+ "cwd": tmp.path(),
+ "body": body,
+ }),
+ );
+ process(state, envelope, None, Vec::new()).await.unwrap();
+ }
+ session
+ }
+
+ let control_tmp = TempDir::new().unwrap();
+ let control_state = Arc::new(make_state(&control_tmp).await);
+ let control_session = replay_session(&control_state, &control_tmp).await;
+ let checkpoints = control_state.wiki.recent_checkpoints(10).unwrap();
+ assert!(
+ checkpoints.iter().any(|c| c.summary.starts_with(&format!(
+ "session {}",
+ short_id(&control_session.to_string())
+ ))),
+ "default SessionEnd must auto-commit; got {:?}",
+ checkpoints
+ .iter()
+ .map(|c| c.summary.as_str())
+ .collect::<Vec<_>>()
+ );
+
+ let skip_tmp = TempDir::new().unwrap();
+ let mut skip_state = make_state(&skip_tmp).await;
+ skip_state.skip_wiki_checkpoints = true;
+ let skip_state = Arc::new(skip_state);
+ let skip_session = replay_session(&skip_state, &skip_tmp).await;
+
+ // Everything retrieval reads still landed.
+ let pages = skip_state
+ .reader
+ .recent_pages_for_project(skip_state.workspace_id, skip_state.project_id, 20)
+ .await
+ .unwrap();
+ assert!(
+ pages
+ .iter()
+ .any(|p| p.path.as_str().starts_with("sessions/")),
+ "the session summary page must still be written with checkpoints skipped"
+ );
+ let disposition = skip_state
+ .reader
+ .session_end_disposition(
+ skip_session,
+ skip_state.workspace_id,
+ skip_state.project_id,
+ AgentKind::ClaudeCode,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ disposition,
+ ai_memory_store::SessionEndDisposition::AlreadyEnded,
+ "the session row must still be ended with checkpoints skipped"
+ );
+ // And the git tree was not committed.
+ let checkpoints = skip_state.wiki.recent_checkpoints(10).unwrap();
+ assert!(
+ checkpoints.is_empty(),
+ "the benchmark escape must leave no session checkpoint; got {:?}",
+ checkpoints
+ .iter()
+ .map(|c| c.summary.as_str())
+ .collect::<Vec<_>>()
+ );
+ }
+
#[tokio::test]
async fn session_end_with_create_on_session_end_disabled_writes_summary_page_without_handoff() {
let tmp = TempDir::new().unwrap();
diff --git a/crates/ai-memory-hooks/tests/suite/idle_end.rs b/crates/ai-memory-hooks/tests/suite/idle_end.rs
index de3c5317..3b62c4e1 100644
--- a/crates/ai-memory-hooks/tests/suite/idle_end.rs
+++ b/crates/ai-memory-hooks/tests/suite/idle_end.rs
@@ -85,6 +85,7 @@ async fn harness() -> (TempDir, Store, Wiki, HookState) {
mid_session_routing: ai_memory_core::MidSessionRouting::default(),
profile: ai_memory_core::profile::ProfileSettings::default(),
ingest_drop_list: crate::drop_list::IngestDropList::default(),
+ skip_wiki_checkpoints: false,
};
(tmp, store, wiki, state)
}
diff --git a/crates/ai-memory-mcp/tests/suite/admin_metrics.rs b/crates/ai-memory-mcp/tests/suite/admin_metrics.rs
index c11650eb..3de346d0 100644
--- a/crates/ai-memory-mcp/tests/suite/admin_metrics.rs
+++ b/crates/ai-memory-mcp/tests/suite/admin_metrics.rs
@@ -92,6 +92,7 @@ async fn harness() -> Harness {
mid_session_routing: ai_memory_core::MidSessionRouting::default(),
profile: ai_memory_core::profile::ProfileSettings::default(),
ingest_drop_list: ai_memory_hooks::IngestDropList::default(),
+ skip_wiki_checkpoints: false,
subagent_sessions: Arc::new(tokio::sync::Mutex::new(SubagentSessionSet::default())),
ingest_rate: Arc::new(tokio::sync::Mutex::new(
ai_memory_hooks::IngestRateLimiter::disabled(),
diff --git a/crates/ai-memory-mcp/tests/suite/autoscope_multiuser.rs b/crates/ai-memory-mcp/tests/suite/autoscope_multiuser.rs
index 8d97e9d0..b93854c3 100644
--- a/crates/ai-memory-mcp/tests/suite/autoscope_multiuser.rs
+++ b/crates/ai-memory-mcp/tests/suite/autoscope_multiuser.rs
@@ -184,6 +184,7 @@ impl MultiUserHarness {
mid_session_routing: ai_memory_core::MidSessionRouting::default(),
profile: ai_memory_core::profile::ProfileSettings::default(),
ingest_drop_list: ai_memory_hooks::IngestDropList::default(),
+ skip_wiki_checkpoints: false,
subagent_sessions: Arc::new(tokio::sync::Mutex::new(SubagentSessionSet::default())),
ingest_rate: Arc::new(tokio::sync::Mutex::new(
ai_memory_hooks::IngestRateLimiter::disabled(),
diff --git a/crates/ai-memory-mcp/tests/suite/recap_identity.rs b/crates/ai-memory-mcp/tests/suite/recap_identity.rs
index 711f982b..f5be1009 100644
--- a/crates/ai-memory-mcp/tests/suite/recap_identity.rs
+++ b/crates/ai-memory-mcp/tests/suite/recap_identity.rs
@@ -69,6 +69,7 @@ async fn harness() -> Harness {
ingest_gates: ai_memory_hooks::IngestGates::default(),
cwd_probes: ai_memory_hooks::CwdProbes::default(),
ingest_drop_list: ai_memory_hooks::IngestDropList::default(),
+ skip_wiki_checkpoints: false,
consolidate_on_session_end: false,
session_consolidation_notify: None,
profile_notify: None,
diff --git a/crates/ai-memory-mcp/tests/suite/stress_autoscope.rs b/crates/ai-memory-mcp/tests/suite/stress_autoscope.rs
index aad23642..ef039446 100644
--- a/crates/ai-memory-mcp/tests/suite/stress_autoscope.rs
+++ b/crates/ai-memory-mcp/tests/suite/stress_autoscope.rs
@@ -172,6 +172,7 @@ impl Harness {
mid_session_routing: ai_memory_core::MidSessionRouting::default(),
profile: ai_memory_core::profile::ProfileSettings::default(),
ingest_drop_list: ai_memory_hooks::IngestDropList::default(),
+ skip_wiki_checkpoints: false,
subagent_sessions: Arc::new(tokio::sync::Mutex::new(SubagentSessionSet::default())),
ingest_rate: Arc::new(tokio::sync::Mutex::new(
ai_memory_hooks::IngestRateLimiter::disabled(),
diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md
index 35d17045..d5787366 100644
--- a/docs/benchmarks/README.md
+++ b/docs/benchmarks/README.md
@@ -156,3 +156,75 @@ baseline-vs-baseline determinism check is clean (all accuracy/token deltas `0.00
expectation first keeps a run from being read in whichever way suits it;
a result that misses it still follows the release policy. The idea came
from #1255.
+
+## Running it fast
+
+The full release benchmark is two runs (zero-LLM FTS and local embeddings)
+over all 470 scored questions. Profiling the ingest path in October 2026
+found most wall clock went to wiki git checkpointing, which retrieval
+never reads: every SessionEnd runs `commit_all`, each commit walks a wiki
+tree and object database that keep growing across the run, and the
+per-event `log.md` append queues behind the same commit mutex. On a
+100-question prefix those two phases were 55% and 38% of server-side
+ingest time. The two changes below follow from that, and neither touches
+a published number.
+
+### Skip wiki checkpoints (release runs)
+
+`AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS=1`, passed to the eval server via
+the harness's general `--server-env` escape, makes the server skip the
+SessionEnd wiki auto-commits. Retrieval never reads git history (the
+markdown files and the SQLite index are the source of truth), so
+accuracy, query latency and context tokens are unchanged. Verified on the
+16- and 100-question prefixes: every hit@k and recall@k matched a default
+run to the last digit, and the 100-question FTS wall clock fell from
+155.9 s to 12.1 s. The variable appears in the report's config line
+(`env:AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS=1`), so a run that used it
+says so. Accuracy, latency and context-token numbers stay comparable with
+earlier releases; only ingest wall clock differs, because runs before
+October 2026 paid for the commits.
+
+```bash
+ai-memory-eval retrieval \
+ --server-bin target/release/ai-memory \
+ --concurrency 6 --http-timeout-secs 900 \
+ --server-env AI_MEMORY_RETRIEVAL__OBSERVATION_STREAM=true \
+ --server-env AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS=1
+
+ai-memory-eval retrieval \
+ --server-bin target/release/ai-memory \
+ --concurrency 6 --http-timeout-secs 900 \
+ --embeddings local \
+ --server-env AI_MEMORY_RETRIEVAL__OBSERVATION_STREAM=true \
+ --server-env AI_MEMORY_EVAL_SKIP_WIKI_CHECKPOINTS=1
+```
+
+### Run the two modes back to back, not at the same time
+
+Two harness processes are fully independent (separate server, temp data
+dir and run dir), and concurrent runs score identically to serial ones:
+a concurrent and a serial 100-question pair showed zero ranking
+differences. But with checkpoints skipped the FTS run takes minutes,
+while the local run saturates the CPU with embedding work. Sharing the
+machine slowed the local leg by 20–45% in repeated trials on the
+32-thread benchmark box, so run FTS first, then local. Concurrent
+execution only pays on hardware with real core headroom.
+
+### Iteration runs (never the release comparison)
+
+`--sample N` (first N questions, one slice), `--stratify N` (first N per
+question type, every slice) and `--question-id a,b,c` (chosen questions)
+exist for iteration and smoke runs; `evals/README.md` documents each. A
+working loop is `--stratify 5` (30 scored questions, every slice) for
+direction, then the failing questions by id. Release comparisons stay
+full-470 per the release policy above; a subset number never lands in
+`docs/benchmarks/`.
+
+### What to expect
+
+On the benchmark hardware (Ryzen 9 7950X3D, 32 threads), with checkpoints
+skipped: the FTS run finishes in a couple of minutes (~5,400 items/s
+sustained on a 282-question prefix); the local-embeddings run is bounded
+by the embedder at roughly 19 pages/s and lands near half an hour. Expect
+about ±15% of run-to-run spread on wall clock when the machine has other
+work on it; accuracy noise is the ±0.005 band above, not wall clock.