Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
test(retrieval): pin the abstract embedding paths; embed L0 abstract at write time
evanmaranzanobb446952bb2f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex cd434cd3..50a969fd 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -15,13 +15,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0(×0.77 combined kind/tier), with `session_recall_bonus` (default 0.25)sizing the lift; `abstract_vectors` adds a fifth RRF stream over the new`page_abstract_embeddings` table (V61) — the L0 layer: a page's- frontmatter `abstract:` line is embedded by the same backfill as its- body, and a one-line summary embeds far more sharply than a- multi-thousand-character body. Measured on a 138-query golden set over a+ frontmatter `abstract:` line is embedded on its own, at write time when an+ embedder is attached and by the same backfill as the body otherwise+ (a one-line summary embeds far more sharply than a+ multi-thousand-character body). Measured on a 138-query golden set over aproduction two-year wiki (FTS5 + entity + vector + graph, mis-teiQwen3-Embedding-8B): hit@1 0.609 → 0.746 (+22%), NDCG@10 0.782 → 0.879(+12%) with both enabled; every unrouted category keeps its exact- baseline ordering, and both signals are visible per hit in+ baseline ordering. Cross-checked on an independently built 99-query+ golden set against the same store: hit@1 0.556 → 0.626, NDCG@10+ 0.712 → 0.779 (+9.4%). Both signals are visible per hit in`memory_query(explain=true)` (`intent`, `intent_boost`,`abstract_rank`, `rrf.abstract`).diff --git a/crates/ai-memory-consolidate/tests/suite/abstract_backfill.rs b/crates/ai-memory-consolidate/tests/suite/abstract_backfill.rsnew file mode 100644index 00000000..a8935971--- /dev/null+++ b/crates/ai-memory-consolidate/tests/suite/abstract_backfill.rs@@ -0,0 +1,108 @@+//! The L0 abstract backfill: a page whose frontmatter carries `abstract:`+//! gets that line embedded into `page_abstract_embeddings` by the same+//! backfill pass as its body, so the opt-in `[retrieval] abstract_vectors`+//! stream has data even for pages written before the feature (or with the+//! embedder detached).++use std::sync::Arc;++use ai_memory_core::{PagePath, Tier};+use ai_memory_llm::{Embedder, SyntheticEmbedder};+use ai_memory_store::Store;+use ai_memory_wiki::{Wiki, WritePageRequest};+use tempfile::TempDir;++#[tokio::test]+async fn embedding_backfill_embeds_frontmatter_abstract() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "scratch", None)+ .await+ .unwrap();+ // Write with the embedder detached: the backfill under test owns both+ // the body and the abstract embedding passes.+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();+ wiki.write_page(WritePageRequest {+ workspace_id: ws,+ project_id: proj,+ path: PagePath::new("notes/with-abstract.md").unwrap(),+ frontmatter: serde_json::json!({+ "title": "with abstract",+ "abstract": "one-line summary of the page for the abstract stream"+ }),+ body: "alpha bravo".to_string(),+ tier: Tier::Semantic,+ pinned: false,+ title: None,+ admission_ctx: None,+ author_id: None,+ actor: ai_memory_core::ActorContext::anonymous(),+ })+ .await+ .unwrap();++ let embedder: Arc<dyn Embedder> = Arc::new(SyntheticEmbedder::new(64));+ let counts = ai_memory_consolidate::run_embedding_backfill(+ &store.reader,+ &store.writer,+ &wiki,+ &embedder,+ ws,+ proj,+ ai_memory_consolidate::EmbedBackfillOptions::default(),+ )+ .await+ .unwrap();++ assert_eq!(counts.embedded, 1, "body embedded");+ assert_eq!(counts.abstracts_embedded, 1, "abstract embedded");+ let abstract_ids = store+ .reader+ .abstract_embedded_page_ids(+ ws,+ proj,+ "synthetic".to_string(),+ "bag-of-words-v1".to_string(),+ 64,+ )+ .await+ .unwrap();+ assert_eq!(abstract_ids.len(), 1);++ // A page without the key is untouched by the abstract pass.+ wiki.write_page(WritePageRequest {+ workspace_id: ws,+ project_id: proj,+ path: PagePath::new("notes/plain.md").unwrap(),+ frontmatter: serde_json::json!({"title": "plain"}),+ body: "charlie delta".to_string(),+ tier: Tier::Semantic,+ pinned: false,+ title: None,+ admission_ctx: None,+ author_id: None,+ actor: ai_memory_core::ActorContext::anonymous(),+ })+ .await+ .unwrap();+ let counts = ai_memory_consolidate::run_embedding_backfill(+ &store.reader,+ &store.writer,+ &wiki,+ &embedder,+ ws,+ proj,+ ai_memory_consolidate::EmbedBackfillOptions::default(),+ )+ .await+ .unwrap();+ assert_eq!(counts.embedded, 1, "plain page body embedded");+ assert_eq!(counts.abstracts_embedded, 0, "no abstract to embed");+}diff --git a/crates/ai-memory-consolidate/tests/suite/mod.rs b/crates/ai-memory-consolidate/tests/suite/mod.rsindex 2c1c4825..eceef9ad 100644--- a/crates/ai-memory-consolidate/tests/suite/mod.rs+++ b/crates/ai-memory-consolidate/tests/suite/mod.rs@@ -2,6 +2,7 @@//! test harness (see the `integration` module in `src/lib.rs`), so they cost//! no extra binary; a new file must be declared below.+mod abstract_backfill;mod access_breadth_sweep;mod embed_backfill;mod embeddings;diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex 28b5137e..8047f912 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -1853,6 +1853,17 @@ fn store_embeddings_in_table(Ok(())}+/// Remove a page's L0 abstract embedding row, if any. Called when a page is+/// rewritten without its frontmatter `abstract:` so the abstract stream+/// never ranks on a line the page no longer carries.+pub fn delete_abstract_embedding(conn: &mut Connection, page_id: &PageId) -> StoreResult<()> {+ conn.execute(+ "DELETE FROM page_abstract_embeddings WHERE page_id = ?1",+ params![page_id.as_bytes()],+ )?;+ Ok(())+}+/// Fold per-client MCP tool-call deltas into their `(client, day)`/// buckets. UPSERT per bucket inside one transaction: the buffer layer/// in the MCP server coalesces a minute of calls into a handful ofdiff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rsindex 9e42461c..3eaf2369 100644--- a/crates/ai-memory-store/src/writer.rs+++ b/crates/ai-memory-store/src/writer.rs@@ -315,6 +315,10 @@ pub(crate) enum WriteCmd {embeddings: Vec<EmbeddingWrite>,reply: oneshot::Sender<StoreResult<()>>,},+ DeleteAbstractEmbedding {+ page_id: PageId,+ reply: oneshot::Sender<StoreResult<()>>,+ },DeleteStalePageEmbeddings {workspace_id: WorkspaceId,project_id: Option<ProjectId>,@@ -1314,6 +1318,20 @@ impl WriterHandle {rx.await.map_err(|_| StoreError::WriterClosed)?}+ /// Remove a page's L0 abstract embedding row (`page_abstract_embeddings`),+ /// if any. Called when a page is rewritten without its frontmatter+ /// `abstract:` so the abstract stream never ranks a line the page no+ /// longer carries.+ ///+ /// # Errors+ /// Returns [`StoreError::WriterClosed`] or propagates SQL errors.+ pub async fn delete_abstract_embedding(&self, page_id: PageId) -> StoreResult<()> {+ let (tx, rx) = oneshot::channel();+ self.send(WriteCmd::DeleteAbstractEmbedding { page_id, reply: tx })+ .await?;+ rx.await.map_err(|_| StoreError::WriterClosed)?+ }+/// Remove embedding rows in a workspace/project scope whose triple does not match the configured provider/model/dim.////// Used when re-embedding after a model migration (e.g. Gemini → OpenRouter).@@ -2960,6 +2978,10 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {let result = ops::store_abstract_embeddings(&mut conn, &embeddings);send_or_warn(reply, result, "store_abstract_embeddings");}+ WriteCmd::DeleteAbstractEmbedding { page_id, reply } => {+ let result = ops::delete_abstract_embedding(&mut conn, &page_id);+ send_or_warn(reply, result, "delete_abstract_embedding");+ }WriteCmd::DeleteStalePageEmbeddings {workspace_id,project_id,diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rsindex 88068c8d..37e65f98 100644--- a/crates/ai-memory-wiki/src/wiki.rs+++ b/crates/ai-memory-wiki/src/wiki.rs@@ -1997,6 +1997,9 @@ impl Wiki {crate::markdown::extract_all_links(&markdown.frontmatter, &markdown.body, &path);let expires_at = parse_expires_at(&path, &markdown.frontmatter)?;let entities = parse_entities(&path, &markdown.frontmatter)?;+ // Read before the destructuring move below: the embed step needs the+ // L0 `abstract:` line out of the final frontmatter.+ let abstract_text = frontmatter_abstract(&markdown.frontmatter).map(str::to_owned);let Markdown {frontmatter: final_frontmatter,@@ -2083,6 +2086,38 @@ impl Wiki {.await;}}+ // L0 abstract: the frontmatter `abstract:` line is embedded on+ // its own so the opt-in abstract stream can rank on the sharp+ // one-line summary. A page rewritten without the key has its+ // stale abstract row removed, mirroring the body row's+ // replace-on-write semantics.+ match abstract_text.as_deref() {+ Some(abstract_text) => match embedder.embed_document(abstract_text).await {+ Ok(vec) => {+ self.writer+ .store_abstract_embeddings(vec![ai_memory_store::EmbeddingWrite {+ page_id,+ vector_bytes: f32_vec_to_bytes(&vec),+ provider: embedder.provider().to_string(),+ model: embedder.model().to_string(),+ dim: embedder.dim(),+ }])+ .await?;+ }+ Err(e) => {+ tracing::warn!(error = %e, path = %page_id, "abstract embedding failed; page indexed without it");+ let _ = self+ .writer+ .record_embed_failure(+ page_id,+ ai_memory_store::EmbedOutcome::Failed,+ Some(e.to_string()),+ )+ .await;+ }+ },+ None => self.writer.delete_abstract_embedding(page_id).await?,+ }}// Non-blocking webhooks fire-and-forget only after the page has landed@@ -2289,6 +2324,15 @@ pub(crate) fn parse_expires_at(/// current file's value when nothing but the timestamp would change, so/// an idempotent rewrite emits byte-identical markdown (no git churn,/// and the store's modulo-`generated.at` comparison keeps the row).+/// The frontmatter `abstract:` line, when it is a non-empty string.+fn frontmatter_abstract(frontmatter: &serde_json::Value) -> Option<&str> {+ frontmatter+ .get("abstract")+ .and_then(serde_json::Value::as_str)+ .map(str::trim)+ .filter(|s| !s.is_empty())+}+fn conform_frontmatter_for_disk(abs: &Path, page_path: &str, markdown: &mut Markdown) {ai_memory_core::okf::conform_frontmatter(page_path, &mut markdown.frontmatter);let inherited = std::fs::read_to_string(abs)@@ -2924,6 +2968,68 @@ mod tests {(store, wiki, ws, proj)}+ #[tokio::test]+ async fn write_page_embeds_frontmatter_abstract_and_cleans_stale_rows() {+ let tmp = TempDir::new().unwrap();+ let (store, wiki, ws, proj) = scoped(&tmp).await;+ let embedder: Arc<dyn ai_memory_llm::Embedder> =+ Arc::new(ai_memory_llm::SyntheticEmbedder::new(64));+ let wiki = wiki.with_embedder(embedder);+ let write_abs = |frontmatter: serde_json::Value| {+ wiki.write_page(WritePageRequest {+ workspace_id: ws,+ project_id: proj,+ path: PagePath::new("notes/abs.md").unwrap(),+ frontmatter,+ body: "alpha bravo".to_string(),+ tier: Tier::Semantic,+ pinned: false,+ title: None,+ admission_ctx: None,+ author_id: None,+ actor: ActorContext::anonymous(),+ })+ };++ write_abs(serde_json::json!({"title": "abs", "abstract": "one-line summary"}))+ .await+ .unwrap();+ let ids = store+ .reader+ .abstract_embedded_page_ids(+ ws,+ proj,+ "synthetic".to_string(),+ "bag-of-words-v1".to_string(),+ 64,+ )+ .await+ .unwrap();+ assert_eq!(ids.len(), 1, "abstract embedded at write time");++ // Rewriting without the key leaves the latest version without an+ // abstract row; the superseded version's row stays behind, matching+ // the body embedding's versioning semantics.+ write_abs(serde_json::json!({"title": "abs"}))+ .await+ .unwrap();+ let ids = store+ .reader+ .abstract_embedded_page_ids(+ ws,+ proj,+ "synthetic".to_string(),+ "bag-of-words-v1".to_string(),+ 64,+ )+ .await+ .unwrap();+ assert!(+ ids.is_empty(),+ "latest version must not carry an abstract row"+ );+ }+#[tokio::test]async fn auto_improve_sidecar_writes_non_indexed_review_file() {let tmp = TempDir::new().unwrap();