Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(wiki): embed pages written by apply_batch and checkpoint restore
AkitaOnRailsd64c321bc073
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex c012712f..0a8c0725 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -25,13 +25,16 @@ version in [`docs/changelog/`](docs/changelog/):session index (0.3 s on the same store). The session list and sessiondetail reads behind `/web`, `/api/v1` and `memory_read_session_observations`ran the same per-session probes and are pinned the same way. (#1257)-- Fixed pages written by an auto-improvement approval getting no embedding.- An approval through `pending-writes approve`, the `memory_auto_improve`- tool, `POST /admin/auto-improve` or the scheduler wrote and indexed the- page but never embedded it, so it stayed out of the vector stream of- hybrid search until an embedding backfill ran, which by default is only at- the next server start. The approval now embeds the applied page the same- way `memory_write_page` does (#1258).+- Fixed pages written by an auto-improvement approval, a batch write or a+ checkpoint restore getting no embedding. An approval (through+ `pending-writes approve`, the `memory_auto_improve` tool,+ `POST /admin/auto-improve` or the scheduler), the batch writes behind+ consolidation, the forget sweep, dream and bootstrap, and+ `ai-memory restore-page` wrote and indexed the page but never embedded+ it, so it stayed out of the vector stream of hybrid search until an+ embedding backfill ran, which by default is only at the next server+ start. These paths now embed each written page the same way+ `memory_write_page` does. (#1258)## [3.0.0] - 2026-10-13diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rsindex b2b58019..83267655 100644--- a/crates/ai-memory-wiki/src/wiki.rs+++ b/crates/ai-memory-wiki/src/wiki.rs@@ -296,10 +296,12 @@ impl Wiki {&self.sanitizer}- /// Attach an embedder. When set, `write_page` computes + stores an- /// embedding for the new version synchronously. `apply_batch` keeps- /// the SQL/file fan-out atomic and leaves vector completeness to- /// admin or scheduled embedding backfill. Without an embedder,+ /// Attach an embedder. When set, every page-writing path (`write_page`,+ /// `apply_batch`, auto-improvement approval, checkpoint restore and the+ /// watcher's reindex) computes + stores an embedding for the new version+ /// on the caller's task, after its mutation guard is released; a+ /// provider failure is recorded and leaves the page to admin or+ /// scheduled embedding backfill. Without an embedder,/// vector search is skipped and `ReaderPool::hybrid_search` uses/// FTS5 + entity + graph expansion.#[must_use]@@ -900,29 +902,38 @@ impl Wiki {let title = derive_title(&md.frontmatter, &md.body, &path);let links = crate::markdown::extract_all_links(&md.frontmatter, &md.body, &path);let meta = derive_index_metadata(&path, &md.frontmatter)?;+ let body = md.body.clone();+ let abstract_text = frontmatter_abstract(&md.frontmatter).map(str::to_owned);- let _guard = self.mutation_lock.read().await;- self.ensure_project_workspace(workspace_id, project_id)- .await?;- let abs = self.confined_page_path(workspace_id, project_id, &path, Prepare::Parents)?;- self.git.write_atomic(&abs, raw.as_bytes())?;- let id = self- .writer- .upsert_page(NewPage {- workspace_id,- project_id,- path,- title,- body: md.body,- tier: meta.tier,- frontmatter_json: md.frontmatter,- pinned: meta.pinned,- links,- author_id: None,- expires_at: meta.expires_at,- entities: meta.entities,- evidence: Vec::new(),- })+ let id = {+ let _guard = self.mutation_lock.read().await;+ self.ensure_project_workspace(workspace_id, project_id)+ .await?;+ let abs = self.confined_page_path(workspace_id, project_id, &path, Prepare::Parents)?;+ self.git.write_atomic(&abs, raw.as_bytes())?;+ self.writer+ .upsert_page(NewPage {+ workspace_id,+ project_id,+ path,+ title,+ body: md.body,+ tier: meta.tier,+ frontmatter_json: md.frontmatter,+ pinned: meta.pinned,+ links,+ author_id: None,+ expires_at: meta.expires_at,+ entities: meta.entities,+ evidence: Vec::new(),+ })+ .await?+ };+ // Embed the restored version like `write_page` does; without this it+ // ranked FTS-only until a backfill. The watcher's reindex of the file+ // just written hits the unchanged-content short-circuit and does not+ // embed it either.+ self.embed_page_version(id, &body, abstract_text.as_deref()).await?;Ok(id)}@@ -2591,6 +2602,22 @@ impl Wiki {(ids, dispatches)};+ // Embed each applied version like `write_page` does, after the+ // guards are released. Unlike `write_page`, a store error while+ // persisting a vector is logged rather than returned: the batch has+ // committed, and callers (consolidation, sweep, dream, bootstrap)+ // record per-page outcomes from these ids, so an `Err` here would+ // report written pages as failed. The vector-less page is picked up+ // by the next embedding backfill.+ for (id, (path, frontmatter, body, _)) in ids.iter().zip(&dispatches) {+ if let Err(error) = self+ .embed_page_version(*id, body, frontmatter_abstract(frontmatter))+ .await+ {+ tracing::warn!(%error, path = %path.as_str(), "batch page indexed without its embedding");+ }+ }+if let Some(chain) = &self.admission_chain {for (path, frontmatter, body, ctx) in &dispatches {if let Some(ctx) = ctx {@@ -2889,11 +2916,15 @@ impl Wiki {/// Embed `body` (and its L0 `abstract:` line, if present) into/// `page_id`'s vectors, when an embedder is configured.///- /// Shared by [`Wiki::write_page`] (every version written through the- /// API) and [`Wiki::reindex_page_locked`] (external file writes and- /// rewrites picked up by the watcher), so a page version becomes- /// hybrid-searchable through the same mechanism regardless of which path- /// produced it. Before this was pulled out, only `write_page` embedded —+ /// Called after the upsert, outside the mutation guard, by every path+ /// that writes a page version: [`Wiki::write_page`] (every version+ /// written through the API), [`Wiki::reindex_page`] (external file+ /// writes and rewrites picked up by the watcher), auto-improvement+ /// approval, [`Wiki::apply_batch`] and checkpoint restore. A page version+ /// becomes hybrid-searchable through the same mechanism regardless of+ /// which path produced it; the watcher cannot cover the others, because+ /// its reindex of a file the wiki just wrote hits the unchanged-content+ /// short-circuit. Before this was pulled out, only `write_page` embedded —/// a watcher-driven rewrite got a new page id (the supersession chain/// worked) but no embedding until a manual `ai-memory embed` (issue/// #929): hybrid search silently fell back to FTS-only ranking for@@ -4953,6 +4984,97 @@ mod tests {);}+ /// Body and abstract vector sets for the synthetic test embedder.+ async fn synthetic_embedded(+ store: &Store,+ ws: WorkspaceId,+ proj: ProjectId,+ ) -> (Vec<PageId>, Vec<PageId>) {+ let (provider, model) = ("synthetic".to_string(), "bag-of-words-v1".to_string());+ let body = store+ .reader+ .embedded_page_ids(ws, proj, provider.clone(), model.clone(), 32)+ .await+ .unwrap();+ let abstracts = store+ .reader+ .abstract_embedded_page_ids(ws, proj, provider, model, 32)+ .await+ .unwrap();+ (body, abstracts)+ }++ /// Batch writes (consolidation, sweep, dream, bootstrap) embed every+ /// applied version, abstract included, like `write_page`. Before, they+ /// stayed out of the vector stream until an embedding backfill.+ #[tokio::test]+ async fn apply_batch_embeds_every_applied_page() {+ let tmp = TempDir::new().unwrap();+ let (store, wiki, ws, proj) = scoped(&tmp).await;+ let embedder: Arc<dyn ai_memory_llm::Embedder> =+ Arc::new(ai_memory_llm::SyntheticEmbedder::new(32));+ let wiki = wiki.with_embedder(embedder);+ let ids = wiki+ .apply_batch(vec![+ req(+ ws,+ proj,+ "notes/plain.md",+ "plain batch body",+ serde_json::json!({}),+ ),+ req(+ ws,+ proj,+ "notes/summarized.md",+ "summarized batch body",+ serde_json::json!({"abstract": "one-line batch summary"}),+ ),+ ])+ .await+ .unwrap();+ let (body, abstracts) = synthetic_embedded(&store, ws, proj).await;+ assert!(+ ids.iter().all(|id| body.contains(id)),+ "every batch page must be embedded by the batch itself: {ids:?} vs {body:?}"+ );+ assert_eq!(abstracts, vec![ids[1]], "only the page with an abstract");+ }++ /// A checkpoint restore writes a new latest version and must embed it;+ /// before, the restored version had no vectors until a backfill.+ #[tokio::test]+ async fn restore_from_checkpoint_embeds_the_restored_version() {+ let tmp = TempDir::new().unwrap();+ let (store, wiki, ws, proj) = scoped(&tmp).await;+ let path = "notes/restored.md";+ // Both earlier versions are written with no embedder, so only the+ // restore itself can produce the vectors checked below.+ wiki.write_page(req(+ ws,+ proj,+ path,+ "checkpointed body",+ serde_json::json!({"abstract": "checkpointed summary"}),+ ))+ .await+ .unwrap();+ let rev = wiki.commit_all("checkpoint").unwrap().unwrap().to_string();+ wiki.write_page(req(ws, proj, path, "later body", serde_json::json!({})))+ .await+ .unwrap();+ let embedder: Arc<dyn ai_memory_llm::Embedder> =+ Arc::new(ai_memory_llm::SyntheticEmbedder::new(32));+ let wiki = wiki.with_embedder(embedder);+ let id = wiki+ .restore_page_from_checkpoint(ws, proj, PagePath::new(path).unwrap(), &rev)+ .await+ .unwrap();+ let (body, abstracts) = synthetic_embedded(&store, ws, proj).await;+ assert_eq!(body, vec![id], "the restored version is embedded");+ assert_eq!(abstracts, vec![id], "with its restored abstract");+ }+#[tokio::test]async fn auto_improve_approval_admission_mutation_is_persisted_and_scrubbed() {use axum::http::StatusCode;diff --git a/docs/prior-art-implementation-findings.md b/docs/prior-art-implementation-findings.mdindex 15c988f6..61dc06db 100644--- a/docs/prior-art-implementation-findings.md+++ b/docs/prior-art-implementation-findings.md@@ -177,6 +177,12 @@ Options evaluated:The implemented choice is status plus scheduled backfill. It preserves thedurable FTS path, exposes vector gaps, and keeps backfill opt-in.+Since #1258, `apply_batch`, auto-improvement approvals and+checkpoint restores also embed each written version after the batch commits+and the mutation guard is released, through the same best-effort helper as+`write_page`. Backfill now covers only provider failures and configuration+changes, not every page a batch writes.+### Implemented: Graph/Link RetrievalThe schema has a `links` table with nullable `to_page_id`. Markdown