Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(wiki): embed pages written by apply_batch and checkpoint restore

AkitaOnRailsd64c321bc073
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c012712f..0a8c0725 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,13 +25,16 @@ version in [`docs/changelog/`](docs/changelog/):
session index (0.3 s on the same store). The session list and session
detail reads behind `/web`, `/api/v1` and `memory_read_session_observations`
ran the same per-session probes and are pinned the same way. (#1257)
-- Fixed pages written by an auto-improvement approval getting no embedding.
- An approval through `pending-writes approve`, the `memory_auto_improve`
- tool, `POST /admin/auto-improve` or the scheduler wrote and indexed the
- page but never embedded it, so it stayed out of the vector stream of
- hybrid search until an embedding backfill ran, which by default is only at
- the next server start. The approval now embeds the applied page the same
- way `memory_write_page` does (#1258).
+- Fixed pages written by an auto-improvement approval, a batch write or a
+ checkpoint restore getting no embedding. An approval (through
+ `pending-writes approve`, the `memory_auto_improve` tool,
+ `POST /admin/auto-improve` or the scheduler), the batch writes behind
+ consolidation, the forget sweep, dream and bootstrap, and
+ `ai-memory restore-page` wrote and indexed the page but never embedded
+ it, so it stayed out of the vector stream of hybrid search until an
+ embedding backfill ran, which by default is only at the next server
+ start. These paths now embed each written page the same way
+ `memory_write_page` does. (#1258)
## [3.0.0] - 2026-10-13
diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rs
index b2b58019..83267655 100644
--- a/crates/ai-memory-wiki/src/wiki.rs
+++ b/crates/ai-memory-wiki/src/wiki.rs
@@ -296,10 +296,12 @@ impl Wiki {
&self.sanitizer
}
- /// Attach an embedder. When set, `write_page` computes + stores an
- /// embedding for the new version synchronously. `apply_batch` keeps
- /// the SQL/file fan-out atomic and leaves vector completeness to
- /// admin or scheduled embedding backfill. Without an embedder,
+ /// Attach an embedder. When set, every page-writing path (`write_page`,
+ /// `apply_batch`, auto-improvement approval, checkpoint restore and the
+ /// watcher's reindex) computes + stores an embedding for the new version
+ /// on the caller's task, after its mutation guard is released; a
+ /// provider failure is recorded and leaves the page to admin or
+ /// scheduled embedding backfill. Without an embedder,
/// vector search is skipped and `ReaderPool::hybrid_search` uses
/// FTS5 + entity + graph expansion.
#[must_use]
@@ -900,29 +902,38 @@ impl Wiki {
let title = derive_title(&md.frontmatter, &md.body, &path);
let links = crate::markdown::extract_all_links(&md.frontmatter, &md.body, &path);
let meta = derive_index_metadata(&path, &md.frontmatter)?;
+ let body = md.body.clone();
+ let abstract_text = frontmatter_abstract(&md.frontmatter).map(str::to_owned);
- let _guard = self.mutation_lock.read().await;
- self.ensure_project_workspace(workspace_id, project_id)
- .await?;
- let abs = self.confined_page_path(workspace_id, project_id, &path, Prepare::Parents)?;
- self.git.write_atomic(&abs, raw.as_bytes())?;
- let id = self
- .writer
- .upsert_page(NewPage {
- workspace_id,
- project_id,
- path,
- title,
- body: md.body,
- tier: meta.tier,
- frontmatter_json: md.frontmatter,
- pinned: meta.pinned,
- links,
- author_id: None,
- expires_at: meta.expires_at,
- entities: meta.entities,
- evidence: Vec::new(),
- })
+ let id = {
+ let _guard = self.mutation_lock.read().await;
+ self.ensure_project_workspace(workspace_id, project_id)
+ .await?;
+ let abs = self.confined_page_path(workspace_id, project_id, &path, Prepare::Parents)?;
+ self.git.write_atomic(&abs, raw.as_bytes())?;
+ self.writer
+ .upsert_page(NewPage {
+ workspace_id,
+ project_id,
+ path,
+ title,
+ body: md.body,
+ tier: meta.tier,
+ frontmatter_json: md.frontmatter,
+ pinned: meta.pinned,
+ links,
+ author_id: None,
+ expires_at: meta.expires_at,
+ entities: meta.entities,
+ evidence: Vec::new(),
+ })
+ .await?
+ };
+ // Embed the restored version like `write_page` does; without this it
+ // ranked FTS-only until a backfill. The watcher's reindex of the file
+ // just written hits the unchanged-content short-circuit and does not
+ // embed it either.
+ self.embed_page_version(id, &body, abstract_text.as_deref())
.await?;
Ok(id)
}
@@ -2591,6 +2602,22 @@ impl Wiki {
(ids, dispatches)
};
+ // Embed each applied version like `write_page` does, after the
+ // guards are released. Unlike `write_page`, a store error while
+ // persisting a vector is logged rather than returned: the batch has
+ // committed, and callers (consolidation, sweep, dream, bootstrap)
+ // record per-page outcomes from these ids, so an `Err` here would
+ // report written pages as failed. The vector-less page is picked up
+ // by the next embedding backfill.
+ for (id, (path, frontmatter, body, _)) in ids.iter().zip(&dispatches) {
+ if let Err(error) = self
+ .embed_page_version(*id, body, frontmatter_abstract(frontmatter))
+ .await
+ {
+ tracing::warn!(%error, path = %path.as_str(), "batch page indexed without its embedding");
+ }
+ }
+
if let Some(chain) = &self.admission_chain {
for (path, frontmatter, body, ctx) in &dispatches {
if let Some(ctx) = ctx {
@@ -2889,11 +2916,15 @@ impl Wiki {
/// Embed `body` (and its L0 `abstract:` line, if present) into
/// `page_id`'s vectors, when an embedder is configured.
///
- /// Shared by [`Wiki::write_page`] (every version written through the
- /// API) and [`Wiki::reindex_page_locked`] (external file writes and
- /// rewrites picked up by the watcher), so a page version becomes
- /// hybrid-searchable through the same mechanism regardless of which path
- /// produced it. Before this was pulled out, only `write_page` embedded —
+ /// Called after the upsert, outside the mutation guard, by every path
+ /// that writes a page version: [`Wiki::write_page`] (every version
+ /// written through the API), [`Wiki::reindex_page`] (external file
+ /// writes and rewrites picked up by the watcher), auto-improvement
+ /// approval, [`Wiki::apply_batch`] and checkpoint restore. A page version
+ /// becomes hybrid-searchable through the same mechanism regardless of
+ /// which path produced it; the watcher cannot cover the others, because
+ /// its reindex of a file the wiki just wrote hits the unchanged-content
+ /// short-circuit. Before this was pulled out, only `write_page` embedded —
/// a watcher-driven rewrite got a new page id (the supersession chain
/// worked) but no embedding until a manual `ai-memory embed` (issue
/// #929): hybrid search silently fell back to FTS-only ranking for
@@ -4953,6 +4984,97 @@ mod tests {
);
}
+ /// Body and abstract vector sets for the synthetic test embedder.
+ async fn synthetic_embedded(
+ store: &Store,
+ ws: WorkspaceId,
+ proj: ProjectId,
+ ) -> (Vec<PageId>, Vec<PageId>) {
+ let (provider, model) = ("synthetic".to_string(), "bag-of-words-v1".to_string());
+ let body = store
+ .reader
+ .embedded_page_ids(ws, proj, provider.clone(), model.clone(), 32)
+ .await
+ .unwrap();
+ let abstracts = store
+ .reader
+ .abstract_embedded_page_ids(ws, proj, provider, model, 32)
+ .await
+ .unwrap();
+ (body, abstracts)
+ }
+
+ /// Batch writes (consolidation, sweep, dream, bootstrap) embed every
+ /// applied version, abstract included, like `write_page`. Before, they
+ /// stayed out of the vector stream until an embedding backfill.
+ #[tokio::test]
+ async fn apply_batch_embeds_every_applied_page() {
+ let tmp = TempDir::new().unwrap();
+ let (store, wiki, ws, proj) = scoped(&tmp).await;
+ let embedder: Arc<dyn ai_memory_llm::Embedder> =
+ Arc::new(ai_memory_llm::SyntheticEmbedder::new(32));
+ let wiki = wiki.with_embedder(embedder);
+ let ids = wiki
+ .apply_batch(vec![
+ req(
+ ws,
+ proj,
+ "notes/plain.md",
+ "plain batch body",
+ serde_json::json!({}),
+ ),
+ req(
+ ws,
+ proj,
+ "notes/summarized.md",
+ "summarized batch body",
+ serde_json::json!({"abstract": "one-line batch summary"}),
+ ),
+ ])
+ .await
+ .unwrap();
+ let (body, abstracts) = synthetic_embedded(&store, ws, proj).await;
+ assert!(
+ ids.iter().all(|id| body.contains(id)),
+ "every batch page must be embedded by the batch itself: {ids:?} vs {body:?}"
+ );
+ assert_eq!(abstracts, vec![ids[1]], "only the page with an abstract");
+ }
+
+ /// A checkpoint restore writes a new latest version and must embed it;
+ /// before, the restored version had no vectors until a backfill.
+ #[tokio::test]
+ async fn restore_from_checkpoint_embeds_the_restored_version() {
+ let tmp = TempDir::new().unwrap();
+ let (store, wiki, ws, proj) = scoped(&tmp).await;
+ let path = "notes/restored.md";
+ // Both earlier versions are written with no embedder, so only the
+ // restore itself can produce the vectors checked below.
+ wiki.write_page(req(
+ ws,
+ proj,
+ path,
+ "checkpointed body",
+ serde_json::json!({"abstract": "checkpointed summary"}),
+ ))
+ .await
+ .unwrap();
+ let rev = wiki.commit_all("checkpoint").unwrap().unwrap().to_string();
+ wiki.write_page(req(ws, proj, path, "later body", serde_json::json!({})))
+ .await
+ .unwrap();
+ let embedder: Arc<dyn ai_memory_llm::Embedder> =
+ Arc::new(ai_memory_llm::SyntheticEmbedder::new(32));
+ let wiki = wiki.with_embedder(embedder);
+ let id = wiki
+ .restore_page_from_checkpoint(ws, proj, PagePath::new(path).unwrap(), &rev)
+ .await
+ .unwrap();
+ let (body, abstracts) = synthetic_embedded(&store, ws, proj).await;
+ assert_eq!(body, vec![id], "the restored version is embedded");
+ assert_eq!(abstracts, vec![id], "with its restored abstract");
+ }
+
#[tokio::test]
async fn auto_improve_approval_admission_mutation_is_persisted_and_scrubbed() {
use axum::http::StatusCode;
diff --git a/docs/prior-art-implementation-findings.md b/docs/prior-art-implementation-findings.md
index 15c988f6..61dc06db 100644
--- a/docs/prior-art-implementation-findings.md
+++ b/docs/prior-art-implementation-findings.md
@@ -177,6 +177,12 @@ Options evaluated:
The implemented choice is status plus scheduled backfill. It preserves the
durable FTS path, exposes vector gaps, and keeps backfill opt-in.
+Since #1258, `apply_batch`, auto-improvement approvals and
+checkpoint restores also embed each written version after the batch commits
+and the mutation guard is released, through the same best-effort helper as
+`write_page`. Backfill now covers only provider failures and configuration
+changes, not every page a batch writes.
+
### Implemented: Graph/Link Retrieval
The schema has a `links` table with nullable `to_page_id`. Markdown