Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(wiki): embed page versions re-indexed by the watcher

Maxsuel Einstein748fb79faaa1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 900dbc50..c9e7c7f3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Fixed
+- A page file rewritten under `wiki/<ws>/<project>/` (the OKF import path)
+ now gets its new version embedded the same way a brand-new file does.
+ The watcher's `reindex_page` upserted the new version and stopped —
+ embedding only ever ran on the `write_page` API path — so a rewrite left
+ hybrid search silently degraded to FTS-only ranking for that page until
+ someone ran `ai-memory embed` by hand. (#958)
- `[capture] ignore_paths` now covers shell commands. Shell tools (`Bash`,
`shell`, `execute_bash`, `terminal`, …) were classified as non-file and always
kept, so `cat docs/adr/*.md` stored the ignored file's full text in the
diff --git a/crates/ai-memory-wiki/src/watcher.rs b/crates/ai-memory-wiki/src/watcher.rs
index 25a60f3b..95381518 100644
--- a/crates/ai-memory-wiki/src/watcher.rs
+++ b/crates/ai-memory-wiki/src/watcher.rs
@@ -14,6 +14,16 @@
//! overflow, hidden-dir globs, etc.). Hidden-directory paths are
//! explicitly NOT skipped (#798 lesson).
//!
+//! Deletions are not reconciled (see #929): the watcher only handles
+//! create/modify events, so a page whose file disappears stays indexed until
+//! `ai-memory delete-page` removes it explicitly.
+//!
+//! A reindexed page version — new file or rewrite alike — is embedded the
+//! same way `write_page` embeds one, via `Wiki::embed_page_version` (#929):
+//! before, only pages written through the API got embeddings, and a
+//! watcher-driven rewrite's new version silently had none until a manual
+//! `ai-memory embed`.
+//!
//! The watcher never *writes* to disk — that loop would be unbounded.
//! External writes drive store updates; internal writes drive disk +
//! store updates via [`Wiki::write_page`].
@@ -619,6 +629,7 @@ fn page_path_relative_to(root: &Path, abs: &Path) -> Option<PagePath> {
mod tests {
use super::*;
use ai_memory_store::Store;
+ use std::sync::Arc;
use tempfile::TempDir;
#[cfg(windows)]
@@ -652,7 +663,15 @@ mod tests {
.get_or_create_project(ws, "scratch", None)
.await
.unwrap();
- let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();
+ // Every production wiki the watcher actually clones carries a store
+ // reader (`serve.rs` attaches one before the embedder or the
+ // watcher) — attached here too so tests exercise the same
+ // reader-dependent path `reindex_page` uses to decide whether a
+ // reindexed page is eligible for embedding (#929), not the fail-closed
+ // no-reader fallback.
+ let wiki = Wiki::new(tmp.path(), store.writer.clone())
+ .unwrap()
+ .with_store_reader(store.reader.clone());
(tmp, store, wiki, ws, proj)
}
@@ -1405,4 +1424,86 @@ mod tests {
.unwrap();
assert!(hits.is_empty(), "direct symlink event must not be indexed");
}
+
+ // --- #929: watcher-driven rewrites get embedded ---
+
+ /// A page rewritten through the watcher's `reindex_page` path (an
+ /// external editor overwriting an already-imported file) must get its
+ /// new version embedded the same way a brand-new file does — without a
+ /// manual `ai-memory embed`. Before the fix, only `Wiki::write_page`
+ /// (the API path) embedded; `reindex_page_locked` upserted the new
+ /// version and stopped.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn reindex_embeds_a_rewritten_page_without_manual_embed() {
+ let (tmp, store, wiki, ws, proj) = setup().await;
+ let embedder: Arc<dyn ai_memory_llm::Embedder> =
+ Arc::new(ai_memory_llm::SyntheticEmbedder::new(32));
+ let wiki = wiki.with_embedder(embedder);
+
+ let proj_dir = tmp
+ .path()
+ .join("wiki")
+ .join(ws.to_string())
+ .join(proj.to_string());
+ std::fs::create_dir_all(&proj_dir).unwrap();
+ let target = proj_dir.join("rewrite.md");
+ std::fs::write(&target, "alpha bravo original\n").unwrap();
+
+ let path = PagePath::new("rewrite.md").unwrap();
+ let id1 = wiki.reindex_page(ws, proj, path.clone()).await.unwrap();
+ let embedded = store
+ .reader
+ .embedded_page_ids(ws, proj, "synthetic".into(), "bag-of-words-v1".into(), 32)
+ .await
+ .unwrap();
+ assert!(
+ embedded.contains(&id1),
+ "a new file indexed by the watcher must be embedded"
+ );
+
+ // Rewrite: same path, new body. `upsert_page` supersedes (mints a
+ // new id) rather than short-circuiting, because the body changed.
+ std::fs::write(&target, "charlie delta rewritten\n").unwrap();
+ let id2 = wiki.reindex_page(ws, proj, path.clone()).await.unwrap();
+ assert_ne!(id1, id2, "a body change must supersede, not short-circuit");
+
+ let embedded = store
+ .reader
+ .embedded_page_ids(ws, proj, "synthetic".into(), "bag-of-words-v1".into(), 32)
+ .await
+ .unwrap();
+ assert!(
+ embedded.contains(&id2),
+ "the rewritten version must be embedded without a manual `ai-memory embed`"
+ );
+ }
+
+ /// A no-op reindex (unchanged content, which is what every 30s
+ /// reconcile pass does for a stable tree) must NOT re-embed. Otherwise
+ /// every configured embedder would pay for the whole wiki's embedding
+ /// cost every `RECONCILE_INTERVAL`, forever.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn reindex_does_not_reembed_an_unchanged_page() {
+ let (tmp, _store, wiki, ws, proj) = setup().await;
+ let embedder: Arc<dyn ai_memory_llm::Embedder> =
+ Arc::new(ai_memory_llm::SyntheticEmbedder::new(32));
+ let wiki = wiki.with_embedder(embedder);
+
+ let proj_dir = tmp
+ .path()
+ .join("wiki")
+ .join(ws.to_string())
+ .join(proj.to_string());
+ std::fs::create_dir_all(&proj_dir).unwrap();
+ let target = proj_dir.join("stable.md");
+ std::fs::write(&target, "stabletoken unchanged\n").unwrap();
+
+ let path = PagePath::new("stable.md").unwrap();
+ let id1 = wiki.reindex_page(ws, proj, path.clone()).await.unwrap();
+ let id2 = wiki.reindex_page(ws, proj, path.clone()).await.unwrap();
+ assert_eq!(
+ id1, id2,
+ "unchanged content must short-circuit to the same id"
+ );
+ }
}
diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rs
index b347548a..fe77491d 100644
--- a/crates/ai-memory-wiki/src/wiki.rs
+++ b/crates/ai-memory-wiki/src/wiki.rs
@@ -93,6 +93,16 @@ enum PageStoreRemoval {
},
}
+/// Body + optional L0 abstract text a caller of `reindex_page_locked` should
+/// embed once it is safe to do so (i.e. after releasing whatever lock it
+/// took to call in). Captured before `upsert_page` moves `md.body` /
+/// `md.frontmatter`, and only constructed when the reindex actually produced
+/// a new page version worth embedding.
+struct PendingEmbed {
+ body: String,
+ abstract_text: Option<String>,
+}
+
/// What [`Wiki::move_session_page`] did with the on-disk
/// `sessions/<session_id>.md` file of the moved session.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -1006,10 +1016,18 @@ impl Wiki {
.await?;
let abs = self.abs_path(workspace_id, project_id, path);
if current_latest.is_none() && abs.try_exists()? {
- current_latest = Some(
- self.reindex_page_locked(workspace_id, project_id, path.clone())
- .await?,
- );
+ // Deliberately drops any `PendingEmbed`: this whole function runs
+ // under the exclusive write lock (`_guard` above, held until
+ // return), and embedding must never happen while that lock is
+ // held (#929) — it can block on the configured provider for an
+ // unbounded time, stalling every other wiki write meanwhile. The
+ // resurrected page's embedding (if any) is recovered later by
+ // `ai-memory embed` / the scheduled backfill, same as any other
+ // gap in `page_embeddings`.
+ let (id, _pending_embed) = self
+ .reindex_page_locked(workspace_id, project_id, path.clone())
+ .await?;
+ current_latest = Some(id);
}
self.writer
@@ -1539,20 +1557,43 @@ impl Wiki {
"refusing to index pending proposal sidecar",
));
}
- let _guard = self.mutation_lock.read().await;
- self.ensure_project_workspace(workspace_id, project_id)
- .await?;
-
- self.reindex_page_locked(workspace_id, project_id, path)
- .await
+ // Scoped so the mutation guard is dropped before the embed call
+ // below — same shape as `write_page` (#607/#929): embedding calls
+ // out to the configured provider, and holding the guard across that
+ // would block every other write in the wiki for its duration.
+ let (id, pending_embed) = {
+ let _guard = self.mutation_lock.read().await;
+ self.ensure_project_workspace(workspace_id, project_id)
+ .await?;
+ self.reindex_page_locked(workspace_id, project_id, path)
+ .await?
+ };
+ // A failed `store_embedding` here is caught and recorded inside
+ // `embed_page_version` (`record_embed_failure`), not retried by the
+ // watcher; `ai-memory embed` (manual) or the scheduled backfill tick
+ // is what recovers it.
+ if let Some(pending) = pending_embed {
+ self.embed_page_version(id, &pending.body, pending.abstract_text.as_deref())
+ .await?;
+ }
+ Ok(id)
}
+ /// Reindex `path` from disk into the store, returning the resulting page
+ /// id and — when the reindex produced a new version worth embedding —
+ /// the inputs for that embed. Embedding itself is deferred to the
+ /// caller, which must not call [`Wiki::embed_page_version`] while
+ /// holding a lock this function was called under: it may block on the
+ /// configured embedder for an arbitrary amount of time, and this path
+ /// has two very different callers with two different locking needs
+ /// (`reindex_page`'s brief read guard; `hard_delete_decay_tombstone`'s
+ /// long-lived write guard, which must never embed at all — see there).
async fn reindex_page_locked(
&self,
workspace_id: WorkspaceId,
project_id: ProjectId,
path: PagePath,
- ) -> WikiResult<PageId> {
+ ) -> WikiResult<(PageId, Option<PendingEmbed>)> {
let abs = self.abs_path(workspace_id, project_id, &path);
if std::fs::symlink_metadata(&abs)?.file_type().is_symlink() {
return Err(WikiError::Io(std::io::Error::other(format!(
@@ -1566,6 +1607,39 @@ impl Wiki {
// Markdown is the source of truth: preserve explicit tier/pinned
// metadata on reindex instead of forcing every page back to semantic.
let meta = derive_index_metadata(&path, &md.frontmatter)?;
+
+ // The watcher's own writes never go through `write_page` (it must
+ // never touch disk — see `crate::watcher`'s module docs), so this is
+ // the only place that can embed a page an external editor created or
+ // rewrote. Resolve the pre-upsert latest id (when an embedder and a
+ // store reader are both available) so a no-op reindex — the
+ // reconcile pass walks every page every 30s — never re-embeds a
+ // stable tree; only a genuine new/rewritten version, recognised by
+ // `upsert_page` minting a fresh id, does.
+ //
+ // Fails CLOSED, unlike most of this crate's reconcile-adjacent
+ // lookups: without a reader there is no cheap way to tell "new
+ // version" from "unchanged", and guessing wrong the open way (embed
+ // unconditionally) would re-embed the entire tree every
+ // `RECONCILE_INTERVAL` for as long as no reader is attached, which
+ // is worse than the rare bare `Wiki::new` fixture simply not getting
+ // watcher-driven embeddings. Skipped up front when no embedder is
+ // configured too, so the zero-LLM default path (invariant #13)
+ // never pays for the extra lookup.
+ let previous_id_and_body = match (&self.embedder, &self.store_reader) {
+ (Some(_), Some(reader)) => {
+ let previous_id = reader
+ .latest_page_id_by_ids(workspace_id, project_id, path.as_str().to_string())
+ .await?;
+ Some((
+ previous_id,
+ md.body.clone(),
+ frontmatter_abstract(&md.frontmatter).map(str::to_owned),
+ ))
+ }
+ _ => None,
+ };
+
let id = self
.writer
.upsert_page(NewPage {
@@ -1584,7 +1658,14 @@ impl Wiki {
evidence: Vec::new(),
})
.await?;
- Ok(id)
+
+ let pending_embed = previous_id_and_body.and_then(|(previous_id, body, abstract_text)| {
+ (previous_id != Some(id)).then_some(PendingEmbed {
+ body,
+ abstract_text,
+ })
+ });
+ Ok((id, pending_embed))
}
/// Read a `_meta.md` scope-manifest's frontmatter from `dir`.
@@ -2244,26 +2325,102 @@ impl Wiki {
// tool reply still happens "indexes commit in the same
// transaction" (basic-memory #763 lesson): no fire-and-forget
// background embedding.
- if let Some(embedder) = &self.embedder {
- match embedder.embed_document(&final_body).await {
+ self.embed_page_version(page_id, &final_body, abstract_text.as_deref())
+ .await?;
+
+ // Non-blocking webhooks fire-and-forget only after the page has landed
+ // on disk and the DB/index write has succeeded. They observe the final
+ // persisted page and cannot mutate or reject it.
+ if let (Some(chain), Some(ctx)) = (&self.admission_chain, &resolved_ctx) {
+ chain.dispatch_async(
+ Some(path_for_dispatch.as_str()),
+ &frontmatter_for_dispatch,
+ &final_body,
+ ctx,
+ );
+ }
+ Ok(page_id)
+ }
+
+ /// Embed `body` (and its L0 `abstract:` line, if present) into
+ /// `page_id`'s vectors, when an embedder is configured.
+ ///
+ /// Shared by [`Wiki::write_page`] (every version written through the
+ /// API) and [`Wiki::reindex_page_locked`] (external file writes and
+ /// rewrites picked up by the watcher), so a page version becomes
+ /// hybrid-searchable through the same mechanism regardless of which path
+ /// produced it. Before this was pulled out, only `write_page` embedded —
+ /// a watcher-driven rewrite got a new page id (the supersession chain
+ /// worked) but no embedding until a manual `ai-memory embed` (issue
+ /// #929): hybrid search silently fell back to FTS-only ranking for
+ /// every rewritten page.
+ ///
+ /// Best-effort: an embedder failure is logged and recorded via
+ /// `record_embed_failure` but never propagated, so a working page write
+ /// is never undone by an embedding hiccup.
+ ///
+ /// # Errors
+ /// Returns [`WikiError`] only for a store error while persisting a
+ /// successfully computed vector; provider/embedder failures are caught
+ /// and recorded instead.
+ async fn embed_page_version(
+ &self,
+ page_id: PageId,
+ body: &str,
+ abstract_text: Option<&str>,
+ ) -> WikiResult<()> {
+ let Some(embedder) = &self.embedder else {
+ return Ok(());
+ };
+ match embedder.embed_document(body).await {
+ Ok(vec) => {
+ let bytes = f32_vec_to_bytes(&vec);
+ self.writer
+ .store_embedding(
+ page_id,
+ bytes,
+ embedder.provider().to_string(),
+ embedder.model().to_string(),
+ embedder.dim(),
+ )
+ .await?;
+ }
+ Err(e) => {
+ tracing::warn!(error = %e, path = %page_id, "embedding failed; page indexed without it");
+ // The warning alone dies with the container. Record it so
+ // the page is attributable later (#528); best-effort,
+ // because failing to note a failure must not fail the
+ // write that already succeeded.
+ let _ = self
+ .writer
+ .record_embed_failure(
+ page_id,
+ ai_memory_store::EmbedOutcome::Failed,
+ Some(e.to_string()),
+ )
+ .await;
+ }
+ }
+ // L0 abstract: the frontmatter `abstract:` line is embedded on
+ // its own so the opt-in abstract stream can rank on the sharp
+ // one-line summary. A page rewritten without the key has its
+ // stale abstract row removed, mirroring the body row's
+ // replace-on-write semantics.
+ match abstract_text {
+ Some(abstract_text) => match embedder.embed_document(abstract_text).await {
Ok(vec) => {
- let bytes = f32_vec_to_bytes(&vec);
self.writer
- .store_embedding(
+ .store_abstract_embeddings(vec![ai_memory_store::EmbeddingWrite {
page_id,
- bytes,
- embedder.provider().to_string(),
- embedder.model().to_string(),
- embedder.dim(),
- )
+ vector_bytes: f32_vec_to_bytes(&vec),
+ provider: embedder.provider().to_string(),
+ model: embedder.model().to_string(),
+ dim: embedder.dim(),
+ }])
.await?;
}
Err(e) => {
- tracing::warn!(error = %e, path = %page_id, "embedding failed; page indexed without it");
- // The warning alone dies with the container. Record it so
- // the page is attributable later (#528); best-effort,
- // because failing to note a failure must not fail the
- // write that already succeeded.
+ tracing::warn!(error = %e, path = %page_id, "abstract embedding failed; page indexed without it");
let _ = self
.writer
.record_embed_failure(
@@ -2273,53 +2430,10 @@ impl Wiki {
)
.await;
}
- }
- // L0 abstract: the frontmatter `abstract:` line is embedded on
- // its own so the opt-in abstract stream can rank on the sharp
- // one-line summary. A page rewritten without the key has its
- // stale abstract row removed, mirroring the body row's
- // replace-on-write semantics.
- match abstract_text.as_deref() {
- Some(abstract_text) => match embedder.embed_document(abstract_text).await {
- Ok(vec) => {
- self.writer
- .store_abstract_embeddings(vec![ai_memory_store::EmbeddingWrite {
- page_id,
- vector_bytes: f32_vec_to_bytes(&vec),
- provider: embedder.provider().to_string(),
- model: embedder.model().to_string(),
- dim: embedder.dim(),
- }])
- .await?;
- }
- Err(e) => {
- tracing::warn!(error = %e, path = %page_id, "abstract embedding failed; page indexed without it");
- let _ = self
- .writer
- .record_embed_failure(
- page_id,
- ai_memory_store::EmbedOutcome::Failed,
- Some(e.to_string()),
- )
- .await;
- }
- },
- None => self.writer.delete_abstract_embedding(page_id).await?,
- }
- }
-
- // Non-blocking webhooks fire-and-forget only after the page has landed
- // on disk and the DB/index write has succeeded. They observe the final
- // persisted page and cannot mutate or reject it.
- if let (Some(chain), Some(ctx)) = (&self.admission_chain, &resolved_ctx) {
- chain.dispatch_async(
- Some(path_for_dispatch.as_str()),
- &frontmatter_for_dispatch,
- &final_body,
- ctx,
- );
+ },
+ None => self.writer.delete_abstract_embedding(page_id).await?,
}
- Ok(page_id)
+ Ok(())
}
}
@@ -4251,6 +4365,72 @@ mod tests {
);
}
+ /// #929: the reindex `hard_delete_decay_tombstone` performs when a
+ /// decayed page's file survived and was rewritten externally must NOT
+ /// embed the resurrected version. That whole function runs under the
+ /// exclusive write lock (`mutation_lock.write()`, held for the entire
+ /// call), and embedding can block on the configured provider for an
+ /// unbounded time — doing it there would stall every other wiki write
+ /// for as long as the provider call takes.
+ #[tokio::test]
+ async fn decay_cleanup_reindex_does_not_embed_under_the_exclusive_lock() {
+ let tmp = TempDir::new().unwrap();
+ let (store, wiki, ws, proj) = scoped(&tmp).await;
+ let embedder: Arc<dyn ai_memory_llm::Embedder> =
+ Arc::new(ai_memory_llm::SyntheticEmbedder::new(32));
+ let wiki = wiki.with_embedder(embedder);
+
+ let path = PagePath::new("sessions/decay-embed.md").unwrap();
+ let page_id = wiki
+ .write_page(req(
+ ws,
+ proj,
+ path.as_str(),
+ "original decayed body",
+ serde_json::json!({}),
+ ))
+ .await
+ .unwrap();
+ assert!(
+ wiki.evict_page_if_latest(ws, proj, &path, page_id, None)
+ .await
+ .unwrap(),
+ "precondition: the page is tombstoned by decay"
+ );
+
+ // The file survives the tombstone (`evict_page_if_latest` never
+ // touches disk) and is rewritten externally before the cleanup
+ // runs, so `hard_delete_decay_tombstone` must reindex it fresh
+ // rather than reuse the tombstoned version.
+ std::fs::write(wiki.abs_path(ws, proj, &path), "resurrected body content\n").unwrap();
+
+ wiki.hard_delete_decay_tombstone(ws, proj, &path, page_id, i64::MAX)
+ .await
+ .unwrap();
+
+ let resurrected_id = store
+ .reader
+ .latest_page_id_by_ids(ws, proj, path.as_str().to_string())
+ .await
+ .unwrap()
+ .expect("the resurrected version must be indexed");
+ assert_ne!(
+ resurrected_id, page_id,
+ "the rewrite must have produced a new version, not reused the tombstone"
+ );
+
+ let embedded = store
+ .reader
+ .embedded_page_ids(ws, proj, "synthetic".into(), "bag-of-words-v1".into(), 32)
+ .await
+ .unwrap();
+ assert!(
+ !embedded.contains(&resurrected_id),
+ "reindexing inside hard_delete_decay_tombstone must not embed \
+ while the exclusive lock is held"
+ );
+ }
+
#[tokio::test]
async fn write_page_rolls_back_file_when_store_upsert_fails() {
let tmp = TempDir::new().unwrap();
diff --git a/docs/okf.md b/docs/okf.md
index 33e7f925..25611239 100644
--- a/docs/okf.md
+++ b/docs/okf.md
@@ -159,5 +159,10 @@ so any later rewrite of an affected page (a restore, a hand edit, a
the export). Import has no dedicated command by design: the format is
native, so unpacking a bundle's concept files into a project's wiki
directory and letting the watcher (or `reindex`) ingest them IS the
- import path.
+ import path. Overwriting an already-imported concept file gets its new
+ version embedded the same way a brand-new file does — no manual
+ `ai-memory embed` needed. Deleting one does not yet remove it from the
+ index: the watcher only reconciles create/modify events, so a deleted
+ concept file still needs an explicit `ai-memory delete-page` (tracked in
+ #929).
- Retrieval regression: LongMemEval baseline re-run; no material drop.