Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(wiki): coordinate reindex with page mutations

Éverton Toffanetto378b51bc8d61
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 762d7fbb..05336780 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -127,6 +127,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
labelled `nix` / `full-ci`. (#989)
### Fixed
+- Fixed watcher reindexing racing with writes and batches to the same page by
+ sharing their per-page mutex from disk read through SQLite upsert. Both
+ mutation guards were released before embedding; external editors remained
+ outside this coordination.
- Fixed completed retries and no-op session endings advancing
`last_persisted_ms` without a durable write. Recovery still advances the
timestamp when it commits a new page or terminal effect. (#1015)
diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rs
index dd5d3188..be7eeb17 100644
--- a/crates/ai-memory-wiki/src/wiki.rs
+++ b/crates/ai-memory-wiki/src/wiki.rs
@@ -178,7 +178,10 @@ pub struct Wiki {
/// while different paths still run concurrently. The map is GC'd
/// opportunistically — entries no writer currently holds are dropped on the
/// next acquisition — so it stays bounded to currently-contended paths.
+ /// Reindexes share it from disk read through SQL upsert.
page_locks: Arc<std::sync::Mutex<HashMap<PageKey, Arc<tokio::sync::Mutex<()>>>>>,
+ #[cfg(test)]
+ reindex_read_barrier: Option<Arc<tokio::sync::Barrier>>,
/// Scopes this process has already materialized `_meta.md` manifests for.
/// Keeps [`Wiki::ensure_scope_manifests`] to one hash lookup per page
/// write after the scope's first — the store query and the two manifest
@@ -216,6 +219,8 @@ impl Wiki {
store_reader: None,
mutation_lock: Arc::new(RwLock::new(())),
page_locks: Arc::new(std::sync::Mutex::new(HashMap::new())),
+ #[cfg(test)]
+ reindex_read_barrier: None,
manifested_scopes: Arc::new(Mutex::new(HashSet::new())),
reconcile_tombstones_deleted_pages: false,
})
@@ -1696,7 +1701,7 @@ impl Wiki {
"refusing to index pending proposal sidecar",
));
}
- // Scoped so the mutation guard is dropped before the embed call
+ // Scoped so both mutation guards are dropped before the embed call
// below — same shape as `write_page` (#607/#929): embedding calls
// out to the configured provider, and holding the guard across that
// would block every other write in the wiki for its duration.
@@ -1727,12 +1732,15 @@ impl Wiki {
/// has two very different callers with two different locking needs
/// (`reindex_page`'s brief read guard; `hard_delete_decay_tombstone`'s
/// long-lived write guard, which must never embed at all — see there).
+ /// The caller already holds the global mutation guard (shared or
+ /// exclusive); acquire only the page mutex here, always after that guard.
async fn reindex_page_locked(
&self,
workspace_id: WorkspaceId,
project_id: ProjectId,
path: PagePath,
) -> WikiResult<(PageId, Option<PendingEmbed>)> {
+ let _page_guard = self.lock_page(workspace_id, project_id, &path).await;
let abs = self.abs_path(workspace_id, project_id, &path);
if std::fs::symlink_metadata(&abs)?.file_type().is_symlink() {
return Err(WikiError::Io(std::io::Error::other(format!(
@@ -1779,6 +1787,12 @@ impl Wiki {
_ => None,
};
+ #[cfg(test)]
+ if let Some(barrier) = &self.reindex_read_barrier {
+ barrier.wait().await;
+ barrier.wait().await;
+ }
+
let id = self
.writer
.upsert_page(NewPage {
@@ -5879,6 +5893,352 @@ mod tests {
);
}
+ async fn finish_lock_test<F: std::future::Future>(future: F) -> F::Output {
+ tokio::time::timeout(std::time::Duration::from_secs(5), future)
+ .await
+ .expect("wiki lock operation must finish without deadlock")
+ }
+
+ // Poll the real operation until it queues behind the held page mutex.
+ // Unlike a delay, this proves it reached that lock before we release it.
+ async fn wait_for_page_waiter<F: std::future::Future>(
+ wiki: &Wiki,
+ key: &PageKey,
+ mut future: std::pin::Pin<&mut F>,
+ ) {
+ finish_lock_test(std::future::poll_fn(|cx| {
+ assert!(
+ future.as_mut().poll(cx).is_pending(),
+ "operation bypassed the held page lock"
+ );
+ let locks = wiki.page_locks.lock().unwrap();
+ if locks
+ .get(key)
+ .is_some_and(|lock| Arc::strong_count(lock) >= 3)
+ {
+ std::task::Poll::Ready(())
+ } else {
+ std::task::Poll::Pending
+ }
+ }))
+ .await;
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn reindex_page_lock_waits_before_reading_and_allows_another_path() {
+ let tmp = TempDir::new().unwrap();
+ let (store, wiki, ws, proj) = scoped(&tmp).await;
+ let path = PagePath::new("notes/held.md").unwrap();
+ let other = PagePath::new("notes/other.md").unwrap();
+ let original = wiki
+ .write_page(req(
+ ws,
+ proj,
+ path.as_str(),
+ "original",
+ serde_json::json!({}),
+ ))
+ .await
+ .unwrap();
+ wiki.write_page(req(
+ ws,
+ proj,
+ other.as_str(),
+ "control",
+ serde_json::json!({}),
+ ))
+ .await
+ .unwrap();
+
+ let global = wiki.mutation_lock.read().await;
+ let page = wiki.lock_page(ws, proj, &path).await;
+ let abs = wiki.abs_path(ws, proj, &path);
+ std::fs::write(&abs, "transientsnapshot\n").unwrap();
+ let reindex = wiki.reindex_page(ws, proj, path.clone());
+ tokio::pin!(reindex);
+ wait_for_page_waiter(&wiki, &(ws, proj, path.clone()), reindex.as_mut()).await;
+ assert_eq!(
+ store
+ .reader
+ .latest_page_id_by_ids(ws, proj, path.as_str().into())
+ .await
+ .unwrap(),
+ Some(original),
+ "the transient disk snapshot must not enter SQL"
+ );
+
+ let other_abs = wiki.abs_path(ws, proj, &other);
+ std::fs::write(&other_abs, "independentcontrol\n").unwrap();
+ finish_lock_test(wiki.reindex_page(ws, proj, other.clone()))
+ .await
+ .unwrap();
+ assert_eq!(
+ store
+ .reader
+ .page_body_by_ids(ws, proj, other.as_str())
+ .await
+ .unwrap()
+ .unwrap()
+ .body,
+ "independentcontrol\n"
+ );
+ assert_eq!(
+ std::fs::read_to_string(&other_abs).unwrap(),
+ "independentcontrol\n"
+ );
+
+ std::fs::write(&abs, "committedwinner\n").unwrap();
+ drop(page);
+ drop(global);
+ let id = finish_lock_test(reindex).await.unwrap();
+ assert_ne!(id, original);
+ assert_eq!(
+ store
+ .reader
+ .page_body_by_ids(ws, proj, path.as_str())
+ .await
+ .unwrap()
+ .unwrap()
+ .body,
+ "committedwinner\n"
+ );
+ assert_eq!(std::fs::read_to_string(&abs).unwrap(), "committedwinner\n");
+ assert!(
+ store
+ .reader
+ .search_pages("transientsnapshot".into(), 10, None)
+ .await
+ .unwrap()
+ .is_empty()
+ );
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn reindex_page_lock_serializes_write_through_the_upsert() {
+ let tmp = TempDir::new().unwrap();
+ let (store, wiki, ws, proj) = scoped(&tmp).await;
+ let path = PagePath::new("notes/race.md").unwrap();
+ wiki.write_page(req(
+ ws,
+ proj,
+ path.as_str(),
+ "original",
+ serde_json::json!({}),
+ ))
+ .await
+ .unwrap();
+ std::fs::write(wiki.abs_path(ws, proj, &path), "watcher snapshot\n").unwrap();
+ let barrier = Arc::new(tokio::sync::Barrier::new(2));
+ let mut watcher = wiki.clone();
+ watcher.reindex_read_barrier = Some(barrier.clone());
+ let reindex_path = path.clone();
+ let reindex =
+ tokio::spawn(async move { watcher.reindex_page(ws, proj, reindex_path).await });
+ finish_lock_test(barrier.wait()).await;
+
+ let write = wiki.write_page(req(
+ ws,
+ proj,
+ path.as_str(),
+ "writer winner",
+ serde_json::json!({}),
+ ));
+ tokio::pin!(write);
+ wait_for_page_waiter(&wiki, &(ws, proj, path.clone()), write.as_mut()).await;
+ assert_eq!(
+ wiki.read_page(ws, proj, &path).unwrap().body,
+ "watcher snapshot\n"
+ );
+ finish_lock_test(barrier.wait()).await;
+ let watcher_id = finish_lock_test(reindex).await.unwrap().unwrap();
+ let writer_id = finish_lock_test(write).await.unwrap();
+ assert_ne!(writer_id, watcher_id);
+ assert_eq!(
+ wiki.read_page(ws, proj, &path).unwrap().body.trim(),
+ "writer winner"
+ );
+ assert_eq!(
+ store
+ .reader
+ .page_body_by_ids(ws, proj, path.as_str())
+ .await
+ .unwrap()
+ .unwrap()
+ .body,
+ "writer winner"
+ );
+ store
+ .reader
+ .with_conn(move |conn| {
+ let (body, supersedes): (String, Vec<u8>) = conn.query_row(
+ "SELECT body, supersedes FROM pages WHERE id = ?1 AND is_latest = 1",
+ [writer_id.as_bytes()],
+ |row| Ok((row.get(0)?, row.get(1)?)),
+ )?;
+ assert_eq!(body, "writer winner");
+ assert_eq!(supersedes, watcher_id.as_bytes());
+ let prior: String = conn.query_row(
+ "SELECT body FROM pages WHERE id = ?1",
+ [watcher_id.as_bytes()],
+ |row| row.get(0),
+ )?;
+ assert_eq!(
+ prior, "watcher snapshot\n",
+ "the earlier version stays reachable"
+ );
+ Ok(())
+ })
+ .await
+ .unwrap();
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn reindex_page_lock_and_reversed_duplicate_batches_do_not_deadlock() {
+ let tmp = TempDir::new().unwrap();
+ let (store, wiki, ws, proj) = scoped(&tmp).await;
+ let a = PagePath::new("notes/a.md").unwrap();
+ let z = PagePath::new("notes/z.md").unwrap();
+ let batch = |paths: &[&PagePath], body: &str| {
+ paths
+ .iter()
+ .map(|path| req(ws, proj, path.as_str(), body, serde_json::json!({})))
+ .collect()
+ };
+ wiki.apply_batch(batch(&[&a, &z], "original"))
+ .await
+ .unwrap();
+ std::fs::write(wiki.abs_path(ws, proj, &z), "watcher snapshot\n").unwrap();
+ let barrier = Arc::new(tokio::sync::Barrier::new(2));
+ let mut watcher = wiki.clone();
+ watcher.reindex_read_barrier = Some(barrier.clone());
+ let reindex_path = z.clone();
+ let reindex =
+ tokio::spawn(async move { watcher.reindex_page(ws, proj, reindex_path).await });
+ finish_lock_test(barrier.wait()).await;
+
+ let reverse = wiki.apply_batch(batch(&[&z, &a, &a], "first batch"));
+ tokio::pin!(reverse);
+ wait_for_page_waiter(&wiki, &(ws, proj, z.clone()), reverse.as_mut()).await;
+ let forward = wiki.apply_batch(batch(&[&a, &z], "second batch winner"));
+ tokio::pin!(forward);
+ wait_for_page_waiter(&wiki, &(ws, proj, a.clone()), forward.as_mut()).await;
+ finish_lock_test(barrier.wait()).await;
+ let watcher_id = finish_lock_test(reindex).await.unwrap().unwrap();
+ let first = finish_lock_test(reverse).await.unwrap();
+ let second = finish_lock_test(forward).await.unwrap();
+ assert_eq!(
+ first[1], first[2],
+ "duplicate paths share one lock and no-op version"
+ );
+ for (path, id) in [(&a, second[0]), (&z, second[1])] {
+ assert_eq!(
+ wiki.read_page(ws, proj, path).unwrap().body.trim(),
+ "second batch winner"
+ );
+ assert_eq!(
+ store
+ .reader
+ .latest_page_id_by_ids(ws, proj, path.as_str().into())
+ .await
+ .unwrap(),
+ Some(id)
+ );
+ assert_eq!(
+ store
+ .reader
+ .page_body_by_ids(ws, proj, path.as_str())
+ .await
+ .unwrap()
+ .unwrap()
+ .body,
+ "second batch winner"
+ );
+ }
+ store
+ .reader
+ .with_conn(move |conn| {
+ let ancestor: Vec<u8> = conn.query_row(
+ "SELECT supersedes FROM pages WHERE id = ?1",
+ [second[1].as_bytes()],
+ |row| row.get(0),
+ )?;
+ assert_eq!(ancestor, first[0].as_bytes());
+ let ancestor: Vec<u8> = conn.query_row(
+ "SELECT supersedes FROM pages WHERE id = ?1",
+ [first[0].as_bytes()],
+ |row| row.get(0),
+ )?;
+ assert_eq!(ancestor, watcher_id.as_bytes());
+ Ok(())
+ })
+ .await
+ .unwrap();
+ }
+
+ #[tokio::test]
+ async fn reindex_page_lock_releases_both_guards_before_embedding() {
+ struct LockCheckingEmbedder {
+ wiki: Wiki,
+ key: PageKey,
+ calls: std::sync::atomic::AtomicUsize,
+ inner: ai_memory_llm::SyntheticEmbedder,
+ }
+ #[async_trait::async_trait]
+ impl Embedder for LockCheckingEmbedder {
+ fn provider(&self) -> &'static str {
+ self.inner.provider()
+ }
+ fn model(&self) -> &str {
+ self.inner.model()
+ }
+ fn dim(&self) -> u32 {
+ self.inner.dim()
+ }
+ async fn embed(&self, text: &str) -> ai_memory_llm::LlmResult<Vec<f32>> {
+ let global = self
+ .wiki
+ .mutation_lock
+ .try_write()
+ .expect("embedding must run outside the global lock");
+ let page =
+ finish_lock_test(self.wiki.lock_page(self.key.0, self.key.1, &self.key.2))
+ .await;
+ drop(page);
+ drop(global);
+ self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ self.inner.embed(text).await
+ }
+ }
+ let tmp = TempDir::new().unwrap();
+ let (store, wiki, ws, proj) = scoped(&tmp).await;
+ let path = PagePath::new("notes/embed.md").unwrap();
+ std::fs::create_dir_all(wiki.abs_path(ws, proj, &path).parent().unwrap()).unwrap();
+ std::fs::write(
+ wiki.abs_path(ws, proj, &path),
+ "---\nabstract: short summary\n---\nembedded body\n",
+ )
+ .unwrap();
+ let embedder = Arc::new(LockCheckingEmbedder {
+ wiki: wiki.clone(),
+ key: (ws, proj, path.clone()),
+ calls: std::sync::atomic::AtomicUsize::new(0),
+ inner: ai_memory_llm::SyntheticEmbedder::new(32),
+ });
+ let wiki = wiki.with_embedder(embedder.clone());
+ let id = finish_lock_test(wiki.reindex_page(ws, proj, path))
+ .await
+ .unwrap();
+ assert_eq!(embedder.calls.load(std::sync::atomic::Ordering::SeqCst), 2);
+ assert!(
+ store
+ .reader
+ .embedded_page_ids(ws, proj, "synthetic".into(), "bag-of-words-v1".into(), 32)
+ .await
+ .unwrap()
+ .contains(&id)
+ );
+ }
+
/// #607 item 3: two concurrent writes to the SAME page path must not
/// interleave their file-rename and DB-upsert. Per-path serialization
/// guarantees the winning writer's body lands in both the on-disk file and
diff --git a/docs/security-boundaries.md b/docs/security-boundaries.md
index 53774ace..8663942c 100644
--- a/docs/security-boundaries.md
+++ b/docs/security-boundaries.md
@@ -37,6 +37,7 @@ boundary not yet built.
| 4h | Offer-mode SessionStart never consumes or injects handoff-controlled text | `ai-memory-hooks/src/router.rs` `fetch_and_accept_handoff_at` branches on the startup configuration before admission/claim and `render_handoff_notice` emits only typed metadata (`HandoffId`, `AgentKind`, computed age); the normal writer CAS remains the only later claim path | `router.rs` `handoff_notice_is_metadata_only_and_never_carries_the_summary` plants hostile summary/questions/steps/files; `offer_mode_leaves_the_baton_open_and_notice_omits_the_summary` exercises the production hook route, asserts the row remains open, then proves the legitimate exact-id writer claim still consumes it (#1030) | STRONG |
| 5a | Pages shared: `author_id` is never a read filter (invariant #16) | `ai-memory-store/src/reader.rs` `search_pages`/`page_body_by_ids` — `author_id` is an attribution JOIN only, never a WHERE term | `multi_session.rs` — a page with a **non-null** `author_id` (operator A) is readable by operator B in the same project | STRONG |
| 5b | Page supersession (loser stays reachable) | `ai-memory-store/src/ops.rs` `upsert_page_in_tx` — demote `is_latest=0` (never delete) + `supersedes` chain | `multi_session.rs` `concurrent_writes_to_one_path_supersede_rather_than_destroy`; `retrieval_superseded.rs` | STRONG |
+| 5c | Watcher reindex and Wiki writes share per-page serialization (invariant #16) | `ai-memory-wiki/src/wiki.rs` `reindex_page` takes `mutation_lock.read` before `reindex_page_locked` takes `lock_page`, covering disk read through the single-writer upsert; callers already holding the exclusive global guard do not reacquire it. `write_page` uses the same page mutex and `apply_batch` retains sorted, deduplicated locks; embedding runs after both guards are released | `wiki.rs` `reindex_page_lock_waits_before_reading_and_allows_another_path` (transient snapshot refused while locked, another path progresses, release indexes the committed bytes without rewriting disk), `reindex_page_lock_serializes_write_through_the_upsert` and `reindex_page_lock_and_reversed_duplicate_batches_do_not_deadlock` (real concurrent operations, final disk/SQL winner and supersession ancestry preserved); all three fail with the reindex page guard removed. `reindex_page_lock_releases_both_guards_before_embedding` checks both body and abstract embedding; `decay_cleanup_reindex_does_not_embed_under_the_exclusive_lock` covers the exclusive caller | STRONG for a shared Wiki handle and its clones. External editors and independently constructed Wiki handles do not participate in these locks |
| 6 | Active-project pointer (PerActor, no clobber) | `ai-memory-core/src/active_project.rs` `set_for`/`lookup_for` (fail-closed on `Mismatch`) | `active_project.rs` `parallel_harnesses_of_one_user_keep_separate_pointers`, `two_operators_never_read_each_others_pointer`, `a_session_mismatch_fails_closed_once_anything_has_been_keyed` | STRONG |
| 6b | Native `_meta` routing coordinate is routing-only (never grants identity) | `ai-memory-mcp/src/server.rs` `attach_native_session` reads only `_meta["ai.opencode/sessionID"]` into `ActorKey.session_id` (trimmed, non-empty-string-validated); `user` still comes from the authenticated `ActorContext`, so a forged `_meta` cannot cross a user/scope boundary — it only selects a session slot behind the existing `active_project` guard | `server.rs` `native_session_metadata_is_routing_only_and_preserves_precedence`, `native_session_metadata_validates_strings_without_granting_identity`; `autoscope_multiuser.rs` `native_metadata_routes_concurrent_reads_and_writes_to_hook_workspaces` (#864) | STRONG |
| 7 | Sanitizer trust boundary (invariant #6) | `ai-memory-core/src/sanitize.rs` `Sanitized<T>` (private field, only `sanitize()` ctor); `Sanitized::new` scrubs the title *before* the 80-char display cap; `WriterHandle::insert_observation` requires `Sanitized`; `ops` crate-private | Structural (compile-time) + `store/src/lib.rs` `insert_observation_boundary_scrubs_before_disk` + sanitizer scrub unit tests + `ai-memory-core::sanitize` `title_is_scrubbed_before_the_80_char_cap_runs` + `ai-memory-hooks::router` `issue_980_user_prompt_title_is_sanitized_before_truncated` (a secret straddling the 80-char boundary is redacted, not stored truncated; #980) | STRONG (structural) |
diff --git a/docs/usage.md b/docs/usage.md
index 5069b213..a86d93e5 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -221,6 +221,14 @@ Markdown (`reindex` requires a clean derived database). `explain: true` exposes
and the entity RRF contribution. Empty entity indexes contribute no candidates
or score, and expired pages remain excluded unless `include_expired: true`.
+Watcher reindexing shares the per-page mutex used by writes and batches on the
+same Wiki handle and its clones. It waits before reading the file, holds that
+mutex through the SQLite upsert, and releases both page and global mutation
+guards before embedding. Different paths can proceed concurrently, and the
+watcher never rewrites the page. External editors and independently constructed
+Wiki handles do not take these locks; external edits can still race with a
+server write or reindex.
+
## Install the routing snippet and Agent Skills
From an agent, say: