Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(wiki): coordinate reindex with page mutations
Éverton Toffanetto378b51bc8d61
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 762d7fbb..05336780 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -127,6 +127,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0labelled `nix` / `full-ci`. (#989)### Fixed+- Fixed watcher reindexing racing with writes and batches to the same page by+ sharing their per-page mutex from disk read through SQLite upsert. Both+ mutation guards were released before embedding; external editors remained+ outside this coordination.- Fixed completed retries and no-op session endings advancing`last_persisted_ms` without a durable write. Recovery still advances thetimestamp when it commits a new page or terminal effect. (#1015)diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rsindex dd5d3188..be7eeb17 100644--- a/crates/ai-memory-wiki/src/wiki.rs+++ b/crates/ai-memory-wiki/src/wiki.rs@@ -178,7 +178,10 @@ pub struct Wiki {/// while different paths still run concurrently. The map is GC'd/// opportunistically — entries no writer currently holds are dropped on the/// next acquisition — so it stays bounded to currently-contended paths.+ /// Reindexes share it from disk read through SQL upsert.page_locks: Arc<std::sync::Mutex<HashMap<PageKey, Arc<tokio::sync::Mutex<()>>>>>,+ #[cfg(test)]+ reindex_read_barrier: Option<Arc<tokio::sync::Barrier>>,/// Scopes this process has already materialized `_meta.md` manifests for./// Keeps [`Wiki::ensure_scope_manifests`] to one hash lookup per page/// write after the scope's first — the store query and the two manifest@@ -216,6 +219,8 @@ impl Wiki {store_reader: None,mutation_lock: Arc::new(RwLock::new(())),page_locks: Arc::new(std::sync::Mutex::new(HashMap::new())),+ #[cfg(test)]+ reindex_read_barrier: None,manifested_scopes: Arc::new(Mutex::new(HashSet::new())),reconcile_tombstones_deleted_pages: false,})@@ -1696,7 +1701,7 @@ impl Wiki {"refusing to index pending proposal sidecar",));}- // Scoped so the mutation guard is dropped before the embed call+ // Scoped so both mutation guards are dropped before the embed call// below — same shape as `write_page` (#607/#929): embedding calls// out to the configured provider, and holding the guard across that// would block every other write in the wiki for its duration.@@ -1727,12 +1732,15 @@ impl Wiki {/// has two very different callers with two different locking needs/// (`reindex_page`'s brief read guard; `hard_delete_decay_tombstone`'s/// long-lived write guard, which must never embed at all — see there).+ /// The caller already holds the global mutation guard (shared or+ /// exclusive); acquire only the page mutex here, always after that guard.async fn reindex_page_locked(&self,workspace_id: WorkspaceId,project_id: ProjectId,path: PagePath,) -> WikiResult<(PageId, Option<PendingEmbed>)> {+ let _page_guard = self.lock_page(workspace_id, project_id, &path).await;let abs = self.abs_path(workspace_id, project_id, &path);if std::fs::symlink_metadata(&abs)?.file_type().is_symlink() {return Err(WikiError::Io(std::io::Error::other(format!(@@ -1779,6 +1787,12 @@ impl Wiki {_ => None,};+ #[cfg(test)]+ if let Some(barrier) = &self.reindex_read_barrier {+ barrier.wait().await;+ barrier.wait().await;+ }+let id = self.writer.upsert_page(NewPage {@@ -5879,6 +5893,352 @@ mod tests {);}+ async fn finish_lock_test<F: std::future::Future>(future: F) -> F::Output {+ tokio::time::timeout(std::time::Duration::from_secs(5), future)+ .await+ .expect("wiki lock operation must finish without deadlock")+ }++ // Poll the real operation until it queues behind the held page mutex.+ // Unlike a delay, this proves it reached that lock before we release it.+ async fn wait_for_page_waiter<F: std::future::Future>(+ wiki: &Wiki,+ key: &PageKey,+ mut future: std::pin::Pin<&mut F>,+ ) {+ finish_lock_test(std::future::poll_fn(|cx| {+ assert!(+ future.as_mut().poll(cx).is_pending(),+ "operation bypassed the held page lock"+ );+ let locks = wiki.page_locks.lock().unwrap();+ if locks+ .get(key)+ .is_some_and(|lock| Arc::strong_count(lock) >= 3)+ {+ std::task::Poll::Ready(())+ } else {+ std::task::Poll::Pending+ }+ }))+ .await;+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn reindex_page_lock_waits_before_reading_and_allows_another_path() {+ let tmp = TempDir::new().unwrap();+ let (store, wiki, ws, proj) = scoped(&tmp).await;+ let path = PagePath::new("notes/held.md").unwrap();+ let other = PagePath::new("notes/other.md").unwrap();+ let original = wiki+ .write_page(req(+ ws,+ proj,+ path.as_str(),+ "original",+ serde_json::json!({}),+ ))+ .await+ .unwrap();+ wiki.write_page(req(+ ws,+ proj,+ other.as_str(),+ "control",+ serde_json::json!({}),+ ))+ .await+ .unwrap();++ let global = wiki.mutation_lock.read().await;+ let page = wiki.lock_page(ws, proj, &path).await;+ let abs = wiki.abs_path(ws, proj, &path);+ std::fs::write(&abs, "transientsnapshot\n").unwrap();+ let reindex = wiki.reindex_page(ws, proj, path.clone());+ tokio::pin!(reindex);+ wait_for_page_waiter(&wiki, &(ws, proj, path.clone()), reindex.as_mut()).await;+ assert_eq!(+ store+ .reader+ .latest_page_id_by_ids(ws, proj, path.as_str().into())+ .await+ .unwrap(),+ Some(original),+ "the transient disk snapshot must not enter SQL"+ );++ let other_abs = wiki.abs_path(ws, proj, &other);+ std::fs::write(&other_abs, "independentcontrol\n").unwrap();+ finish_lock_test(wiki.reindex_page(ws, proj, other.clone()))+ .await+ .unwrap();+ assert_eq!(+ store+ .reader+ .page_body_by_ids(ws, proj, other.as_str())+ .await+ .unwrap()+ .unwrap()+ .body,+ "independentcontrol\n"+ );+ assert_eq!(+ std::fs::read_to_string(&other_abs).unwrap(),+ "independentcontrol\n"+ );++ std::fs::write(&abs, "committedwinner\n").unwrap();+ drop(page);+ drop(global);+ let id = finish_lock_test(reindex).await.unwrap();+ assert_ne!(id, original);+ assert_eq!(+ store+ .reader+ .page_body_by_ids(ws, proj, path.as_str())+ .await+ .unwrap()+ .unwrap()+ .body,+ "committedwinner\n"+ );+ assert_eq!(std::fs::read_to_string(&abs).unwrap(), "committedwinner\n");+ assert!(+ store+ .reader+ .search_pages("transientsnapshot".into(), 10, None)+ .await+ .unwrap()+ .is_empty()+ );+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn reindex_page_lock_serializes_write_through_the_upsert() {+ let tmp = TempDir::new().unwrap();+ let (store, wiki, ws, proj) = scoped(&tmp).await;+ let path = PagePath::new("notes/race.md").unwrap();+ wiki.write_page(req(+ ws,+ proj,+ path.as_str(),+ "original",+ serde_json::json!({}),+ ))+ .await+ .unwrap();+ std::fs::write(wiki.abs_path(ws, proj, &path), "watcher snapshot\n").unwrap();+ let barrier = Arc::new(tokio::sync::Barrier::new(2));+ let mut watcher = wiki.clone();+ watcher.reindex_read_barrier = Some(barrier.clone());+ let reindex_path = path.clone();+ let reindex =+ tokio::spawn(async move { watcher.reindex_page(ws, proj, reindex_path).await });+ finish_lock_test(barrier.wait()).await;++ let write = wiki.write_page(req(+ ws,+ proj,+ path.as_str(),+ "writer winner",+ serde_json::json!({}),+ ));+ tokio::pin!(write);+ wait_for_page_waiter(&wiki, &(ws, proj, path.clone()), write.as_mut()).await;+ assert_eq!(+ wiki.read_page(ws, proj, &path).unwrap().body,+ "watcher snapshot\n"+ );+ finish_lock_test(barrier.wait()).await;+ let watcher_id = finish_lock_test(reindex).await.unwrap().unwrap();+ let writer_id = finish_lock_test(write).await.unwrap();+ assert_ne!(writer_id, watcher_id);+ assert_eq!(+ wiki.read_page(ws, proj, &path).unwrap().body.trim(),+ "writer winner"+ );+ assert_eq!(+ store+ .reader+ .page_body_by_ids(ws, proj, path.as_str())+ .await+ .unwrap()+ .unwrap()+ .body,+ "writer winner"+ );+ store+ .reader+ .with_conn(move |conn| {+ let (body, supersedes): (String, Vec<u8>) = conn.query_row(+ "SELECT body, supersedes FROM pages WHERE id = ?1 AND is_latest = 1",+ [writer_id.as_bytes()],+ |row| Ok((row.get(0)?, row.get(1)?)),+ )?;+ assert_eq!(body, "writer winner");+ assert_eq!(supersedes, watcher_id.as_bytes());+ let prior: String = conn.query_row(+ "SELECT body FROM pages WHERE id = ?1",+ [watcher_id.as_bytes()],+ |row| row.get(0),+ )?;+ assert_eq!(+ prior, "watcher snapshot\n",+ "the earlier version stays reachable"+ );+ Ok(())+ })+ .await+ .unwrap();+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn reindex_page_lock_and_reversed_duplicate_batches_do_not_deadlock() {+ let tmp = TempDir::new().unwrap();+ let (store, wiki, ws, proj) = scoped(&tmp).await;+ let a = PagePath::new("notes/a.md").unwrap();+ let z = PagePath::new("notes/z.md").unwrap();+ let batch = |paths: &[&PagePath], body: &str| {+ paths+ .iter()+ .map(|path| req(ws, proj, path.as_str(), body, serde_json::json!({})))+ .collect()+ };+ wiki.apply_batch(batch(&[&a, &z], "original"))+ .await+ .unwrap();+ std::fs::write(wiki.abs_path(ws, proj, &z), "watcher snapshot\n").unwrap();+ let barrier = Arc::new(tokio::sync::Barrier::new(2));+ let mut watcher = wiki.clone();+ watcher.reindex_read_barrier = Some(barrier.clone());+ let reindex_path = z.clone();+ let reindex =+ tokio::spawn(async move { watcher.reindex_page(ws, proj, reindex_path).await });+ finish_lock_test(barrier.wait()).await;++ let reverse = wiki.apply_batch(batch(&[&z, &a, &a], "first batch"));+ tokio::pin!(reverse);+ wait_for_page_waiter(&wiki, &(ws, proj, z.clone()), reverse.as_mut()).await;+ let forward = wiki.apply_batch(batch(&[&a, &z], "second batch winner"));+ tokio::pin!(forward);+ wait_for_page_waiter(&wiki, &(ws, proj, a.clone()), forward.as_mut()).await;+ finish_lock_test(barrier.wait()).await;+ let watcher_id = finish_lock_test(reindex).await.unwrap().unwrap();+ let first = finish_lock_test(reverse).await.unwrap();+ let second = finish_lock_test(forward).await.unwrap();+ assert_eq!(+ first[1], first[2],+ "duplicate paths share one lock and no-op version"+ );+ for (path, id) in [(&a, second[0]), (&z, second[1])] {+ assert_eq!(+ wiki.read_page(ws, proj, path).unwrap().body.trim(),+ "second batch winner"+ );+ assert_eq!(+ store+ .reader+ .latest_page_id_by_ids(ws, proj, path.as_str().into())+ .await+ .unwrap(),+ Some(id)+ );+ assert_eq!(+ store+ .reader+ .page_body_by_ids(ws, proj, path.as_str())+ .await+ .unwrap()+ .unwrap()+ .body,+ "second batch winner"+ );+ }+ store+ .reader+ .with_conn(move |conn| {+ let ancestor: Vec<u8> = conn.query_row(+ "SELECT supersedes FROM pages WHERE id = ?1",+ [second[1].as_bytes()],+ |row| row.get(0),+ )?;+ assert_eq!(ancestor, first[0].as_bytes());+ let ancestor: Vec<u8> = conn.query_row(+ "SELECT supersedes FROM pages WHERE id = ?1",+ [first[0].as_bytes()],+ |row| row.get(0),+ )?;+ assert_eq!(ancestor, watcher_id.as_bytes());+ Ok(())+ })+ .await+ .unwrap();+ }++ #[tokio::test]+ async fn reindex_page_lock_releases_both_guards_before_embedding() {+ struct LockCheckingEmbedder {+ wiki: Wiki,+ key: PageKey,+ calls: std::sync::atomic::AtomicUsize,+ inner: ai_memory_llm::SyntheticEmbedder,+ }+ #[async_trait::async_trait]+ impl Embedder for LockCheckingEmbedder {+ fn provider(&self) -> &'static str {+ self.inner.provider()+ }+ fn model(&self) -> &str {+ self.inner.model()+ }+ fn dim(&self) -> u32 {+ self.inner.dim()+ }+ async fn embed(&self, text: &str) -> ai_memory_llm::LlmResult<Vec<f32>> {+ let global = self+ .wiki+ .mutation_lock+ .try_write()+ .expect("embedding must run outside the global lock");+ let page =+ finish_lock_test(self.wiki.lock_page(self.key.0, self.key.1, &self.key.2))+ .await;+ drop(page);+ drop(global);+ self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ self.inner.embed(text).await+ }+ }+ let tmp = TempDir::new().unwrap();+ let (store, wiki, ws, proj) = scoped(&tmp).await;+ let path = PagePath::new("notes/embed.md").unwrap();+ std::fs::create_dir_all(wiki.abs_path(ws, proj, &path).parent().unwrap()).unwrap();+ std::fs::write(+ wiki.abs_path(ws, proj, &path),+ "---\nabstract: short summary\n---\nembedded body\n",+ )+ .unwrap();+ let embedder = Arc::new(LockCheckingEmbedder {+ wiki: wiki.clone(),+ key: (ws, proj, path.clone()),+ calls: std::sync::atomic::AtomicUsize::new(0),+ inner: ai_memory_llm::SyntheticEmbedder::new(32),+ });+ let wiki = wiki.with_embedder(embedder.clone());+ let id = finish_lock_test(wiki.reindex_page(ws, proj, path))+ .await+ .unwrap();+ assert_eq!(embedder.calls.load(std::sync::atomic::Ordering::SeqCst), 2);+ assert!(+ store+ .reader+ .embedded_page_ids(ws, proj, "synthetic".into(), "bag-of-words-v1".into(), 32)+ .await+ .unwrap()+ .contains(&id)+ );+ }+/// #607 item 3: two concurrent writes to the SAME page path must not/// interleave their file-rename and DB-upsert. Per-path serialization/// guarantees the winning writer's body lands in both the on-disk file anddiff --git a/docs/security-boundaries.md b/docs/security-boundaries.mdindex 53774ace..8663942c 100644--- a/docs/security-boundaries.md+++ b/docs/security-boundaries.md@@ -37,6 +37,7 @@ boundary not yet built.| 4h | Offer-mode SessionStart never consumes or injects handoff-controlled text | `ai-memory-hooks/src/router.rs` `fetch_and_accept_handoff_at` branches on the startup configuration before admission/claim and `render_handoff_notice` emits only typed metadata (`HandoffId`, `AgentKind`, computed age); the normal writer CAS remains the only later claim path | `router.rs` `handoff_notice_is_metadata_only_and_never_carries_the_summary` plants hostile summary/questions/steps/files; `offer_mode_leaves_the_baton_open_and_notice_omits_the_summary` exercises the production hook route, asserts the row remains open, then proves the legitimate exact-id writer claim still consumes it (#1030) | STRONG || 5a | Pages shared: `author_id` is never a read filter (invariant #16) | `ai-memory-store/src/reader.rs` `search_pages`/`page_body_by_ids` — `author_id` is an attribution JOIN only, never a WHERE term | `multi_session.rs` — a page with a **non-null** `author_id` (operator A) is readable by operator B in the same project | STRONG || 5b | Page supersession (loser stays reachable) | `ai-memory-store/src/ops.rs` `upsert_page_in_tx` — demote `is_latest=0` (never delete) + `supersedes` chain | `multi_session.rs` `concurrent_writes_to_one_path_supersede_rather_than_destroy`; `retrieval_superseded.rs` | STRONG |+| 5c | Watcher reindex and Wiki writes share per-page serialization (invariant #16) | `ai-memory-wiki/src/wiki.rs` `reindex_page` takes `mutation_lock.read` before `reindex_page_locked` takes `lock_page`, covering disk read through the single-writer upsert; callers already holding the exclusive global guard do not reacquire it. `write_page` uses the same page mutex and `apply_batch` retains sorted, deduplicated locks; embedding runs after both guards are released | `wiki.rs` `reindex_page_lock_waits_before_reading_and_allows_another_path` (transient snapshot refused while locked, another path progresses, release indexes the committed bytes without rewriting disk), `reindex_page_lock_serializes_write_through_the_upsert` and `reindex_page_lock_and_reversed_duplicate_batches_do_not_deadlock` (real concurrent operations, final disk/SQL winner and supersession ancestry preserved); all three fail with the reindex page guard removed. `reindex_page_lock_releases_both_guards_before_embedding` checks both body and abstract embedding; `decay_cleanup_reindex_does_not_embed_under_the_exclusive_lock` covers the exclusive caller | STRONG for a shared Wiki handle and its clones. External editors and independently constructed Wiki handles do not participate in these locks || 6 | Active-project pointer (PerActor, no clobber) | `ai-memory-core/src/active_project.rs` `set_for`/`lookup_for` (fail-closed on `Mismatch`) | `active_project.rs` `parallel_harnesses_of_one_user_keep_separate_pointers`, `two_operators_never_read_each_others_pointer`, `a_session_mismatch_fails_closed_once_anything_has_been_keyed` | STRONG || 6b | Native `_meta` routing coordinate is routing-only (never grants identity) | `ai-memory-mcp/src/server.rs` `attach_native_session` reads only `_meta["ai.opencode/sessionID"]` into `ActorKey.session_id` (trimmed, non-empty-string-validated); `user` still comes from the authenticated `ActorContext`, so a forged `_meta` cannot cross a user/scope boundary — it only selects a session slot behind the existing `active_project` guard | `server.rs` `native_session_metadata_is_routing_only_and_preserves_precedence`, `native_session_metadata_validates_strings_without_granting_identity`; `autoscope_multiuser.rs` `native_metadata_routes_concurrent_reads_and_writes_to_hook_workspaces` (#864) | STRONG || 7 | Sanitizer trust boundary (invariant #6) | `ai-memory-core/src/sanitize.rs` `Sanitized<T>` (private field, only `sanitize()` ctor); `Sanitized::new` scrubs the title *before* the 80-char display cap; `WriterHandle::insert_observation` requires `Sanitized`; `ops` crate-private | Structural (compile-time) + `store/src/lib.rs` `insert_observation_boundary_scrubs_before_disk` + sanitizer scrub unit tests + `ai-memory-core::sanitize` `title_is_scrubbed_before_the_80_char_cap_runs` + `ai-memory-hooks::router` `issue_980_user_prompt_title_is_sanitized_before_truncated` (a secret straddling the 80-char boundary is redacted, not stored truncated; #980) | STRONG (structural) |diff --git a/docs/usage.md b/docs/usage.mdindex 5069b213..a86d93e5 100644--- a/docs/usage.md+++ b/docs/usage.md@@ -221,6 +221,14 @@ Markdown (`reindex` requires a clean derived database). `explain: true` exposesand the entity RRF contribution. Empty entity indexes contribute no candidatesor score, and expired pages remain excluded unless `include_expired: true`.+Watcher reindexing shares the per-page mutex used by writes and batches on the+same Wiki handle and its clones. It waits before reading the file, holds that+mutex through the SQLite upsert, and releases both page and global mutation+guards before embedding. Different paths can proceed concurrently, and the+watcher never rewrites the page. External editors and independently constructed+Wiki handles do not take these locks; external edits can still race with a+server write or reindex.+## Install the routing snippet and Agent SkillsFrom an agent, say: