Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
harden(store,wiki): prevent purge resurrection + serialize same-path writes (#607)
AkitaOnRails740606cc63bb
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8dc82984..723e9d94 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -93,6 +93,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0workspace mismatch to hunt for; the disambiguating lookup runs only on thefailure path, so the common case still pays a single query.+### Security+- A purged project or workspace can no longer be resurrected by a later+ `reindex` (#607, data-layer audit follow-up, item 2). `purge_project` /+ `delete_workspace` commit the DB deletion first and remove on-disk files+ afterward, best-effort; a crash or failure in that window left the markdown+ directory (`_meta.md` included) on disk with no row, and `reindex` rebuilt+ the scope from that manifest — silently undoing the purge. Each purge now+ writes a `purged_scopes` tombstone in the same transaction as the deletion+ (a whole-workspace delete uses a `zeroblob(16)` sentinel project id), and+ `reindex_all` skips any tombstoned scope instead of recreating it. The+ inert files are left for a later purge or manual cleanup; reindex stays+ non-destructive.+- Concurrent writes to the *same* page path are now serialized (#607, item 3).+ Per-page writes take the shared side of the wiki mutation lock, so two+ writes to one `(workspace, project, path)` could interleave their+ file-rename and DB-upsert and transiently leave the on-disk markdown+ disagreeing with the DB `is_latest` row (self-healing on reindex, no data+ loss). A per-path async lock now serializes same-path writers while+ different paths still proceed concurrently; batches acquire their paths in a+ fixed global order so they cannot deadlock.+## [2.0.2] - 2026-09-03### Fixeddiff --git a/crates/ai-memory-store/migrations/V59__purged_scopes_tombstones.sql b/crates/ai-memory-store/migrations/V59__purged_scopes_tombstones.sqlnew file mode 100644index 00000000..f3216a2f--- /dev/null+++ b/crates/ai-memory-store/migrations/V59__purged_scopes_tombstones.sql@@ -0,0 +1,34 @@+-- Tombstones for project/workspace scopes removed by `purge_project` /+-- `delete_workspace` (#607, item 2).+--+-- A purge commits the DB deletion first, then removes the on-disk wiki files+-- best-effort. A crash (or a file-removal failure) in that window leaves the+-- project's markdown directory — `_meta.md` included — on disk with no row in+-- `projects`. A later `ai-memory reindex` rebuilds the DB from markdown and+-- calls `ensure_project_with_id` / `ensure_workspace_with_id` from each+-- `_meta.md`, which would recreate the scope and resurrect its pages, silently+-- undoing the purge.+--+-- The tombstone makes the deletion terminal: `reindex_all` checks it and+-- refuses to recreate a scope that was purged. Unlike `purged_sessions`, this+-- table carries NO foreign keys — the workspace/project rows it names are the+-- ones being deleted, so an `ON DELETE CASCADE` reference would either fail or+-- cascade the tombstone away with them.+--+-- Keyed by (workspace_id, project_id). A project purge inserts the real+-- project id. A workspace delete inserts a 16-byte all-zero project id+-- (`zeroblob(16)`) as a whole-workspace tombstone, which never collides with a+-- real UUIDv7 project id. Ids are terminal: a legitimately recreated+-- workspace/project gets a fresh UUID, so a tombstone never blocks new work.+--+-- `purged_at` exists so the ledger can be pruned; a tombstone only has to+-- outlive any surviving on-disk directory that a reindex might otherwise+-- resurrect.+CREATE TABLE purged_scopes (+ workspace_id BLOB NOT NULL,+ project_id BLOB NOT NULL,+ purged_at INTEGER NOT NULL,+ PRIMARY KEY (workspace_id, project_id)+);++CREATE INDEX idx_purged_scopes_at ON purged_scopes(purged_at);diff --git a/crates/ai-memory-store/src/api_credentials.rs b/crates/ai-memory-store/src/api_credentials.rsindex d1d90786..0a5763d6 100644--- a/crates/ai-memory-store/src/api_credentials.rs+++ b/crates/ai-memory-store/src/api_credentials.rs@@ -411,7 +411,7 @@ mod tests {|row| row.get(0),).unwrap();- assert_eq!(version, 58, "update the pin when adding a migration");+ assert_eq!(version, 59, "update the pin when adding a migration");let cols: i64 = conn.query_row("SELECT COUNT(*) FROM pragma_table_info('users') WHERE name = 'token_hash'",diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex 49f98b97..654aed6a 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -3627,6 +3627,30 @@ pub fn purge_session(/// Returns [`StoreError::ManagedRunActive`] when a managed run's lease is/// still live and `force` is false, or [`StoreError`] if any SQL statement/// fails. The transaction is rolled back automatically on error.+/// Whether `(workspace_id, project_id)` — or the whole workspace — was purged+/// and tombstoned by [`purge_project`] / [`delete_workspace`] (#607).+///+/// Returns true when either an exact project tombstone exists or a+/// whole-workspace tombstone (the `zeroblob(16)` sentinel project id) covers+/// the workspace. `reindex` calls this before recreating a scope from on-disk+/// `_meta.md`, so a purge whose file removal crashed cannot be resurrected.+pub fn scope_is_purged(+ conn: &Connection,+ workspace_id: &WorkspaceId,+ project_id: &ProjectId,+) -> StoreResult<bool> {+ let found: Option<i64> = conn+ .query_row(+ "SELECT 1 FROM purged_scopes \+ WHERE workspace_id = ?1 AND (project_id = ?2 OR project_id = zeroblob(16)) \+ LIMIT 1",+ rusqlite::params![workspace_id.as_bytes(), project_id.as_bytes()],+ |row| row.get(0),+ )+ .optional()?;+ Ok(found.is_some())+}+pub fn purge_project(conn: &mut Connection,workspace_id: &WorkspaceId,@@ -3749,6 +3773,17 @@ pub fn purge_project(rusqlite::params![&pid[..], workspace_id.as_bytes()],)?;+ // Tombstone the scope so a `reindex` cannot resurrect it from an on-disk+ // directory the post-commit file removal failed to (or crashed before)+ // deleting. Written in the same transaction as the DELETE so the deletion+ // and its terminality commit atomically (#607). `INSERT OR REPLACE` keeps+ // a repeated purge idempotent and refreshes `purged_at`.+ tx.execute(+ "INSERT OR REPLACE INTO purged_scopes (workspace_id, project_id, purged_at) \+ VALUES (?1, ?2, ?3)",+ rusqlite::params![workspace_id.as_bytes(), &pid[..], now],+ )?;+// Attributed audit trail for the destructive purge. `page_id` is None// (the whole project is gone); the operator identity comes from the// authenticated request (NULL when single-user / unauthenticated).@@ -3854,6 +3889,18 @@ pub fn delete_workspace(if removed == 0 {return Err(StoreError::NotFound("workspace".into()));}++ // Whole-workspace tombstone (#607): a 16-byte all-zero project id marks the+ // entire workspace as purged so `reindex` cannot recreate it — or any of+ // its projects — from on-disk `_meta.md` that the post-commit directory+ // removal failed to delete. Same transaction as the DELETE for atomic+ // terminality.+ tx.execute(+ "INSERT OR REPLACE INTO purged_scopes (workspace_id, project_id, purged_at) \+ VALUES (?1, zeroblob(16), ?2)",+ rusqlite::params![&wid[..], Timestamp::now().as_microsecond()],+ )?;+tx.commit()?;if compaction == Compaction::Reclaim {@@ -5039,6 +5086,52 @@ pub(crate) mod tests {/// docs and `docs/lifecycle-ops.md` cannot drift away from the behaviour:/// if this starts failing, byte-level removal became the default and all/// three need updating together.+ #[test]+ fn purge_project_tombstones_the_scope_against_resurrection() {+ let (_tmp, mut conn, ws, proj) = fresh_db();+ seed_session(&mut conn, ws, proj, "canaryproj");++ assert!(+ !scope_is_purged(&conn, &ws, &proj).unwrap(),+ "a live project is not tombstoned"+ );++ purge_project(+ &mut conn,+ &ws,+ &proj,+ "default/scratch",+ None,+ false,+ Compaction::Skip,+ )+ .unwrap();++ assert!(+ scope_is_purged(&conn, &ws, &proj).unwrap(),+ "purge must tombstone the scope so reindex cannot resurrect it"+ );+ // A different, un-purged project in the same workspace is unaffected.+ assert!(+ !scope_is_purged(&conn, &ws, &ai_memory_core::ProjectId::new()).unwrap(),+ "the tombstone is scoped to the purged project id only"+ );+ }++ #[test]+ fn delete_workspace_tombstones_the_whole_workspace() {+ let (_tmp, mut conn, ws, proj) = fresh_db();++ delete_workspace(&mut conn, &ws, true, Compaction::Skip).unwrap();++ // The whole-workspace tombstone covers every project id in that ws,+ // including ones whose rows are already gone via cascade.+ assert!(scope_is_purged(&conn, &ws, &proj).unwrap());+ assert!(scope_is_purged(&conn, &ws, &ai_memory_core::ProjectId::new()).unwrap());+ // A different workspace is not affected by the sentinel.+ assert!(!scope_is_purged(&conn, &ai_memory_core::WorkspaceId::new(), &proj).unwrap());+ }+#[test]fn purge_project_is_a_logical_delete_by_default() {let (tmp, mut conn, ws, proj) = fresh_db();diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rsindex b2a44330..8d8c27d5 100644--- a/crates/ai-memory-store/src/writer.rs+++ b/crates/ai-memory-store/src/writer.rs@@ -77,6 +77,11 @@ pub(crate) enum WriteCmd {repo_path: Option<String>,reply: oneshot::Sender<StoreResult<()>>,},+ ScopeIsPurged {+ workspace_id: WorkspaceId,+ project_id: ProjectId,+ reply: oneshot::Sender<StoreResult<bool>>,+ },UpsertPage {page: NewPage,reply: oneshot::Sender<StoreResult<PageId>>,@@ -728,6 +733,26 @@ impl WriterHandle {rx.await.map_err(|_| StoreError::WriterClosed)?}+ /// Whether the scope was purged by `purge_project` / `delete_workspace` and+ /// tombstoned. `reindex` consults this before recreating a scope from+ /// on-disk `_meta.md` so a purge that crashed before its files were removed+ /// cannot be silently undone (#607). Routed through the writer actor+ /// because it owns the connection and is always present.+ pub async fn scope_is_purged(+ &self,+ workspace_id: WorkspaceId,+ project_id: ProjectId,+ ) -> StoreResult<bool> {+ let (tx, rx) = oneshot::channel();+ self.send(WriteCmd::ScopeIsPurged {+ workspace_id,+ project_id,+ reply: tx,+ })+ .await?;+ rx.await.map_err(|_| StoreError::WriterClosed)?+ }+/// Begin a session (idempotent on the supplied id).////// # Errors@@ -2445,6 +2470,14 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {);send_or_warn(reply, result, "ensure_project_with_id");}+ WriteCmd::ScopeIsPurged {+ workspace_id,+ project_id,+ reply,+ } => {+ let result = ops::scope_is_purged(&conn, &workspace_id, &project_id);+ send_or_warn(reply, result, "scope_is_purged");+ }WriteCmd::UpsertPage { page, reply } => {let result = ops::upsert_page(&mut conn, &page);send_or_warn(reply, result, "upsert_page");diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rsindex fcad38cb..17f7d4f0 100644--- a/crates/ai-memory-wiki/src/wiki.rs+++ b/crates/ai-memory-wiki/src/wiki.rs@@ -1,5 +1,6 @@//! [`Wiki`] — the only correct write path for the markdown source-of-truth.+use std::collections::HashMap;use std::path::{Path, PathBuf};use std::sync::Arc;@@ -31,6 +32,10 @@ pub struct ReindexSummary {pub projects: usize,/// Pages reindexed from the wiki tree.pub pages: usize,+ /// Project directories skipped because their scope was purged and+ /// tombstoned (#607) — a purge whose on-disk removal did not complete.+ /// Their pages are deliberately not resurrected.+ pub skipped_purged: usize,}enum PageStoreRemoval {@@ -110,8 +115,20 @@ pub struct Wiki {/// the directory rename and SQLite re-stamp so stale writes cannot land/// files under the old workspace while the project is in flight.mutation_lock: Arc<RwLock<()>>,+ /// Per-page write serialization (#607, item 3). Two concurrent writes to+ /// the *same* `(ws, proj, path)` would otherwise interleave their+ /// file-rename and DB-upsert under the shared `mutation_lock` read guard,+ /// transiently leaving on-disk markdown disagreeing with the DB `latest`+ /// row. Each path gets its own async mutex, so same-path writers serialize+ /// while different paths still run concurrently. The map is GC'd+ /// opportunistically — entries no writer currently holds are dropped on the+ /// next acquisition — so it stays bounded to currently-contended paths.+ page_locks: Arc<std::sync::Mutex<HashMap<PageKey, Arc<tokio::sync::Mutex<()>>>>>,}+/// Key uniquely identifying a page for per-path write serialization.+type PageKey = (WorkspaceId, ProjectId, PagePath);+impl Wiki {/// Construct a wiki handle rooted at `<data_dir>/wiki/`. Creates the/// directory if absent and initialises a git repo inside it.@@ -132,6 +149,7 @@ impl Wiki {admission_chain: None,store_reader: None,mutation_lock: Arc::new(RwLock::new(())),+ page_locks: Arc::new(std::sync::Mutex::new(HashMap::new())),})}@@ -1392,6 +1410,21 @@ impl Wiki {let mut seen_ws = std::collections::HashSet::new();for (ws, proj, proj_root) in project_dirs {+ // A purged scope must not be resurrected from an on-disk directory+ // its purge failed to remove (#607). The tombstone is terminal:+ // skip the whole directory rather than recreating the scope row and+ // reindexing its pages. The files stay inert (the watcher skips+ // them too, #613) until a later purge or manual cleanup removes+ // them; reindex stays non-destructive.+ if self.writer.scope_is_purged(ws, proj).await? {+ tracing::debug!(+ workspace = %ws,+ project = %proj,+ "skipping reindex of a purged (tombstoned) scope",+ );+ summary.skipped_purged += 1;+ continue;+ }if seen_ws.insert(ws) {let ws_dir = proj_root.parent()@@ -1598,6 +1631,14 @@ impl Wiki {let (ids, dispatches) = {let _guard = self.mutation_lock.read().await;+ // Serialize every path in this batch against concurrent single-page+ // or batch writers to the same path (#607). Sorted acquisition+ // keeps a batch deadlock-free against any other writer.+ let batch_keys: Vec<PageKey> = staged+ .iter()+ .map(|(req, _, _, _)| (req.workspace_id, req.project_id, req.path.clone()))+ .collect();+ let _page_guards = self.lock_pages(&batch_keys).await;let mut staged_files: Vec<(WritePageRequest,tempfile::NamedTempFile,@@ -1685,6 +1726,53 @@ impl Wiki {Ok(ids)}+ /// Acquire the per-path write lock for one page, serializing concurrent+ /// writers to the same `(ws, proj, path)` while different paths run+ /// concurrently. See [`Wiki::page_locks`].+ async fn lock_page(+ &self,+ workspace_id: WorkspaceId,+ project_id: ProjectId,+ path: &PagePath,+ ) -> tokio::sync::OwnedMutexGuard<()> {+ let lock = {+ let mut map = self.page_locks.lock().expect("page_locks mutex poisoned");+ // Opportunistic GC under the same short critical section that hands+ // out locks: an entry whose only remaining strong ref is the map's+ // own (count == 1) has no live guard and no waiter, so drop it.+ map.retain(|_, v| Arc::strong_count(v) > 1);+ Arc::clone(+ map.entry((workspace_id, project_id, path.clone()))+ .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))),+ )+ };+ lock.lock_owned().await+ }++ /// Acquire per-path write locks for a batch of pages in a deterministic+ /// global order (sorted by id/path bytes, deduped) so a batch write and any+ /// concurrent single-page or batch writer can never deadlock. See+ /// [`Wiki::page_locks`].+ async fn lock_pages(&self, keys: &[PageKey]) -> Vec<tokio::sync::OwnedMutexGuard<()>> {+ let mut ordered: Vec<PageKey> = keys.to_vec();+ // The id newtypes are not `Ord`, so order by their byte / string+ // representation — any total order works, it only has to be consistent+ // across all acquirers.+ ordered.sort_by(|a, b| {+ (a.0.as_bytes(), a.1.as_bytes(), a.2.as_str()).cmp(&(+ b.0.as_bytes(),+ b.1.as_bytes(),+ b.2.as_str(),+ ))+ });+ ordered.dedup();+ let mut guards = Vec::with_capacity(ordered.len());+ for (ws, proj, path) in ordered {+ guards.push(self.lock_page(ws, proj, &path).await);+ }+ guards+ }+/// Write `body` (with optional `frontmatter`) atomically to/// `<wiki_root>/<workspace_id>/<project_id>/<path>` and upsert the/// matching page row in the store.@@ -1799,6 +1887,11 @@ impl Wiki {let page_id = {let _guard = self.mutation_lock.read().await;+ // Serialize with any concurrent write to this same path, so the+ // file-rename and DB-upsert below cannot interleave with another+ // writer's and leave disk disagreeing with the DB `latest` row+ // (#607). Different paths still proceed concurrently.+ let _page_guard = self.lock_page(workspace_id, project_id, &path).await;self.ensure_project_workspace(workspace_id, project_id).await?;let abs = self.abs_path(workspace_id, project_id, &path);@@ -4198,6 +4291,156 @@ mod tests {let _ = meta;}+ /// #607 item 2: a purge commits the DB deletion but removes on-disk files+ /// only afterward (best-effort). If that removal is interrupted, the+ /// markdown directory — `_meta.md` included — survives with no `projects`+ /// row. `reindex_all` must NOT resurrect it: the tombstone written in the+ /// purge transaction makes the deletion terminal.+ #[tokio::test]+ async fn reindex_does_not_resurrect_a_purged_scope_whose_files_survived() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let wiki = Wiki::new(tmp.path(), store.writer.clone())+ .unwrap()+ .with_store_reader(store.reader.clone());+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "scratch", None)+ .await+ .unwrap();++ let path = PagePath::new("notes/keep.md").unwrap();+ wiki.write_page(WritePageRequest {+ workspace_id: ws,+ project_id: proj,+ path: PagePath::new("notes/keep.md").unwrap(),+ frontmatter: serde_json::json!({"title": "Keep"}),+ body: "survivor".into(),+ tier: Tier::Semantic,+ pinned: false,+ title: None,+ admission_ctx: None,+ author_id: None,+ actor: ai_memory_core::ActorContext::anonymous(),+ })+ .await+ .unwrap();+ // Manifests reindex reads to recreate scopes from disk.+ wiki.backfill_scope_manifests().await.unwrap();++ // Purge the DB only (store side); the on-disk files intentionally+ // survive — exactly the crash-before-file-removal window.+ store+ .writer+ .purge_project(+ ws,+ proj,+ "default/scratch",+ None,+ false,+ ai_memory_store::Compaction::Skip,+ )+ .await+ .unwrap();+ let meta_on_disk = tmp+ .path()+ .join("wiki")+ .join(ws.to_string())+ .join(proj.to_string())+ .join("_meta.md");+ assert!(+ meta_on_disk.exists(),+ "the purge left the manifest on disk (the crash window this guards)"+ );++ let summary = wiki.reindex_all().await.unwrap();++ assert_eq!(+ summary.skipped_purged, 1,+ "the purged scope must be skipped"+ );+ assert_eq!(+ summary.projects, 0,+ "the purged project must not be recreated"+ );+ assert!(+ store+ .reader+ .page_body_by_ids(ws, proj, path.as_str())+ .await+ .unwrap()+ .is_none(),+ "a purged page must not reappear in the index"+ );+ }++ /// #607 item 3: two concurrent writes to the SAME page path must not+ /// interleave their file-rename and DB-upsert. Per-path serialization+ /// guarantees the winning writer's body lands in both the on-disk file and+ /// the DB `is_latest` row — they never disagree about who won.+ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]+ async fn concurrent_same_path_writes_keep_file_and_db_consistent() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "scratch", None)+ .await+ .unwrap();++ let path = PagePath::new("notes/hot.md").unwrap();+ let mk = |tag: char| WritePageRequest {+ workspace_id: ws,+ project_id: proj,+ path: PagePath::new("notes/hot.md").unwrap(),+ frontmatter: serde_json::json!({"title": "Hot"}),+ body: format!("body-{tag}"),+ tier: Tier::Semantic,+ pinned: false,+ title: None,+ admission_ctx: None,+ author_id: None,+ actor: ai_memory_core::ActorContext::anonymous(),+ };++ // Many rounds so any interleave window is likely to be exercised.+ for _ in 0..25 {+ let (wa, wb) = (wiki.clone(), wiki.clone());+ let (ra, rb) = (mk('a'), mk('b'));+ let ha = tokio::spawn(async move { wa.write_page(ra).await });+ let hb = tokio::spawn(async move { wb.write_page(rb).await });+ ha.await.unwrap().unwrap();+ hb.await.unwrap().unwrap();++ let on_disk = wiki.read_page(ws, proj, &path).unwrap().body;+ let in_db = store+ .reader+ .page_body_by_ids(ws, proj, path.as_str())+ .await+ .unwrap()+ .expect("a latest row exists")+ .body;+ // Compare who won, not exact bytes (emit/parse normalizes newlines):+ // the file and the DB latest row must reflect the same writer.+ assert_eq!(+ on_disk.contains("body-a"),+ in_db.contains("body-a"),+ "file ({on_disk:?}) and DB latest ({in_db:?}) disagree on the winning writer"+ );+ }+ }+/// Backward-compat: anonymous writes do not add attribution frontmatter.#[tokio::test]async fn write_page_with_anonymous_actor_omits_attribution_frontmatter() {