Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

harden(store,wiki): prevent purge resurrection + serialize same-path writes (#607)

AkitaOnRails740606cc63bb
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8dc82984..723e9d94 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -93,6 +93,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
workspace mismatch to hunt for; the disambiguating lookup runs only on the
failure path, so the common case still pays a single query.
+### Security
+- A purged project or workspace can no longer be resurrected by a later
+ `reindex` (#607, data-layer audit follow-up, item 2). `purge_project` /
+ `delete_workspace` commit the DB deletion first and remove on-disk files
+ afterward, best-effort; a crash or failure in that window left the markdown
+ directory (`_meta.md` included) on disk with no row, and `reindex` rebuilt
+ the scope from that manifest — silently undoing the purge. Each purge now
+ writes a `purged_scopes` tombstone in the same transaction as the deletion
+ (a whole-workspace delete uses a `zeroblob(16)` sentinel project id), and
+ `reindex_all` skips any tombstoned scope instead of recreating it. The
+ inert files are left for a later purge or manual cleanup; reindex stays
+ non-destructive.
+- Concurrent writes to the *same* page path are now serialized (#607, item 3).
+ Per-page writes take the shared side of the wiki mutation lock, so two
+ writes to one `(workspace, project, path)` could interleave their
+ file-rename and DB-upsert and transiently leave the on-disk markdown
+ disagreeing with the DB `is_latest` row (self-healing on reindex, no data
+ loss). A per-path async lock now serializes same-path writers while
+ different paths still proceed concurrently; batches acquire their paths in a
+ fixed global order so they cannot deadlock.
+
## [2.0.2] - 2026-09-03
### Fixed
diff --git a/crates/ai-memory-store/migrations/V59__purged_scopes_tombstones.sql b/crates/ai-memory-store/migrations/V59__purged_scopes_tombstones.sql
new file mode 100644
index 00000000..f3216a2f
--- /dev/null
+++ b/crates/ai-memory-store/migrations/V59__purged_scopes_tombstones.sql
@@ -0,0 +1,34 @@
+-- Tombstones for project/workspace scopes removed by `purge_project` /
+-- `delete_workspace` (#607, item 2).
+--
+-- A purge commits the DB deletion first, then removes the on-disk wiki files
+-- best-effort. A crash (or a file-removal failure) in that window leaves the
+-- project's markdown directory — `_meta.md` included — on disk with no row in
+-- `projects`. A later `ai-memory reindex` rebuilds the DB from markdown and
+-- calls `ensure_project_with_id` / `ensure_workspace_with_id` from each
+-- `_meta.md`, which would recreate the scope and resurrect its pages, silently
+-- undoing the purge.
+--
+-- The tombstone makes the deletion terminal: `reindex_all` checks it and
+-- refuses to recreate a scope that was purged. Unlike `purged_sessions`, this
+-- table carries NO foreign keys — the workspace/project rows it names are the
+-- ones being deleted, so an `ON DELETE CASCADE` reference would either fail or
+-- cascade the tombstone away with them.
+--
+-- Keyed by (workspace_id, project_id). A project purge inserts the real
+-- project id. A workspace delete inserts a 16-byte all-zero project id
+-- (`zeroblob(16)`) as a whole-workspace tombstone, which never collides with a
+-- real UUIDv7 project id. Ids are terminal: a legitimately recreated
+-- workspace/project gets a fresh UUID, so a tombstone never blocks new work.
+--
+-- `purged_at` exists so the ledger can be pruned; a tombstone only has to
+-- outlive any surviving on-disk directory that a reindex might otherwise
+-- resurrect.
+CREATE TABLE purged_scopes (
+ workspace_id BLOB NOT NULL,
+ project_id BLOB NOT NULL,
+ purged_at INTEGER NOT NULL,
+ PRIMARY KEY (workspace_id, project_id)
+);
+
+CREATE INDEX idx_purged_scopes_at ON purged_scopes(purged_at);
diff --git a/crates/ai-memory-store/src/api_credentials.rs b/crates/ai-memory-store/src/api_credentials.rs
index d1d90786..0a5763d6 100644
--- a/crates/ai-memory-store/src/api_credentials.rs
+++ b/crates/ai-memory-store/src/api_credentials.rs
@@ -411,7 +411,7 @@ mod tests {
|row| row.get(0),
)
.unwrap();
- assert_eq!(version, 58, "update the pin when adding a migration");
+ assert_eq!(version, 59, "update the pin when adding a migration");
let cols: i64 = conn
.query_row(
"SELECT COUNT(*) FROM pragma_table_info('users') WHERE name = 'token_hash'",
diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rs
index 49f98b97..654aed6a 100644
--- a/crates/ai-memory-store/src/ops.rs
+++ b/crates/ai-memory-store/src/ops.rs
@@ -3627,6 +3627,30 @@ pub fn purge_session(
/// Returns [`StoreError::ManagedRunActive`] when a managed run's lease is
/// still live and `force` is false, or [`StoreError`] if any SQL statement
/// fails. The transaction is rolled back automatically on error.
+/// Whether `(workspace_id, project_id)` — or the whole workspace — was purged
+/// and tombstoned by [`purge_project`] / [`delete_workspace`] (#607).
+///
+/// Returns true when either an exact project tombstone exists or a
+/// whole-workspace tombstone (the `zeroblob(16)` sentinel project id) covers
+/// the workspace. `reindex` calls this before recreating a scope from on-disk
+/// `_meta.md`, so a purge whose file removal crashed cannot be resurrected.
+pub fn scope_is_purged(
+ conn: &Connection,
+ workspace_id: &WorkspaceId,
+ project_id: &ProjectId,
+) -> StoreResult<bool> {
+ let found: Option<i64> = conn
+ .query_row(
+ "SELECT 1 FROM purged_scopes \
+ WHERE workspace_id = ?1 AND (project_id = ?2 OR project_id = zeroblob(16)) \
+ LIMIT 1",
+ rusqlite::params![workspace_id.as_bytes(), project_id.as_bytes()],
+ |row| row.get(0),
+ )
+ .optional()?;
+ Ok(found.is_some())
+}
+
pub fn purge_project(
conn: &mut Connection,
workspace_id: &WorkspaceId,
@@ -3749,6 +3773,17 @@ pub fn purge_project(
rusqlite::params![&pid[..], workspace_id.as_bytes()],
)?;
+ // Tombstone the scope so a `reindex` cannot resurrect it from an on-disk
+ // directory the post-commit file removal failed to (or crashed before)
+ // deleting. Written in the same transaction as the DELETE so the deletion
+ // and its terminality commit atomically (#607). `INSERT OR REPLACE` keeps
+ // a repeated purge idempotent and refreshes `purged_at`.
+ tx.execute(
+ "INSERT OR REPLACE INTO purged_scopes (workspace_id, project_id, purged_at) \
+ VALUES (?1, ?2, ?3)",
+ rusqlite::params![workspace_id.as_bytes(), &pid[..], now],
+ )?;
+
// Attributed audit trail for the destructive purge. `page_id` is None
// (the whole project is gone); the operator identity comes from the
// authenticated request (NULL when single-user / unauthenticated).
@@ -3854,6 +3889,18 @@ pub fn delete_workspace(
if removed == 0 {
return Err(StoreError::NotFound("workspace".into()));
}
+
+ // Whole-workspace tombstone (#607): a 16-byte all-zero project id marks the
+ // entire workspace as purged so `reindex` cannot recreate it — or any of
+ // its projects — from on-disk `_meta.md` that the post-commit directory
+ // removal failed to delete. Same transaction as the DELETE for atomic
+ // terminality.
+ tx.execute(
+ "INSERT OR REPLACE INTO purged_scopes (workspace_id, project_id, purged_at) \
+ VALUES (?1, zeroblob(16), ?2)",
+ rusqlite::params![&wid[..], Timestamp::now().as_microsecond()],
+ )?;
+
tx.commit()?;
if compaction == Compaction::Reclaim {
@@ -5039,6 +5086,52 @@ pub(crate) mod tests {
/// docs and `docs/lifecycle-ops.md` cannot drift away from the behaviour:
/// if this starts failing, byte-level removal became the default and all
/// three need updating together.
+ #[test]
+ fn purge_project_tombstones_the_scope_against_resurrection() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ seed_session(&mut conn, ws, proj, "canaryproj");
+
+ assert!(
+ !scope_is_purged(&conn, &ws, &proj).unwrap(),
+ "a live project is not tombstoned"
+ );
+
+ purge_project(
+ &mut conn,
+ &ws,
+ &proj,
+ "default/scratch",
+ None,
+ false,
+ Compaction::Skip,
+ )
+ .unwrap();
+
+ assert!(
+ scope_is_purged(&conn, &ws, &proj).unwrap(),
+ "purge must tombstone the scope so reindex cannot resurrect it"
+ );
+ // A different, un-purged project in the same workspace is unaffected.
+ assert!(
+ !scope_is_purged(&conn, &ws, &ai_memory_core::ProjectId::new()).unwrap(),
+ "the tombstone is scoped to the purged project id only"
+ );
+ }
+
+ #[test]
+ fn delete_workspace_tombstones_the_whole_workspace() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+
+ delete_workspace(&mut conn, &ws, true, Compaction::Skip).unwrap();
+
+ // The whole-workspace tombstone covers every project id in that ws,
+ // including ones whose rows are already gone via cascade.
+ assert!(scope_is_purged(&conn, &ws, &proj).unwrap());
+ assert!(scope_is_purged(&conn, &ws, &ai_memory_core::ProjectId::new()).unwrap());
+ // A different workspace is not affected by the sentinel.
+ assert!(!scope_is_purged(&conn, &ai_memory_core::WorkspaceId::new(), &proj).unwrap());
+ }
+
#[test]
fn purge_project_is_a_logical_delete_by_default() {
let (tmp, mut conn, ws, proj) = fresh_db();
diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rs
index b2a44330..8d8c27d5 100644
--- a/crates/ai-memory-store/src/writer.rs
+++ b/crates/ai-memory-store/src/writer.rs
@@ -77,6 +77,11 @@ pub(crate) enum WriteCmd {
repo_path: Option<String>,
reply: oneshot::Sender<StoreResult<()>>,
},
+ ScopeIsPurged {
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ reply: oneshot::Sender<StoreResult<bool>>,
+ },
UpsertPage {
page: NewPage,
reply: oneshot::Sender<StoreResult<PageId>>,
@@ -728,6 +733,26 @@ impl WriterHandle {
rx.await.map_err(|_| StoreError::WriterClosed)?
}
+ /// Whether the scope was purged by `purge_project` / `delete_workspace` and
+ /// tombstoned. `reindex` consults this before recreating a scope from
+ /// on-disk `_meta.md` so a purge that crashed before its files were removed
+ /// cannot be silently undone (#607). Routed through the writer actor
+ /// because it owns the connection and is always present.
+ pub async fn scope_is_purged(
+ &self,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ ) -> StoreResult<bool> {
+ let (tx, rx) = oneshot::channel();
+ self.send(WriteCmd::ScopeIsPurged {
+ workspace_id,
+ project_id,
+ reply: tx,
+ })
+ .await?;
+ rx.await.map_err(|_| StoreError::WriterClosed)?
+ }
+
/// Begin a session (idempotent on the supplied id).
///
/// # Errors
@@ -2445,6 +2470,14 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {
);
send_or_warn(reply, result, "ensure_project_with_id");
}
+ WriteCmd::ScopeIsPurged {
+ workspace_id,
+ project_id,
+ reply,
+ } => {
+ let result = ops::scope_is_purged(&conn, &workspace_id, &project_id);
+ send_or_warn(reply, result, "scope_is_purged");
+ }
WriteCmd::UpsertPage { page, reply } => {
let result = ops::upsert_page(&mut conn, &page);
send_or_warn(reply, result, "upsert_page");
diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rs
index fcad38cb..17f7d4f0 100644
--- a/crates/ai-memory-wiki/src/wiki.rs
+++ b/crates/ai-memory-wiki/src/wiki.rs
@@ -1,5 +1,6 @@
//! [`Wiki`] — the only correct write path for the markdown source-of-truth.
+use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@@ -31,6 +32,10 @@ pub struct ReindexSummary {
pub projects: usize,
/// Pages reindexed from the wiki tree.
pub pages: usize,
+ /// Project directories skipped because their scope was purged and
+ /// tombstoned (#607) — a purge whose on-disk removal did not complete.
+ /// Their pages are deliberately not resurrected.
+ pub skipped_purged: usize,
}
enum PageStoreRemoval {
@@ -110,8 +115,20 @@ pub struct Wiki {
/// the directory rename and SQLite re-stamp so stale writes cannot land
/// files under the old workspace while the project is in flight.
mutation_lock: Arc<RwLock<()>>,
+ /// Per-page write serialization (#607, item 3). Two concurrent writes to
+ /// the *same* `(ws, proj, path)` would otherwise interleave their
+ /// file-rename and DB-upsert under the shared `mutation_lock` read guard,
+ /// transiently leaving on-disk markdown disagreeing with the DB `latest`
+ /// row. Each path gets its own async mutex, so same-path writers serialize
+ /// while different paths still run concurrently. The map is GC'd
+ /// opportunistically — entries no writer currently holds are dropped on the
+ /// next acquisition — so it stays bounded to currently-contended paths.
+ page_locks: Arc<std::sync::Mutex<HashMap<PageKey, Arc<tokio::sync::Mutex<()>>>>>,
}
+/// Key uniquely identifying a page for per-path write serialization.
+type PageKey = (WorkspaceId, ProjectId, PagePath);
+
impl Wiki {
/// Construct a wiki handle rooted at `<data_dir>/wiki/`. Creates the
/// directory if absent and initialises a git repo inside it.
@@ -132,6 +149,7 @@ impl Wiki {
admission_chain: None,
store_reader: None,
mutation_lock: Arc::new(RwLock::new(())),
+ page_locks: Arc::new(std::sync::Mutex::new(HashMap::new())),
})
}
@@ -1392,6 +1410,21 @@ impl Wiki {
let mut seen_ws = std::collections::HashSet::new();
for (ws, proj, proj_root) in project_dirs {
+ // A purged scope must not be resurrected from an on-disk directory
+ // its purge failed to remove (#607). The tombstone is terminal:
+ // skip the whole directory rather than recreating the scope row and
+ // reindexing its pages. The files stay inert (the watcher skips
+ // them too, #613) until a later purge or manual cleanup removes
+ // them; reindex stays non-destructive.
+ if self.writer.scope_is_purged(ws, proj).await? {
+ tracing::debug!(
+ workspace = %ws,
+ project = %proj,
+ "skipping reindex of a purged (tombstoned) scope",
+ );
+ summary.skipped_purged += 1;
+ continue;
+ }
if seen_ws.insert(ws) {
let ws_dir = proj_root
.parent()
@@ -1598,6 +1631,14 @@ impl Wiki {
let (ids, dispatches) = {
let _guard = self.mutation_lock.read().await;
+ // Serialize every path in this batch against concurrent single-page
+ // or batch writers to the same path (#607). Sorted acquisition
+ // keeps a batch deadlock-free against any other writer.
+ let batch_keys: Vec<PageKey> = staged
+ .iter()
+ .map(|(req, _, _, _)| (req.workspace_id, req.project_id, req.path.clone()))
+ .collect();
+ let _page_guards = self.lock_pages(&batch_keys).await;
let mut staged_files: Vec<(
WritePageRequest,
tempfile::NamedTempFile,
@@ -1685,6 +1726,53 @@ impl Wiki {
Ok(ids)
}
+ /// Acquire the per-path write lock for one page, serializing concurrent
+ /// writers to the same `(ws, proj, path)` while different paths run
+ /// concurrently. See [`Wiki::page_locks`].
+ async fn lock_page(
+ &self,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ path: &PagePath,
+ ) -> tokio::sync::OwnedMutexGuard<()> {
+ let lock = {
+ let mut map = self.page_locks.lock().expect("page_locks mutex poisoned");
+ // Opportunistic GC under the same short critical section that hands
+ // out locks: an entry whose only remaining strong ref is the map's
+ // own (count == 1) has no live guard and no waiter, so drop it.
+ map.retain(|_, v| Arc::strong_count(v) > 1);
+ Arc::clone(
+ map.entry((workspace_id, project_id, path.clone()))
+ .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))),
+ )
+ };
+ lock.lock_owned().await
+ }
+
+ /// Acquire per-path write locks for a batch of pages in a deterministic
+ /// global order (sorted by id/path bytes, deduped) so a batch write and any
+ /// concurrent single-page or batch writer can never deadlock. See
+ /// [`Wiki::page_locks`].
+ async fn lock_pages(&self, keys: &[PageKey]) -> Vec<tokio::sync::OwnedMutexGuard<()>> {
+ let mut ordered: Vec<PageKey> = keys.to_vec();
+ // The id newtypes are not `Ord`, so order by their byte / string
+ // representation — any total order works, it only has to be consistent
+ // across all acquirers.
+ ordered.sort_by(|a, b| {
+ (a.0.as_bytes(), a.1.as_bytes(), a.2.as_str()).cmp(&(
+ b.0.as_bytes(),
+ b.1.as_bytes(),
+ b.2.as_str(),
+ ))
+ });
+ ordered.dedup();
+ let mut guards = Vec::with_capacity(ordered.len());
+ for (ws, proj, path) in ordered {
+ guards.push(self.lock_page(ws, proj, &path).await);
+ }
+ guards
+ }
+
/// Write `body` (with optional `frontmatter`) atomically to
/// `<wiki_root>/<workspace_id>/<project_id>/<path>` and upsert the
/// matching page row in the store.
@@ -1799,6 +1887,11 @@ impl Wiki {
let page_id = {
let _guard = self.mutation_lock.read().await;
+ // Serialize with any concurrent write to this same path, so the
+ // file-rename and DB-upsert below cannot interleave with another
+ // writer's and leave disk disagreeing with the DB `latest` row
+ // (#607). Different paths still proceed concurrently.
+ let _page_guard = self.lock_page(workspace_id, project_id, &path).await;
self.ensure_project_workspace(workspace_id, project_id)
.await?;
let abs = self.abs_path(workspace_id, project_id, &path);
@@ -4198,6 +4291,156 @@ mod tests {
let _ = meta;
}
+ /// #607 item 2: a purge commits the DB deletion but removes on-disk files
+ /// only afterward (best-effort). If that removal is interrupted, the
+ /// markdown directory — `_meta.md` included — survives with no `projects`
+ /// row. `reindex_all` must NOT resurrect it: the tombstone written in the
+ /// purge transaction makes the deletion terminal.
+ #[tokio::test]
+ async fn reindex_does_not_resurrect_a_purged_scope_whose_files_survived() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let wiki = Wiki::new(tmp.path(), store.writer.clone())
+ .unwrap()
+ .with_store_reader(store.reader.clone());
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "scratch", None)
+ .await
+ .unwrap();
+
+ let path = PagePath::new("notes/keep.md").unwrap();
+ wiki.write_page(WritePageRequest {
+ workspace_id: ws,
+ project_id: proj,
+ path: PagePath::new("notes/keep.md").unwrap(),
+ frontmatter: serde_json::json!({"title": "Keep"}),
+ body: "survivor".into(),
+ tier: Tier::Semantic,
+ pinned: false,
+ title: None,
+ admission_ctx: None,
+ author_id: None,
+ actor: ai_memory_core::ActorContext::anonymous(),
+ })
+ .await
+ .unwrap();
+ // Manifests reindex reads to recreate scopes from disk.
+ wiki.backfill_scope_manifests().await.unwrap();
+
+ // Purge the DB only (store side); the on-disk files intentionally
+ // survive — exactly the crash-before-file-removal window.
+ store
+ .writer
+ .purge_project(
+ ws,
+ proj,
+ "default/scratch",
+ None,
+ false,
+ ai_memory_store::Compaction::Skip,
+ )
+ .await
+ .unwrap();
+ let meta_on_disk = tmp
+ .path()
+ .join("wiki")
+ .join(ws.to_string())
+ .join(proj.to_string())
+ .join("_meta.md");
+ assert!(
+ meta_on_disk.exists(),
+ "the purge left the manifest on disk (the crash window this guards)"
+ );
+
+ let summary = wiki.reindex_all().await.unwrap();
+
+ assert_eq!(
+ summary.skipped_purged, 1,
+ "the purged scope must be skipped"
+ );
+ assert_eq!(
+ summary.projects, 0,
+ "the purged project must not be recreated"
+ );
+ assert!(
+ store
+ .reader
+ .page_body_by_ids(ws, proj, path.as_str())
+ .await
+ .unwrap()
+ .is_none(),
+ "a purged page must not reappear in the index"
+ );
+ }
+
+ /// #607 item 3: two concurrent writes to the SAME page path must not
+ /// interleave their file-rename and DB-upsert. Per-path serialization
+ /// guarantees the winning writer's body lands in both the on-disk file and
+ /// the DB `is_latest` row — they never disagree about who won.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+ async fn concurrent_same_path_writes_keep_file_and_db_consistent() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "scratch", None)
+ .await
+ .unwrap();
+
+ let path = PagePath::new("notes/hot.md").unwrap();
+ let mk = |tag: char| WritePageRequest {
+ workspace_id: ws,
+ project_id: proj,
+ path: PagePath::new("notes/hot.md").unwrap(),
+ frontmatter: serde_json::json!({"title": "Hot"}),
+ body: format!("body-{tag}"),
+ tier: Tier::Semantic,
+ pinned: false,
+ title: None,
+ admission_ctx: None,
+ author_id: None,
+ actor: ai_memory_core::ActorContext::anonymous(),
+ };
+
+ // Many rounds so any interleave window is likely to be exercised.
+ for _ in 0..25 {
+ let (wa, wb) = (wiki.clone(), wiki.clone());
+ let (ra, rb) = (mk('a'), mk('b'));
+ let ha = tokio::spawn(async move { wa.write_page(ra).await });
+ let hb = tokio::spawn(async move { wb.write_page(rb).await });
+ ha.await.unwrap().unwrap();
+ hb.await.unwrap().unwrap();
+
+ let on_disk = wiki.read_page(ws, proj, &path).unwrap().body;
+ let in_db = store
+ .reader
+ .page_body_by_ids(ws, proj, path.as_str())
+ .await
+ .unwrap()
+ .expect("a latest row exists")
+ .body;
+ // Compare who won, not exact bytes (emit/parse normalizes newlines):
+ // the file and the DB latest row must reflect the same writer.
+ assert_eq!(
+ on_disk.contains("body-a"),
+ in_db.contains("body-a"),
+ "file ({on_disk:?}) and DB latest ({in_db:?}) disagree on the winning writer"
+ );
+ }
+ }
+
/// Backward-compat: anonymous writes do not add attribution frontmatter.
#[tokio::test]
async fn write_page_with_anonymous_actor_omits_attribution_frontmatter() {