Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(wiki): write a scope's manifest with its first page (#643)
Samir Hanna Verzac12dede815db
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a6f92c69..2f46beb3 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Fixed+- A project that first materializes while the server is running is now+ self-describing immediately, instead of only after the next startup+ backfill (#643). Scope manifests (`_meta.md`) were written at startup and on+ the rename/move admin paths, so a session in a checkout the server had not+ seen before produced a scope directory with pages but no manifest. Stop the+ server in that window and `reindex` could not rebuild that tree — the one+ situation where an operator most needs the rebuild to work. The manifest is+ now written with the scope's first page, one store lookup per scope per+ process, byte-identical to what the backfill writes so restarts still do not+ churn the wiki's git history. The startup backfill is unchanged and remains+ the repair path for trees written by older releases.+## [2.0.3] - 2026-09-04### Changeddiff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rsindex dae39c5e..380e44f1 100644--- a/crates/ai-memory-store/src/reader.rs+++ b/crates/ai-memory-store/src/reader.rs@@ -5840,6 +5840,49 @@ impl ReaderPool {.await}+ /// Return one `(workspace, project)` scope by id, with the names and+ /// `repo_path` its `_meta.md` manifest is written from. `None` when the+ /// pair has no row.+ ///+ /// The single-scope counterpart of [`list_all_scopes`]: the wiki+ /// materializes a manifest for one scope the first time it writes into+ /// it, on a path where enumerating every scope in the store would be an+ /// N+1 over the whole tree.+ ///+ /// # Errors+ /// Propagates any SQL or pool error.+ pub async fn scope_row_by_ids(+ &self,+ workspace_id: WorkspaceId,+ project_id: ProjectId,+ ) -> StoreResult<Option<ScopeRow>> {+ // (ws_name, proj_name, repo_path) — the ids are already known.+ type RawScope = (String, String, Option<String>);+ let raw: Option<RawScope> = self+ .with_conn(move |conn| {+ let row = conn+ .query_row(+ "SELECT w.name, p.name, p.repo_path \+ FROM projects p JOIN workspaces w ON w.id = p.workspace_id \+ WHERE p.id = ?1 AND p.workspace_id = ?2",+ params![project_id.as_bytes(), workspace_id.as_bytes()],+ |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),+ )+ .optional()?;+ Ok(row)+ })+ .await?;+ Ok(+ raw.map(|(workspace_name, project_name, repo_path)| ScopeRow {+ workspace_id,+ workspace_name,+ project_id,+ project_name,+ repo_path,+ }),+ )+ }+/// Return every `(workspace, project)` scope with its ids, names and/// `repo_path` — the data needed to write each scope's self-describing/// `_meta.md` manifest. Unlike [`list_projects_with_stats`], this carries@@ -8496,6 +8539,53 @@ mod tests {assert!(free > 0);}+ /// The single-scope manifest lookup the wiki resolves a new scope's+ /// `_meta.md` names from. It carries `repo_path`, and it is keyed by the+ /// full pair: a project id offered under the wrong workspace resolves to+ /// nothing rather than leaking the other workspace's name into a+ /// manifest.+ #[tokio::test]+ async fn scope_row_by_ids_returns_manifest_names_and_isolates_workspaces() {+ let tmp = tempfile::TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store.writer.get_or_create_workspace("acme").await.unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "webapp", Some("/repo/webapp".into()))+ .await+ .unwrap();+ let other_ws = store.writer.get_or_create_workspace("other").await.unwrap();++ let row = store+ .reader+ .scope_row_by_ids(ws, proj)+ .await+ .unwrap()+ .expect("the scope exists");+ assert_eq!(row.workspace_name, "acme");+ assert_eq!(row.project_name, "webapp");+ assert_eq!(row.repo_path.as_deref(), Some("/repo/webapp"));++ assert!(+ store+ .reader+ .scope_row_by_ids(other_ws, proj)+ .await+ .unwrap()+ .is_none(),+ "a project id under the wrong workspace resolves to nothing"+ );+ assert!(+ store+ .reader+ .scope_row_by_ids(ws, ai_memory_core::ProjectId::new())+ .await+ .unwrap()+ .is_none(),+ "an unknown project id is None, not an error"+ );+ }+use super::{DESCRIPTOR_MAX_CHARS, StorageStatus, entity_query_tokens, handoff_listing_sql, like_escape,page_descriptor, page_descriptor_expr,diff --git a/crates/ai-memory-wiki/src/watcher.rs b/crates/ai-memory-wiki/src/watcher.rsindex 1e7dfd75..57a27cfb 100644--- a/crates/ai-memory-wiki/src/watcher.rs+++ b/crates/ai-memory-wiki/src/watcher.rs@@ -317,7 +317,11 @@ async fn reconcile(wiki: &Wiki) -> WikiResult<ReconcileStats> {// directory and skip the whole thing at debug, instead of warning per// page indefinitely. If the row later appears (project recreated), the// check passes and the directory indexes normally on the next pass.- if let Err(e) = wiki.ensure_project_workspace(ws, proj).await {+ // Rows only: reconcile runs outside the mutation guard, so it must+ // not write a `_meta.md` into a directory a concurrent project move+ // may be renaming away. These directories already have their+ // manifests — written with their first page, or by the backfill.+ if let Err(e) = wiki.ensure_project_scope_rows(ws, proj).await {debug!(workspace = %ws,project = %proj,diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rsindex 07c3da74..a7f93dec 100644--- a/crates/ai-memory-wiki/src/wiki.rs+++ b/crates/ai-memory-wiki/src/wiki.rs@@ -1,7 +1,8 @@//! [`Wiki`] — the only correct write path for the markdown source-of-truth.+use std::collections::HashSet;use std::path::{Path, PathBuf};-use std::sync::Arc;+use std::sync::{Arc, Mutex};use ai_memory_core::{ActorContext, AutoImproveProposalId, NewPage, PageId, PagePath, ProjectId, Sanitizer,@@ -110,6 +111,11 @@ pub struct Wiki {/// the directory rename and SQLite re-stamp so stale writes cannot land/// files under the old workspace while the project is in flight.mutation_lock: Arc<RwLock<()>>,+ /// Scopes this process has already materialized `_meta.md` manifests for.+ /// Keeps [`Wiki::ensure_scope_manifests`] to one hash lookup per page+ /// write after the scope's first — the store query and the two manifest+ /// reads happen once per scope per process, never per page.+ manifested_scopes: Arc<Mutex<HashSet<(WorkspaceId, ProjectId)>>>,}impl Wiki {@@ -132,6 +138,7 @@ impl Wiki {admission_chain: None,store_reader: None,mutation_lock: Arc::new(RwLock::new(())),+ manifested_scopes: Arc::new(Mutex::new(HashSet::new())),})}@@ -154,9 +161,12 @@ impl Wiki {/// external webhooks must fall back to header introspection or use/// `_unscoped` placeholders.///- /// The reader is only invoked when the chain is configured AND would- /// actually fire; tests and CLI paths that don't wire a chain pay- /// nothing for setting (or omitting) this.+ /// The reader is also what makes a scope self-describing: the names a+ /// `_meta.md` manifest carries are resolved through it, so without one+ /// neither [`Self::backfill_scope_manifests`] nor the manifest written+ /// with a scope's first page can run, and both become no-ops. For the+ /// admission chain specifically the reader is still only consulted when+ /// a chain is configured and would actually fire.#[must_use]pub fn with_store_reader(mut self, reader: ReaderPool) -> Self {self.store_reader = Some(reader);@@ -474,10 +484,32 @@ impl Wiki {}}+ /// Ensure the store rows for a scope exist **and** that the scope is+ /// self-describing on disk. Every wiki write path funnels through here,+ /// which is what guarantees a project directory never outlives its+ /// `_meta.md`: the directory comes into existence with the first page+ /// written into it, and this runs first.pub(crate) async fn ensure_project_workspace(&self,workspace_id: WorkspaceId,project_id: ProjectId,+ ) -> WikiResult<()> {+ self.ensure_project_scope_rows(workspace_id, project_id)+ .await?;+ self.ensure_scope_manifests(workspace_id, project_id).await;+ Ok(())+ }++ /// Ensure the store rows for a scope exist, without touching the wiki+ /// tree. Only the watcher's reconcile pass wants this: it walks+ /// directories that already exist (so their manifests were written when+ /// their first page landed, or by the startup backfill), and it runs+ /// outside the mutation guard — creating files there could drop a+ /// manifest into a directory a concurrent project move is renaming away.+ pub(crate) async fn ensure_project_scope_rows(+ &self,+ workspace_id: WorkspaceId,+ project_id: ProjectId,) -> WikiResult<()> {self.writer.ensure_project_workspace(workspace_id, project_id)@@ -485,6 +517,61 @@ impl Wiki {Ok(())}+ /// Write the workspace and project `_meta.md` manifests the first time+ /// this process writes into a scope, so a project that first materializes+ /// *while the server is up* is rebuildable immediately instead of only+ /// after the next startup backfill (#643). In that window `reindex` could+ /// not rebuild the scope at all: it walks the directories that exist on+ /// disk, and the directory exists from the first page write onward.+ ///+ /// Best-effort on purpose. A manifest that cannot be written must not+ /// fail the page write that triggered it — the page is the operator's+ /// data, the manifest is derived, and+ /// [`Self::backfill_scope_manifests`] rewrites it on the next start.+ async fn ensure_scope_manifests(&self, workspace_id: WorkspaceId, project_id: ProjectId) {+ let Some(reader) = &self.store_reader else {+ return;+ };+ let key = (workspace_id, project_id);+ if self+ .manifested_scopes+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .contains(&key)+ {+ return;+ }+ let scope = match reader.scope_row_by_ids(workspace_id, project_id).await {+ Ok(Some(scope)) => scope,+ // No row to describe: the scope was purged or moved between the+ // ensure above and this lookup. The next write re-checks.+ Ok(None) => return,+ Err(e) => {+ tracing::warn!(error = %e, "scope-manifest lookup failed (non-fatal)");+ return;+ }+ };+ let ws_dir = self.root.join(workspace_id.to_string());+ let mut project_fm = serde_json::json!({ "project": scope.project_name });+ if let Some(repo_path) = scope.repo_path {+ project_fm["repo_path"] = serde_json::Value::String(repo_path);+ }+ let written = Self::write_scope_manifest(+ &ws_dir,+ serde_json::json!({ "workspace": scope.workspace_name }),+ )+ .and_then(|_| Self::write_scope_manifest(&ws_dir.join(project_id.to_string()), project_fm));+ match written {+ Ok(_) => {+ self.manifested_scopes+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .insert(key);+ }+ Err(e) => tracing::warn!(error = %e, "scope-manifest write failed (non-fatal)"),+ }+ }+/// Absolute on-disk path for a page within a specific project.#[must_use]pub fn abs_path(@@ -1380,7 +1467,10 @@ impl Wiki {/// # Errors/// Returns [`WikiError`] for filesystem/parse/store errors, including a/// scope directory that lacks its `_meta.md` (the wiki is not- /// self-describing — newer engines write the manifest on scope creation).+ /// self-describing). Scopes written by this engine always have one: it+ /// is materialized with the scope's first page, and+ /// [`Self::backfill_scope_manifests`] repairs trees written by older+ /// ones on every start.pub async fn reindex_all(&self) -> WikiResult<ReindexSummary> {let root = self.root().to_path_buf();let project_dirs =@@ -4253,10 +4343,12 @@ mod tests {}/// End-to-end "DB is rebuildable from files": `backfill_scope_manifests`- /// makes the wiki self-describing, then `reindex_all` on a FRESH store- /// (no DB carried over) recreates the named scopes + all pages from the- /// wiki tree alone — including a page that lives at the reserved name- /// `log.md` (kept because it has frontmatter).+ /// repairs a tree whose manifests are missing (one written by a release+ /// before scopes described themselves from their first page), then+ /// `reindex_all` on a FRESH store (no DB carried over) recreates the+ /// named scopes + all pages from the wiki tree alone — including a page+ /// that lives at the reserved name `log.md` (kept because it has+ /// frontmatter).#[tokio::test]async fn backfill_then_reindex_rebuilds_from_wiki_alone() {// Source store: a named scope with two pages (one at `log.md`).@@ -4290,10 +4382,17 @@ mod tests {.await.unwrap();- // Make the wiki self-describing.+ // Simulate a tree written by an engine that had no manifests at all+ // (pre-#643 releases), so the backfill is exercised as the repair+ // path it now is — the writes above already made this scope+ // self-describing on their own.+ let ws_dir = src.path().join("wiki").join(ws.to_string());+ std::fs::remove_file(ws_dir.join("_meta.md")).unwrap();+ std::fs::remove_file(ws_dir.join(proj.to_string()).join("_meta.md")).unwrap();++ // Make the wiki self-describing again.let written = w1.backfill_scope_manifests().await.unwrap();assert!(written >= 2, "ws + proj manifests written, got {written}");- let ws_dir = src.path().join("wiki").join(ws.to_string());assert!(ws_dir.join("_meta.md").is_file());assert!(ws_dir.join(proj.to_string()).join("_meta.md").is_file());drop(s1);@@ -4372,6 +4471,132 @@ mod tests {assert!(meta.contains("workspace: empty-ws"));}+ /// #643: a scope that first materializes *while the server is up* is+ /// self-describing from its first page, without waiting for the next+ /// startup backfill. The manifests must be byte-identical to the ones+ /// the backfill would have written, or every restart would rewrite them+ /// and churn the wiki's git history.+ #[tokio::test]+ async fn first_write_into_a_new_scope_writes_its_manifests() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let wiki = Wiki::new(tmp.path(), store.writer.clone())+ .unwrap()+ .with_store_reader(store.reader.clone());++ // The scope appears after startup — no backfill has run for it.+ let ws = store.writer.get_or_create_workspace("acme").await.unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "webapp", Some("/repo/webapp".into()))+ .await+ .unwrap();+ wiki.write_page(req(ws, proj, "notes/a.md", "alpha", serde_json::json!({})))+ .await+ .unwrap();++ let ws_dir = tmp.path().join("wiki").join(ws.to_string());+ let ws_meta = std::fs::read_to_string(ws_dir.join("_meta.md")).unwrap();+ assert!(ws_meta.contains("workspace: acme"), "{ws_meta}");+ let proj_meta =+ std::fs::read_to_string(ws_dir.join(proj.to_string()).join("_meta.md")).unwrap();+ assert!(proj_meta.contains("project: webapp"), "{proj_meta}");+ assert!(+ proj_meta.contains("repo_path: /repo/webapp"),+ "repo_path is carried, as the backfill carries it: {proj_meta}"+ );+ assert!(proj_meta.contains("type: Scope Manifest"), "{proj_meta}");++ assert_eq!(+ wiki.backfill_scope_manifests().await.unwrap(),+ 0,+ "a later backfill finds nothing to write; the two emitters agree byte for byte"+ );+ }++ /// #643 end-to-end: stop the server inside the window the issue+ /// describes — a project created after startup, so no backfill has ever+ /// seen it — and `reindex` still rebuilds it from the wiki tree alone.+ /// Before the manifest was written with the first page, this aborted with+ /// a bare `No such file or directory (os error 2)`.+ #[tokio::test]+ async fn scope_created_after_startup_reindexes_without_a_restart() {+ let src = TempDir::new().unwrap();+ let s1 = Store::open(src.path()).unwrap();+ let w1 = Wiki::new(src.path(), s1.writer.clone())+ .unwrap()+ .with_store_reader(s1.reader.clone());+ // Startup backfill: the server has seen nothing yet, so it writes+ // nothing. The scope below is created afterwards, mid-run.+ assert_eq!(w1.backfill_scope_manifests().await.unwrap(), 0);++ let ws = s1.writer.get_or_create_workspace("acme").await.unwrap();+ let proj = s1+ .writer+ .get_or_create_project(ws, "webapp", None)+ .await+ .unwrap();+ w1.write_page(req(+ ws,+ proj,+ "notes/a.md",+ "alpha uniquetoken",+ serde_json::json!({}),+ ))+ .await+ .unwrap();+ drop(s1);++ // Fresh store, wiki tree only — the recovery the operator runs.+ let dst = TempDir::new().unwrap();+ let s2 = Store::open(dst.path()).unwrap();+ copy_tree(&src.path().join("wiki"), &dst.path().join("wiki"));+ let w2 = Wiki::new(dst.path(), s2.writer.clone()).unwrap();+ let summary = w2.reindex_all().await.unwrap();++ assert_eq!(summary.projects, 1);+ assert_eq!(summary.pages, 1);+ assert_eq!(+ s2.reader.workspace_name_by_id(ws).await.unwrap().as_deref(),+ Some("acme"),+ "workspace name recovered from the manifest written mid-run"+ );+ assert_eq!(+ s2.reader+ .project_name_by_id(ws, proj)+ .await+ .unwrap()+ .as_deref(),+ Some("webapp"),+ );+ }++ /// The watcher's reconcile pre-check runs outside the mutation guard, so+ /// it deliberately ensures store rows ONLY. Writing a manifest there+ /// could drop a file into a directory a concurrent project move is+ /// renaming away, and buys nothing: a directory it can see already got+ /// its manifest with its first page, or from the backfill.+ #[tokio::test]+ async fn reconcile_scope_check_does_not_write_manifests() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let wiki = Wiki::new(tmp.path(), store.writer.clone())+ .unwrap()+ .with_store_reader(store.reader.clone());+ let ws = store.writer.get_or_create_workspace("acme").await.unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "webapp", None)+ .await+ .unwrap();++ wiki.ensure_project_scope_rows(ws, proj).await.unwrap();++ let ws_dir = tmp.path().join("wiki").join(ws.to_string());+ assert!(!ws_dir.join("_meta.md").exists());+ assert!(!ws_dir.join(proj.to_string()).join("_meta.md").exists());+ }+/// Post-audit regression: manifests are OKF-typed at the writer/// choke point, and a typeless manifest (the tug-of-war era, or a/// hand edit) is HEALED by the next backfill instead of revertingdiff --git a/docs/lifecycle-ops.md b/docs/lifecycle-ops.mdindex 65161ed3..731d76a5 100644--- a/docs/lifecycle-ops.md+++ b/docs/lifecycle-ops.md@@ -755,6 +755,15 @@ What is not rebuilt:- Sessions, observations, handoffs, users/tokens, audit rows, access counters,and embeddings. Those are DB-only state; keep a backup if you need them.+Every scope directory carries the `_meta.md` manifest `reindex` reads its+workspace/project name from. The manifest is written with the scope's first+page, so a project that first appears while the server is running is+rebuildable from that moment on — no restart required. The startup backfill+still runs on every boot and repairs a tree written by an older release, or one+whose manifests were removed by hand. If `reindex` reports a missing manifest,+start the server once against that data directory and let the backfill write+it, then stop the server and reindex again.+## Operator workflows### "Fresh start" (wipe everything)