Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(wiki): write a scope's manifest with its first page (#643)

Samir Hanna Verzac12dede815db
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a6f92c69..2f46beb3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Fixed
+- A project that first materializes while the server is running is now
+ self-describing immediately, instead of only after the next startup
+ backfill (#643). Scope manifests (`_meta.md`) were written at startup and on
+ the rename/move admin paths, so a session in a checkout the server had not
+ seen before produced a scope directory with pages but no manifest. Stop the
+ server in that window and `reindex` could not rebuild that tree — the one
+ situation where an operator most needs the rebuild to work. The manifest is
+ now written with the scope's first page, one store lookup per scope per
+ process, byte-identical to what the backfill writes so restarts still do not
+ churn the wiki's git history. The startup backfill is unchanged and remains
+ the repair path for trees written by older releases.
+
## [2.0.3] - 2026-09-04
### Changed
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index dae39c5e..380e44f1 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -5840,6 +5840,49 @@ impl ReaderPool {
.await
}
+ /// Return one `(workspace, project)` scope by id, with the names and
+ /// `repo_path` its `_meta.md` manifest is written from. `None` when the
+ /// pair has no row.
+ ///
+ /// The single-scope counterpart of [`list_all_scopes`]: the wiki
+ /// materializes a manifest for one scope the first time it writes into
+ /// it, on a path where enumerating every scope in the store would be an
+ /// N+1 over the whole tree.
+ ///
+ /// # Errors
+ /// Propagates any SQL or pool error.
+ pub async fn scope_row_by_ids(
+ &self,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ ) -> StoreResult<Option<ScopeRow>> {
+ // (ws_name, proj_name, repo_path) — the ids are already known.
+ type RawScope = (String, String, Option<String>);
+ let raw: Option<RawScope> = self
+ .with_conn(move |conn| {
+ let row = conn
+ .query_row(
+ "SELECT w.name, p.name, p.repo_path \
+ FROM projects p JOIN workspaces w ON w.id = p.workspace_id \
+ WHERE p.id = ?1 AND p.workspace_id = ?2",
+ params![project_id.as_bytes(), workspace_id.as_bytes()],
+ |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
+ )
+ .optional()?;
+ Ok(row)
+ })
+ .await?;
+ Ok(
+ raw.map(|(workspace_name, project_name, repo_path)| ScopeRow {
+ workspace_id,
+ workspace_name,
+ project_id,
+ project_name,
+ repo_path,
+ }),
+ )
+ }
+
/// Return every `(workspace, project)` scope with its ids, names and
/// `repo_path` — the data needed to write each scope's self-describing
/// `_meta.md` manifest. Unlike [`list_projects_with_stats`], this carries
@@ -8496,6 +8539,53 @@ mod tests {
assert!(free > 0);
}
+ /// The single-scope manifest lookup the wiki resolves a new scope's
+ /// `_meta.md` names from. It carries `repo_path`, and it is keyed by the
+ /// full pair: a project id offered under the wrong workspace resolves to
+ /// nothing rather than leaking the other workspace's name into a
+ /// manifest.
+ #[tokio::test]
+ async fn scope_row_by_ids_returns_manifest_names_and_isolates_workspaces() {
+ let tmp = tempfile::TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store.writer.get_or_create_workspace("acme").await.unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "webapp", Some("/repo/webapp".into()))
+ .await
+ .unwrap();
+ let other_ws = store.writer.get_or_create_workspace("other").await.unwrap();
+
+ let row = store
+ .reader
+ .scope_row_by_ids(ws, proj)
+ .await
+ .unwrap()
+ .expect("the scope exists");
+ assert_eq!(row.workspace_name, "acme");
+ assert_eq!(row.project_name, "webapp");
+ assert_eq!(row.repo_path.as_deref(), Some("/repo/webapp"));
+
+ assert!(
+ store
+ .reader
+ .scope_row_by_ids(other_ws, proj)
+ .await
+ .unwrap()
+ .is_none(),
+ "a project id under the wrong workspace resolves to nothing"
+ );
+ assert!(
+ store
+ .reader
+ .scope_row_by_ids(ws, ai_memory_core::ProjectId::new())
+ .await
+ .unwrap()
+ .is_none(),
+ "an unknown project id is None, not an error"
+ );
+ }
+
use super::{
DESCRIPTOR_MAX_CHARS, StorageStatus, entity_query_tokens, handoff_listing_sql, like_escape,
page_descriptor, page_descriptor_expr,
diff --git a/crates/ai-memory-wiki/src/watcher.rs b/crates/ai-memory-wiki/src/watcher.rs
index 1e7dfd75..57a27cfb 100644
--- a/crates/ai-memory-wiki/src/watcher.rs
+++ b/crates/ai-memory-wiki/src/watcher.rs
@@ -317,7 +317,11 @@ async fn reconcile(wiki: &Wiki) -> WikiResult<ReconcileStats> {
// directory and skip the whole thing at debug, instead of warning per
// page indefinitely. If the row later appears (project recreated), the
// check passes and the directory indexes normally on the next pass.
- if let Err(e) = wiki.ensure_project_workspace(ws, proj).await {
+ // Rows only: reconcile runs outside the mutation guard, so it must
+ // not write a `_meta.md` into a directory a concurrent project move
+ // may be renaming away. These directories already have their
+ // manifests — written with their first page, or by the backfill.
+ if let Err(e) = wiki.ensure_project_scope_rows(ws, proj).await {
debug!(
workspace = %ws,
project = %proj,
diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rs
index 07c3da74..a7f93dec 100644
--- a/crates/ai-memory-wiki/src/wiki.rs
+++ b/crates/ai-memory-wiki/src/wiki.rs
@@ -1,7 +1,8 @@
//! [`Wiki`] — the only correct write path for the markdown source-of-truth.
+use std::collections::HashSet;
use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
use ai_memory_core::{
ActorContext, AutoImproveProposalId, NewPage, PageId, PagePath, ProjectId, Sanitizer,
@@ -110,6 +111,11 @@ pub struct Wiki {
/// the directory rename and SQLite re-stamp so stale writes cannot land
/// files under the old workspace while the project is in flight.
mutation_lock: Arc<RwLock<()>>,
+ /// Scopes this process has already materialized `_meta.md` manifests for.
+ /// Keeps [`Wiki::ensure_scope_manifests`] to one hash lookup per page
+ /// write after the scope's first — the store query and the two manifest
+ /// reads happen once per scope per process, never per page.
+ manifested_scopes: Arc<Mutex<HashSet<(WorkspaceId, ProjectId)>>>,
}
impl Wiki {
@@ -132,6 +138,7 @@ impl Wiki {
admission_chain: None,
store_reader: None,
mutation_lock: Arc::new(RwLock::new(())),
+ manifested_scopes: Arc::new(Mutex::new(HashSet::new())),
})
}
@@ -154,9 +161,12 @@ impl Wiki {
/// external webhooks must fall back to header introspection or use
/// `_unscoped` placeholders.
///
- /// The reader is only invoked when the chain is configured AND would
- /// actually fire; tests and CLI paths that don't wire a chain pay
- /// nothing for setting (or omitting) this.
+ /// The reader is also what makes a scope self-describing: the names a
+ /// `_meta.md` manifest carries are resolved through it, so without one
+ /// neither [`Self::backfill_scope_manifests`] nor the manifest written
+ /// with a scope's first page can run, and both become no-ops. For the
+ /// admission chain specifically the reader is still only consulted when
+ /// a chain is configured and would actually fire.
#[must_use]
pub fn with_store_reader(mut self, reader: ReaderPool) -> Self {
self.store_reader = Some(reader);
@@ -474,10 +484,32 @@ impl Wiki {
}
}
+ /// Ensure the store rows for a scope exist **and** that the scope is
+ /// self-describing on disk. Every wiki write path funnels through here,
+ /// which is what guarantees a project directory never outlives its
+ /// `_meta.md`: the directory comes into existence with the first page
+ /// written into it, and this runs first.
pub(crate) async fn ensure_project_workspace(
&self,
workspace_id: WorkspaceId,
project_id: ProjectId,
+ ) -> WikiResult<()> {
+ self.ensure_project_scope_rows(workspace_id, project_id)
+ .await?;
+ self.ensure_scope_manifests(workspace_id, project_id).await;
+ Ok(())
+ }
+
+ /// Ensure the store rows for a scope exist, without touching the wiki
+ /// tree. Only the watcher's reconcile pass wants this: it walks
+ /// directories that already exist (so their manifests were written when
+ /// their first page landed, or by the startup backfill), and it runs
+ /// outside the mutation guard — creating files there could drop a
+ /// manifest into a directory a concurrent project move is renaming away.
+ pub(crate) async fn ensure_project_scope_rows(
+ &self,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
) -> WikiResult<()> {
self.writer
.ensure_project_workspace(workspace_id, project_id)
@@ -485,6 +517,61 @@ impl Wiki {
Ok(())
}
+ /// Write the workspace and project `_meta.md` manifests the first time
+ /// this process writes into a scope, so a project that first materializes
+ /// *while the server is up* is rebuildable immediately instead of only
+ /// after the next startup backfill (#643). In that window `reindex` could
+ /// not rebuild the scope at all: it walks the directories that exist on
+ /// disk, and the directory exists from the first page write onward.
+ ///
+ /// Best-effort on purpose. A manifest that cannot be written must not
+ /// fail the page write that triggered it — the page is the operator's
+ /// data, the manifest is derived, and
+ /// [`Self::backfill_scope_manifests`] rewrites it on the next start.
+ async fn ensure_scope_manifests(&self, workspace_id: WorkspaceId, project_id: ProjectId) {
+ let Some(reader) = &self.store_reader else {
+ return;
+ };
+ let key = (workspace_id, project_id);
+ if self
+ .manifested_scopes
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .contains(&key)
+ {
+ return;
+ }
+ let scope = match reader.scope_row_by_ids(workspace_id, project_id).await {
+ Ok(Some(scope)) => scope,
+ // No row to describe: the scope was purged or moved between the
+ // ensure above and this lookup. The next write re-checks.
+ Ok(None) => return,
+ Err(e) => {
+ tracing::warn!(error = %e, "scope-manifest lookup failed (non-fatal)");
+ return;
+ }
+ };
+ let ws_dir = self.root.join(workspace_id.to_string());
+ let mut project_fm = serde_json::json!({ "project": scope.project_name });
+ if let Some(repo_path) = scope.repo_path {
+ project_fm["repo_path"] = serde_json::Value::String(repo_path);
+ }
+ let written = Self::write_scope_manifest(
+ &ws_dir,
+ serde_json::json!({ "workspace": scope.workspace_name }),
+ )
+ .and_then(|_| Self::write_scope_manifest(&ws_dir.join(project_id.to_string()), project_fm));
+ match written {
+ Ok(_) => {
+ self.manifested_scopes
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .insert(key);
+ }
+ Err(e) => tracing::warn!(error = %e, "scope-manifest write failed (non-fatal)"),
+ }
+ }
+
/// Absolute on-disk path for a page within a specific project.
#[must_use]
pub fn abs_path(
@@ -1380,7 +1467,10 @@ impl Wiki {
/// # Errors
/// Returns [`WikiError`] for filesystem/parse/store errors, including a
/// scope directory that lacks its `_meta.md` (the wiki is not
- /// self-describing — newer engines write the manifest on scope creation).
+ /// self-describing). Scopes written by this engine always have one: it
+ /// is materialized with the scope's first page, and
+ /// [`Self::backfill_scope_manifests`] repairs trees written by older
+ /// ones on every start.
pub async fn reindex_all(&self) -> WikiResult<ReindexSummary> {
let root = self.root().to_path_buf();
let project_dirs =
@@ -4253,10 +4343,12 @@ mod tests {
}
/// End-to-end "DB is rebuildable from files": `backfill_scope_manifests`
- /// makes the wiki self-describing, then `reindex_all` on a FRESH store
- /// (no DB carried over) recreates the named scopes + all pages from the
- /// wiki tree alone — including a page that lives at the reserved name
- /// `log.md` (kept because it has frontmatter).
+ /// repairs a tree whose manifests are missing (one written by a release
+ /// before scopes described themselves from their first page), then
+ /// `reindex_all` on a FRESH store (no DB carried over) recreates the
+ /// named scopes + all pages from the wiki tree alone — including a page
+ /// that lives at the reserved name `log.md` (kept because it has
+ /// frontmatter).
#[tokio::test]
async fn backfill_then_reindex_rebuilds_from_wiki_alone() {
// Source store: a named scope with two pages (one at `log.md`).
@@ -4290,10 +4382,17 @@ mod tests {
.await
.unwrap();
- // Make the wiki self-describing.
+ // Simulate a tree written by an engine that had no manifests at all
+ // (pre-#643 releases), so the backfill is exercised as the repair
+ // path it now is — the writes above already made this scope
+ // self-describing on their own.
+ let ws_dir = src.path().join("wiki").join(ws.to_string());
+ std::fs::remove_file(ws_dir.join("_meta.md")).unwrap();
+ std::fs::remove_file(ws_dir.join(proj.to_string()).join("_meta.md")).unwrap();
+
+ // Make the wiki self-describing again.
let written = w1.backfill_scope_manifests().await.unwrap();
assert!(written >= 2, "ws + proj manifests written, got {written}");
- let ws_dir = src.path().join("wiki").join(ws.to_string());
assert!(ws_dir.join("_meta.md").is_file());
assert!(ws_dir.join(proj.to_string()).join("_meta.md").is_file());
drop(s1);
@@ -4372,6 +4471,132 @@ mod tests {
assert!(meta.contains("workspace: empty-ws"));
}
+ /// #643: a scope that first materializes *while the server is up* is
+ /// self-describing from its first page, without waiting for the next
+ /// startup backfill. The manifests must be byte-identical to the ones
+ /// the backfill would have written, or every restart would rewrite them
+ /// and churn the wiki's git history.
+ #[tokio::test]
+ async fn first_write_into_a_new_scope_writes_its_manifests() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let wiki = Wiki::new(tmp.path(), store.writer.clone())
+ .unwrap()
+ .with_store_reader(store.reader.clone());
+
+ // The scope appears after startup — no backfill has run for it.
+ let ws = store.writer.get_or_create_workspace("acme").await.unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "webapp", Some("/repo/webapp".into()))
+ .await
+ .unwrap();
+ wiki.write_page(req(ws, proj, "notes/a.md", "alpha", serde_json::json!({})))
+ .await
+ .unwrap();
+
+ let ws_dir = tmp.path().join("wiki").join(ws.to_string());
+ let ws_meta = std::fs::read_to_string(ws_dir.join("_meta.md")).unwrap();
+ assert!(ws_meta.contains("workspace: acme"), "{ws_meta}");
+ let proj_meta =
+ std::fs::read_to_string(ws_dir.join(proj.to_string()).join("_meta.md")).unwrap();
+ assert!(proj_meta.contains("project: webapp"), "{proj_meta}");
+ assert!(
+ proj_meta.contains("repo_path: /repo/webapp"),
+ "repo_path is carried, as the backfill carries it: {proj_meta}"
+ );
+ assert!(proj_meta.contains("type: Scope Manifest"), "{proj_meta}");
+
+ assert_eq!(
+ wiki.backfill_scope_manifests().await.unwrap(),
+ 0,
+ "a later backfill finds nothing to write; the two emitters agree byte for byte"
+ );
+ }
+
+ /// #643 end-to-end: stop the server inside the window the issue
+ /// describes — a project created after startup, so no backfill has ever
+ /// seen it — and `reindex` still rebuilds it from the wiki tree alone.
+ /// Before the manifest was written with the first page, this aborted with
+ /// a bare `No such file or directory (os error 2)`.
+ #[tokio::test]
+ async fn scope_created_after_startup_reindexes_without_a_restart() {
+ let src = TempDir::new().unwrap();
+ let s1 = Store::open(src.path()).unwrap();
+ let w1 = Wiki::new(src.path(), s1.writer.clone())
+ .unwrap()
+ .with_store_reader(s1.reader.clone());
+ // Startup backfill: the server has seen nothing yet, so it writes
+ // nothing. The scope below is created afterwards, mid-run.
+ assert_eq!(w1.backfill_scope_manifests().await.unwrap(), 0);
+
+ let ws = s1.writer.get_or_create_workspace("acme").await.unwrap();
+ let proj = s1
+ .writer
+ .get_or_create_project(ws, "webapp", None)
+ .await
+ .unwrap();
+ w1.write_page(req(
+ ws,
+ proj,
+ "notes/a.md",
+ "alpha uniquetoken",
+ serde_json::json!({}),
+ ))
+ .await
+ .unwrap();
+ drop(s1);
+
+ // Fresh store, wiki tree only — the recovery the operator runs.
+ let dst = TempDir::new().unwrap();
+ let s2 = Store::open(dst.path()).unwrap();
+ copy_tree(&src.path().join("wiki"), &dst.path().join("wiki"));
+ let w2 = Wiki::new(dst.path(), s2.writer.clone()).unwrap();
+ let summary = w2.reindex_all().await.unwrap();
+
+ assert_eq!(summary.projects, 1);
+ assert_eq!(summary.pages, 1);
+ assert_eq!(
+ s2.reader.workspace_name_by_id(ws).await.unwrap().as_deref(),
+ Some("acme"),
+ "workspace name recovered from the manifest written mid-run"
+ );
+ assert_eq!(
+ s2.reader
+ .project_name_by_id(ws, proj)
+ .await
+ .unwrap()
+ .as_deref(),
+ Some("webapp"),
+ );
+ }
+
+ /// The watcher's reconcile pre-check runs outside the mutation guard, so
+ /// it deliberately ensures store rows ONLY. Writing a manifest there
+ /// could drop a file into a directory a concurrent project move is
+ /// renaming away, and buys nothing: a directory it can see already got
+ /// its manifest with its first page, or from the backfill.
+ #[tokio::test]
+ async fn reconcile_scope_check_does_not_write_manifests() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let wiki = Wiki::new(tmp.path(), store.writer.clone())
+ .unwrap()
+ .with_store_reader(store.reader.clone());
+ let ws = store.writer.get_or_create_workspace("acme").await.unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "webapp", None)
+ .await
+ .unwrap();
+
+ wiki.ensure_project_scope_rows(ws, proj).await.unwrap();
+
+ let ws_dir = tmp.path().join("wiki").join(ws.to_string());
+ assert!(!ws_dir.join("_meta.md").exists());
+ assert!(!ws_dir.join(proj.to_string()).join("_meta.md").exists());
+ }
+
/// Post-audit regression: manifests are OKF-typed at the writer
/// choke point, and a typeless manifest (the tug-of-war era, or a
/// hand edit) is HEALED by the next backfill instead of reverting
diff --git a/docs/lifecycle-ops.md b/docs/lifecycle-ops.md
index 65161ed3..731d76a5 100644
--- a/docs/lifecycle-ops.md
+++ b/docs/lifecycle-ops.md
@@ -755,6 +755,15 @@ What is not rebuilt:
- Sessions, observations, handoffs, users/tokens, audit rows, access counters,
and embeddings. Those are DB-only state; keep a backup if you need them.
+Every scope directory carries the `_meta.md` manifest `reindex` reads its
+workspace/project name from. The manifest is written with the scope's first
+page, so a project that first appears while the server is running is
+rebuildable from that moment on — no restart required. The startup backfill
+still runs on every boot and repairs a tree written by an older release, or one
+whose manifests were removed by hand. If `reindex` reports a missing manifest,
+start the server once against that data directory and let the backfill write
+it, then stop the server and reindex again.
+
## Operator workflows
### "Fresh start" (wipe everything)