Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(wiki): reindex-from-wiki — rebuild the index from self-describing markdown
Djalma Júniore622f33140d8
diff --git a/crates/ai-memory-cli/src/cli.rs b/crates/ai-memory-cli/src/cli.rsindex b0d66a91..d9a11d10 100644--- a/crates/ai-memory-cli/src/cli.rs+++ b/crates/ai-memory-cli/src/cli.rs@@ -52,6 +52,11 @@ pub enum Command {Backup(BackupArgs),/// Restore a backup tarball into the data directory.Restore(RestoreArgs),+ /// Rebuild the SQLite index from the wiki/ markdown (the "DB is+ /// rebuildable from files" guarantee). Recreates workspaces/projects from+ /// each scope's `_meta.md` manifest and reindexes every page. Run with the+ /// server stopped, against a freshly-migrated (clean) data dir.+ Reindex(ReindexArgs),/// Print (or apply) lifecycle-hook configuration for an agent CLI.InstallHooks(InstallHooksArgs),/// Emit a single lifecycle hook natively (reads the event payload@@ -625,6 +630,10 @@ pub struct RestoreArgs {pub force: bool,}+/// Arguments for `reindex`.+#[derive(Debug, Args)]+pub struct ReindexArgs {}+/// Agent CLI to install hooks/extensions for. For MCP-only clients/// (Claude Desktop), use `install-mcp --client <name>` instead.#[derive(Debug, Clone, Copy, clap::ValueEnum)]diff --git a/crates/ai-memory-cli/src/commands/mod.rs b/crates/ai-memory-cli/src/commands/mod.rsindex 54c541a1..9b13ca4c 100644--- a/crates/ai-memory-cli/src/commands/mod.rs+++ b/crates/ai-memory-cli/src/commands/mod.rs@@ -26,6 +26,7 @@ pub mod move_project;pub mod openclaw_plugin;pub mod purge_project;pub mod read_page;+pub mod reindex;pub mod rename_project;pub mod render_shared;pub mod reorg;diff --git a/crates/ai-memory-cli/src/commands/reindex.rs b/crates/ai-memory-cli/src/commands/reindex.rsnew file mode 100644index 00000000..493529cb--- /dev/null+++ b/crates/ai-memory-cli/src/commands/reindex.rs@@ -0,0 +1,66 @@+//! `ai-memory reindex` — rebuild the SQLite index from the wiki/ markdown.+//!+//! Walks the wiki tree, recreates workspaces/projects from each scope's+//! self-describing `_meta.md` manifest (preserving the ids the tree is keyed+//! by), and reindexes every page. This is the "DB is rebuildable from files"+//! guarantee made operable — e.g. to move a data dir onto a clean migration+//! lineage: drop the old `db/`, then `reindex` rebuilds it from the markdown.+//!+//! # Exception to invariant §16+//!+//! Like `reset`/`restore`, `reindex` is a documented exception to the+//! thin-HTTP-client rule: it opens the store directly and must run with the+//! server stopped (a live SQLite writer would race the rebuild). The sysinfo+//! guard enforces this.+//!+//! Episodic DB-only state (sessions, observations, handoffs, decay counters)+//! is NOT in the markdown and is not reconstructed; embeddings can be+//! recomputed afterwards via `ai-memory embed`.++use ai_memory_store::Store;+use ai_memory_wiki::Wiki;+use anyhow::{Context, Result, bail};+use tracing::info;++use crate::cli::ReindexArgs;+use crate::config::Config;+use crate::process_guard::{busy_message, sibling_processes};++/// Run the `reindex` subcommand.+///+/// # Errors+/// Returns an error if another `ai-memory` process is running, the store+/// fails to open (e.g. a divergent migration lineage — rebuild onto a fresh+/// `db/` instead), or a scope directory lacks its `_meta.md` manifest.+pub async fn run(config: &Config, _args: ReindexArgs) -> Result<()> {+ let siblings = sibling_processes();+ if !siblings.is_empty() {+ bail!(busy_message("reindex", &siblings));+ }++ let store = Store::open(&config.data_dir).context("opening store for reindex")?;+ let wiki = Wiki::new(&config.data_dir, store.writer.clone())+ .context("opening wiki")?+ // Reindex is index-rebuild only; embeddings are recomputed separately+ // via `embed` so a missing/paid provider never blocks a rebuild.+ .without_embedder();++ let summary = wiki+ .reindex_all()+ .await+ .context("rebuilding index from wiki/")?;+ info!(+ workspaces = summary.workspaces,+ projects = summary.projects,+ pages = summary.pages,+ "reindex complete"+ );+ println!(+ "reindexed {} pages across {} project(s) in {} workspace(s) from {}",+ summary.pages,+ summary.projects,+ summary.workspaces,+ config.data_dir.join("wiki").display(),+ );+ Ok(())+}diff --git a/crates/ai-memory-cli/src/main.rs b/crates/ai-memory-cli/src/main.rsindex 81598bf5..b2092465 100644--- a/crates/ai-memory-cli/src/main.rs+++ b/crates/ai-memory-cli/src/main.rs@@ -62,6 +62,7 @@ async fn main() -> Result<()> {Command::Reset(args) => commands::reset::run(&config, args),Command::Backup(args) => commands::backup::run(&config, args).await,Command::Restore(args) => commands::restore::run(&config, args),+ Command::Reindex(args) => commands::reindex::run(&config, args).await,Command::InstallHooks(args) => commands::install_hooks::run(&config, args),// `Hook` is handled in the fast-path above (before config/tracing).Command::Hook(_) => unreachable!("hook handled before config load"),diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex 1e440843..13b8196d 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -142,6 +142,48 @@ pub fn get_or_create_project(Ok(id)}+/// Insert a workspace with an **explicit id**, idempotent. Unlike+/// [`get_or_create_workspace`] (which mints a fresh id), this preserves the id+/// the caller already holds — used by `reindex`, which recovers the id from the+/// wiki directory name so the rebuilt index keys pages by the same+/// `(workspace_id, project_id)` the on-disk tree is laid out under. Re-running+/// is a no-op (`ON CONFLICT(id)`). `created_at` is the rebuild time.+pub fn ensure_workspace_with_id(+ conn: &mut Connection,+ id: ai_memory_core::WorkspaceId,+ name: &str,+) -> StoreResult<()> {+ conn.execute(+ "INSERT INTO workspaces (id, name, created_at) VALUES (?1, ?2, ?3) \+ ON CONFLICT(id) DO NOTHING",+ params![id.as_bytes(), name, Timestamp::now().as_microsecond()],+ )?;+ Ok(())+}++/// Insert a project with an **explicit id** under `workspace_id`, idempotent.+/// The reindex counterpart of [`ensure_workspace_with_id`].+pub fn ensure_project_with_id(+ conn: &mut Connection,+ id: ai_memory_core::ProjectId,+ workspace_id: ai_memory_core::WorkspaceId,+ name: &str,+ repo_path: Option<&str>,+) -> StoreResult<()> {+ conn.execute(+ "INSERT INTO projects (id, workspace_id, name, repo_path, created_at) \+ VALUES (?1, ?2, ?3, ?4, ?5) ON CONFLICT(id) DO NOTHING",+ params![+ id.as_bytes(),+ workspace_id.as_bytes(),+ name,+ repo_path,+ Timestamp::now().as_microsecond()+ ],+ )?;+ Ok(())+}+/// Assert that `project_id` currently belongs to `workspace_id`.////// Wiki writes call this before touching the filesystem so a stale hook/cachediff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rsindex 8989b3ce..c12aba56 100644--- a/crates/ai-memory-store/src/writer.rs+++ b/crates/ai-memory-store/src/writer.rs@@ -38,6 +38,18 @@ pub(crate) enum WriteCmd {project_id: ProjectId,reply: oneshot::Sender<StoreResult<()>>,},+ EnsureWorkspaceWithId {+ id: WorkspaceId,+ name: String,+ reply: oneshot::Sender<StoreResult<()>>,+ },+ EnsureProjectWithId {+ id: ProjectId,+ workspace_id: WorkspaceId,+ name: String,+ repo_path: Option<String>,+ reply: oneshot::Sender<StoreResult<()>>,+ },UpsertPage {page: NewPage,reply: oneshot::Sender<StoreResult<PageId>>,@@ -269,6 +281,51 @@ impl WriterHandle {rx.await.map_err(|_| StoreError::WriterClosed)?}+ /// Insert a workspace with an **explicit id** (idempotent). Used by+ /// `reindex` to recreate a scope under the id recovered from the wiki+ /// directory name. See [`ops::ensure_workspace_with_id`].+ ///+ /// # Errors+ /// Returns [`StoreError::WriterClosed`] or propagates SQL errors.+ pub async fn ensure_workspace_with_id(+ &self,+ id: WorkspaceId,+ name: impl Into<String>,+ ) -> StoreResult<()> {+ let (tx, rx) = oneshot::channel();+ self.send(WriteCmd::EnsureWorkspaceWithId {+ id,+ name: name.into(),+ reply: tx,+ })+ .await?;+ rx.await.map_err(|_| StoreError::WriterClosed)?+ }++ /// Insert a project with an **explicit id** under `workspace_id`+ /// (idempotent). See [`ops::ensure_project_with_id`].+ ///+ /// # Errors+ /// Returns [`StoreError::WriterClosed`] or propagates SQL errors.+ pub async fn ensure_project_with_id(+ &self,+ id: ProjectId,+ workspace_id: WorkspaceId,+ name: impl Into<String>,+ repo_path: Option<String>,+ ) -> StoreResult<()> {+ let (tx, rx) = oneshot::channel();+ self.send(WriteCmd::EnsureProjectWithId {+ id,+ workspace_id,+ name: name.into(),+ repo_path,+ reply: tx,+ })+ .await?;+ rx.await.map_err(|_| StoreError::WriterClosed)?+ }+/// Begin a session (idempotent on the supplied id).////// # Errors@@ -786,6 +843,26 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {let result = ops::ensure_project_workspace(&conn, &workspace_id, &project_id);send_or_warn(reply, result, "ensure_project_workspace");}+ WriteCmd::EnsureWorkspaceWithId { id, name, reply } => {+ let result = ops::ensure_workspace_with_id(&mut conn, id, &name);+ send_or_warn(reply, result, "ensure_workspace_with_id");+ }+ WriteCmd::EnsureProjectWithId {+ id,+ workspace_id,+ name,+ repo_path,+ reply,+ } => {+ let result = ops::ensure_project_with_id(+ &mut conn,+ id,+ workspace_id,+ &name,+ repo_path.as_deref(),+ );+ send_or_warn(reply, result, "ensure_project_with_id");+ }WriteCmd::UpsertPage { page, reply } => {let result = ops::upsert_page(&mut conn, &page);send_or_warn(reply, result, "upsert_page");diff --git a/crates/ai-memory-wiki/src/watcher.rs b/crates/ai-memory-wiki/src/watcher.rsindex d87e4bf8..4c151065 100644--- a/crates/ai-memory-wiki/src/watcher.rs+++ b/crates/ai-memory-wiki/src/watcher.rs@@ -219,7 +219,7 @@ async fn handle_event(wiki: &Wiki, event: notify_debouncer_full::DebouncedEvent)let Some((ws, proj, page_path)) = extract_project_ids(wiki.root(), raw_path) else {continue;};- if is_reserved_filename(&page_path) {+ if is_reserved_page_file(raw_path, &page_path) {continue;}if !raw_path.is_file() {@@ -284,7 +284,7 @@ async fn reconcile(wiki: &Wiki) -> WikiResult<()> {/// Walk `<wiki_root>` and return all `(WorkspaceId, ProjectId, proj_root)` tuples/// whose first two path segments parse as valid UUIDs.-fn walk_project_dirs(+pub(crate) fn walk_project_dirs(wiki_root: &Path,) -> WikiResult<Vec<(WorkspaceId, ProjectId, std::path::PathBuf)>> {let mut out = Vec::new();@@ -376,7 +376,7 @@ fn extract_project_dir_ids(Some((ws_id, proj_id, wiki_root.join(ws_seg).join(proj_seg)))}-fn walk_markdown(root: &Path) -> WikiResult<Vec<PagePath>> {+pub(crate) fn walk_markdown(root: &Path) -> WikiResult<Vec<PagePath>> {let mut out = Vec::new();let mut stack = vec![root.to_path_buf()];while let Some(dir) = stack.pop() {@@ -404,7 +404,7 @@ fn walk_markdown(root: &Path) -> WikiResult<Vec<PagePath>> {&& is_markdown(&path)&& !is_tempfile(&path)&& let Some(pp) = page_path_relative_to(root, &path)- && !is_reserved_filename(&pp)+ && !is_reserved_page_file(&path, &pp){out.push(pp);}@@ -423,21 +423,55 @@ fn is_tempfile(path: &Path) -> bool {.is_some_and(|n| n.starts_with(".ai-memory-tmp."))}-/// Returns `true` for the per-project reserved files that are NOT wiki-/// pages: `log-YYYY-MM.md` (per-month rolling event ledger — see-/// `ai-memory-hooks::log::log_filename_for`), the legacy `log.md`-/// filename for backwards compatibility, and `bootstrap.md` (manifest).-/// The watcher must skip them to avoid supersession loops — every-/// `append_event` write triggers a watcher event, triggering an-/// `upsert_page`, creating a spurious new supersession version.-fn is_reserved_filename(page_path: &PagePath) -> bool {+/// `_meta.md` is the per-scope manifest the engine writes (workspace/project+/// name + repo_path) so the wiki tree is self-describing. It describes the+/// scope, it is never a wiki page.+fn is_manifest_filename(page_path: &PagePath) -> bool {+ page_path+ .as_str()+ .rsplit('/')+ .next()+ .is_some_and(|name| name == "_meta.md")+}++/// `log.md` / `log-YYYY-MM.md` are the raw per-project event ledger the hooks+/// append to (see `ai-memory-hooks::log::log_filename_for`): `## [ts] ...`+/// entries, never YAML frontmatter.+fn is_log_ledger_filename(page_path: &PagePath) -> bool {let s = page_path.as_str();s == "log.md"- || s == "bootstrap.md"// log-YYYY-MM.md — 7 chars between the dash and the dot.|| (s.starts_with("log-") && s.ends_with(".md") && s.len() == "log-YYYY-MM.md".len())}+/// Cheap peek: does the file open with a `---` YAML frontmatter fence?+/// Used to tell a real page apart from the raw event ledger.+fn opens_with_frontmatter(abs: &Path) -> bool {+ use std::io::{BufRead, BufReader};+ let Ok(file) = std::fs::File::open(abs) else {+ return false;+ };+ let mut line = String::new();+ BufReader::new(file).read_line(&mut line).is_ok() && line.trim_end() == "---"+}++/// Returns `true` for markdown files that are NOT wiki pages and must be+/// skipped by the indexer:+/// - `_meta.md` (the self-describing scope manifest) and `bootstrap.md` —+/// always; and+/// - the raw event ledger (`log.md` / `log-YYYY-MM.md`) — skipping which+/// avoids supersession loops, since every `append_event` write triggers a+/// watcher event. **Unless** a file that merely collides with a ledger name+/// actually carries YAML frontmatter: then it is a genuine page (e.g. a page+/// that happens to live at `log.md`) and IS indexed, so it is never silently+/// dropped on a reindex.+fn is_reserved_page_file(abs: &Path, page_path: &PagePath) -> bool {+ if is_manifest_filename(page_path) || page_path.as_str() == "bootstrap.md" {+ return true;+ }+ is_log_ledger_filename(page_path) && !opens_with_frontmatter(abs)+}+fn page_path_relative_to(root: &Path, abs: &Path) -> Option<PagePath> {let rel: &Path = abs.strip_prefix(root).ok()?;let s = rel.to_string_lossy().replace('\\', "/");@@ -723,6 +757,80 @@ mod tests {drop(store);}+ /// A page that *collides* with a reserved ledger name (`log.md`) but+ /// carries YAML frontmatter is a real page and MUST be indexed — not+ /// silently dropped. Regression for a prod data anomaly (a page lived at+ /// `log.md`) that a filename-only skip would lose on every reindex. The+ /// `_meta.md` manifest, by contrast, must NEVER be indexed even though it+ /// also has frontmatter.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn reindex_page_by_content_not_filename() {+ let (tmp, store, wiki, ws, proj) = setup().await;+ let proj_dir = tmp+ .path()+ .join("wiki")+ .join(ws.to_string())+ .join(proj.to_string());+ std::fs::create_dir_all(&proj_dir).unwrap();++ // A genuine page that happens to live at `log.md` (has frontmatter).+ std::fs::write(+ proj_dir.join("log.md"),+ "---\ntitle: Collides With Ledger\n---\nframmaticpage uniquetoken\n",+ )+ .unwrap();+ // The self-describing manifest — never a page, even with frontmatter.+ std::fs::write(+ proj_dir.join("_meta.md"),+ "---\nworkspace: default\nproject: scratch\n---\nmanifesttoken here\n",+ )+ .unwrap();+ // A raw ledger (no frontmatter) — still skipped.+ std::fs::write(+ proj_dir.join("log-2026-06.md"),+ "## [t] evt | x\nrawledgertoken\n",+ )+ .unwrap();++ let handle = WatcherHandle::start(wiki.clone()).unwrap();+ reconcile(&wiki).await.unwrap();++ let page_hits = store+ .reader+ .search_pages("uniquetoken".into(), 5)+ .await+ .unwrap();+ assert_eq!(+ page_hits.len(),+ 1,+ "frontmatter page named log.md must be indexed"+ );+ assert_eq!(page_hits[0].path.as_str(), "log.md");++ let meta_hits = store+ .reader+ .search_pages("manifesttoken".into(), 5)+ .await+ .unwrap();+ assert!(+ meta_hits.is_empty(),+ "_meta.md manifest must not be indexed"+ );++ let ledger_hits = store+ .reader+ .search_pages("rawledgertoken".into(), 5)+ .await+ .unwrap();+ assert!(+ ledger_hits.is_empty(),+ "raw ledger (no frontmatter) must not be indexed"+ );++ handle.shutdown().await;+ drop(store);+ }+/// Defence: an attacker who can write to wiki/ shouldn't be able/// to make the watcher index arbitrary files via symlinks.#[tokio::test(flavor = "multi_thread", worker_threads = 2)]diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rsindex 82dd279b..34224095 100644--- a/crates/ai-memory-wiki/src/wiki.rs+++ b/crates/ai-memory-wiki/src/wiki.rs@@ -10,10 +10,21 @@ use tokio::sync::RwLock;use crate::admission::{AdmissionChain, AdmissionContext, AdmissionOp};use crate::atomic;-use crate::error::WikiResult;+use crate::error::{WikiError, WikiResult};use crate::git::GitAdapter;use crate::markdown::{Markdown, derive_title, emit, extract_links, parse};+/// Summary of a [`Wiki::reindex_all`] run.+#[derive(Debug, Default, Clone)]+pub struct ReindexSummary {+ /// Workspaces recreated from `_meta.md`.+ pub workspaces: usize,+ /// Projects recreated from `_meta.md`.+ pub projects: usize,+ /// Pages reindexed from the wiki tree.+ pub pages: usize,+}+/// Wiki filesystem handle.////// Owns the path of the wiki root (`<data_dir>/wiki/`) and a cloneable@@ -513,6 +524,92 @@ impl Wiki {Ok(id)}+ /// Read a `_meta.md` scope-manifest's frontmatter from `dir`.+ fn read_scope_meta(dir: &Path) -> WikiResult<serde_json::Value> {+ let raw = std::fs::read_to_string(dir.join("_meta.md"))?;+ Ok(parse(&raw)?.frontmatter)+ }++ /// Rebuild the **entire** store index from the on-disk wiki tree — the+ /// "DB is rebuildable from files" guarantee made concrete. Walks every+ /// `<ws-uuid>/<proj-uuid>/` directory, recreates the workspace/project rows+ /// from each dir's self-describing `_meta.md` manifest (preserving the ids+ /// the tree is keyed by, via [`WriterHandle::ensure_workspace_with_id`] /+ /// [`ensure_project_with_id`]), then reindexes every page. Pages are+ /// detected by content (a frontmatter file named `log.md` is a page; the+ /// raw `## [..]` ledger, `_meta.md` and `bootstrap.md` are skipped).+ ///+ /// Intended for a freshly-migrated (clean) store, e.g. to move a data dir+ /// onto a different migration lineage without carrying the old+ /// `refinery_schema_history`. DB-only episodic state (sessions,+ /// observations, handoffs, decay counters) is NOT reconstructed — it is not+ /// in the markdown; embeddings can be recomputed separately via `embed`.+ ///+ /// # Errors+ /// Returns [`WikiError`] for filesystem/parse/store errors, including a+ /// scope directory that lacks its `_meta.md` (the wiki is not+ /// self-describing — newer engines write the manifest on scope creation).+ pub async fn reindex_all(&self) -> WikiResult<ReindexSummary> {+ let root = self.root().to_path_buf();+ let project_dirs =+ tokio::task::spawn_blocking(move || crate::watcher::walk_project_dirs(&root))+ .await+ .map_err(|e| WikiError::Io(std::io::Error::other(e.to_string())))??;++ let mut summary = ReindexSummary::default();+ let mut seen_ws = std::collections::HashSet::new();++ for (ws, proj, proj_root) in project_dirs {+ if seen_ws.insert(ws) {+ let ws_dir = proj_root+ .parent()+ .unwrap_or(proj_root.as_path())+ .to_path_buf();+ let meta = Self::read_scope_meta(&ws_dir)?;+ let name = meta+ .get("workspace")+ .and_then(|v| v.as_str())+ .ok_or_else(|| {+ WikiError::Io(std::io::Error::other(format!(+ "{}/_meta.md is missing the `workspace` name",+ ws_dir.display()+ )))+ })?;+ self.writer.ensure_workspace_with_id(ws, name).await?;+ summary.workspaces += 1;+ }++ let meta = Self::read_scope_meta(&proj_root)?;+ let name = meta+ .get("project")+ .and_then(|v| v.as_str())+ .ok_or_else(|| {+ WikiError::Io(std::io::Error::other(format!(+ "{}/_meta.md is missing the `project` name",+ proj_root.display()+ )))+ })?;+ let repo_path = meta+ .get("repo_path")+ .and_then(|v| v.as_str())+ .map(String::from);+ self.writer+ .ensure_project_with_id(proj, ws, name, repo_path)+ .await?;+ summary.projects += 1;++ let pr = proj_root.clone();+ let pages = tokio::task::spawn_blocking(move || crate::watcher::walk_markdown(&pr))+ .await+ .map_err(|e| WikiError::Io(std::io::Error::other(e.to_string())))??;+ for path in pages {+ self.reindex_page(ws, proj, path).await?;+ summary.pages += 1;+ }+ }+ Ok(summary)+ }+/// Atomically apply a batch of page writes. Either all pages land/// (one SQL transaction) and their files are renamed into place,/// or no DB row changes and tempfiles are dropped.