Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(wiki): self-describing _meta.md manifests (backfill on startup)

Djalma Júnior1a2edf2533df
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rs
index b08014dd..ae7c3de0 100644
--- a/crates/ai-memory-cli/src/commands/serve.rs
+++ b/crates/ai-memory-cli/src/commands/serve.rs
@@ -96,7 +96,12 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {
// operators discover misconfiguration immediately.
let sanitizer = Sanitizer::new(&config.sanitize)
.context("compiling sanitizer.extra_patterns from config")?;
- let wiki = Wiki::new(&config.data_dir, store.writer.clone())?.with_sanitizer(sanitizer.clone());
+ let wiki = Wiki::new(&config.data_dir, store.writer.clone())?
+ .with_sanitizer(sanitizer.clone())
+ // Reader attached unconditionally: admission name-resolution uses it
+ // when a chain is configured, and the startup scope-manifest backfill
+ // (below) always needs it to enumerate scopes.
+ .with_store_reader(store.reader.clone());
// Attach the admission webhook chain (operator-configured via
// `[[admission_webhooks]]` in config.toml or `AI_MEMORY_ADMISSION_WEBHOOKS__N__*`
// env vars). Empty config = no chain attached, zero overhead. The store
@@ -117,6 +122,15 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {
let provider_health = ProviderHealth::default();
let (wiki, embedder) = configure_embedder(config, &store, wiki, &provider_health).await?;
+ // Make the wiki tree self-describing: write each scope's `_meta.md`
+ // (workspace/project name + repo_path) if missing, so the markdown alone
+ // can rebuild the index via `ai-memory reindex`. Idempotent; non-fatal.
+ match wiki.backfill_scope_manifests().await {
+ Ok(0) => {}
+ Ok(n) => tracing::info!(count = n, "wrote _meta.md scope manifests"),
+ Err(e) => tracing::warn!(error = %e, "scope-manifest backfill failed (non-fatal)"),
+ }
+
// Keep the guard alive for the lifetime of `serve`.
let _watcher = start_watcher(&args, &wiki)?;
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index c93bf828..22dbc733 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -30,7 +30,7 @@ pub use reader::{
ActivityWindow, BriefingPage, BriefingSnapshot, DecayCandidate, DerivedIndexStatus,
EmbeddingTripleCount, HealthDetail, HealthPage, ObservationHit, PageAuthor, PageHit,
PageHitWithMeta, PageLinks, PageMeta, PageSummary, ProjectSummary, ReaderPool, RelatedPage,
- StatusCounts, StoredEmbedding, StoredPageBody, WorkspaceSummary, f32_vec_to_bytes,
+ ScopeRow, StatusCounts, StoredEmbedding, StoredPageBody, WorkspaceSummary, f32_vec_to_bytes,
};
pub use users::{TOKEN_HASH_LEN, TOKEN_RAW_LEN, TokenPepper, generate_token, hash_token};
pub use writer::WriterHandle;
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index 1447624d..026a049f 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -230,6 +230,23 @@ pub struct ProjectSummary {
pub last_updated: Option<String>,
}
+/// One `(workspace, project)` scope with the ids + repo_path needed to write
+/// its self-describing `_meta.md` manifest. Returned by
+/// [`ReaderPool::list_all_scopes`]; consumed by `Wiki::backfill_scope_manifests`.
+#[derive(Debug, Clone)]
+pub struct ScopeRow {
+ /// Workspace id — matches the level-1 wiki directory name.
+ pub workspace_id: WorkspaceId,
+ /// Human-readable workspace name.
+ pub workspace_name: String,
+ /// Project id — matches the level-2 wiki directory name.
+ pub project_id: ProjectId,
+ /// Human-readable project name.
+ pub project_name: String,
+ /// Filesystem path the project's cwd-based routing resolves to, if any.
+ pub repo_path: Option<String>,
+}
+
/// One row per workspace with aggregate stats.
/// Returned by [`ReaderPool::list_workspaces_with_stats`].
#[derive(Debug, Clone, Serialize)]
@@ -2465,6 +2482,48 @@ impl ReaderPool {
.await
}
+ /// Return every `(workspace, project)` scope with its ids, names and
+ /// `repo_path` — the data needed to write each scope's self-describing
+ /// `_meta.md` manifest. Unlike [`list_projects_with_stats`], this carries
+ /// the surrogate ids (not just names), so a rebuild can key the wiki tree.
+ ///
+ /// # Errors
+ /// Propagates any SQL or pool error.
+ pub async fn list_all_scopes(&self) -> StoreResult<Vec<ScopeRow>> {
+ // (ws_id, ws_name, proj_id, proj_name, repo_path) as raw SQL columns.
+ type RawScope = (Vec<u8>, String, Vec<u8>, String, Option<String>);
+ let raw: Vec<RawScope> = self
+ .with_conn(|conn| {
+ let mut stmt = conn.prepare(
+ "SELECT w.id, w.name, p.id, p.name, p.repo_path \
+ FROM projects p JOIN workspaces w ON w.id = p.workspace_id",
+ )?;
+ let rows = stmt.query_map([], |row| {
+ Ok((
+ row.get(0)?,
+ row.get(1)?,
+ row.get(2)?,
+ row.get(3)?,
+ row.get(4)?,
+ ))
+ })?;
+ let out: rusqlite::Result<Vec<_>> = rows.collect();
+ Ok(out?)
+ })
+ .await?;
+ raw.into_iter()
+ .map(|(wi, wn, pi, pn, rp)| {
+ Ok(ScopeRow {
+ workspace_id: WorkspaceId::from_slice(&wi)?,
+ workspace_name: wn,
+ project_id: ProjectId::from_slice(&pi)?,
+ project_name: pn,
+ repo_path: rp,
+ })
+ })
+ .collect()
+ }
+
/// Return one row per workspace with project/page-count and
/// last-updated aggregates. Used by custom frontends that need a
/// workspace chooser before narrowing into projects.
diff --git a/crates/ai-memory-wiki/src/watcher.rs.bak b/crates/ai-memory-wiki/src/watcher.rs.bak
new file mode 100644
index 00000000..cc3cb958
--- /dev/null
+++ b/crates/ai-memory-wiki/src/watcher.rs.bak
@@ -0,0 +1,837 @@
+//! Filesystem watcher with debouncing and a periodic reconciliation pass.
+//!
+//! Two parts work together:
+//!
+//! 1. **Debounced events** via [`notify_debouncer_full`]. When a markdown
+//! file under the wiki root is created or modified, we read it from
+//! disk, parse the frontmatter, and `reindex_page` against the store.
+//! Own-writes are absorbed by the store's sha256 short-circuit, so
+//! the loop terminates after one no-op reindex.
+//! 2. **Reconciliation tick** every 30s walks the entire wiki tree and
+//! reindexes every markdown file. Catches any events the OS dropped
+//! (basic-memory #580 — file watchers go stale under FSEvents buffer
+//! overflow, hidden-dir globs, etc.). Hidden-directory paths are
+//! explicitly NOT skipped (#798 lesson).
+//!
+//! The watcher never *writes* to disk — that loop would be unbounded.
+//! External writes drive store updates; internal writes drive disk +
+//! store updates via [`Wiki::write_page`].
+
+use std::path::Path;
+use std::str::FromStr;
+use std::time::Duration;
+
+use ai_memory_core::{PagePath, ProjectId, WorkspaceId};
+use notify::{EventKind, RecursiveMode};
+use notify_debouncer_full::{DebounceEventResult, Debouncer, RecommendedCache, new_debouncer_opt};
+use tokio::sync::mpsc;
+use tracing::{debug, info, warn};
+
+use crate::error::{WikiError, WikiResult};
+use crate::wiki::Wiki;
+
+/// Reconciliation tick interval.
+pub const RECONCILE_INTERVAL: Duration = Duration::from_secs(30);
+
+/// Debounce window for filesystem events.
+pub const DEBOUNCE_WINDOW: Duration = Duration::from_millis(300);
+
+#[cfg(all(test, target_os = "macos"))]
+type PlatformWatcher = notify::PollWatcher;
+#[cfg(not(all(test, target_os = "macos")))]
+type PlatformWatcher = notify::RecommendedWatcher;
+
+/// Handle representing an active watcher; drop to stop.
+pub struct WatcherHandle {
+ _debouncer: Debouncer<PlatformWatcher, RecommendedCache>,
+ shutdown: Option<tokio::sync::oneshot::Sender<()>>,
+ task: Option<tokio::task::JoinHandle<()>>,
+}
+
+impl WatcherHandle {
+ /// Start watching `wiki.root()` recursively. Spawns one tokio task
+ /// that consumes debounced events and runs the reconciliation timer.
+ ///
+ /// Events are attributed to their `(workspace_id, project_id)` by
+ /// parsing the first two path segments as UUIDs. Events outside the
+ /// `<ws_uuid>/<proj_uuid>/...` layout are silently ignored.
+ ///
+ /// # Errors
+ /// Propagates any notify error encountered when installing the OS
+ /// watcher.
+ pub fn start(wiki: Wiki) -> WikiResult<Self> {
+ let (event_tx, event_rx) = mpsc::unbounded_channel();
+
+ let mut debouncer = new_debouncer_opt::<_, PlatformWatcher, RecommendedCache>(
+ DEBOUNCE_WINDOW,
+ None,
+ move |result: DebounceEventResult| match result {
+ Ok(events) => {
+ for event in events {
+ let _ = event_tx.send(event);
+ }
+ }
+ Err(errors) => {
+ for e in errors {
+ warn!(error = %e, "notify error");
+ }
+ }
+ },
+ RecommendedCache::new(),
+ watcher_config(),
+ )
+ .map_err(|e| WikiError::Io(std::io::Error::other(e.to_string())))?;
+
+ debouncer
+ .watch(wiki.root(), RecursiveMode::Recursive)
+ .map_err(|e| WikiError::Io(std::io::Error::other(e.to_string())))?;
+
+ let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();
+ let task = tokio::spawn(run_loop(wiki, event_rx, shutdown_rx));
+
+ Ok(Self {
+ _debouncer: debouncer,
+ shutdown: Some(shutdown_tx),
+ task: Some(task),
+ })
+ }
+
+ /// Stop the watcher and wait for the event loop to drain.
+ pub async fn shutdown(mut self) {
+ if let Some(tx) = self.shutdown.take() {
+ let _ = tx.send(());
+ }
+ if let Some(handle) = self.task.take() {
+ let _ = handle.await;
+ }
+ }
+}
+
+impl Drop for WatcherHandle {
+ fn drop(&mut self) {
+ if let Some(tx) = self.shutdown.take() {
+ let _ = tx.send(());
+ }
+ }
+}
+
+fn watcher_config() -> notify::Config {
+ let config = notify::Config::default();
+ #[cfg(all(test, target_os = "macos"))]
+ {
+ // GitHub macOS runners have flaky FSEvents delivery for tempdir unit
+ // tests. Use the polling backend there so the test covers our watcher
+ // loop without depending on runner-specific FSEvents behavior.
+ config.with_poll_interval(DEBOUNCE_WINDOW)
+ }
+ #[cfg(not(all(test, target_os = "macos")))]
+ {
+ config
+ }
+}
+
+async fn run_loop(
+ wiki: Wiki,
+ mut rx: mpsc::UnboundedReceiver<notify_debouncer_full::DebouncedEvent>,
+ mut shutdown: tokio::sync::oneshot::Receiver<()>,
+) {
+ let mut tick = tokio::time::interval(RECONCILE_INTERVAL);
+ tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+ // First tick fires immediately; consume it so we don't reconcile at boot.
+ tick.tick().await;
+
+ // Track consecutive failures of the reconciliation pass so we can
+ // surface a clear "watcher is degraded" event after a streak, in
+ // addition to the per-failure error log. Without this, a broken
+ // disk → store bridge can stay broken indefinitely with only a
+ // line per 30s in the warn stream — easy to miss in busy logs.
+ let mut consecutive_failures: u32 = 0;
+ const DEGRADED_AFTER: u32 = 5;
+
+ loop {
+ tokio::select! {
+ biased;
+ _ = &mut shutdown => {
+ debug!("watcher shutting down");
+ return;
+ }
+ Some(event) = rx.recv() => {
+ handle_event(&wiki, event).await;
+ }
+ _ = tick.tick() => {
+ match reconcile(&wiki).await {
+ Ok(()) => {
+ if consecutive_failures > 0 {
+ tracing::info!(
+ prior_failures = consecutive_failures,
+ "reconciliation recovered after consecutive failures",
+ );
+ consecutive_failures = 0;
+ }
+ }
+ Err(e) => {
+ consecutive_failures += 1;
+ tracing::error!(
+ error = %e,
+ consecutive_failures,
+ "reconciliation failed",
+ );
+ if consecutive_failures == DEGRADED_AFTER {
+ tracing::error!(
+ consecutive_failures,
+ event = "watcher_degraded",
+ "wiki↔store reconciliation has failed {DEGRADED_AFTER} \
+ times in a row; the disk and SQLite index may now be \
+ out of sync. Investigate disk permissions, DB lock \
+ contention, or filesystem health. The watcher will \
+ keep retrying every {RECONCILE_INTERVAL:?}.",
+ );
+ }
+ }
+ }
+ }
+ else => return,
+ }
+ }
+}
+
+async fn handle_event(wiki: &Wiki, event: notify_debouncer_full::DebouncedEvent) {
+ if !matches!(
+ event.kind,
+ EventKind::Create(_) | EventKind::Modify(_) | EventKind::Other
+ ) {
+ return;
+ }
+ for raw_path in &event.paths {
+ if raw_path.is_dir() {
+ let Some((ws, proj, proj_root)) = extract_project_dir_ids(wiki.root(), raw_path) else {
+ continue;
+ };
+ reindex_project_dir(wiki, ws, proj, proj_root).await;
+ continue;
+ }
+ if !is_markdown(raw_path) {
+ continue;
+ }
+ if is_tempfile(raw_path) {
+ continue;
+ }
+ let Some((ws, proj, page_path)) = extract_project_ids(wiki.root(), raw_path) else {
+ continue;
+ };
+ if is_reserved_page_file(raw_path, &page_path) {
+ continue;
+ }
+ if !raw_path.is_file() {
+ // Likely a transient state (mv, atomic rename in flight).
+ continue;
+ }
+ match wiki.reindex_page(ws, proj, page_path.clone()).await {
+ Ok(_) => debug!(path = %page_path, "reindexed via watcher"),
+ Err(e) => warn!(path = %page_path, error = %e, "watcher reindex failed"),
+ }
+ }
+}
+
+async fn reindex_project_dir(
+ wiki: &Wiki,
+ ws: WorkspaceId,
+ proj: ProjectId,
+ proj_root: std::path::PathBuf,
+) {
+ let pages = match tokio::task::spawn_blocking(move || walk_markdown(&proj_root)).await {
+ Ok(Ok(pages)) => pages,
+ Ok(Err(e)) => {
+ warn!(error = %e, "watcher directory walk failed");
+ return;
+ }
+ Err(e) => {
+ warn!(error = %e, "watcher directory walk task failed");
+ return;
+ }
+ };
+
+ for path in pages {
+ match wiki.reindex_page(ws, proj, path.clone()).await {
+ Ok(_) => debug!(path = %path, "reindexed via watcher directory event"),
+ Err(e) => warn!(path = %path, error = %e, "watcher directory reindex failed"),
+ }
+ }
+}
+
+async fn reconcile(wiki: &Wiki) -> WikiResult<()> {
+ let root = wiki.root().to_path_buf();
+ // Walk all per-project subdirectories: <ws_uuid>/<proj_uuid>/
+ let project_dirs = tokio::task::spawn_blocking(move || walk_project_dirs(&root))
+ .await
+ .map_err(|e| WikiError::Io(std::io::Error::other(e.to_string())))??;
+
+ let mut total = 0_usize;
+ for (ws, proj, proj_root) in project_dirs {
+ let pages = tokio::task::spawn_blocking(move || walk_markdown(&proj_root))
+ .await
+ .map_err(|e| WikiError::Io(std::io::Error::other(e.to_string())))??;
+ total += pages.len();
+ for path in pages {
+ if let Err(e) = wiki.reindex_page(ws, proj, path.clone()).await {
+ warn!(path = %path, error = %e, "reconcile reindex failed");
+ }
+ }
+ }
+ info!(count = total, "reconciliation pass complete");
+ Ok(())
+}
+
+/// Walk `<wiki_root>` and return all `(WorkspaceId, ProjectId, proj_root)` tuples
+/// whose first two path segments parse as valid UUIDs.
+fn walk_project_dirs(
+ wiki_root: &Path,
+) -> WikiResult<Vec<(WorkspaceId, ProjectId, std::path::PathBuf)>> {
+ let mut out = Vec::new();
+ let ws_read = match std::fs::read_dir(wiki_root) {
+ Ok(r) => r,
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(out),
+ Err(e) => return Err(WikiError::Io(e)),
+ };
+ for ws_entry in ws_read {
+ let ws_entry = ws_entry?;
+ if !ws_entry.file_type()?.is_dir() {
+ continue;
+ }
+ let ws_name = ws_entry.file_name();
+ let Some(ws_str) = ws_name.to_str() else {
+ continue;
+ };
+ let Ok(ws_id) = WorkspaceId::from_str(ws_str) else {
+ continue;
+ };
+ let proj_read = match std::fs::read_dir(ws_entry.path()) {
+ Ok(r) => r,
+ Err(_) => continue,
+ };
+ for proj_entry in proj_read {
+ let proj_entry = proj_entry?;
+ if !proj_entry.file_type()?.is_dir() {
+ continue;
+ }
+ let proj_name = proj_entry.file_name();
+ let Some(proj_str) = proj_name.to_str() else {
+ continue;
+ };
+ let Ok(proj_id) = ProjectId::from_str(proj_str) else {
+ continue;
+ };
+ out.push((ws_id, proj_id, proj_entry.path()));
+ }
+ }
+ Ok(out)
+}
+
+/// Parse `(WorkspaceId, ProjectId, PagePath)` from a filesystem event path.
+///
+/// Expects the path to have the structure:
+/// `<wiki_root>/<ws_uuid>/<proj_uuid>/<page-path...>`
+///
+/// Returns `None` when:
+/// - The path does not start with `wiki_root`.
+/// - The first segment is not a valid UUID (`WorkspaceId`).
+/// - The second segment is not a valid UUID (`ProjectId`).
+/// - There are no remaining segments (the page path would be empty).
+pub(crate) fn extract_project_ids(
+ wiki_root: &Path,
+ event_path: &Path,
+) -> Option<(WorkspaceId, ProjectId, PagePath)> {
+ let rel = event_path.strip_prefix(wiki_root).ok()?;
+ let mut components = rel.components();
+
+ let ws_seg = components.next()?.as_os_str().to_str()?;
+ let ws_id = WorkspaceId::from_str(ws_seg).ok()?;
+
+ let proj_seg = components.next()?.as_os_str().to_str()?;
+ let proj_id = ProjectId::from_str(proj_seg).ok()?;
+
+ // Rejoin remaining segments as the page path.
+ let page_rel: std::path::PathBuf = components.collect();
+ let page_str = page_rel.to_string_lossy().replace('\\', "/");
+ if page_str.is_empty() {
+ return None;
+ }
+ let page_path = PagePath::new(page_str).ok()?;
+ Some((ws_id, proj_id, page_path))
+}
+
+fn extract_project_dir_ids(
+ wiki_root: &Path,
+ event_path: &Path,
+) -> Option<(WorkspaceId, ProjectId, std::path::PathBuf)> {
+ let rel = event_path.strip_prefix(wiki_root).ok()?;
+ let mut components = rel.components();
+
+ let ws_seg = components.next()?.as_os_str().to_str()?;
+ let ws_id = WorkspaceId::from_str(ws_seg).ok()?;
+
+ let proj_seg = components.next()?.as_os_str().to_str()?;
+ let proj_id = ProjectId::from_str(proj_seg).ok()?;
+
+ Some((ws_id, proj_id, wiki_root.join(ws_seg).join(proj_seg)))
+}
+
+fn walk_markdown(root: &Path) -> WikiResult<Vec<PagePath>> {
+ let mut out = Vec::new();
+ let mut stack = vec![root.to_path_buf()];
+ while let Some(dir) = stack.pop() {
+ let read = match std::fs::read_dir(&dir) {
+ Ok(r) => r,
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
+ Err(e) => return Err(WikiError::Io(e)),
+ };
+ for entry in read {
+ let entry = entry?;
+ let path = entry.path();
+ let ft = entry.file_type()?;
+ // Skip symlinks entirely. An attacker with write access to
+ // the wiki/ dir could otherwise plant a symlink to /etc/hosts,
+ // /home/user/.ssh/id_ed25519 etc. and have the watcher
+ // index the target's content. The sanitiser would still
+ // scrub credentials, but we'd be reading files we
+ // shouldn't be reading. (Audit critical #3.)
+ if ft.is_symlink() {
+ continue;
+ }
+ if ft.is_dir() {
+ stack.push(path);
+ } else if ft.is_file()
+ && is_markdown(&path)
+ && !is_tempfile(&path)
+ && let Some(pp) = page_path_relative_to(root, &path)
+ && !is_reserved_page_file(&path, &pp)
+ {
+ out.push(pp);
+ }
+ }
+ }
+ Ok(out)
+}
+
+fn is_markdown(path: &Path) -> bool {
+ path.extension().is_some_and(|e| e == "md")
+}
+
+fn is_tempfile(path: &Path) -> bool {
+ path.file_name()
+ .and_then(|n| n.to_str())
+ .is_some_and(|n| n.starts_with(".ai-memory-tmp."))
+}
+
+/// `_meta.md` is the per-scope manifest the engine writes (workspace/project
+/// name + repo_path) so the wiki tree is self-describing. It describes the
+/// scope, it is never a wiki page.
+fn is_manifest_filename(page_path: &PagePath) -> bool {
+ page_path
+ .as_str()
+ .rsplit('/')
+ .next()
+ .is_some_and(|name| name == "_meta.md")
+}
+
+/// `log.md` / `log-YYYY-MM.md` are the raw per-project event ledger the hooks
+/// append to (see `ai-memory-hooks::log::log_filename_for`): `## [ts] ...`
+/// entries, never YAML frontmatter.
+fn is_log_ledger_filename(page_path: &PagePath) -> bool {
+ let s = page_path.as_str();
+ s == "log.md"
+ // log-YYYY-MM.md — 7 chars between the dash and the dot.
+ || (s.starts_with("log-") && s.ends_with(".md") && s.len() == "log-YYYY-MM.md".len())
+}
+
+/// Cheap peek: does the file open with a `---` YAML frontmatter fence?
+/// Used to tell a real page apart from the raw event ledger.
+fn opens_with_frontmatter(abs: &Path) -> bool {
+ use std::io::{BufRead, BufReader};
+ let Ok(file) = std::fs::File::open(abs) else {
+ return false;
+ };
+ let mut line = String::new();
+ BufReader::new(file).read_line(&mut line).is_ok() && line.trim_end() == "---"
+}
+
+/// Returns `true` for markdown files that are NOT wiki pages and must be
+/// skipped by the indexer:
+/// - `_meta.md` (the self-describing scope manifest) and `bootstrap.md` —
+/// always; and
+/// - the raw event ledger (`log.md` / `log-YYYY-MM.md`) — skipping which
+/// avoids supersession loops, since every `append_event` write triggers a
+/// watcher event. **Unless** a file that merely collides with a ledger name
+/// actually carries YAML frontmatter: then it is a genuine page (e.g. a page
+/// that happens to live at `log.md`) and IS indexed, so it is never silently
+/// dropped on a reindex.
+fn is_reserved_page_file(abs: &Path, page_path: &PagePath) -> bool {
+ if is_manifest_filename(page_path) || page_path.as_str() == "bootstrap.md" {
+ return true;
+ }
+ is_log_ledger_filename(page_path) && !opens_with_frontmatter(abs)
+}
+
+fn page_path_relative_to(root: &Path, abs: &Path) -> Option<PagePath> {
+ let rel: &Path = abs.strip_prefix(root).ok()?;
+ let s = rel.to_string_lossy().replace('\\', "/");
+ PagePath::new(s).ok()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use ai_memory_store::Store;
+ use tempfile::TempDir;
+
+ async fn setup() -> (TempDir, Store, Wiki, WorkspaceId, ProjectId) {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "scratch", None)
+ .await
+ .unwrap();
+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();
+ (tmp, store, wiki, ws, proj)
+ }
+
+ /// `extract_project_ids` must parse a valid `<ws>/<proj>/<path>` triplet.
+ #[test]
+ fn extract_project_ids_valid_path() {
+ let wiki_root = Path::new("/data/wiki");
+ let ws_id = WorkspaceId::new();
+ let proj_id = ProjectId::new();
+ let event_path =
+ std::path::PathBuf::from(format!("/data/wiki/{}/{}/decisions/foo.md", ws_id, proj_id));
+ let result = extract_project_ids(wiki_root, &event_path);
+ assert!(
+ result.is_some(),
+ "must extract IDs from valid namespaced path"
+ );
+ let (ws, proj, pp) = result.unwrap();
+ assert_eq!(ws, ws_id);
+ assert_eq!(proj, proj_id);
+ assert_eq!(pp.as_str(), "decisions/foo.md");
+ }
+
+ /// `extract_project_ids` must return `None` when the first segment is not a UUID.
+ #[test]
+ fn extract_project_ids_garbage_first_segment() {
+ let wiki_root = Path::new("/data/wiki");
+ let event_path = Path::new("/data/wiki/not-a-uuid/some-proj/foo.md");
+ assert!(
+ extract_project_ids(wiki_root, event_path).is_none(),
+ "garbage first segment must return None"
+ );
+ }
+
+ /// `extract_project_ids` must return `None` for flat (non-namespaced) paths.
+ #[test]
+ fn extract_project_ids_flat_path_returns_none() {
+ let wiki_root = Path::new("/data/wiki");
+ let event_path = Path::new("/data/wiki/foo.md");
+ assert!(
+ extract_project_ids(wiki_root, event_path).is_none(),
+ "flat path with no namespace must return None"
+ );
+ }
+
+ /// `extract_project_ids` must return `None` when the second segment is not a valid UUID.
+ #[test]
+ fn extract_rejects_garbage_in_project_segment() {
+ let wiki_root = Path::new("/tmp/wiki");
+ let ws = WorkspaceId::new().to_string();
+ let event_path =
+ std::path::PathBuf::from(format!("/tmp/wiki/{ws}/not-a-uuid/decisions/foo.md"));
+ assert!(
+ extract_project_ids(wiki_root, &event_path).is_none(),
+ "garbage project segment must return None"
+ );
+ }
+
+ /// `extract_project_ids` must return `None` when there is no page path
+ /// after the two UUID segments (would produce an empty `PagePath`).
+ #[test]
+ fn extract_rejects_empty_page_path() {
+ let wiki_root = Path::new("/tmp/wiki");
+ let ws = WorkspaceId::new().to_string();
+ let proj = ProjectId::new().to_string();
+ // Just the project dir itself with no page path beneath.
+ let event_path = std::path::PathBuf::from(format!("/tmp/wiki/{ws}/{proj}"));
+ assert!(
+ extract_project_ids(wiki_root, &event_path).is_none(),
+ "missing page path must return None"
+ );
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn picks_up_externally_created_file() {
+ let (tmp, store, wiki, ws, proj) = setup().await;
+
+ // Create the project directory BEFORE starting the watcher so
+ // the inotify backend adds a watch for it immediately. If we
+ // created it after, there is a race between the new-dir event
+ // and the file-write event that can cause the watcher to miss
+ // the file on slower Linux inotify instances.
+ let proj_dir = tmp
+ .path()
+ .join("wiki")
+ .join(ws.to_string())
+ .join(proj.to_string());
+ std::fs::create_dir_all(&proj_dir).unwrap();
+
+ let handle = WatcherHandle::start(wiki.clone()).unwrap();
+ // FSEvents can report readiness before the recursive watch is fully
+ // settled. Give the backend one debounce window before creating the
+ // file so this test checks event delivery, not watcher-start races.
+ tokio::time::sleep(DEBOUNCE_WINDOW + Duration::from_millis(200)).await;
+
+ // Drop a file inside the per-project directory, bypassing the wiki write API
+ // (simulating an external editor).
+ let target = proj_dir.join("external.md");
+ std::fs::write(&target, "Hello from outside the wiki API.\n").unwrap();
+
+ // Poll for the row to land. Watcher debounces at 300ms; extra
+ // margin for slow CI environments.
+ let deadline = std::time::Instant::now() + Duration::from_secs(15);
+ let mut hits = Vec::new();
+ while std::time::Instant::now() < deadline {
+ hits = store
+ .reader
+ .search_pages("outside".into(), 5)
+ .await
+ .unwrap();
+ if !hits.is_empty() {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ }
+ assert!(!hits.is_empty(), "watcher did not pick up external write");
+ assert_eq!(hits[0].path.as_str(), "external.md");
+ handle.shutdown().await;
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn directory_event_reindexes_project_markdown() {
+ let (tmp, store, wiki, ws, proj) = setup().await;
+ let proj_dir = tmp
+ .path()
+ .join("wiki")
+ .join(ws.to_string())
+ .join(proj.to_string());
+ std::fs::create_dir_all(&proj_dir).unwrap();
+ std::fs::write(
+ proj_dir.join("external.md"),
+ "Directory event should reindex this page.\n",
+ )
+ .unwrap();
+
+ let event = notify_debouncer_full::DebouncedEvent::new(
+ notify::Event::new(EventKind::Modify(notify::event::ModifyKind::Any))
+ .add_path(proj_dir),
+ std::time::Instant::now(),
+ );
+ handle_event(&wiki, event).await;
+
+ let hits = store
+ .reader
+ .search_pages("reindex".into(), 5)
+ .await
+ .unwrap();
+ assert_eq!(hits.len(), 1);
+ assert_eq!(hits[0].path.as_str(), "external.md");
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn reconcile_picks_up_file_added_while_watcher_offline() {
+ let (tmp, store, wiki, ws, proj) = setup().await;
+
+ // Write a file BEFORE starting the watcher — directly in the project dir.
+ let proj_dir = tmp
+ .path()
+ .join("wiki")
+ .join(ws.to_string())
+ .join(proj.to_string());
+ std::fs::create_dir_all(&proj_dir).unwrap();
+ let target = proj_dir.join("preexisting.md");
+ std::fs::write(&target, "I existed first.\n").unwrap();
+
+ let handle = WatcherHandle::start(wiki.clone()).unwrap();
+ // Hit reconcile manually instead of waiting 30s.
+ reconcile(&wiki).await.unwrap();
+
+ let hits = store
+ .reader
+ .search_pages("existed".into(), 5)
+ .await
+ .unwrap();
+ assert_eq!(hits.len(), 1);
+ assert_eq!(hits[0].path.as_str(), "preexisting.md");
+ handle.shutdown().await;
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn ignores_own_atomic_tempfiles() {
+ // Quick unit test: tempfile prefix detection.
+ let p = Path::new("/some/dir/.ai-memory-tmp.abc.md");
+ assert!(is_tempfile(p));
+ let q = Path::new("/some/dir/normal.md");
+ assert!(!is_tempfile(q));
+ }
+
+ /// `walk_markdown` must not return `log.md` or `bootstrap.md`
+ /// (reserved per-project files that must not become wiki pages).
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn walk_markdown_skips_reserved_filenames() {
+ let (tmp, store, wiki, ws, proj) = setup().await;
+
+ let proj_dir = tmp
+ .path()
+ .join("wiki")
+ .join(ws.to_string())
+ .join(proj.to_string());
+ std::fs::create_dir_all(&proj_dir).unwrap();
+
+ // Write a legitimate page plus the reserved files (legacy
+ // `log.md`, the rotated `log-YYYY-MM.md`, and `bootstrap.md`).
+ // Single-word unique tokens so FTS5 (which parses hyphens as
+ // operators) can match them.
+ std::fs::write(proj_dir.join("real.md"), "real content\n").unwrap();
+ std::fs::write(proj_dir.join("log.md"), "sessionstarted logtoken unique\n").unwrap();
+ std::fs::write(
+ proj_dir.join("log-2026-05.md"),
+ "sessionstarted rotatedlogtoken unique\n",
+ )
+ .unwrap();
+ std::fs::write(
+ proj_dir.join("bootstrap.md"),
+ "bootstrapmanifest boottoken unique\n",
+ )
+ .unwrap();
+
+ let handle = WatcherHandle::start(wiki.clone()).unwrap();
+ // Trigger the reconciliation pass directly.
+ reconcile(&wiki).await.unwrap();
+
+ // Only `real.md` should land in the index.
+ let hits = store
+ .reader
+ .search_pages("real content".into(), 5)
+ .await
+ .unwrap();
+ assert_eq!(hits.len(), 1, "only the real page should be indexed");
+ assert_eq!(hits[0].path.as_str(), "real.md");
+
+ // Neither reserved file should be searchable.
+ let log_hits = store
+ .reader
+ .search_pages("logtoken".into(), 5)
+ .await
+ .unwrap();
+ assert!(log_hits.is_empty(), "log.md must not be indexed");
+
+ let rotated_hits = store
+ .reader
+ .search_pages("rotatedlogtoken".into(), 5)
+ .await
+ .unwrap();
+ assert!(
+ rotated_hits.is_empty(),
+ "log-YYYY-MM.md (rotated) must not be indexed"
+ );
+
+ let boot_hits = store
+ .reader
+ .search_pages("boottoken".into(), 5)
+ .await
+ .unwrap();
+ assert!(boot_hits.is_empty(), "bootstrap.md must not be indexed");
+
+ handle.shutdown().await;
+ drop(store);
+ }
+
+ /// A page that *collides* with a reserved ledger name (`log.md`) but
+ /// carries YAML frontmatter is a real page and MUST be indexed — not
+ /// silently dropped. Regression for a prod data anomaly (a page lived at
+ /// `log.md`) that a filename-only skip would lose on every reindex. The
+ /// `_meta.md` manifest, by contrast, must NEVER be indexed even though it
+ /// also has frontmatter.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn reindex_page_by_content_not_filename() {
+ let (tmp, store, wiki, ws, proj) = setup().await;
+ let proj_dir = tmp
+ .path()
+ .join("wiki")
+ .join(ws.to_string())
+ .join(proj.to_string());
+ std::fs::create_dir_all(&proj_dir).unwrap();
+
+ // A genuine page that happens to live at `log.md` (has frontmatter).
+ std::fs::write(
+ proj_dir.join("log.md"),
+ "---\ntitle: Collides With Ledger\n---\nframmaticpage uniquetoken\n",
+ )
+ .unwrap();
+ // The self-describing manifest — never a page, even with frontmatter.
+ std::fs::write(
+ proj_dir.join("_meta.md"),
+ "---\nworkspace: default\nproject: scratch\n---\nmanifesttoken here\n",
+ )
+ .unwrap();
+ // A raw ledger (no frontmatter) — still skipped.
+ std::fs::write(proj_dir.join("log-2026-06.md"), "## [t] evt | x\nrawledgertoken\n").unwrap();
+
+ let handle = WatcherHandle::start(wiki.clone()).unwrap();
+ reconcile(&wiki).await.unwrap();
+
+ let page_hits = store.reader.search_pages("uniquetoken".into(), 5).await.unwrap();
+ assert_eq!(page_hits.len(), 1, "frontmatter page named log.md must be indexed");
+ assert_eq!(page_hits[0].path.as_str(), "log.md");
+
+ let meta_hits = store.reader.search_pages("manifesttoken".into(), 5).await.unwrap();
+ assert!(meta_hits.is_empty(), "_meta.md manifest must not be indexed");
+
+ let ledger_hits = store.reader.search_pages("rawledgertoken".into(), 5).await.unwrap();
+ assert!(ledger_hits.is_empty(), "raw ledger (no frontmatter) must not be indexed");
+
+ handle.shutdown().await;
+ drop(store);
+ }
+
+ /// Defence: an attacker who can write to wiki/ shouldn't be able
+ /// to make the watcher index arbitrary files via symlinks.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn walk_markdown_skips_symlinks() {
+ let tmp = TempDir::new().unwrap();
+ let proj_root = tmp.path().join("proj");
+ std::fs::create_dir_all(&proj_root).unwrap();
+
+ // A real file (should be picked up).
+ std::fs::write(proj_root.join("real.md"), "real content\n").unwrap();
+
+ // A "secret" file outside the project root.
+ let secret = tmp.path().join("secret.md");
+ std::fs::write(&secret, "this is sensitive\n").unwrap();
+
+ // Plant a symlink inside proj/ pointing at the outside file.
+ #[cfg(unix)]
+ std::os::unix::fs::symlink(&secret, proj_root.join("symlinked.md")).unwrap();
+ #[cfg(windows)]
+ std::os::windows::fs::symlink_file(&secret, proj_root.join("symlinked.md")).unwrap();
+
+ let found = walk_markdown(&proj_root).unwrap();
+ let names: Vec<_> = found.iter().map(|p| p.as_str().to_string()).collect();
+ assert!(names.contains(&"real.md".to_string()), "real file present");
+ assert!(
+ !names.contains(&"symlinked.md".to_string()),
+ "symlink to outside file must be skipped; got: {names:?}"
+ );
+ }
+}
diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rs
index 34224095..4f6285a7 100644
--- a/crates/ai-memory-wiki/src/wiki.rs
+++ b/crates/ai-memory-wiki/src/wiki.rs
@@ -610,6 +610,59 @@ impl Wiki {
Ok(summary)
}
+ /// Write a `_meta.md` scope manifest under `dir` from `frontmatter`,
+ /// idempotently — unchanged content is left untouched so a startup
+ /// backfill never churns the wiki git history. Returns `true` if written.
+ fn write_scope_manifest(dir: &Path, frontmatter: serde_json::Value) -> WikiResult<bool> {
+ let content = emit(&Markdown {
+ frontmatter,
+ body: String::new(),
+ })?;
+ let path = dir.join("_meta.md");
+ if std::fs::read_to_string(&path).is_ok_and(|existing| existing == content) {
+ return Ok(false);
+ }
+ std::fs::create_dir_all(dir)?;
+ crate::atomic::write_atomic(&path, content.as_bytes())?;
+ Ok(true)
+ }
+
+ /// Ensure every workspace/project scope has its self-describing `_meta.md`
+ /// manifest on disk (`workspace`/`project` name + `repo_path`), so the wiki
+ /// tree alone is enough to rebuild the index via [`Self::reindex_all`] —
+ /// the "DB is rebuildable from files" guarantee. Idempotent; safe to run on
+ /// every startup. No-op without a store reader. Returns the count written.
+ ///
+ /// # Errors
+ /// Returns [`WikiError`] for store or filesystem errors.
+ pub async fn backfill_scope_manifests(&self) -> WikiResult<usize> {
+ let Some(reader) = &self.store_reader else {
+ return Ok(0);
+ };
+ let scopes = reader.list_all_scopes().await?;
+ let mut written = 0;
+ let mut seen_ws = std::collections::HashSet::new();
+ for s in scopes {
+ let ws_dir = self.root().join(s.workspace_id.to_string());
+ if seen_ws.insert(s.workspace_id)
+ && Self::write_scope_manifest(
+ &ws_dir,
+ serde_json::json!({ "workspace": s.workspace_name }),
+ )?
+ {
+ written += 1;
+ }
+ let mut fm = serde_json::json!({ "project": s.project_name });
+ if let Some(rp) = s.repo_path {
+ fm["repo_path"] = serde_json::Value::String(rp);
+ }
+ if Self::write_scope_manifest(&ws_dir.join(s.project_id.to_string()), fm)? {
+ written += 1;
+ }
+ }
+ Ok(written)
+ }
+
/// Atomically apply a batch of page writes. Either all pages land
/// (one SQL transaction) and their files are renamed into place,
/// or no DB row changes and tempfiles are dropped.
@@ -1751,4 +1804,90 @@ mod tests {
);
assert_eq!(md.frontmatter["title"], "Anon");
}
+
+ fn copy_tree(src: &Path, dst: &Path) {
+ std::fs::create_dir_all(dst).unwrap();
+ for entry in std::fs::read_dir(src).unwrap() {
+ let entry = entry.unwrap();
+ let to = dst.join(entry.file_name());
+ if entry.file_type().unwrap().is_dir() {
+ copy_tree(&entry.path(), &to);
+ } else {
+ std::fs::copy(entry.path(), &to).unwrap();
+ }
+ }
+ }
+
+ /// End-to-end "DB is rebuildable from files": `backfill_scope_manifests`
+ /// makes the wiki self-describing, then `reindex_all` on a FRESH store
+ /// (no DB carried over) recreates the named scopes + all pages from the
+ /// wiki tree alone — including a page that lives at the reserved name
+ /// `log.md` (kept because it has frontmatter).
+ #[tokio::test]
+ async fn backfill_then_reindex_rebuilds_from_wiki_alone() {
+ // Source store: a named scope with two pages (one at `log.md`).
+ let src = TempDir::new().unwrap();
+ let s1 = Store::open(src.path()).unwrap();
+ let ws = s1.writer.get_or_create_workspace("acme").await.unwrap();
+ let proj = s1
+ .writer
+ .get_or_create_project(ws, "webapp", Some("/repo/webapp".into()))
+ .await
+ .unwrap();
+ let w1 = Wiki::new(src.path(), s1.writer.clone())
+ .unwrap()
+ .with_store_reader(s1.reader.clone());
+ w1.apply_batch(vec![
+ req(
+ ws,
+ proj,
+ "notes/a.md",
+ "alpha uniquetoken",
+ serde_json::json!({}),
+ ),
+ req(
+ ws,
+ proj,
+ "log.md",
+ "a page that lives at the reserved log name",
+ serde_json::json!({ "title": "Log Page" }),
+ ),
+ ])
+ .await
+ .unwrap();
+
+ // Make the wiki self-describing.
+ let written = w1.backfill_scope_manifests().await.unwrap();
+ assert!(written >= 2, "ws + proj manifests written, got {written}");
+ let ws_dir = src.path().join("wiki").join(ws.to_string());
+ assert!(ws_dir.join("_meta.md").is_file());
+ assert!(ws_dir.join(proj.to_string()).join("_meta.md").is_file());
+ drop(s1);
+
+ // Fresh store; copy ONLY the wiki tree (no db/); rebuild from it.
+ let dst = TempDir::new().unwrap();
+ let s2 = Store::open(dst.path()).unwrap();
+ copy_tree(&src.path().join("wiki"), &dst.path().join("wiki"));
+ let w2 = Wiki::new(dst.path(), s2.writer.clone()).unwrap();
+ let summary = w2.reindex_all().await.unwrap();
+
+ assert_eq!(summary.projects, 1);
+ assert_eq!(summary.pages, 2, "both pages incl. log.md reconstructed");
+ assert_eq!(
+ s2.reader.workspace_name_by_id(ws).await.unwrap().as_deref(),
+ Some("acme"),
+ "workspace name recovered from _meta.md"
+ );
+ let hits = s2
+ .reader
+ .search_pages("uniquetoken".into(), 5)
+ .await
+ .unwrap();
+ assert_eq!(
+ hits.len(),
+ 1,
+ "reindexed page is searchable in the fresh store"
+ );
+ drop(s2);
+ }
}