Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(wiki): self-describing _meta.md manifests (backfill on startup)
Djalma Júnior1a2edf2533df
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rsindex b08014dd..ae7c3de0 100644--- a/crates/ai-memory-cli/src/commands/serve.rs+++ b/crates/ai-memory-cli/src/commands/serve.rs@@ -96,7 +96,12 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {// operators discover misconfiguration immediately.let sanitizer = Sanitizer::new(&config.sanitize).context("compiling sanitizer.extra_patterns from config")?;- let wiki = Wiki::new(&config.data_dir, store.writer.clone())?.with_sanitizer(sanitizer.clone());+ let wiki = Wiki::new(&config.data_dir, store.writer.clone())?+ .with_sanitizer(sanitizer.clone())+ // Reader attached unconditionally: admission name-resolution uses it+ // when a chain is configured, and the startup scope-manifest backfill+ // (below) always needs it to enumerate scopes.+ .with_store_reader(store.reader.clone());// Attach the admission webhook chain (operator-configured via// `[[admission_webhooks]]` in config.toml or `AI_MEMORY_ADMISSION_WEBHOOKS__N__*`// env vars). Empty config = no chain attached, zero overhead. The store@@ -117,6 +122,15 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {let provider_health = ProviderHealth::default();let (wiki, embedder) = configure_embedder(config, &store, wiki, &provider_health).await?;+ // Make the wiki tree self-describing: write each scope's `_meta.md`+ // (workspace/project name + repo_path) if missing, so the markdown alone+ // can rebuild the index via `ai-memory reindex`. Idempotent; non-fatal.+ match wiki.backfill_scope_manifests().await {+ Ok(0) => {}+ Ok(n) => tracing::info!(count = n, "wrote _meta.md scope manifests"),+ Err(e) => tracing::warn!(error = %e, "scope-manifest backfill failed (non-fatal)"),+ }+// Keep the guard alive for the lifetime of `serve`.let _watcher = start_watcher(&args, &wiki)?;diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rsindex c93bf828..22dbc733 100644--- a/crates/ai-memory-store/src/lib.rs+++ b/crates/ai-memory-store/src/lib.rs@@ -30,7 +30,7 @@ pub use reader::{ActivityWindow, BriefingPage, BriefingSnapshot, DecayCandidate, DerivedIndexStatus,EmbeddingTripleCount, HealthDetail, HealthPage, ObservationHit, PageAuthor, PageHit,PageHitWithMeta, PageLinks, PageMeta, PageSummary, ProjectSummary, ReaderPool, RelatedPage,- StatusCounts, StoredEmbedding, StoredPageBody, WorkspaceSummary, f32_vec_to_bytes,+ ScopeRow, StatusCounts, StoredEmbedding, StoredPageBody, WorkspaceSummary, f32_vec_to_bytes,};pub use users::{TOKEN_HASH_LEN, TOKEN_RAW_LEN, TokenPepper, generate_token, hash_token};pub use writer::WriterHandle;diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rsindex 1447624d..026a049f 100644--- a/crates/ai-memory-store/src/reader.rs+++ b/crates/ai-memory-store/src/reader.rs@@ -230,6 +230,23 @@ pub struct ProjectSummary {pub last_updated: Option<String>,}+/// One `(workspace, project)` scope with the ids + repo_path needed to write+/// its self-describing `_meta.md` manifest. Returned by+/// [`ReaderPool::list_all_scopes`]; consumed by `Wiki::backfill_scope_manifests`.+#[derive(Debug, Clone)]+pub struct ScopeRow {+ /// Workspace id — matches the level-1 wiki directory name.+ pub workspace_id: WorkspaceId,+ /// Human-readable workspace name.+ pub workspace_name: String,+ /// Project id — matches the level-2 wiki directory name.+ pub project_id: ProjectId,+ /// Human-readable project name.+ pub project_name: String,+ /// Filesystem path the project's cwd-based routing resolves to, if any.+ pub repo_path: Option<String>,+}+/// One row per workspace with aggregate stats./// Returned by [`ReaderPool::list_workspaces_with_stats`].#[derive(Debug, Clone, Serialize)]@@ -2465,6 +2482,48 @@ impl ReaderPool {.await}+ /// Return every `(workspace, project)` scope with its ids, names and+ /// `repo_path` — the data needed to write each scope's self-describing+ /// `_meta.md` manifest. Unlike [`list_projects_with_stats`], this carries+ /// the surrogate ids (not just names), so a rebuild can key the wiki tree.+ ///+ /// # Errors+ /// Propagates any SQL or pool error.+ pub async fn list_all_scopes(&self) -> StoreResult<Vec<ScopeRow>> {+ // (ws_id, ws_name, proj_id, proj_name, repo_path) as raw SQL columns.+ type RawScope = (Vec<u8>, String, Vec<u8>, String, Option<String>);+ let raw: Vec<RawScope> = self+ .with_conn(|conn| {+ let mut stmt = conn.prepare(+ "SELECT w.id, w.name, p.id, p.name, p.repo_path \+ FROM projects p JOIN workspaces w ON w.id = p.workspace_id",+ )?;+ let rows = stmt.query_map([], |row| {+ Ok((+ row.get(0)?,+ row.get(1)?,+ row.get(2)?,+ row.get(3)?,+ row.get(4)?,+ ))+ })?;+ let out: rusqlite::Result<Vec<_>> = rows.collect();+ Ok(out?)+ })+ .await?;+ raw.into_iter()+ .map(|(wi, wn, pi, pn, rp)| {+ Ok(ScopeRow {+ workspace_id: WorkspaceId::from_slice(&wi)?,+ workspace_name: wn,+ project_id: ProjectId::from_slice(&pi)?,+ project_name: pn,+ repo_path: rp,+ })+ })+ .collect()+ }+/// Return one row per workspace with project/page-count and/// last-updated aggregates. Used by custom frontends that need a/// workspace chooser before narrowing into projects.diff --git a/crates/ai-memory-wiki/src/watcher.rs.bak b/crates/ai-memory-wiki/src/watcher.rs.baknew file mode 100644index 00000000..cc3cb958--- /dev/null+++ b/crates/ai-memory-wiki/src/watcher.rs.bak@@ -0,0 +1,837 @@+//! Filesystem watcher with debouncing and a periodic reconciliation pass.+//!+//! Two parts work together:+//!+//! 1. **Debounced events** via [`notify_debouncer_full`]. When a markdown+//! file under the wiki root is created or modified, we read it from+//! disk, parse the frontmatter, and `reindex_page` against the store.+//! Own-writes are absorbed by the store's sha256 short-circuit, so+//! the loop terminates after one no-op reindex.+//! 2. **Reconciliation tick** every 30s walks the entire wiki tree and+//! reindexes every markdown file. Catches any events the OS dropped+//! (basic-memory #580 — file watchers go stale under FSEvents buffer+//! overflow, hidden-dir globs, etc.). Hidden-directory paths are+//! explicitly NOT skipped (#798 lesson).+//!+//! The watcher never *writes* to disk — that loop would be unbounded.+//! External writes drive store updates; internal writes drive disk ++//! store updates via [`Wiki::write_page`].++use std::path::Path;+use std::str::FromStr;+use std::time::Duration;++use ai_memory_core::{PagePath, ProjectId, WorkspaceId};+use notify::{EventKind, RecursiveMode};+use notify_debouncer_full::{DebounceEventResult, Debouncer, RecommendedCache, new_debouncer_opt};+use tokio::sync::mpsc;+use tracing::{debug, info, warn};++use crate::error::{WikiError, WikiResult};+use crate::wiki::Wiki;++/// Reconciliation tick interval.+pub const RECONCILE_INTERVAL: Duration = Duration::from_secs(30);++/// Debounce window for filesystem events.+pub const DEBOUNCE_WINDOW: Duration = Duration::from_millis(300);++#[cfg(all(test, target_os = "macos"))]+type PlatformWatcher = notify::PollWatcher;+#[cfg(not(all(test, target_os = "macos")))]+type PlatformWatcher = notify::RecommendedWatcher;++/// Handle representing an active watcher; drop to stop.+pub struct WatcherHandle {+ _debouncer: Debouncer<PlatformWatcher, RecommendedCache>,+ shutdown: Option<tokio::sync::oneshot::Sender<()>>,+ task: Option<tokio::task::JoinHandle<()>>,+}++impl WatcherHandle {+ /// Start watching `wiki.root()` recursively. Spawns one tokio task+ /// that consumes debounced events and runs the reconciliation timer.+ ///+ /// Events are attributed to their `(workspace_id, project_id)` by+ /// parsing the first two path segments as UUIDs. Events outside the+ /// `<ws_uuid>/<proj_uuid>/...` layout are silently ignored.+ ///+ /// # Errors+ /// Propagates any notify error encountered when installing the OS+ /// watcher.+ pub fn start(wiki: Wiki) -> WikiResult<Self> {+ let (event_tx, event_rx) = mpsc::unbounded_channel();++ let mut debouncer = new_debouncer_opt::<_, PlatformWatcher, RecommendedCache>(+ DEBOUNCE_WINDOW,+ None,+ move |result: DebounceEventResult| match result {+ Ok(events) => {+ for event in events {+ let _ = event_tx.send(event);+ }+ }+ Err(errors) => {+ for e in errors {+ warn!(error = %e, "notify error");+ }+ }+ },+ RecommendedCache::new(),+ watcher_config(),+ )+ .map_err(|e| WikiError::Io(std::io::Error::other(e.to_string())))?;++ debouncer+ .watch(wiki.root(), RecursiveMode::Recursive)+ .map_err(|e| WikiError::Io(std::io::Error::other(e.to_string())))?;++ let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>();+ let task = tokio::spawn(run_loop(wiki, event_rx, shutdown_rx));++ Ok(Self {+ _debouncer: debouncer,+ shutdown: Some(shutdown_tx),+ task: Some(task),+ })+ }++ /// Stop the watcher and wait for the event loop to drain.+ pub async fn shutdown(mut self) {+ if let Some(tx) = self.shutdown.take() {+ let _ = tx.send(());+ }+ if let Some(handle) = self.task.take() {+ let _ = handle.await;+ }+ }+}++impl Drop for WatcherHandle {+ fn drop(&mut self) {+ if let Some(tx) = self.shutdown.take() {+ let _ = tx.send(());+ }+ }+}++fn watcher_config() -> notify::Config {+ let config = notify::Config::default();+ #[cfg(all(test, target_os = "macos"))]+ {+ // GitHub macOS runners have flaky FSEvents delivery for tempdir unit+ // tests. Use the polling backend there so the test covers our watcher+ // loop without depending on runner-specific FSEvents behavior.+ config.with_poll_interval(DEBOUNCE_WINDOW)+ }+ #[cfg(not(all(test, target_os = "macos")))]+ {+ config+ }+}++async fn run_loop(+ wiki: Wiki,+ mut rx: mpsc::UnboundedReceiver<notify_debouncer_full::DebouncedEvent>,+ mut shutdown: tokio::sync::oneshot::Receiver<()>,+) {+ let mut tick = tokio::time::interval(RECONCILE_INTERVAL);+ tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);+ // First tick fires immediately; consume it so we don't reconcile at boot.+ tick.tick().await;++ // Track consecutive failures of the reconciliation pass so we can+ // surface a clear "watcher is degraded" event after a streak, in+ // addition to the per-failure error log. Without this, a broken+ // disk → store bridge can stay broken indefinitely with only a+ // line per 30s in the warn stream — easy to miss in busy logs.+ let mut consecutive_failures: u32 = 0;+ const DEGRADED_AFTER: u32 = 5;++ loop {+ tokio::select! {+ biased;+ _ = &mut shutdown => {+ debug!("watcher shutting down");+ return;+ }+ Some(event) = rx.recv() => {+ handle_event(&wiki, event).await;+ }+ _ = tick.tick() => {+ match reconcile(&wiki).await {+ Ok(()) => {+ if consecutive_failures > 0 {+ tracing::info!(+ prior_failures = consecutive_failures,+ "reconciliation recovered after consecutive failures",+ );+ consecutive_failures = 0;+ }+ }+ Err(e) => {+ consecutive_failures += 1;+ tracing::error!(+ error = %e,+ consecutive_failures,+ "reconciliation failed",+ );+ if consecutive_failures == DEGRADED_AFTER {+ tracing::error!(+ consecutive_failures,+ event = "watcher_degraded",+ "wiki↔store reconciliation has failed {DEGRADED_AFTER} \+ times in a row; the disk and SQLite index may now be \+ out of sync. Investigate disk permissions, DB lock \+ contention, or filesystem health. The watcher will \+ keep retrying every {RECONCILE_INTERVAL:?}.",+ );+ }+ }+ }+ }+ else => return,+ }+ }+}++async fn handle_event(wiki: &Wiki, event: notify_debouncer_full::DebouncedEvent) {+ if !matches!(+ event.kind,+ EventKind::Create(_) | EventKind::Modify(_) | EventKind::Other+ ) {+ return;+ }+ for raw_path in &event.paths {+ if raw_path.is_dir() {+ let Some((ws, proj, proj_root)) = extract_project_dir_ids(wiki.root(), raw_path) else {+ continue;+ };+ reindex_project_dir(wiki, ws, proj, proj_root).await;+ continue;+ }+ if !is_markdown(raw_path) {+ continue;+ }+ if is_tempfile(raw_path) {+ continue;+ }+ let Some((ws, proj, page_path)) = extract_project_ids(wiki.root(), raw_path) else {+ continue;+ };+ if is_reserved_page_file(raw_path, &page_path) {+ continue;+ }+ if !raw_path.is_file() {+ // Likely a transient state (mv, atomic rename in flight).+ continue;+ }+ match wiki.reindex_page(ws, proj, page_path.clone()).await {+ Ok(_) => debug!(path = %page_path, "reindexed via watcher"),+ Err(e) => warn!(path = %page_path, error = %e, "watcher reindex failed"),+ }+ }+}++async fn reindex_project_dir(+ wiki: &Wiki,+ ws: WorkspaceId,+ proj: ProjectId,+ proj_root: std::path::PathBuf,+) {+ let pages = match tokio::task::spawn_blocking(move || walk_markdown(&proj_root)).await {+ Ok(Ok(pages)) => pages,+ Ok(Err(e)) => {+ warn!(error = %e, "watcher directory walk failed");+ return;+ }+ Err(e) => {+ warn!(error = %e, "watcher directory walk task failed");+ return;+ }+ };++ for path in pages {+ match wiki.reindex_page(ws, proj, path.clone()).await {+ Ok(_) => debug!(path = %path, "reindexed via watcher directory event"),+ Err(e) => warn!(path = %path, error = %e, "watcher directory reindex failed"),+ }+ }+}++async fn reconcile(wiki: &Wiki) -> WikiResult<()> {+ let root = wiki.root().to_path_buf();+ // Walk all per-project subdirectories: <ws_uuid>/<proj_uuid>/+ let project_dirs = tokio::task::spawn_blocking(move || walk_project_dirs(&root))+ .await+ .map_err(|e| WikiError::Io(std::io::Error::other(e.to_string())))??;++ let mut total = 0_usize;+ for (ws, proj, proj_root) in project_dirs {+ let pages = tokio::task::spawn_blocking(move || walk_markdown(&proj_root))+ .await+ .map_err(|e| WikiError::Io(std::io::Error::other(e.to_string())))??;+ total += pages.len();+ for path in pages {+ if let Err(e) = wiki.reindex_page(ws, proj, path.clone()).await {+ warn!(path = %path, error = %e, "reconcile reindex failed");+ }+ }+ }+ info!(count = total, "reconciliation pass complete");+ Ok(())+}++/// Walk `<wiki_root>` and return all `(WorkspaceId, ProjectId, proj_root)` tuples+/// whose first two path segments parse as valid UUIDs.+fn walk_project_dirs(+ wiki_root: &Path,+) -> WikiResult<Vec<(WorkspaceId, ProjectId, std::path::PathBuf)>> {+ let mut out = Vec::new();+ let ws_read = match std::fs::read_dir(wiki_root) {+ Ok(r) => r,+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(out),+ Err(e) => return Err(WikiError::Io(e)),+ };+ for ws_entry in ws_read {+ let ws_entry = ws_entry?;+ if !ws_entry.file_type()?.is_dir() {+ continue;+ }+ let ws_name = ws_entry.file_name();+ let Some(ws_str) = ws_name.to_str() else {+ continue;+ };+ let Ok(ws_id) = WorkspaceId::from_str(ws_str) else {+ continue;+ };+ let proj_read = match std::fs::read_dir(ws_entry.path()) {+ Ok(r) => r,+ Err(_) => continue,+ };+ for proj_entry in proj_read {+ let proj_entry = proj_entry?;+ if !proj_entry.file_type()?.is_dir() {+ continue;+ }+ let proj_name = proj_entry.file_name();+ let Some(proj_str) = proj_name.to_str() else {+ continue;+ };+ let Ok(proj_id) = ProjectId::from_str(proj_str) else {+ continue;+ };+ out.push((ws_id, proj_id, proj_entry.path()));+ }+ }+ Ok(out)+}++/// Parse `(WorkspaceId, ProjectId, PagePath)` from a filesystem event path.+///+/// Expects the path to have the structure:+/// `<wiki_root>/<ws_uuid>/<proj_uuid>/<page-path...>`+///+/// Returns `None` when:+/// - The path does not start with `wiki_root`.+/// - The first segment is not a valid UUID (`WorkspaceId`).+/// - The second segment is not a valid UUID (`ProjectId`).+/// - There are no remaining segments (the page path would be empty).+pub(crate) fn extract_project_ids(+ wiki_root: &Path,+ event_path: &Path,+) -> Option<(WorkspaceId, ProjectId, PagePath)> {+ let rel = event_path.strip_prefix(wiki_root).ok()?;+ let mut components = rel.components();++ let ws_seg = components.next()?.as_os_str().to_str()?;+ let ws_id = WorkspaceId::from_str(ws_seg).ok()?;++ let proj_seg = components.next()?.as_os_str().to_str()?;+ let proj_id = ProjectId::from_str(proj_seg).ok()?;++ // Rejoin remaining segments as the page path.+ let page_rel: std::path::PathBuf = components.collect();+ let page_str = page_rel.to_string_lossy().replace('\\', "/");+ if page_str.is_empty() {+ return None;+ }+ let page_path = PagePath::new(page_str).ok()?;+ Some((ws_id, proj_id, page_path))+}++fn extract_project_dir_ids(+ wiki_root: &Path,+ event_path: &Path,+) -> Option<(WorkspaceId, ProjectId, std::path::PathBuf)> {+ let rel = event_path.strip_prefix(wiki_root).ok()?;+ let mut components = rel.components();++ let ws_seg = components.next()?.as_os_str().to_str()?;+ let ws_id = WorkspaceId::from_str(ws_seg).ok()?;++ let proj_seg = components.next()?.as_os_str().to_str()?;+ let proj_id = ProjectId::from_str(proj_seg).ok()?;++ Some((ws_id, proj_id, wiki_root.join(ws_seg).join(proj_seg)))+}++fn walk_markdown(root: &Path) -> WikiResult<Vec<PagePath>> {+ let mut out = Vec::new();+ let mut stack = vec![root.to_path_buf()];+ while let Some(dir) = stack.pop() {+ let read = match std::fs::read_dir(&dir) {+ Ok(r) => r,+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,+ Err(e) => return Err(WikiError::Io(e)),+ };+ for entry in read {+ let entry = entry?;+ let path = entry.path();+ let ft = entry.file_type()?;+ // Skip symlinks entirely. An attacker with write access to+ // the wiki/ dir could otherwise plant a symlink to /etc/hosts,+ // /home/user/.ssh/id_ed25519 etc. and have the watcher+ // index the target's content. The sanitiser would still+ // scrub credentials, but we'd be reading files we+ // shouldn't be reading. (Audit critical #3.)+ if ft.is_symlink() {+ continue;+ }+ if ft.is_dir() {+ stack.push(path);+ } else if ft.is_file()+ && is_markdown(&path)+ && !is_tempfile(&path)+ && let Some(pp) = page_path_relative_to(root, &path)+ && !is_reserved_page_file(&path, &pp)+ {+ out.push(pp);+ }+ }+ }+ Ok(out)+}++fn is_markdown(path: &Path) -> bool {+ path.extension().is_some_and(|e| e == "md")+}++fn is_tempfile(path: &Path) -> bool {+ path.file_name()+ .and_then(|n| n.to_str())+ .is_some_and(|n| n.starts_with(".ai-memory-tmp."))+}++/// `_meta.md` is the per-scope manifest the engine writes (workspace/project+/// name + repo_path) so the wiki tree is self-describing. It describes the+/// scope, it is never a wiki page.+fn is_manifest_filename(page_path: &PagePath) -> bool {+ page_path+ .as_str()+ .rsplit('/')+ .next()+ .is_some_and(|name| name == "_meta.md")+}++/// `log.md` / `log-YYYY-MM.md` are the raw per-project event ledger the hooks+/// append to (see `ai-memory-hooks::log::log_filename_for`): `## [ts] ...`+/// entries, never YAML frontmatter.+fn is_log_ledger_filename(page_path: &PagePath) -> bool {+ let s = page_path.as_str();+ s == "log.md"+ // log-YYYY-MM.md — 7 chars between the dash and the dot.+ || (s.starts_with("log-") && s.ends_with(".md") && s.len() == "log-YYYY-MM.md".len())+}++/// Cheap peek: does the file open with a `---` YAML frontmatter fence?+/// Used to tell a real page apart from the raw event ledger.+fn opens_with_frontmatter(abs: &Path) -> bool {+ use std::io::{BufRead, BufReader};+ let Ok(file) = std::fs::File::open(abs) else {+ return false;+ };+ let mut line = String::new();+ BufReader::new(file).read_line(&mut line).is_ok() && line.trim_end() == "---"+}++/// Returns `true` for markdown files that are NOT wiki pages and must be+/// skipped by the indexer:+/// - `_meta.md` (the self-describing scope manifest) and `bootstrap.md` —+/// always; and+/// - the raw event ledger (`log.md` / `log-YYYY-MM.md`) — skipping which+/// avoids supersession loops, since every `append_event` write triggers a+/// watcher event. **Unless** a file that merely collides with a ledger name+/// actually carries YAML frontmatter: then it is a genuine page (e.g. a page+/// that happens to live at `log.md`) and IS indexed, so it is never silently+/// dropped on a reindex.+fn is_reserved_page_file(abs: &Path, page_path: &PagePath) -> bool {+ if is_manifest_filename(page_path) || page_path.as_str() == "bootstrap.md" {+ return true;+ }+ is_log_ledger_filename(page_path) && !opens_with_frontmatter(abs)+}++fn page_path_relative_to(root: &Path, abs: &Path) -> Option<PagePath> {+ let rel: &Path = abs.strip_prefix(root).ok()?;+ let s = rel.to_string_lossy().replace('\\', "/");+ PagePath::new(s).ok()+}++#[cfg(test)]+mod tests {+ use super::*;+ use ai_memory_store::Store;+ use tempfile::TempDir;++ async fn setup() -> (TempDir, Store, Wiki, WorkspaceId, ProjectId) {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "scratch", None)+ .await+ .unwrap();+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();+ (tmp, store, wiki, ws, proj)+ }++ /// `extract_project_ids` must parse a valid `<ws>/<proj>/<path>` triplet.+ #[test]+ fn extract_project_ids_valid_path() {+ let wiki_root = Path::new("/data/wiki");+ let ws_id = WorkspaceId::new();+ let proj_id = ProjectId::new();+ let event_path =+ std::path::PathBuf::from(format!("/data/wiki/{}/{}/decisions/foo.md", ws_id, proj_id));+ let result = extract_project_ids(wiki_root, &event_path);+ assert!(+ result.is_some(),+ "must extract IDs from valid namespaced path"+ );+ let (ws, proj, pp) = result.unwrap();+ assert_eq!(ws, ws_id);+ assert_eq!(proj, proj_id);+ assert_eq!(pp.as_str(), "decisions/foo.md");+ }++ /// `extract_project_ids` must return `None` when the first segment is not a UUID.+ #[test]+ fn extract_project_ids_garbage_first_segment() {+ let wiki_root = Path::new("/data/wiki");+ let event_path = Path::new("/data/wiki/not-a-uuid/some-proj/foo.md");+ assert!(+ extract_project_ids(wiki_root, event_path).is_none(),+ "garbage first segment must return None"+ );+ }++ /// `extract_project_ids` must return `None` for flat (non-namespaced) paths.+ #[test]+ fn extract_project_ids_flat_path_returns_none() {+ let wiki_root = Path::new("/data/wiki");+ let event_path = Path::new("/data/wiki/foo.md");+ assert!(+ extract_project_ids(wiki_root, event_path).is_none(),+ "flat path with no namespace must return None"+ );+ }++ /// `extract_project_ids` must return `None` when the second segment is not a valid UUID.+ #[test]+ fn extract_rejects_garbage_in_project_segment() {+ let wiki_root = Path::new("/tmp/wiki");+ let ws = WorkspaceId::new().to_string();+ let event_path =+ std::path::PathBuf::from(format!("/tmp/wiki/{ws}/not-a-uuid/decisions/foo.md"));+ assert!(+ extract_project_ids(wiki_root, &event_path).is_none(),+ "garbage project segment must return None"+ );+ }++ /// `extract_project_ids` must return `None` when there is no page path+ /// after the two UUID segments (would produce an empty `PagePath`).+ #[test]+ fn extract_rejects_empty_page_path() {+ let wiki_root = Path::new("/tmp/wiki");+ let ws = WorkspaceId::new().to_string();+ let proj = ProjectId::new().to_string();+ // Just the project dir itself with no page path beneath.+ let event_path = std::path::PathBuf::from(format!("/tmp/wiki/{ws}/{proj}"));+ assert!(+ extract_project_ids(wiki_root, &event_path).is_none(),+ "missing page path must return None"+ );+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn picks_up_externally_created_file() {+ let (tmp, store, wiki, ws, proj) = setup().await;++ // Create the project directory BEFORE starting the watcher so+ // the inotify backend adds a watch for it immediately. If we+ // created it after, there is a race between the new-dir event+ // and the file-write event that can cause the watcher to miss+ // the file on slower Linux inotify instances.+ let proj_dir = tmp+ .path()+ .join("wiki")+ .join(ws.to_string())+ .join(proj.to_string());+ std::fs::create_dir_all(&proj_dir).unwrap();++ let handle = WatcherHandle::start(wiki.clone()).unwrap();+ // FSEvents can report readiness before the recursive watch is fully+ // settled. Give the backend one debounce window before creating the+ // file so this test checks event delivery, not watcher-start races.+ tokio::time::sleep(DEBOUNCE_WINDOW + Duration::from_millis(200)).await;++ // Drop a file inside the per-project directory, bypassing the wiki write API+ // (simulating an external editor).+ let target = proj_dir.join("external.md");+ std::fs::write(&target, "Hello from outside the wiki API.\n").unwrap();++ // Poll for the row to land. Watcher debounces at 300ms; extra+ // margin for slow CI environments.+ let deadline = std::time::Instant::now() + Duration::from_secs(15);+ let mut hits = Vec::new();+ while std::time::Instant::now() < deadline {+ hits = store+ .reader+ .search_pages("outside".into(), 5)+ .await+ .unwrap();+ if !hits.is_empty() {+ break;+ }+ tokio::time::sleep(Duration::from_millis(100)).await;+ }+ assert!(!hits.is_empty(), "watcher did not pick up external write");+ assert_eq!(hits[0].path.as_str(), "external.md");+ handle.shutdown().await;+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn directory_event_reindexes_project_markdown() {+ let (tmp, store, wiki, ws, proj) = setup().await;+ let proj_dir = tmp+ .path()+ .join("wiki")+ .join(ws.to_string())+ .join(proj.to_string());+ std::fs::create_dir_all(&proj_dir).unwrap();+ std::fs::write(+ proj_dir.join("external.md"),+ "Directory event should reindex this page.\n",+ )+ .unwrap();++ let event = notify_debouncer_full::DebouncedEvent::new(+ notify::Event::new(EventKind::Modify(notify::event::ModifyKind::Any))+ .add_path(proj_dir),+ std::time::Instant::now(),+ );+ handle_event(&wiki, event).await;++ let hits = store+ .reader+ .search_pages("reindex".into(), 5)+ .await+ .unwrap();+ assert_eq!(hits.len(), 1);+ assert_eq!(hits[0].path.as_str(), "external.md");+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn reconcile_picks_up_file_added_while_watcher_offline() {+ let (tmp, store, wiki, ws, proj) = setup().await;++ // Write a file BEFORE starting the watcher — directly in the project dir.+ let proj_dir = tmp+ .path()+ .join("wiki")+ .join(ws.to_string())+ .join(proj.to_string());+ std::fs::create_dir_all(&proj_dir).unwrap();+ let target = proj_dir.join("preexisting.md");+ std::fs::write(&target, "I existed first.\n").unwrap();++ let handle = WatcherHandle::start(wiki.clone()).unwrap();+ // Hit reconcile manually instead of waiting 30s.+ reconcile(&wiki).await.unwrap();++ let hits = store+ .reader+ .search_pages("existed".into(), 5)+ .await+ .unwrap();+ assert_eq!(hits.len(), 1);+ assert_eq!(hits[0].path.as_str(), "preexisting.md");+ handle.shutdown().await;+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn ignores_own_atomic_tempfiles() {+ // Quick unit test: tempfile prefix detection.+ let p = Path::new("/some/dir/.ai-memory-tmp.abc.md");+ assert!(is_tempfile(p));+ let q = Path::new("/some/dir/normal.md");+ assert!(!is_tempfile(q));+ }++ /// `walk_markdown` must not return `log.md` or `bootstrap.md`+ /// (reserved per-project files that must not become wiki pages).+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn walk_markdown_skips_reserved_filenames() {+ let (tmp, store, wiki, ws, proj) = setup().await;++ let proj_dir = tmp+ .path()+ .join("wiki")+ .join(ws.to_string())+ .join(proj.to_string());+ std::fs::create_dir_all(&proj_dir).unwrap();++ // Write a legitimate page plus the reserved files (legacy+ // `log.md`, the rotated `log-YYYY-MM.md`, and `bootstrap.md`).+ // Single-word unique tokens so FTS5 (which parses hyphens as+ // operators) can match them.+ std::fs::write(proj_dir.join("real.md"), "real content\n").unwrap();+ std::fs::write(proj_dir.join("log.md"), "sessionstarted logtoken unique\n").unwrap();+ std::fs::write(+ proj_dir.join("log-2026-05.md"),+ "sessionstarted rotatedlogtoken unique\n",+ )+ .unwrap();+ std::fs::write(+ proj_dir.join("bootstrap.md"),+ "bootstrapmanifest boottoken unique\n",+ )+ .unwrap();++ let handle = WatcherHandle::start(wiki.clone()).unwrap();+ // Trigger the reconciliation pass directly.+ reconcile(&wiki).await.unwrap();++ // Only `real.md` should land in the index.+ let hits = store+ .reader+ .search_pages("real content".into(), 5)+ .await+ .unwrap();+ assert_eq!(hits.len(), 1, "only the real page should be indexed");+ assert_eq!(hits[0].path.as_str(), "real.md");++ // Neither reserved file should be searchable.+ let log_hits = store+ .reader+ .search_pages("logtoken".into(), 5)+ .await+ .unwrap();+ assert!(log_hits.is_empty(), "log.md must not be indexed");++ let rotated_hits = store+ .reader+ .search_pages("rotatedlogtoken".into(), 5)+ .await+ .unwrap();+ assert!(+ rotated_hits.is_empty(),+ "log-YYYY-MM.md (rotated) must not be indexed"+ );++ let boot_hits = store+ .reader+ .search_pages("boottoken".into(), 5)+ .await+ .unwrap();+ assert!(boot_hits.is_empty(), "bootstrap.md must not be indexed");++ handle.shutdown().await;+ drop(store);+ }++ /// A page that *collides* with a reserved ledger name (`log.md`) but+ /// carries YAML frontmatter is a real page and MUST be indexed — not+ /// silently dropped. Regression for a prod data anomaly (a page lived at+ /// `log.md`) that a filename-only skip would lose on every reindex. The+ /// `_meta.md` manifest, by contrast, must NEVER be indexed even though it+ /// also has frontmatter.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn reindex_page_by_content_not_filename() {+ let (tmp, store, wiki, ws, proj) = setup().await;+ let proj_dir = tmp+ .path()+ .join("wiki")+ .join(ws.to_string())+ .join(proj.to_string());+ std::fs::create_dir_all(&proj_dir).unwrap();++ // A genuine page that happens to live at `log.md` (has frontmatter).+ std::fs::write(+ proj_dir.join("log.md"),+ "---\ntitle: Collides With Ledger\n---\nframmaticpage uniquetoken\n",+ )+ .unwrap();+ // The self-describing manifest — never a page, even with frontmatter.+ std::fs::write(+ proj_dir.join("_meta.md"),+ "---\nworkspace: default\nproject: scratch\n---\nmanifesttoken here\n",+ )+ .unwrap();+ // A raw ledger (no frontmatter) — still skipped.+ std::fs::write(proj_dir.join("log-2026-06.md"), "## [t] evt | x\nrawledgertoken\n").unwrap();++ let handle = WatcherHandle::start(wiki.clone()).unwrap();+ reconcile(&wiki).await.unwrap();++ let page_hits = store.reader.search_pages("uniquetoken".into(), 5).await.unwrap();+ assert_eq!(page_hits.len(), 1, "frontmatter page named log.md must be indexed");+ assert_eq!(page_hits[0].path.as_str(), "log.md");++ let meta_hits = store.reader.search_pages("manifesttoken".into(), 5).await.unwrap();+ assert!(meta_hits.is_empty(), "_meta.md manifest must not be indexed");++ let ledger_hits = store.reader.search_pages("rawledgertoken".into(), 5).await.unwrap();+ assert!(ledger_hits.is_empty(), "raw ledger (no frontmatter) must not be indexed");++ handle.shutdown().await;+ drop(store);+ }++ /// Defence: an attacker who can write to wiki/ shouldn't be able+ /// to make the watcher index arbitrary files via symlinks.+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn walk_markdown_skips_symlinks() {+ let tmp = TempDir::new().unwrap();+ let proj_root = tmp.path().join("proj");+ std::fs::create_dir_all(&proj_root).unwrap();++ // A real file (should be picked up).+ std::fs::write(proj_root.join("real.md"), "real content\n").unwrap();++ // A "secret" file outside the project root.+ let secret = tmp.path().join("secret.md");+ std::fs::write(&secret, "this is sensitive\n").unwrap();++ // Plant a symlink inside proj/ pointing at the outside file.+ #[cfg(unix)]+ std::os::unix::fs::symlink(&secret, proj_root.join("symlinked.md")).unwrap();+ #[cfg(windows)]+ std::os::windows::fs::symlink_file(&secret, proj_root.join("symlinked.md")).unwrap();++ let found = walk_markdown(&proj_root).unwrap();+ let names: Vec<_> = found.iter().map(|p| p.as_str().to_string()).collect();+ assert!(names.contains(&"real.md".to_string()), "real file present");+ assert!(+ !names.contains(&"symlinked.md".to_string()),+ "symlink to outside file must be skipped; got: {names:?}"+ );+ }+}diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rsindex 34224095..4f6285a7 100644--- a/crates/ai-memory-wiki/src/wiki.rs+++ b/crates/ai-memory-wiki/src/wiki.rs@@ -610,6 +610,59 @@ impl Wiki {Ok(summary)}+ /// Write a `_meta.md` scope manifest under `dir` from `frontmatter`,+ /// idempotently — unchanged content is left untouched so a startup+ /// backfill never churns the wiki git history. Returns `true` if written.+ fn write_scope_manifest(dir: &Path, frontmatter: serde_json::Value) -> WikiResult<bool> {+ let content = emit(&Markdown {+ frontmatter,+ body: String::new(),+ })?;+ let path = dir.join("_meta.md");+ if std::fs::read_to_string(&path).is_ok_and(|existing| existing == content) {+ return Ok(false);+ }+ std::fs::create_dir_all(dir)?;+ crate::atomic::write_atomic(&path, content.as_bytes())?;+ Ok(true)+ }++ /// Ensure every workspace/project scope has its self-describing `_meta.md`+ /// manifest on disk (`workspace`/`project` name + `repo_path`), so the wiki+ /// tree alone is enough to rebuild the index via [`Self::reindex_all`] —+ /// the "DB is rebuildable from files" guarantee. Idempotent; safe to run on+ /// every startup. No-op without a store reader. Returns the count written.+ ///+ /// # Errors+ /// Returns [`WikiError`] for store or filesystem errors.+ pub async fn backfill_scope_manifests(&self) -> WikiResult<usize> {+ let Some(reader) = &self.store_reader else {+ return Ok(0);+ };+ let scopes = reader.list_all_scopes().await?;+ let mut written = 0;+ let mut seen_ws = std::collections::HashSet::new();+ for s in scopes {+ let ws_dir = self.root().join(s.workspace_id.to_string());+ if seen_ws.insert(s.workspace_id)+ && Self::write_scope_manifest(+ &ws_dir,+ serde_json::json!({ "workspace": s.workspace_name }),+ )?+ {+ written += 1;+ }+ let mut fm = serde_json::json!({ "project": s.project_name });+ if let Some(rp) = s.repo_path {+ fm["repo_path"] = serde_json::Value::String(rp);+ }+ if Self::write_scope_manifest(&ws_dir.join(s.project_id.to_string()), fm)? {+ written += 1;+ }+ }+ Ok(written)+ }+/// Atomically apply a batch of page writes. Either all pages land/// (one SQL transaction) and their files are renamed into place,/// or no DB row changes and tempfiles are dropped.@@ -1751,4 +1804,90 @@ mod tests {);assert_eq!(md.frontmatter["title"], "Anon");}++ fn copy_tree(src: &Path, dst: &Path) {+ std::fs::create_dir_all(dst).unwrap();+ for entry in std::fs::read_dir(src).unwrap() {+ let entry = entry.unwrap();+ let to = dst.join(entry.file_name());+ if entry.file_type().unwrap().is_dir() {+ copy_tree(&entry.path(), &to);+ } else {+ std::fs::copy(entry.path(), &to).unwrap();+ }+ }+ }++ /// End-to-end "DB is rebuildable from files": `backfill_scope_manifests`+ /// makes the wiki self-describing, then `reindex_all` on a FRESH store+ /// (no DB carried over) recreates the named scopes + all pages from the+ /// wiki tree alone — including a page that lives at the reserved name+ /// `log.md` (kept because it has frontmatter).+ #[tokio::test]+ async fn backfill_then_reindex_rebuilds_from_wiki_alone() {+ // Source store: a named scope with two pages (one at `log.md`).+ let src = TempDir::new().unwrap();+ let s1 = Store::open(src.path()).unwrap();+ let ws = s1.writer.get_or_create_workspace("acme").await.unwrap();+ let proj = s1+ .writer+ .get_or_create_project(ws, "webapp", Some("/repo/webapp".into()))+ .await+ .unwrap();+ let w1 = Wiki::new(src.path(), s1.writer.clone())+ .unwrap()+ .with_store_reader(s1.reader.clone());+ w1.apply_batch(vec![+ req(+ ws,+ proj,+ "notes/a.md",+ "alpha uniquetoken",+ serde_json::json!({}),+ ),+ req(+ ws,+ proj,+ "log.md",+ "a page that lives at the reserved log name",+ serde_json::json!({ "title": "Log Page" }),+ ),+ ])+ .await+ .unwrap();++ // Make the wiki self-describing.+ let written = w1.backfill_scope_manifests().await.unwrap();+ assert!(written >= 2, "ws + proj manifests written, got {written}");+ let ws_dir = src.path().join("wiki").join(ws.to_string());+ assert!(ws_dir.join("_meta.md").is_file());+ assert!(ws_dir.join(proj.to_string()).join("_meta.md").is_file());+ drop(s1);++ // Fresh store; copy ONLY the wiki tree (no db/); rebuild from it.+ let dst = TempDir::new().unwrap();+ let s2 = Store::open(dst.path()).unwrap();+ copy_tree(&src.path().join("wiki"), &dst.path().join("wiki"));+ let w2 = Wiki::new(dst.path(), s2.writer.clone()).unwrap();+ let summary = w2.reindex_all().await.unwrap();++ assert_eq!(summary.projects, 1);+ assert_eq!(summary.pages, 2, "both pages incl. log.md reconstructed");+ assert_eq!(+ s2.reader.workspace_name_by_id(ws).await.unwrap().as_deref(),+ Some("acme"),+ "workspace name recovered from _meta.md"+ );+ let hits = s2+ .reader+ .search_pages("uniquetoken".into(), 5)+ .await+ .unwrap();+ assert_eq!(+ hits.len(),+ 1,+ "reindexed page is searchable in the fresh store"+ );+ drop(s2);+ }}