Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(wiki): skip OKF-conformed event ledgers in the indexer
Marco Frasson1a4efb7b6bd0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a050f732..d5d9d3e9 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -24,6 +24,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.028s to 19s on a 32-thread Windows box.### Fixed+- OKF-conformed event ledgers are skipped by the indexer again, so a migrated+ store stops growing without bound. The reserved-file check treated any+ `log.md` / `log-YYYY-MM.md` carrying YAML frontmatter as an ordinary page,+ but the OKF v0.2 migration stamps frontmatter on every `.md` under `wiki/`,+ ledgers included. A migrated store therefore indexed its ledgers, and each+ hook `append_event` superseded them: one `pages` row per appended line,+ holding the whole multi-megabyte ledger body. One affected store reached+ 6,539 page versions and 14 GB from 101 live pages within four days of+ migrating. The check now reads past the frontmatter fence and classifies on+ the first body line, so a real page that happens to be named `log.md` is+ still indexed.- Authentication-disabled HTTP servers now ignore stale or unexpected Bearerheaders and preserve anonymous access. Previously, a client retaining an old`AI_MEMORY_AUTH_TOKEN` received `401 Unauthorized` even though the serverdiff --git a/crates/ai-memory-wiki/src/watcher.rs b/crates/ai-memory-wiki/src/watcher.rsindex 086c63ee..3b863e83 100644--- a/crates/ai-memory-wiki/src/watcher.rs+++ b/crates/ai-memory-wiki/src/watcher.rs@@ -540,31 +540,63 @@ fn is_rotated_log_filename(s: &str) -> bool {&& bytes[5..].iter().all(|b| b.is_ascii_digit())}-/// Cheap peek: does the file open with a `---` YAML frontmatter fence?-/// Used to tell a real page apart from the raw event ledger.-fn opens_with_frontmatter(abs: &Path) -> bool {- use std::io::{BufRead, BufReader};- let Ok(file) = std::fs::File::open(abs) else {- return false;- };- let mut line = String::new();- BufReader::new(file).read_line(&mut line).is_ok() && line.trim_end() == "---"-}--/// Cheap check for the raw hook event ledger shape. Real page markdown can be-/// frontmatter-free; a reserved-looking filename is only a ledger when the-/// content starts with the hook log prefix.+/// Cheap check for the raw hook event ledger shape. A reserved-looking+/// filename is only a ledger when its first body line is a hook log entry+/// (`## [ts] ...`).+///+/// The body may sit under a YAML frontmatter block: the OKF v0.2 migration+/// conforms every `.md` under `wiki/`, ledgers included, so a migrated store+/// has `type: Note` stamped on top of each `log-YYYY-MM.md`. Stopping at the+/// fence would classify those ledgers as ordinary pages, and each hook+/// `append_event` would then supersede a multi-megabyte page row.fn opens_with_log_ledger(abs: &Path) -> bool {use std::io::{BufRead, BufReader};let Ok(file) = std::fs::File::open(abs) else {return false;};+ let mut reader = BufReader::new(file);let mut line = String::new();- BufReader::new(file)- .read_line(&mut line)- .is_ok_and(|_| line.starts_with("## ["))+ if reader.read_line(&mut line).is_err() {+ return false;+ }+ if line.trim_end() != "---" {+ return line.starts_with("## [");+ }+ // Walk past the frontmatter block. The bound keeps a pathological file+ // (a lone opening fence in a multi-gigabyte log) from a full scan.+ let mut closed = false;+ for _ in 0..MAX_FRONTMATTER_LINES {+ line.clear();+ match reader.read_line(&mut line) {+ Ok(0) | Err(_) => return false,+ Ok(_) => {}+ }+ if line.trim_end() == "---" {+ closed = true;+ break;+ }+ }+ if !closed {+ return false;+ }+ // First non-blank line after the fence decides.+ loop {+ line.clear();+ match reader.read_line(&mut line) {+ Ok(0) | Err(_) => return false,+ Ok(_) => {}+ }+ if !line.trim().is_empty() {+ return line.starts_with("## [");+ }+ }}+/// Upper bound on frontmatter lines scanned by [`opens_with_log_ledger`].+/// Conformant OKF frontmatter is a handful of keys; this is slack, not a+/// format limit.+const MAX_FRONTMATTER_LINES: usize = 64;+/// Returns `true` for markdown files that are NOT wiki pages and must be/// skipped by the indexer:/// - `_meta.md` (the self-describing scope manifest) and `bootstrap.md` —@@ -572,13 +604,13 @@ fn opens_with_log_ledger(abs: &Path) -> bool {/// - the raw event ledger (`log.md` / exact `log-YYYY-MM.md`) — skipping which/// avoids supersession loops, since every `append_event` write triggers a/// watcher event. A reserved-looking filename is skipped only when its-/// content opens with the raw hook log prefix; ordinary markdown pages with-/// those names are indexed.+/// first body line is a raw hook log entry; ordinary markdown pages with+/// those names are indexed, frontmatter or not.fn is_reserved_page_file(abs: &Path, page_path: &PagePath) -> bool {if is_manifest_filename(page_path) || page_path.as_str() == "bootstrap.md" {return true;}- is_log_ledger_filename(page_path) && !opens_with_frontmatter(abs) && opens_with_log_ledger(abs)+ is_log_ledger_filename(page_path) && opens_with_log_ledger(abs)}fn page_path_relative_to(root: &Path, abs: &Path) -> Option<PagePath> {@@ -1083,6 +1115,14 @@ mod tests {"## [t] evt | x\nrawledgertoken\n",).unwrap();+ // An OKF-conformed ledger: the migration stamps frontmatter on+ // every .md, ledgers included. Still a ledger, still skipped.+ std::fs::write(+ proj_dir.join("log-2026-07.md"),+ "---\ntype: Note\ngenerated:\n by: process:ai-memory/2.0.0\n---\n\+ ## [t] evt | x\nstampedledgertoken\n",+ )+ .unwrap();let handle = WatcherHandle::start(wiki.clone()).unwrap();reconcile(&wiki).await.unwrap();@@ -1119,6 +1159,21 @@ mod tests {"raw ledger (no frontmatter) must not be indexed");+ // Regression: before this check looked past the frontmatter fence,+ // an OKF-migrated ledger was indexed as a page. Every hook+ // `append_event` then superseded it, writing the whole (ever+ // growing) ledger body as a new `pages` row — a store that grew+ // into the gigabytes within days.+ let stamped_hits = store+ .reader+ .search_pages("stampedledgertoken".into(), 5)+ .await+ .unwrap();+ assert!(+ stamped_hits.is_empty(),+ "OKF-conformed ledger (frontmatter + log entries) must not be indexed"+ );+handle.shutdown().await;drop(store);}