Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(wiki): skip log ledgers in the OKF nonconformance scan (#669)
AkitaOnRails052df2052a65
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 96e5047a..e07c1d64 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -65,6 +65,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0re-emitted the mark after the closing `---` fence, where it is a strayzero-width no-break space rather than a byte-order mark. Both paths now dropthe leading BOM, which is what the frontmatter path already did. (#663)+- `serve` no longer re-archives the whole data dir on every start once a+ monthly log ledger exists. The OKF conformance migration's+ `nonconformant_files` scan flagged every frontmatter-less `log-YYYY-MM.md`+ / `log.md` event ledger as a pre-OKF page (`okf::is_conformant` requires a+ `type` key and a ledger has none), so the pre-migration backup gate saw a+ non-empty pending list and took a full `tar.gz` snapshot on every boot —+ the flip side of #660, which taught only the watcher's indexer to skip+ ledgers by content. The migration scan now shares that same content-gated+ check (a reserved-looking filename is excluded only when its first body+ line is a hook log entry), moved into a shared `ledger` module so both+ call sites stay in sync; a page literally named `log-2026-09.md` whose+ body is prose is still migrated. (#669)## [2.1.0] - 2026-09-06diff --git a/crates/ai-memory-wiki/src/ledger.rs b/crates/ai-memory-wiki/src/ledger.rsnew file mode 100644index 00000000..b388467c--- /dev/null+++ b/crates/ai-memory-wiki/src/ledger.rs@@ -0,0 +1,87 @@+//! Shared helpers for recognizing the raw hook event ledger+//! (`log.md` / `log-YYYY-MM.md`) by filename shape and by content.+//!+//! Both the watcher's indexer (#660) and the OKF conformance migration+//! (#669) must treat these files identically: a reserved-looking filename+//! is only a ledger when its content opens with a hook log entry, never on+//! filename alone (an ordinary page can be named `log-2026-09.md`).++use std::path::Path;++use ai_memory_core::PagePath;++/// `log.md` / `log-YYYY-MM.md` are the raw per-project event ledger the hooks+/// append to (see `ai-memory-hooks::log::log_filename_for`): `## [ts] ...`+/// entries, never YAML frontmatter.+pub(crate) fn is_log_ledger_filename(page_path: &PagePath) -> bool {+ let s = page_path.as_str();+ s == "log.md" || is_rotated_log_filename(s)+}++pub(crate) fn is_rotated_log_filename(s: &str) -> bool {+ let Some(stem) = s.strip_prefix("log-").and_then(|v| v.strip_suffix(".md")) else {+ return false;+ };+ let bytes = stem.as_bytes();+ bytes.len() == "YYYY-MM".len()+ && bytes[4] == b'-'+ && bytes[..4].iter().all(|b| b.is_ascii_digit())+ && bytes[5..].iter().all(|b| b.is_ascii_digit())+}++/// Cheap check for the raw hook event ledger shape. A reserved-looking+/// filename is only a ledger when its first body line is a hook log entry+/// (`## [ts] ...`).+///+/// The body may sit under a YAML frontmatter block: the OKF v0.2 migration+/// conforms every `.md` under `wiki/`, ledgers included, so a migrated store+/// has `type: Note` stamped on top of each `log-YYYY-MM.md`. Stopping at the+/// fence would classify those ledgers as ordinary pages, and each hook+/// `append_event` would then supersede a multi-megabyte page row.+pub(crate) fn opens_with_log_ledger(abs: &Path) -> bool {+ use std::io::{BufRead, BufReader};+ let Ok(file) = std::fs::File::open(abs) else {+ return false;+ };+ let mut reader = BufReader::new(file);+ let mut line = String::new();+ if reader.read_line(&mut line).is_err() {+ return false;+ }+ if line.trim_end() != "---" {+ return line.starts_with("## [");+ }+ // Walk past the frontmatter block. The bound keeps a pathological file+ // (a lone opening fence in a multi-gigabyte log) from a full scan.+ let mut closed = false;+ for _ in 0..MAX_FRONTMATTER_LINES {+ line.clear();+ match reader.read_line(&mut line) {+ Ok(0) | Err(_) => return false,+ Ok(_) => {}+ }+ if line.trim_end() == "---" {+ closed = true;+ break;+ }+ }+ if !closed {+ return false;+ }+ // First non-blank line after the fence decides.+ loop {+ line.clear();+ match reader.read_line(&mut line) {+ Ok(0) | Err(_) => return false,+ Ok(_) => {}+ }+ if !line.trim().is_empty() {+ return line.starts_with("## [");+ }+ }+}++/// Upper bound on frontmatter lines scanned by [`opens_with_log_ledger`].+/// Conformant OKF frontmatter is a handful of keys; this is slack, not a+/// format limit.+pub(crate) const MAX_FRONTMATTER_LINES: usize = 64;diff --git a/crates/ai-memory-wiki/src/lib.rs b/crates/ai-memory-wiki/src/lib.rsindex 37aea528..b0d10c4c 100644--- a/crates/ai-memory-wiki/src/lib.rs+++ b/crates/ai-memory-wiki/src/lib.rs@@ -10,6 +10,7 @@ mod atomic;pub mod backup;mod error;mod git;+mod ledger;mod markdown;pub mod migrations;mod watcher;diff --git a/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs b/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rsindex 1429aae7..eed55476 100644--- a/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs+++ b/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs@@ -210,6 +210,7 @@ fn nonconformant_files(wiki_root: &Path) -> WikiResult<Vec<PathBuf>> {} else if ft.is_file()&& path.extension().is_some_and(|e| e == "md")&& path.file_name().is_none_or(|n| n != "index.md")+ && !is_ledger_file(&path){let Ok(raw) = std::fs::read_to_string(&path) else {continue;@@ -235,6 +236,26 @@ fn is_meta_manifest(path: &Path) -> bool {path.file_name().is_some_and(|n| n == "_meta.md")}+/// The raw hook event ledger (`log.md` / `log-YYYY-MM.md`) never carries OKF+/// frontmatter and must not be migrated: stamping `type: Note` onto it would+/// recreate the frontmatter #660 already taught the watcher's indexer to+/// treat as an ordinary (huge) page, and it would make every `serve` boot+/// re-archive the whole data dir here (#669), since this scan feeds the+/// pre-migration backup gate.+///+/// Content-gated, matching the watcher's own check (#660): a reserved-looking+/// filename is only excluded when its first body line is a hook log entry, so+/// an ordinary page literally named `log-2026-09.md` is still migrated.+fn is_ledger_file(path: &Path) -> bool {+ let Some(name) = path.file_name().and_then(|n| n.to_str()) else {+ return false;+ };+ let Ok(page_path) = ai_memory_core::PagePath::new(name) else {+ return false;+ };+ crate::ledger::is_log_ledger_filename(&page_path) && crate::ledger::opens_with_log_ledger(path)+}+/// Conform one file in place: same body, frontmatter filled. Manifests/// (`_meta.md`) get their `type` only — they are identity records, not/// concept pages with provenance.@@ -732,4 +753,148 @@ mod tests {"files were touched despite the failed backup");}++ // ---- #669: a raw event ledger must never look like a pending 1.x file ----++ /// A fully OKF-conformant wiki plus a frontmatter-less monthly ledger+ /// must not be treated as a 1.x store: the ledger is excluded from+ /// `nonconformant_files` by content (#660's own check), so `serve`+ /// never re-archives the whole data dir just because a ledger exists.+ #[tokio::test]+ async fn a_conformant_store_with_a_ledger_skips_the_backup() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store.writer.get_or_create_workspace("w").await.unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "p", None)+ .await+ .unwrap();+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();+ wiki.write_page(WritePageRequest {+ workspace_id: ws,+ project_id: proj,+ path: PagePath::new("notes/setup.md").unwrap(),+ frontmatter: serde_json::json!({"title": "setup"}),+ body: "how it was set up".into(),+ tier: Tier::Semantic,+ pinned: false,+ title: None,+ admission_ctx: None,+ author_id: None,+ actor: ai_memory_core::ActorContext::anonymous(),+ })+ .await+ .unwrap();++ let proj_dir = tmp+ .path()+ .join("wiki")+ .join(ws.to_string())+ .join(proj.to_string());+ std::fs::write(+ proj_dir.join("log-2026-09.md"),+ "## [2026-09-01T00:00:00Z] session started\n",+ )+ .unwrap();++ let wiki_root = tmp.path().join("wiki");+ let pending = nonconformant_files(&wiki_root).unwrap();+ assert!(+ pending.is_empty(),+ "the ledger must not be listed as nonconformant: {pending:?}"+ );++ let dest = TempDir::new().unwrap();+ let receipt = snapshot_before_db_migration(tmp.path(), Some(dest.path())).unwrap();+ assert!(+ receipt.is_none(),+ "a conformant store plus a ledger must not trigger a backup"+ );+ assert_eq!(+ std::fs::read_dir(dest.path()).unwrap().count(),+ 0,+ "the ledger caused a full data-dir archive"+ );+ }++ /// Control: an ordinary page still missing its `type` frontmatter must+ /// still be flagged, so the ledger exclusion is not swallowing real 1.x+ /// pages.+ #[tokio::test]+ async fn a_page_missing_type_frontmatter_is_still_flagged_alongside_a_ledger() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store.writer.get_or_create_workspace("w").await.unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "p", None)+ .await+ .unwrap();+ let proj_dir = tmp+ .path()+ .join("wiki")+ .join(ws.to_string())+ .join(proj.to_string());+ std::fs::create_dir_all(&proj_dir).unwrap();+ std::fs::write(+ proj_dir.join("notes.md"),+ "# Notes\n\nno frontmatter at all\n",+ )+ .unwrap();+ std::fs::write(+ proj_dir.join("log-2026-09.md"),+ "## [2026-09-01T00:00:00Z] session started\n",+ )+ .unwrap();++ let wiki_root = tmp.path().join("wiki");+ let pending = nonconformant_files(&wiki_root).unwrap();+ assert!(+ pending+ .iter()+ .any(|p| p.file_name().is_some_and(|n| n == "notes.md")),+ "a real pre-OKF page must still be flagged: {pending:?}"+ );+ assert!(+ pending+ .iter()+ .all(|p| p.file_name().is_none_or(|n| n != "log-2026-09.md")),+ "the ledger must not be flagged even alongside a real pre-OKF page: {pending:?}"+ );+ }++ /// Content-gating, not filename-only: a page that merely happens to be+ /// named like a ledger but whose body is prose must still migrate.+ #[tokio::test]+ async fn a_prose_page_named_like_a_ledger_is_still_flagged() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store.writer.get_or_create_workspace("w").await.unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "p", None)+ .await+ .unwrap();+ let proj_dir = tmp+ .path()+ .join("wiki")+ .join(ws.to_string())+ .join(proj.to_string());+ std::fs::create_dir_all(&proj_dir).unwrap();+ std::fs::write(+ proj_dir.join("log-2026-09.md"),+ "# September retro\n\nJust an ordinary page someone named this way.\n",+ )+ .unwrap();++ let wiki_root = tmp.path().join("wiki");+ let pending = nonconformant_files(&wiki_root).unwrap();+ assert!(+ pending+ .iter()+ .any(|p| p.file_name().is_some_and(|n| n == "log-2026-09.md")),+ "a prose page named like a ledger must still migrate: {pending:?}"+ );+ }}diff --git a/crates/ai-memory-wiki/src/watcher.rs b/crates/ai-memory-wiki/src/watcher.rsindex 3b863e83..2daed57f 100644--- a/crates/ai-memory-wiki/src/watcher.rs+++ b/crates/ai-memory-wiki/src/watcher.rs@@ -521,82 +521,6 @@ fn is_manifest_filename(page_path: &PagePath) -> bool {.is_some_and(|name| name == "_meta.md")}-/// `log.md` / `log-YYYY-MM.md` are the raw per-project event ledger the hooks-/// append to (see `ai-memory-hooks::log::log_filename_for`): `## [ts] ...`-/// entries, never YAML frontmatter.-fn is_log_ledger_filename(page_path: &PagePath) -> bool {- let s = page_path.as_str();- s == "log.md" || is_rotated_log_filename(s)-}--fn is_rotated_log_filename(s: &str) -> bool {- let Some(stem) = s.strip_prefix("log-").and_then(|v| v.strip_suffix(".md")) else {- return false;- };- let bytes = stem.as_bytes();- bytes.len() == "YYYY-MM".len()- && bytes[4] == b'-'- && bytes[..4].iter().all(|b| b.is_ascii_digit())- && bytes[5..].iter().all(|b| b.is_ascii_digit())-}--/// Cheap check for the raw hook event ledger shape. A reserved-looking-/// filename is only a ledger when its first body line is a hook log entry-/// (`## [ts] ...`).-///-/// The body may sit under a YAML frontmatter block: the OKF v0.2 migration-/// conforms every `.md` under `wiki/`, ledgers included, so a migrated store-/// has `type: Note` stamped on top of each `log-YYYY-MM.md`. Stopping at the-/// fence would classify those ledgers as ordinary pages, and each hook-/// `append_event` would then supersede a multi-megabyte page row.-fn opens_with_log_ledger(abs: &Path) -> bool {- use std::io::{BufRead, BufReader};- let Ok(file) = std::fs::File::open(abs) else {- return false;- };- let mut reader = BufReader::new(file);- let mut line = String::new();- if reader.read_line(&mut line).is_err() {- return false;- }- if line.trim_end() != "---" {- return line.starts_with("## [");- }- // Walk past the frontmatter block. The bound keeps a pathological file- // (a lone opening fence in a multi-gigabyte log) from a full scan.- let mut closed = false;- for _ in 0..MAX_FRONTMATTER_LINES {- line.clear();- match reader.read_line(&mut line) {- Ok(0) | Err(_) => return false,- Ok(_) => {}- }- if line.trim_end() == "---" {- closed = true;- break;- }- }- if !closed {- return false;- }- // First non-blank line after the fence decides.- loop {- line.clear();- match reader.read_line(&mut line) {- Ok(0) | Err(_) => return false,- Ok(_) => {}- }- if !line.trim().is_empty() {- return line.starts_with("## [");- }- }-}--/// Upper bound on frontmatter lines scanned by [`opens_with_log_ledger`].-/// Conformant OKF frontmatter is a handful of keys; this is slack, not a-/// format limit.-const MAX_FRONTMATTER_LINES: usize = 64;-/// Returns `true` for markdown files that are NOT wiki pages and must be/// skipped by the indexer:/// - `_meta.md` (the self-describing scope manifest) and `bootstrap.md` —@@ -610,7 +534,7 @@ fn is_reserved_page_file(abs: &Path, page_path: &PagePath) -> bool {if is_manifest_filename(page_path) || page_path.as_str() == "bootstrap.md" {return true;}- is_log_ledger_filename(page_path) && opens_with_log_ledger(abs)+ crate::ledger::is_log_ledger_filename(page_path) && crate::ledger::opens_with_log_ledger(abs)}fn page_path_relative_to(root: &Path, abs: &Path) -> Option<PagePath> {