Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(wiki): skip log ledgers in the OKF nonconformance scan (#669)

AkitaOnRails052df2052a65
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 96e5047a..e07c1d64 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -65,6 +65,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
re-emitted the mark after the closing `---` fence, where it is a stray
zero-width no-break space rather than a byte-order mark. Both paths now drop
the leading BOM, which is what the frontmatter path already did. (#663)
+- `serve` no longer re-archives the whole data dir on every start once a
+ monthly log ledger exists. The OKF conformance migration's
+ `nonconformant_files` scan flagged every frontmatter-less `log-YYYY-MM.md`
+ / `log.md` event ledger as a pre-OKF page (`okf::is_conformant` requires a
+ `type` key and a ledger has none), so the pre-migration backup gate saw a
+ non-empty pending list and took a full `tar.gz` snapshot on every boot —
+ the flip side of #660, which taught only the watcher's indexer to skip
+ ledgers by content. The migration scan now shares that same content-gated
+ check (a reserved-looking filename is excluded only when its first body
+ line is a hook log entry), moved into a shared `ledger` module so both
+ call sites stay in sync; a page literally named `log-2026-09.md` whose
+ body is prose is still migrated. (#669)
## [2.1.0] - 2026-09-06
diff --git a/crates/ai-memory-wiki/src/ledger.rs b/crates/ai-memory-wiki/src/ledger.rs
new file mode 100644
index 00000000..b388467c
--- /dev/null
+++ b/crates/ai-memory-wiki/src/ledger.rs
@@ -0,0 +1,87 @@
+//! Shared helpers for recognizing the raw hook event ledger
+//! (`log.md` / `log-YYYY-MM.md`) by filename shape and by content.
+//!
+//! Both the watcher's indexer (#660) and the OKF conformance migration
+//! (#669) must treat these files identically: a reserved-looking filename
+//! is only a ledger when its content opens with a hook log entry, never on
+//! filename alone (an ordinary page can be named `log-2026-09.md`).
+
+use std::path::Path;
+
+use ai_memory_core::PagePath;
+
+/// `log.md` / `log-YYYY-MM.md` are the raw per-project event ledger the hooks
+/// append to (see `ai-memory-hooks::log::log_filename_for`): `## [ts] ...`
+/// entries, never YAML frontmatter.
+pub(crate) fn is_log_ledger_filename(page_path: &PagePath) -> bool {
+ let s = page_path.as_str();
+ s == "log.md" || is_rotated_log_filename(s)
+}
+
+pub(crate) fn is_rotated_log_filename(s: &str) -> bool {
+ let Some(stem) = s.strip_prefix("log-").and_then(|v| v.strip_suffix(".md")) else {
+ return false;
+ };
+ let bytes = stem.as_bytes();
+ bytes.len() == "YYYY-MM".len()
+ && bytes[4] == b'-'
+ && bytes[..4].iter().all(|b| b.is_ascii_digit())
+ && bytes[5..].iter().all(|b| b.is_ascii_digit())
+}
+
+/// Cheap check for the raw hook event ledger shape. A reserved-looking
+/// filename is only a ledger when its first body line is a hook log entry
+/// (`## [ts] ...`).
+///
+/// The body may sit under a YAML frontmatter block: the OKF v0.2 migration
+/// conforms every `.md` under `wiki/`, ledgers included, so a migrated store
+/// has `type: Note` stamped on top of each `log-YYYY-MM.md`. Stopping at the
+/// fence would classify those ledgers as ordinary pages, and each hook
+/// `append_event` would then supersede a multi-megabyte page row.
+pub(crate) fn opens_with_log_ledger(abs: &Path) -> bool {
+ use std::io::{BufRead, BufReader};
+ let Ok(file) = std::fs::File::open(abs) else {
+ return false;
+ };
+ let mut reader = BufReader::new(file);
+ let mut line = String::new();
+ if reader.read_line(&mut line).is_err() {
+ return false;
+ }
+ if line.trim_end() != "---" {
+ return line.starts_with("## [");
+ }
+ // Walk past the frontmatter block. The bound keeps a pathological file
+ // (a lone opening fence in a multi-gigabyte log) from a full scan.
+ let mut closed = false;
+ for _ in 0..MAX_FRONTMATTER_LINES {
+ line.clear();
+ match reader.read_line(&mut line) {
+ Ok(0) | Err(_) => return false,
+ Ok(_) => {}
+ }
+ if line.trim_end() == "---" {
+ closed = true;
+ break;
+ }
+ }
+ if !closed {
+ return false;
+ }
+ // First non-blank line after the fence decides.
+ loop {
+ line.clear();
+ match reader.read_line(&mut line) {
+ Ok(0) | Err(_) => return false,
+ Ok(_) => {}
+ }
+ if !line.trim().is_empty() {
+ return line.starts_with("## [");
+ }
+ }
+}
+
+/// Upper bound on frontmatter lines scanned by [`opens_with_log_ledger`].
+/// Conformant OKF frontmatter is a handful of keys; this is slack, not a
+/// format limit.
+pub(crate) const MAX_FRONTMATTER_LINES: usize = 64;
diff --git a/crates/ai-memory-wiki/src/lib.rs b/crates/ai-memory-wiki/src/lib.rs
index 37aea528..b0d10c4c 100644
--- a/crates/ai-memory-wiki/src/lib.rs
+++ b/crates/ai-memory-wiki/src/lib.rs
@@ -10,6 +10,7 @@ mod atomic;
pub mod backup;
mod error;
mod git;
+mod ledger;
mod markdown;
pub mod migrations;
mod watcher;
diff --git a/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs b/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs
index 1429aae7..eed55476 100644
--- a/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs
+++ b/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs
@@ -210,6 +210,7 @@ fn nonconformant_files(wiki_root: &Path) -> WikiResult<Vec<PathBuf>> {
} else if ft.is_file()
&& path.extension().is_some_and(|e| e == "md")
&& path.file_name().is_none_or(|n| n != "index.md")
+ && !is_ledger_file(&path)
{
let Ok(raw) = std::fs::read_to_string(&path) else {
continue;
@@ -235,6 +236,26 @@ fn is_meta_manifest(path: &Path) -> bool {
path.file_name().is_some_and(|n| n == "_meta.md")
}
+/// The raw hook event ledger (`log.md` / `log-YYYY-MM.md`) never carries OKF
+/// frontmatter and must not be migrated: stamping `type: Note` onto it would
+/// recreate the frontmatter #660 already taught the watcher's indexer to
+/// treat as an ordinary (huge) page, and it would make every `serve` boot
+/// re-archive the whole data dir here (#669), since this scan feeds the
+/// pre-migration backup gate.
+///
+/// Content-gated, matching the watcher's own check (#660): a reserved-looking
+/// filename is only excluded when its first body line is a hook log entry, so
+/// an ordinary page literally named `log-2026-09.md` is still migrated.
+fn is_ledger_file(path: &Path) -> bool {
+ let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
+ return false;
+ };
+ let Ok(page_path) = ai_memory_core::PagePath::new(name) else {
+ return false;
+ };
+ crate::ledger::is_log_ledger_filename(&page_path) && crate::ledger::opens_with_log_ledger(path)
+}
+
/// Conform one file in place: same body, frontmatter filled. Manifests
/// (`_meta.md`) get their `type` only — they are identity records, not
/// concept pages with provenance.
@@ -732,4 +753,148 @@ mod tests {
"files were touched despite the failed backup"
);
}
+
+ // ---- #669: a raw event ledger must never look like a pending 1.x file ----
+
+ /// A fully OKF-conformant wiki plus a frontmatter-less monthly ledger
+ /// must not be treated as a 1.x store: the ledger is excluded from
+ /// `nonconformant_files` by content (#660's own check), so `serve`
+ /// never re-archives the whole data dir just because a ledger exists.
+ #[tokio::test]
+ async fn a_conformant_store_with_a_ledger_skips_the_backup() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store.writer.get_or_create_workspace("w").await.unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "p", None)
+ .await
+ .unwrap();
+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();
+ wiki.write_page(WritePageRequest {
+ workspace_id: ws,
+ project_id: proj,
+ path: PagePath::new("notes/setup.md").unwrap(),
+ frontmatter: serde_json::json!({"title": "setup"}),
+ body: "how it was set up".into(),
+ tier: Tier::Semantic,
+ pinned: false,
+ title: None,
+ admission_ctx: None,
+ author_id: None,
+ actor: ai_memory_core::ActorContext::anonymous(),
+ })
+ .await
+ .unwrap();
+
+ let proj_dir = tmp
+ .path()
+ .join("wiki")
+ .join(ws.to_string())
+ .join(proj.to_string());
+ std::fs::write(
+ proj_dir.join("log-2026-09.md"),
+ "## [2026-09-01T00:00:00Z] session started\n",
+ )
+ .unwrap();
+
+ let wiki_root = tmp.path().join("wiki");
+ let pending = nonconformant_files(&wiki_root).unwrap();
+ assert!(
+ pending.is_empty(),
+ "the ledger must not be listed as nonconformant: {pending:?}"
+ );
+
+ let dest = TempDir::new().unwrap();
+ let receipt = snapshot_before_db_migration(tmp.path(), Some(dest.path())).unwrap();
+ assert!(
+ receipt.is_none(),
+ "a conformant store plus a ledger must not trigger a backup"
+ );
+ assert_eq!(
+ std::fs::read_dir(dest.path()).unwrap().count(),
+ 0,
+ "the ledger caused a full data-dir archive"
+ );
+ }
+
+ /// Control: an ordinary page still missing its `type` frontmatter must
+ /// still be flagged, so the ledger exclusion is not swallowing real 1.x
+ /// pages.
+ #[tokio::test]
+ async fn a_page_missing_type_frontmatter_is_still_flagged_alongside_a_ledger() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store.writer.get_or_create_workspace("w").await.unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "p", None)
+ .await
+ .unwrap();
+ let proj_dir = tmp
+ .path()
+ .join("wiki")
+ .join(ws.to_string())
+ .join(proj.to_string());
+ std::fs::create_dir_all(&proj_dir).unwrap();
+ std::fs::write(
+ proj_dir.join("notes.md"),
+ "# Notes\n\nno frontmatter at all\n",
+ )
+ .unwrap();
+ std::fs::write(
+ proj_dir.join("log-2026-09.md"),
+ "## [2026-09-01T00:00:00Z] session started\n",
+ )
+ .unwrap();
+
+ let wiki_root = tmp.path().join("wiki");
+ let pending = nonconformant_files(&wiki_root).unwrap();
+ assert!(
+ pending
+ .iter()
+ .any(|p| p.file_name().is_some_and(|n| n == "notes.md")),
+ "a real pre-OKF page must still be flagged: {pending:?}"
+ );
+ assert!(
+ pending
+ .iter()
+ .all(|p| p.file_name().is_none_or(|n| n != "log-2026-09.md")),
+ "the ledger must not be flagged even alongside a real pre-OKF page: {pending:?}"
+ );
+ }
+
+ /// Content-gating, not filename-only: a page that merely happens to be
+ /// named like a ledger but whose body is prose must still migrate.
+ #[tokio::test]
+ async fn a_prose_page_named_like_a_ledger_is_still_flagged() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store.writer.get_or_create_workspace("w").await.unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "p", None)
+ .await
+ .unwrap();
+ let proj_dir = tmp
+ .path()
+ .join("wiki")
+ .join(ws.to_string())
+ .join(proj.to_string());
+ std::fs::create_dir_all(&proj_dir).unwrap();
+ std::fs::write(
+ proj_dir.join("log-2026-09.md"),
+ "# September retro\n\nJust an ordinary page someone named this way.\n",
+ )
+ .unwrap();
+
+ let wiki_root = tmp.path().join("wiki");
+ let pending = nonconformant_files(&wiki_root).unwrap();
+ assert!(
+ pending
+ .iter()
+ .any(|p| p.file_name().is_some_and(|n| n == "log-2026-09.md")),
+ "a prose page named like a ledger must still migrate: {pending:?}"
+ );
+ }
}
diff --git a/crates/ai-memory-wiki/src/watcher.rs b/crates/ai-memory-wiki/src/watcher.rs
index 3b863e83..2daed57f 100644
--- a/crates/ai-memory-wiki/src/watcher.rs
+++ b/crates/ai-memory-wiki/src/watcher.rs
@@ -521,82 +521,6 @@ fn is_manifest_filename(page_path: &PagePath) -> bool {
.is_some_and(|name| name == "_meta.md")
}
-/// `log.md` / `log-YYYY-MM.md` are the raw per-project event ledger the hooks
-/// append to (see `ai-memory-hooks::log::log_filename_for`): `## [ts] ...`
-/// entries, never YAML frontmatter.
-fn is_log_ledger_filename(page_path: &PagePath) -> bool {
- let s = page_path.as_str();
- s == "log.md" || is_rotated_log_filename(s)
-}
-
-fn is_rotated_log_filename(s: &str) -> bool {
- let Some(stem) = s.strip_prefix("log-").and_then(|v| v.strip_suffix(".md")) else {
- return false;
- };
- let bytes = stem.as_bytes();
- bytes.len() == "YYYY-MM".len()
- && bytes[4] == b'-'
- && bytes[..4].iter().all(|b| b.is_ascii_digit())
- && bytes[5..].iter().all(|b| b.is_ascii_digit())
-}
-
-/// Cheap check for the raw hook event ledger shape. A reserved-looking
-/// filename is only a ledger when its first body line is a hook log entry
-/// (`## [ts] ...`).
-///
-/// The body may sit under a YAML frontmatter block: the OKF v0.2 migration
-/// conforms every `.md` under `wiki/`, ledgers included, so a migrated store
-/// has `type: Note` stamped on top of each `log-YYYY-MM.md`. Stopping at the
-/// fence would classify those ledgers as ordinary pages, and each hook
-/// `append_event` would then supersede a multi-megabyte page row.
-fn opens_with_log_ledger(abs: &Path) -> bool {
- use std::io::{BufRead, BufReader};
- let Ok(file) = std::fs::File::open(abs) else {
- return false;
- };
- let mut reader = BufReader::new(file);
- let mut line = String::new();
- if reader.read_line(&mut line).is_err() {
- return false;
- }
- if line.trim_end() != "---" {
- return line.starts_with("## [");
- }
- // Walk past the frontmatter block. The bound keeps a pathological file
- // (a lone opening fence in a multi-gigabyte log) from a full scan.
- let mut closed = false;
- for _ in 0..MAX_FRONTMATTER_LINES {
- line.clear();
- match reader.read_line(&mut line) {
- Ok(0) | Err(_) => return false,
- Ok(_) => {}
- }
- if line.trim_end() == "---" {
- closed = true;
- break;
- }
- }
- if !closed {
- return false;
- }
- // First non-blank line after the fence decides.
- loop {
- line.clear();
- match reader.read_line(&mut line) {
- Ok(0) | Err(_) => return false,
- Ok(_) => {}
- }
- if !line.trim().is_empty() {
- return line.starts_with("## [");
- }
- }
-}
-
-/// Upper bound on frontmatter lines scanned by [`opens_with_log_ledger`].
-/// Conformant OKF frontmatter is a handful of keys; this is slack, not a
-/// format limit.
-const MAX_FRONTMATTER_LINES: usize = 64;
-
/// Returns `true` for markdown files that are NOT wiki pages and must be
/// skipped by the indexer:
/// - `_meta.md` (the self-describing scope manifest) and `bootstrap.md` —
@@ -610,7 +534,7 @@ fn is_reserved_page_file(abs: &Path, page_path: &PagePath) -> bool {
if is_manifest_filename(page_path) || page_path.as_str() == "bootstrap.md" {
return true;
}
- is_log_ledger_filename(page_path) && opens_with_log_ledger(abs)
+ crate::ledger::is_log_ledger_filename(page_path) && crate::ledger::opens_with_log_ledger(abs)
}
fn page_path_relative_to(root: &Path, abs: &Path) -> Option<PagePath> {