Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(wiki): parse frontmatter on pages with CRLF line endings
kevin93272edd4177d2ec
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 4bfd236d..7f4b1b5e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -38,6 +38,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0feedback-loop guard stays non-overridable. (#894)### Fixed+- A wiki page with CRLF line endings is parsed as having frontmatter again.+ `markdown::parse` only matched the fence lines with a bare `\n`, so a page+ a Windows editor saved, or one `core.autocrlf=true` checked out, was treated+ as body-only: `reindex`/the watcher indexed it without its `tier`,+ `pinned`, `expires_at` and `entities` (a pinned page became decay-eligible),+ the title came from the filename, and the one-shot OKF file pass wrote a+ second frontmatter block above the authored one. The parser now accepts+ `---\r\n` fences and leaves the body's line endings untouched. (#NNN)- A manual `memory_consolidate` now reconciles the session's durableconsolidation job row. The MCP handler wrote the page directly through theconsolidator without touching `session_consolidation_jobs`, so a sessiondiff --git a/crates/ai-memory-wiki/src/markdown.rs b/crates/ai-memory-wiki/src/markdown.rsindex 837fe2c0..6b28beac 100644--- a/crates/ai-memory-wiki/src/markdown.rs+++ b/crates/ai-memory-wiki/src/markdown.rs@@ -26,7 +26,8 @@ pub struct Markdown {/// Parse markdown text into [`Markdown`].////// Recognises only the canonical `---\n<yaml>\n---\n` block at the very-/// start of the document. Anything else is treated as body.+/// start of the document, with either LF or CRLF line endings on the fence+/// lines. Anything else is treated as body.////// A leading UTF-8 BOM is dropped either way. It only means "this file is/// UTF-8" while it sits at offset zero; carried into `body` it is a@@ -34,16 +35,28 @@ pub struct Markdown {/// from [`derive_title`] and rides into the body a later re-emit writes/// back after the frontmatter fence.///+/// CRLF is what a Windows editor saves, and what `core.autocrlf=true` checks+/// out for every page of a wiki cloned onto Windows. Missing the fence there+/// treats the block as body: the page reindexes with no tier, pin or TTL, and+/// the in-place OKF file pass writes a second frontmatter block above it. The+/// body keeps its line endings untouched either way.+////// # Errors/// Returns [`WikiError::Yaml`] if the frontmatter block exists but does/// not parse as YAML.pub fn parse(input: &str) -> WikiResult<Markdown> {let trimmed = input.strip_prefix('\u{FEFF}').unwrap_or(input);- if let Some(rest) = trimmed.strip_prefix("---\n")- && let Some(end) = rest.find("\n---\n")+ let (rest, newline) = if let Some(rest) = trimmed.strip_prefix("---\r\n") {+ (Some(rest), "\r\n")+ } else {+ (trimmed.strip_prefix("---\n"), "\n")+ };+ let close = format!("\n---{newline}");+ if let Some(rest) = rest+ && let Some(end) = rest.find(&close){- let fm_str = &rest[..end];- let body = rest[end + 5..].to_string();+ let fm_str = rest[..end].trim_end_matches('\r');+ let body = rest[end + close.len()..].to_string();let fm_yaml: serde_yaml::Value = serde_yaml::from_str(fm_str)?;let fm_json: serde_json::Value = serde_json::to_value(fm_yaml)?;return Ok(Markdown {@@ -557,6 +570,22 @@ mod tests {assert_eq!(md.body, "Body\n");}+ /// A page saved with CRLF line endings (a Windows editor, or a wiki+ /// checked out with `core.autocrlf=true`): the fence lines end in+ /// `\r\n`, but they are still the canonical frontmatter block. Treating+ /// the file as body-only drops `pinned`/`tier`/`expires_at` on reindex+ /// and lets the OKF file pass write a second frontmatter block above+ /// the first.+ #[test]+ fn parses_crlf_frontmatter() {+ let src = "---\r\ntitle: Hello\r\npinned: true\r\ntags:\r\n - a\r\n---\r\nBody\r\n";+ let md = parse(src).unwrap();+ assert_eq!(md.frontmatter["title"], "Hello");+ assert_eq!(md.frontmatter["pinned"], true);+ assert_eq!(md.frontmatter["tags"][0], "a");+ assert_eq!(md.body, "Body\r\n");+ }+/// A page a Windows editor saved with a UTF-8 BOM and no frontmatter:/// the mark belongs to the file, not to the first line. Left in `body`/// it sits in front of the `#`, so the H1 stops being a heading and thediff --git a/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs b/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rsindex 90d0a731..27d1de09 100644--- a/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs+++ b/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs@@ -608,6 +608,39 @@ mod tests {assert_eq!(parse(&conformed).unwrap().frontmatter["type"], "Note");}+ /// A CRLF page already has a frontmatter block; the parser must see it+ /// so the pass fills that block instead of writing a fresh one above it.+ /// The duplicate-frontmatter shape is exactly the basic-memory #528 bug+ /// `markdown.rs` exists to avoid, and this pass is one-shot and in place.+ #[test]+ fn conforming_a_crlf_page_does_not_duplicate_its_frontmatter() {+ let tmp = TempDir::new().unwrap();+ let rel = Path::new("w/p/notes/hand-written.md");+ let abs = tmp.path().join(rel);+ std::fs::create_dir_all(abs.parent().unwrap()).unwrap();+ std::fs::write(+ &abs,+ "---\r\ntitle: Hand written\r\npinned: true\r\n---\r\n# Hand written\r\n\r\nBody.\r\n",+ )+ .unwrap();++ let git = crate::git::GitAdapter::open_or_init(tmp.path()).unwrap();+ conform_file(&git, rel, &std::collections::HashMap::new()).unwrap();++ let conformed = std::fs::read_to_string(&abs).unwrap();+ let md = parse(&conformed).unwrap();+ assert_eq!(+ md.frontmatter["title"], "Hand written",+ "the authored frontmatter must survive the pass: {conformed:?}"+ );+ assert_eq!(md.frontmatter["pinned"], true);+ assert_eq!(md.frontmatter["type"], "Note");+ assert_eq!(+ md.body, "# Hand written\r\n\r\nBody.\r\n",+ "the body must not gain a second frontmatter block: {conformed:?}"+ );+ }+// ---- #633: the safety archive must be taken BEFORE the DB migration ----/// The core assertion for #633: the pre-open snapshot captures the DB as itdiff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rsindex 4dac00be..0c4c4efb 100644--- a/crates/ai-memory-wiki/src/wiki.rs+++ b/crates/ai-memory-wiki/src/wiki.rs@@ -3464,6 +3464,36 @@ mod tests {);}+ /// The same page saved with CRLF line endings (a Windows editor, or a+ /// wiki checked out with `core.autocrlf=true`) must index the same way:+ /// its frontmatter is still the source of truth for tier/pinned/title,+ /// not body text ahead of the H1.+ #[tokio::test]+ async fn reindex_page_reads_crlf_frontmatter() {+ let tmp = TempDir::new().unwrap();+ let (store, wiki, ws, proj) = scoped(&tmp).await;++ let path = PagePath::new("sessions/crlf.md").unwrap();+ let abs = wiki.abs_path(ws, proj, &path);+ std::fs::create_dir_all(abs.parent().unwrap()).unwrap();+ std::fs::write(+ &abs,+ "---\r\ntitle: S\r\ntier: episodic\r\npinned: true\r\n---\r\nbody\r\n",+ )+ .unwrap();++ wiki.reindex_page(ws, proj, path.clone()).await.unwrap();+ let meta = store+ .reader+ .page_meta("default", "scratch", path.as_str())+ .await+ .unwrap()+ .unwrap();+ assert_eq!(meta.title, "S", "the frontmatter title must win");+ assert_eq!(meta.tier, "episodic");+ assert!(meta.pinned, "a CRLF `pinned: true` must still pin the page");+ }+#[tokio::test]async fn write_paths_persist_index_metadata_for_reindex_round_trip() {let tmp = TempDir::new().unwrap();