Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(wiki): parse frontmatter on pages with CRLF line endings

kevin93272edd4177d2ec
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4bfd236d..7f4b1b5e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -38,6 +38,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
feedback-loop guard stays non-overridable. (#894)
### Fixed
+- A wiki page with CRLF line endings is parsed as having frontmatter again.
+ `markdown::parse` only matched the fence lines with a bare `\n`, so a page
+ a Windows editor saved, or one `core.autocrlf=true` checked out, was treated
+ as body-only: `reindex`/the watcher indexed it without its `tier`,
+ `pinned`, `expires_at` and `entities` (a pinned page became decay-eligible),
+ the title came from the filename, and the one-shot OKF file pass wrote a
+ second frontmatter block above the authored one. The parser now accepts
+ `---\r\n` fences and leaves the body's line endings untouched. (#NNN)
- A manual `memory_consolidate` now reconciles the session's durable
consolidation job row. The MCP handler wrote the page directly through the
consolidator without touching `session_consolidation_jobs`, so a session
diff --git a/crates/ai-memory-wiki/src/markdown.rs b/crates/ai-memory-wiki/src/markdown.rs
index 837fe2c0..6b28beac 100644
--- a/crates/ai-memory-wiki/src/markdown.rs
+++ b/crates/ai-memory-wiki/src/markdown.rs
@@ -26,7 +26,8 @@ pub struct Markdown {
/// Parse markdown text into [`Markdown`].
///
/// Recognises only the canonical `---\n<yaml>\n---\n` block at the very
-/// start of the document. Anything else is treated as body.
+/// start of the document, with either LF or CRLF line endings on the fence
+/// lines. Anything else is treated as body.
///
/// A leading UTF-8 BOM is dropped either way. It only means "this file is
/// UTF-8" while it sits at offset zero; carried into `body` it is a
@@ -34,16 +35,28 @@ pub struct Markdown {
/// from [`derive_title`] and rides into the body a later re-emit writes
/// back after the frontmatter fence.
///
+/// CRLF is what a Windows editor saves, and what `core.autocrlf=true` checks
+/// out for every page of a wiki cloned onto Windows. Missing the fence there
+/// treats the block as body: the page reindexes with no tier, pin or TTL, and
+/// the in-place OKF file pass writes a second frontmatter block above it. The
+/// body keeps its line endings untouched either way.
+///
/// # Errors
/// Returns [`WikiError::Yaml`] if the frontmatter block exists but does
/// not parse as YAML.
pub fn parse(input: &str) -> WikiResult<Markdown> {
let trimmed = input.strip_prefix('\u{FEFF}').unwrap_or(input);
- if let Some(rest) = trimmed.strip_prefix("---\n")
- && let Some(end) = rest.find("\n---\n")
+ let (rest, newline) = if let Some(rest) = trimmed.strip_prefix("---\r\n") {
+ (Some(rest), "\r\n")
+ } else {
+ (trimmed.strip_prefix("---\n"), "\n")
+ };
+ let close = format!("\n---{newline}");
+ if let Some(rest) = rest
+ && let Some(end) = rest.find(&close)
{
- let fm_str = &rest[..end];
- let body = rest[end + 5..].to_string();
+ let fm_str = rest[..end].trim_end_matches('\r');
+ let body = rest[end + close.len()..].to_string();
let fm_yaml: serde_yaml::Value = serde_yaml::from_str(fm_str)?;
let fm_json: serde_json::Value = serde_json::to_value(fm_yaml)?;
return Ok(Markdown {
@@ -557,6 +570,22 @@ mod tests {
assert_eq!(md.body, "Body\n");
}
+ /// A page saved with CRLF line endings (a Windows editor, or a wiki
+ /// checked out with `core.autocrlf=true`): the fence lines end in
+ /// `\r\n`, but they are still the canonical frontmatter block. Treating
+ /// the file as body-only drops `pinned`/`tier`/`expires_at` on reindex
+ /// and lets the OKF file pass write a second frontmatter block above
+ /// the first.
+ #[test]
+ fn parses_crlf_frontmatter() {
+ let src = "---\r\ntitle: Hello\r\npinned: true\r\ntags:\r\n - a\r\n---\r\nBody\r\n";
+ let md = parse(src).unwrap();
+ assert_eq!(md.frontmatter["title"], "Hello");
+ assert_eq!(md.frontmatter["pinned"], true);
+ assert_eq!(md.frontmatter["tags"][0], "a");
+ assert_eq!(md.body, "Body\r\n");
+ }
+
/// A page a Windows editor saved with a UTF-8 BOM and no frontmatter:
/// the mark belongs to the file, not to the first line. Left in `body`
/// it sits in front of the `#`, so the H1 stops being a heading and the
diff --git a/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs b/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs
index 90d0a731..27d1de09 100644
--- a/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs
+++ b/crates/ai-memory-wiki/src/migrations/m2026_09_okf_conformance.rs
@@ -608,6 +608,39 @@ mod tests {
assert_eq!(parse(&conformed).unwrap().frontmatter["type"], "Note");
}
+ /// A CRLF page already has a frontmatter block; the parser must see it
+ /// so the pass fills that block instead of writing a fresh one above it.
+ /// The duplicate-frontmatter shape is exactly the basic-memory #528 bug
+ /// `markdown.rs` exists to avoid, and this pass is one-shot and in place.
+ #[test]
+ fn conforming_a_crlf_page_does_not_duplicate_its_frontmatter() {
+ let tmp = TempDir::new().unwrap();
+ let rel = Path::new("w/p/notes/hand-written.md");
+ let abs = tmp.path().join(rel);
+ std::fs::create_dir_all(abs.parent().unwrap()).unwrap();
+ std::fs::write(
+ &abs,
+ "---\r\ntitle: Hand written\r\npinned: true\r\n---\r\n# Hand written\r\n\r\nBody.\r\n",
+ )
+ .unwrap();
+
+ let git = crate::git::GitAdapter::open_or_init(tmp.path()).unwrap();
+ conform_file(&git, rel, &std::collections::HashMap::new()).unwrap();
+
+ let conformed = std::fs::read_to_string(&abs).unwrap();
+ let md = parse(&conformed).unwrap();
+ assert_eq!(
+ md.frontmatter["title"], "Hand written",
+ "the authored frontmatter must survive the pass: {conformed:?}"
+ );
+ assert_eq!(md.frontmatter["pinned"], true);
+ assert_eq!(md.frontmatter["type"], "Note");
+ assert_eq!(
+ md.body, "# Hand written\r\n\r\nBody.\r\n",
+ "the body must not gain a second frontmatter block: {conformed:?}"
+ );
+ }
+
// ---- #633: the safety archive must be taken BEFORE the DB migration ----
/// The core assertion for #633: the pre-open snapshot captures the DB as it
diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rs
index 4dac00be..0c4c4efb 100644
--- a/crates/ai-memory-wiki/src/wiki.rs
+++ b/crates/ai-memory-wiki/src/wiki.rs
@@ -3464,6 +3464,36 @@ mod tests {
);
}
+ /// The same page saved with CRLF line endings (a Windows editor, or a
+ /// wiki checked out with `core.autocrlf=true`) must index the same way:
+ /// its frontmatter is still the source of truth for tier/pinned/title,
+ /// not body text ahead of the H1.
+ #[tokio::test]
+ async fn reindex_page_reads_crlf_frontmatter() {
+ let tmp = TempDir::new().unwrap();
+ let (store, wiki, ws, proj) = scoped(&tmp).await;
+
+ let path = PagePath::new("sessions/crlf.md").unwrap();
+ let abs = wiki.abs_path(ws, proj, &path);
+ std::fs::create_dir_all(abs.parent().unwrap()).unwrap();
+ std::fs::write(
+ &abs,
+ "---\r\ntitle: S\r\ntier: episodic\r\npinned: true\r\n---\r\nbody\r\n",
+ )
+ .unwrap();
+
+ wiki.reindex_page(ws, proj, path.clone()).await.unwrap();
+ let meta = store
+ .reader
+ .page_meta("default", "scratch", path.as_str())
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(meta.title, "S", "the frontmatter title must win");
+ assert_eq!(meta.tier, "episodic");
+ assert!(meta.pinned, "a CRLF `pinned: true` must still pin the page");
+ }
+
#[tokio::test]
async fn write_paths_persist_index_metadata_for_reindex_round_trip() {
let tmp = TempDir::new().unwrap();