Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(consolidate): share the matching-fence rule with profile harvesting
AkitaOnRails2be4bd2418c6
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 28e58432..59269ef5 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -34,8 +34,10 @@ version in [`docs/changelog/`](docs/changelog/):block that contains a different fence. A line of three backticks inside a`~~~` block (or a shorter run inside a longer fence) ended the block early,so the headings after it were missed, and a `replace_section` on the- section before them replaced those sections too. A fence now closes only on- a run of the same character that is at least as long. (#1316)+ section before them replaced those sections too. Profile harvesting had the+ same flaw: the code after such a line was read as prose and could be+ harvested as a preference. A fence now closes only on a run of the same+ character that is at least as long. (#1316)- Fixed the thin-client commands ignoring a custom `bind`, which could writeinto the wrong store. `server_url` defaulted to `http://127.0.0.1:49374`and was never derived from `bind`, so `ai-memory --data-dir X status` (anddiff --git a/crates/ai-memory-consolidate/src/auto_improve.rs b/crates/ai-memory-consolidate/src/auto_improve.rsindex 3281a7d3..a41b9379 100644--- a/crates/ai-memory-consolidate/src/auto_improve.rs+++ b/crates/ai-memory-consolidate/src/auto_improve.rs@@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize, de};use sha2::{Digest, Sha256};use thiserror::Error;+use crate::code_fence::CodeFence;use crate::path_sanitize::slugify_page_path;use crate::projection::{ObservationProjectionConfig, cap_text_with_marker, project_observations};@@ -1510,47 +1511,19 @@ fn parse_markdown_sections(body: &str) -> Vec<MarkdownSection> {fn heading_positions(body: &str) -> Vec<(usize, usize, String)> {let mut out = Vec::new();let mut offset = 0;- // The open fence's glyph and run length: only a run of the same glyph,- // at least as long and with no info string, closes it (CommonMark §4.5,- // as `ai_memory_wiki`'s `CodeFence` tracks it). Toggling on any fence- // line let a lone ``` inside a ~~~ block end it early and hide every- // heading after the block.- let mut fence: Option<(char, usize)> = None;+ let mut fence = None;for line in body.split_inclusive('\n') {let trimmed = line.trim_end_matches(['\r', '\n']);- match (fence, fence_run(trimmed)) {- // A backtick fence's info string cannot contain a backtick.- (None, Some((glyph, len, info))) if glyph == '~' || !info.contains('`') => {- fence = Some((glyph, len));- }- (Some((open, open_len)), Some((glyph, len, info)))- if glyph == open && len >= open_len && info.trim().is_empty() =>- {- fence = None;- }- (None, _) => {- if let Some(level) = heading_level(trimmed) {- out.push((offset, level, trimmed.to_string()));- }- }- (Some(_), _) => {}+ let (next_fence, is_code) = CodeFence::step(fence, trimmed);+ fence = next_fence;+ if !is_code && let Some(level) = heading_level(trimmed) {+ out.push((offset, level, trimmed.to_string()));}offset += line.len();}out}-/// The glyph, run length and info string of a fence line, if it is one.-fn fence_run(line: &str) -> Option<(char, usize, &str)> {- let trimmed = line.trim_start();- let glyph = trimmed.chars().next().filter(|&c| matches!(c, '`' | '~'))?;- let len = trimmed.chars().take_while(|&c| c == glyph).count();- if len < 3 {- return None;- }- Some((glyph, len, &trimmed[len..]))-}-fn heading_level(line: &str) -> Option<usize> {let hashes = line.chars().take_while(|c| *c == '#').count();if (1..=6).contains(&hashes) && line.as_bytes().get(hashes) == Some(&b' ') {diff --git a/crates/ai-memory-consolidate/src/code_fence.rs b/crates/ai-memory-consolidate/src/code_fence.rsnew file mode 100644index 00000000..bdc4a593--- /dev/null+++ b/crates/ai-memory-consolidate/src/code_fence.rs@@ -0,0 +1,91 @@+//! Fenced code block tracking for the line scanners in this crate.+//!+//! Auto-improve's heading parser and profile harvesting both walk a page+//! line by line and must skip fenced code. Toggling on any fence line lets+//! a lone ```` ``` ```` inside a `~~~` block (or a shorter run inside a+//! longer fence) end the block early, so code after it reads as headings+//! or prose (#1316). This follows CommonMark §4.5 the way+//! `ai_memory_wiki`'s link scanner does.++/// The open fence's glyph and run length.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub(crate) struct CodeFence {+ glyph: char,+ len: usize,+}++impl CodeFence {+ /// The fence state after `line`, and whether `line` is a fence line or+ /// code inside a fence.+ pub(crate) fn step(current: Option<Self>, line: &str) -> (Option<Self>, bool) {+ let in_fence = current.is_some();+ let trimmed = line.trim_start();+ let glyph = match trimmed.chars().next() {+ Some(c @ ('`' | '~')) => c,+ _ => return (current, in_fence),+ };+ let len = trimmed.chars().take_while(|&c| c == glyph).count();+ if len < 3 {+ return (current, in_fence);+ }+ let info = &trimmed[len..];+ match current {+ // A backtick fence's info string cannot contain a backtick, so+ // ```` ```code``` text ```` is a paragraph with an inline span.+ None if glyph == '`' && info.contains('`') => (None, false),+ None => (Some(Self { glyph, len }), true),+ // Only a run of the same glyph, at least as long and with no+ // info string, closes the fence.+ Some(open) if open.glyph == glyph && len >= open.len && info.trim().is_empty() => {+ (None, true)+ }+ Some(open) => (Some(open), true),+ }+ }+}++#[cfg(test)]+mod tests {+ use super::CodeFence;++ fn code_lines(text: &str) -> Vec<bool> {+ let mut fence = None;+ text.lines()+ .map(|line| {+ let (next, is_code) = CodeFence::step(fence, line);+ fence = next;+ is_code+ })+ .collect()+ }++ #[test]+ fn a_fence_closes_only_on_its_own_glyph_with_a_run_at_least_as_long() {+ assert_eq!(+ code_lines("~~~sh\n```\nx\n~~~\nprose"),+ [true, true, true, true, false]+ );+ assert_eq!(+ code_lines("````md\n```\nx\n```\n````\nprose"),+ [true, true, true, true, true, false]+ );+ assert_eq!(+ code_lines("```\nx\n``` rust\ny\n```\nprose"),+ [true, true, true, true, true, false]+ );+ }++ #[test]+ fn a_backtick_run_with_a_backtick_in_its_info_string_is_not_a_fence() {+ assert_eq!(code_lines("```code``` text\nprose"), [false, false]);+ assert_eq!(+ code_lines("~~~ `ok`\nx\n~~~\nprose"),+ [true, true, true, false]+ );+ }++ #[test]+ fn a_run_shorter_than_three_is_not_a_fence() {+ assert_eq!(code_lines("``\nprose"), [false, false]);+ }+}diff --git a/crates/ai-memory-consolidate/src/lib.rs b/crates/ai-memory-consolidate/src/lib.rsindex f9725c2d..5332df02 100644--- a/crates/ai-memory-consolidate/src/lib.rs+++ b/crates/ai-memory-consolidate/src/lib.rs@@ -12,6 +12,7 @@ pub mod auto_improve;pub mod auto_improve_schedule;pub mod auto_improve_telemetry;pub mod bootstrap;+mod code_fence;pub mod cold_cluster;pub mod compaction;pub mod consolidator;diff --git a/crates/ai-memory-consolidate/src/profile.rs b/crates/ai-memory-consolidate/src/profile.rsindex 7b6c8df0..4c6662d5 100644--- a/crates/ai-memory-consolidate/src/profile.rs+++ b/crates/ai-memory-consolidate/src/profile.rs@@ -46,6 +46,7 @@ use serde::Deserialize;use sha2::{Digest, Sha256};use crate::auto_improve_schedule::AUTO_IMPROVE_AUTO_APPROVE_AGENT;+use crate::code_fence::CodeFence;/// Frontmatter value stamping a page the harvester wrote.pub const PROFILE_GENERATED_BY: &str = "profile-harvest";@@ -591,16 +592,14 @@ fn without_fenced(text: &str, start: &str, end: &str) -> String {/// trimmed and collapsed to one line.fn sentences(text: &str) -> Vec<String> {let mut out = Vec::new();- let mut in_fence = false;+ let mut fence = None;for line in text.lines() {- let trimmed = line.trim_start();- if trimmed.starts_with("```") || trimmed.starts_with("~~~") {- in_fence = !in_fence;- continue;- }- if in_fence || line.starts_with(" ") || line.starts_with('\t') {+ let (next_fence, is_code) = CodeFence::step(fence, line);+ fence = next_fence;+ if is_code || line.starts_with(" ") || line.starts_with('\t') {continue;}+ let trimmed = line.trim_start();let mut current = String::new();let mut chars = trimmed.chars().peekable();while let Some(c) = chars.next() {@@ -2771,6 +2770,21 @@ mod tests {assert!(detect_preferences("The build always fails on Fridays.").is_empty());}+ #[test]+ fn a_different_fence_inside_a_code_block_does_not_end_it() {+ // A lone ``` inside a ~~~ block is code: the line after it is still+ // inside the block and must not be harvested as a sentence.+ let text = "~~~md\n```\nAlways use npm for installs.\n~~~\nPrefer pnpm.";+ assert_eq!(sentences(text), ["Prefer pnpm."]);+ let found = detect_preferences(text);+ assert!(+ found+ .iter()+ .all(|p| !p.statement.contains("npm for installs")),+ "{found:?}"+ );+ }+#[test]fn topic_overlap_groups_a_later_ruling_on_the_same_choice() {let a = topic_tokens("Always use pnpm.");