Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(consolidate): share the matching-fence rule with profile harvesting

AkitaOnRails2be4bd2418c6
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 28e58432..59269ef5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -34,8 +34,10 @@ version in [`docs/changelog/`](docs/changelog/):
block that contains a different fence. A line of three backticks inside a
`~~~` block (or a shorter run inside a longer fence) ended the block early,
so the headings after it were missed, and a `replace_section` on the
- section before them replaced those sections too. A fence now closes only on
- a run of the same character that is at least as long. (#1316)
+ section before them replaced those sections too. Profile harvesting had the
+ same flaw: the code after such a line was read as prose and could be
+ harvested as a preference. A fence now closes only on a run of the same
+ character that is at least as long. (#1316)
- Fixed the thin-client commands ignoring a custom `bind`, which could write
into the wrong store. `server_url` defaulted to `http://127.0.0.1:49374`
and was never derived from `bind`, so `ai-memory --data-dir X status` (and
diff --git a/crates/ai-memory-consolidate/src/auto_improve.rs b/crates/ai-memory-consolidate/src/auto_improve.rs
index 3281a7d3..a41b9379 100644
--- a/crates/ai-memory-consolidate/src/auto_improve.rs
+++ b/crates/ai-memory-consolidate/src/auto_improve.rs
@@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize, de};
use sha2::{Digest, Sha256};
use thiserror::Error;
+use crate::code_fence::CodeFence;
use crate::path_sanitize::slugify_page_path;
use crate::projection::{ObservationProjectionConfig, cap_text_with_marker, project_observations};
@@ -1510,47 +1511,19 @@ fn parse_markdown_sections(body: &str) -> Vec<MarkdownSection> {
fn heading_positions(body: &str) -> Vec<(usize, usize, String)> {
let mut out = Vec::new();
let mut offset = 0;
- // The open fence's glyph and run length: only a run of the same glyph,
- // at least as long and with no info string, closes it (CommonMark §4.5,
- // as `ai_memory_wiki`'s `CodeFence` tracks it). Toggling on any fence
- // line let a lone ``` inside a ~~~ block end it early and hide every
- // heading after the block.
- let mut fence: Option<(char, usize)> = None;
+ let mut fence = None;
for line in body.split_inclusive('\n') {
let trimmed = line.trim_end_matches(['\r', '\n']);
- match (fence, fence_run(trimmed)) {
- // A backtick fence's info string cannot contain a backtick.
- (None, Some((glyph, len, info))) if glyph == '~' || !info.contains('`') => {
- fence = Some((glyph, len));
- }
- (Some((open, open_len)), Some((glyph, len, info)))
- if glyph == open && len >= open_len && info.trim().is_empty() =>
- {
- fence = None;
- }
- (None, _) => {
- if let Some(level) = heading_level(trimmed) {
- out.push((offset, level, trimmed.to_string()));
- }
- }
- (Some(_), _) => {}
+ let (next_fence, is_code) = CodeFence::step(fence, trimmed);
+ fence = next_fence;
+ if !is_code && let Some(level) = heading_level(trimmed) {
+ out.push((offset, level, trimmed.to_string()));
}
offset += line.len();
}
out
}
-/// The glyph, run length and info string of a fence line, if it is one.
-fn fence_run(line: &str) -> Option<(char, usize, &str)> {
- let trimmed = line.trim_start();
- let glyph = trimmed.chars().next().filter(|&c| matches!(c, '`' | '~'))?;
- let len = trimmed.chars().take_while(|&c| c == glyph).count();
- if len < 3 {
- return None;
- }
- Some((glyph, len, &trimmed[len..]))
-}
-
fn heading_level(line: &str) -> Option<usize> {
let hashes = line.chars().take_while(|c| *c == '#').count();
if (1..=6).contains(&hashes) && line.as_bytes().get(hashes) == Some(&b' ') {
diff --git a/crates/ai-memory-consolidate/src/code_fence.rs b/crates/ai-memory-consolidate/src/code_fence.rs
new file mode 100644
index 00000000..bdc4a593
--- /dev/null
+++ b/crates/ai-memory-consolidate/src/code_fence.rs
@@ -0,0 +1,91 @@
+//! Fenced code block tracking for the line scanners in this crate.
+//!
+//! Auto-improve's heading parser and profile harvesting both walk a page
+//! line by line and must skip fenced code. Toggling on any fence line lets
+//! a lone ```` ``` ```` inside a `~~~` block (or a shorter run inside a
+//! longer fence) end the block early, so code after it reads as headings
+//! or prose (#1316). This follows CommonMark §4.5 the way
+//! `ai_memory_wiki`'s link scanner does.
+
+/// The open fence's glyph and run length.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) struct CodeFence {
+ glyph: char,
+ len: usize,
+}
+
+impl CodeFence {
+ /// The fence state after `line`, and whether `line` is a fence line or
+ /// code inside a fence.
+ pub(crate) fn step(current: Option<Self>, line: &str) -> (Option<Self>, bool) {
+ let in_fence = current.is_some();
+ let trimmed = line.trim_start();
+ let glyph = match trimmed.chars().next() {
+ Some(c @ ('`' | '~')) => c,
+ _ => return (current, in_fence),
+ };
+ let len = trimmed.chars().take_while(|&c| c == glyph).count();
+ if len < 3 {
+ return (current, in_fence);
+ }
+ let info = &trimmed[len..];
+ match current {
+ // A backtick fence's info string cannot contain a backtick, so
+ // ```` ```code``` text ```` is a paragraph with an inline span.
+ None if glyph == '`' && info.contains('`') => (None, false),
+ None => (Some(Self { glyph, len }), true),
+ // Only a run of the same glyph, at least as long and with no
+ // info string, closes the fence.
+ Some(open) if open.glyph == glyph && len >= open.len && info.trim().is_empty() => {
+ (None, true)
+ }
+ Some(open) => (Some(open), true),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::CodeFence;
+
+ fn code_lines(text: &str) -> Vec<bool> {
+ let mut fence = None;
+ text.lines()
+ .map(|line| {
+ let (next, is_code) = CodeFence::step(fence, line);
+ fence = next;
+ is_code
+ })
+ .collect()
+ }
+
+ #[test]
+ fn a_fence_closes_only_on_its_own_glyph_with_a_run_at_least_as_long() {
+ assert_eq!(
+ code_lines("~~~sh\n```\nx\n~~~\nprose"),
+ [true, true, true, true, false]
+ );
+ assert_eq!(
+ code_lines("````md\n```\nx\n```\n````\nprose"),
+ [true, true, true, true, true, false]
+ );
+ assert_eq!(
+ code_lines("```\nx\n``` rust\ny\n```\nprose"),
+ [true, true, true, true, true, false]
+ );
+ }
+
+ #[test]
+ fn a_backtick_run_with_a_backtick_in_its_info_string_is_not_a_fence() {
+ assert_eq!(code_lines("```code``` text\nprose"), [false, false]);
+ assert_eq!(
+ code_lines("~~~ `ok`\nx\n~~~\nprose"),
+ [true, true, true, false]
+ );
+ }
+
+ #[test]
+ fn a_run_shorter_than_three_is_not_a_fence() {
+ assert_eq!(code_lines("``\nprose"), [false, false]);
+ }
+}
diff --git a/crates/ai-memory-consolidate/src/lib.rs b/crates/ai-memory-consolidate/src/lib.rs
index f9725c2d..5332df02 100644
--- a/crates/ai-memory-consolidate/src/lib.rs
+++ b/crates/ai-memory-consolidate/src/lib.rs
@@ -12,6 +12,7 @@ pub mod auto_improve;
pub mod auto_improve_schedule;
pub mod auto_improve_telemetry;
pub mod bootstrap;
+mod code_fence;
pub mod cold_cluster;
pub mod compaction;
pub mod consolidator;
diff --git a/crates/ai-memory-consolidate/src/profile.rs b/crates/ai-memory-consolidate/src/profile.rs
index 7b6c8df0..4c6662d5 100644
--- a/crates/ai-memory-consolidate/src/profile.rs
+++ b/crates/ai-memory-consolidate/src/profile.rs
@@ -46,6 +46,7 @@ use serde::Deserialize;
use sha2::{Digest, Sha256};
use crate::auto_improve_schedule::AUTO_IMPROVE_AUTO_APPROVE_AGENT;
+use crate::code_fence::CodeFence;
/// Frontmatter value stamping a page the harvester wrote.
pub const PROFILE_GENERATED_BY: &str = "profile-harvest";
@@ -591,16 +592,14 @@ fn without_fenced(text: &str, start: &str, end: &str) -> String {
/// trimmed and collapsed to one line.
fn sentences(text: &str) -> Vec<String> {
let mut out = Vec::new();
- let mut in_fence = false;
+ let mut fence = None;
for line in text.lines() {
- let trimmed = line.trim_start();
- if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
- in_fence = !in_fence;
- continue;
- }
- if in_fence || line.starts_with(" ") || line.starts_with('\t') {
+ let (next_fence, is_code) = CodeFence::step(fence, line);
+ fence = next_fence;
+ if is_code || line.starts_with(" ") || line.starts_with('\t') {
continue;
}
+ let trimmed = line.trim_start();
let mut current = String::new();
let mut chars = trimmed.chars().peekable();
while let Some(c) = chars.next() {
@@ -2771,6 +2770,21 @@ mod tests {
assert!(detect_preferences("The build always fails on Fridays.").is_empty());
}
+ #[test]
+ fn a_different_fence_inside_a_code_block_does_not_end_it() {
+ // A lone ``` inside a ~~~ block is code: the line after it is still
+ // inside the block and must not be harvested as a sentence.
+ let text = "~~~md\n```\nAlways use npm for installs.\n~~~\nPrefer pnpm.";
+ assert_eq!(sentences(text), ["Prefer pnpm."]);
+ let found = detect_preferences(text);
+ assert!(
+ found
+ .iter()
+ .all(|p| !p.statement.contains("npm for installs")),
+ "{found:?}"
+ );
+ }
+
#[test]
fn topic_overlap_groups_a_later_ruling_on_the_same_choice() {
let a = topic_tokens("Always use pnpm.");