Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust
4.1 KB
Raw Blame
//! Fenced code block tracking for the line scanners in this crate.
//!
//! Auto-improve's heading parser and profile harvesting both walk a page
//! line by line and must skip fenced code. Toggling on any fence line lets
//! a lone ```` ``` ```` inside a `~~~` block (or a shorter run inside a
//! longer fence) end the block early, so code after it reads as headings
//! or prose (#1316). This follows CommonMark §4.5 the way
//! `ai_memory_wiki`'s link scanner does.

/// The open fence's glyph and run length.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct CodeFence {
    glyph: char,
    len: usize,
}

impl CodeFence {
    /// The fence state after `line`, and whether `line` is a fence line or
    /// code inside a fence.
    pub(crate) fn step(current: Option<Self>, line: &str) -> (Option<Self>, bool) {
        let in_fence = current.is_some();
        let leading_spaces = line.as_bytes().iter().take_while(|&&b| b == b' ').count();
        if leading_spaces > 3 {
            return (current, in_fence);
        }
        let trimmed = &line[leading_spaces..];
        let glyph = match trimmed.chars().next() {
            Some(c @ ('`' | '~')) => c,
            _ => return (current, in_fence),
        };
        let len = trimmed.chars().take_while(|&c| c == glyph).count();
        if len < 3 {
            return (current, in_fence);
        }
        let info = &trimmed[len..];
        match current {
            // A backtick fence's info string cannot contain a backtick, so
            // ```` ```code``` text ```` is a paragraph with an inline span.
            None if glyph == '`' && info.contains('`') => (None, false),
            None => (Some(Self { glyph, len }), true),
            // Only a run of the same glyph, at least as long and with no
            // info string, closes the fence.
            Some(open) if open.glyph == glyph && len >= open.len && info.trim().is_empty() => {
                (None, true)
            }
            Some(open) => (Some(open), true),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::CodeFence;

    fn code_lines(text: &str) -> Vec<bool> {
        let mut fence = None;
        text.lines()
            .map(|line| {
                let (next, is_code) = CodeFence::step(fence, line);
                fence = next;
                is_code
            })
            .collect()
    }

    #[test]
    fn a_fence_closes_only_on_its_own_glyph_with_a_run_at_least_as_long() {
        assert_eq!(
            code_lines("~~~sh\n```\nx\n~~~\nprose"),
            [true, true, true, true, false]
        );
        assert_eq!(
            code_lines("````md\n```\nx\n```\n````\nprose"),
            [true, true, true, true, true, false]
        );
        assert_eq!(
            code_lines("```\nx\n``` rust\ny\n```\nprose"),
            [true, true, true, true, true, false]
        );
    }

    #[test]
    fn a_backtick_run_with_a_backtick_in_its_info_string_is_not_a_fence() {
        assert_eq!(code_lines("```code``` text\nprose"), [false, false]);
        assert_eq!(
            code_lines("~~~ `ok`\nx\n~~~\nprose"),
            [true, true, true, false]
        );
    }

    #[test]
    fn a_run_shorter_than_three_is_not_a_fence() {
        assert_eq!(code_lines("``\nprose"), [false, false]);
    }

    #[test]
    fn only_zero_to_three_ascii_spaces_may_prefix_a_fence() {
        for spaces in 0..=3 {
            let prefix = " ".repeat(spaces);
            assert_eq!(
                code_lines(&format!("{prefix}````md\ninside\n{prefix}````\nafter")),
                [true, true, true, false],
                "{spaces} spaces"
            );
        }

        for prefix in ["    ", "\t", "\u{2003}"] {
            assert_eq!(
                code_lines(&format!("{prefix}````md\ninside\n{prefix}````\nafter")),
                [false, false, false, false],
                "prefix {prefix:?} must not delimit a fence"
            );
            assert_eq!(
                code_lines(&format!("````md\n{prefix}````\nstill code\n````\nafter")),
                [true, true, true, true, false],
                "prefix {prefix:?} must not close a fence"
            );
        }
    }
}