Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust
20.4 KB
Raw Blame
//! Replay LongMemEval haystack sessions through the real hook-ingest
//! surface (`POST /hook/batch`), at the same cadence the lifecycle hooks
//! produce in production: `session-start`, then per chat round a
//! `user-prompt-submit` (user turn) and a `stop` carrying the opt-in
//! assistant excerpt (assistant turn), then `session-end`.
//!
//! Fidelity notes, deliberately inherited from the production capture:
//!
//! - excerpts are bounded (~2 KB) at the privacy boundary, so evidence
//!   buried deeper in a single long turn is genuinely out of reach;
//! - each session's original date is prepended to turn text (production
//!   installs see wall-clock time; replayed histories must carry their
//!   own), mirroring what the official LongMemEval harness exposes to
//!   retrievers as session timestamps.
//!
//! Two ablation knobs change that shape for the decomposition (defaults
//! keep every published baseline reproducible):
//!
//! - [`DatePrefix`] moves or drops the per-turn date prefix, which takes
//!   about 40 of the 80 characters a zero-LLM session page keeps per turn;
//! - [`IngestMode`] can skip the hooks and write each haystack session as
//!   one wiki page through `POST /admin/write-page` (the real
//!   `Wiki::write_page` path), which isolates ranking from capture.

use std::time::{Duration, Instant};

use anyhow::{Context, Result, bail};
use serde_json::json;

use super::dataset::Question;
use super::server::EVAL_AUTH_TOKEN;

/// Everything in one benchmark run lives in this workspace.
pub const EVAL_WORKSPACE: &str = "longmemeval";

/// Server-side cap on batch items per request.
const MAX_BATCH: usize = 256;

/// Client-side mirror of the capture excerpt cap; anything longer is
/// truncated by the server anyway, and the assistant marker requires the
/// client to do the capping.
const EXCERPT_MAX_BYTES: usize = 2_000;

/// Where the replay puts each session's original date.
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
pub enum DatePrefix {
    /// `[session date: …]` before every turn (the published baselines).
    Turn,
    /// Once per session, as the `session-start` event's title; turns carry
    /// no prefix.
    SessionStart,
    /// Nowhere.
    None,
}

impl DatePrefix {
    pub fn as_str(self) -> &'static str {
        match self {
            Self::Turn => "turn",
            Self::SessionStart => "session-start",
            Self::None => "none",
        }
    }
}

/// How a question's haystack reaches the server.
#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)]
pub enum IngestMode {
    /// Replay through `/hook/batch` (the shipped capture path).
    Hooks,
    /// One page per session holding the full raw transcript: the
    /// ranking-only upper bound, comparable to whole-session harnesses.
    WholeSession,
    /// One page per session holding exactly the text capture keeps in
    /// observation bodies (user turns up to 16 KiB, assistant excerpts up
    /// to 2 KB, date prefix as configured): what an observation-evidence
    /// stream could rank on, aggregated per session.
    CapturedSession,
}

impl IngestMode {
    pub fn as_str(self) -> &'static str {
        match self {
            Self::Hooks => "hooks",
            Self::WholeSession => "whole-session",
            Self::CapturedSession => "captured-session",
        }
    }
}

/// Server-side user-prompt body cap (`USER_PROMPT_EXCERPT_MAX_BYTES`,
/// equal to the 16 KiB observation backstop).
const USER_PROMPT_MAX_BYTES: usize = 16 * 1024;

/// Timing and volume for one question's ingest.
#[derive(Debug, Clone, Default)]
pub struct IngestStats {
    /// Hook events posted, or pages written.
    pub items: usize,
    /// Wall clock for the whole question.
    pub elapsed: Duration,
    /// Round trip of every HTTP request (batch POST or page write).
    pub request_ms: Vec<u128>,
    /// Pages whose body exceeded the 16 KiB observation backstop (page
    /// modes only; the page write path has no such cap, see the report).
    pub docs_over_16kib: usize,
}

/// Project name for one benchmark question (its private haystack).
pub fn project_for(q: &Question) -> String {
    format!("lme-{}", q.question_id)
}

/// The stored session id for dataset session `raw` in `q`'s haystack.
///
/// LongMemEval reuses session ids across questions: 4,715 of the 23,566
/// scored haystack sessions (8 of them evidence) also sit in an earlier
/// question's haystack. Replayed under its raw id, such a session already
/// exists in the other question's project when the second replay arrives;
/// the store files its turns under the new project (follow-cwd routing)
/// but drops its `session-end` as naming a foreign scope, so the second
/// project never gets the session page. A real harness mints a fresh id
/// per session, so the replay namespaces the id by project and sends the
/// resulting UUID, which the store keeps verbatim.
pub fn stored_session_uuid(q: &Question, raw: &str) -> uuid::Uuid {
    uuid::Uuid::new_v5(
        &uuid::Uuid::NAMESPACE_OID,
        format!("{}\n{raw}", project_for(q)).as_bytes(),
    )
}

/// Truncate on a char boundary so the marker stays valid UTF-8.
fn cap_excerpt(s: &str) -> &str {
    cap_bytes(s, EXCERPT_MAX_BYTES)
}

fn cap_bytes(s: &str, max: usize) -> &str {
    if s.len() <= max {
        return s;
    }
    let mut end = max;
    while !s.is_char_boundary(end) {
        end -= 1;
    }
    &s[..end]
}

/// Turn text as the hook replay sends it under `prefix`.
fn prefixed(prefix: DatePrefix, date: &str, text: &str) -> String {
    match prefix {
        DatePrefix::Turn => format!("[session date: {date}] {text}"),
        DatePrefix::SessionStart | DatePrefix::None => text.to_string(),
    }
}

/// One haystack session as a page body for the page ingest modes. Every
/// turn renders as `[<role>] <text>`, joined by blank lines, under a
/// `[Source session date: …]` line (the same document the grep baseline
/// scores). `CapturedSession` caps each turn the way capture does and
/// places the date exactly where the hook replay under `prefix` would.
pub fn session_document(
    mode: IngestMode,
    prefix: DatePrefix,
    date: &str,
    turns: &[super::dataset::Turn],
) -> String {
    let rendered: Vec<String> = turns
        .iter()
        .map(|t| match mode {
            IngestMode::CapturedSession => {
                let text = match t.role.as_str() {
                    "assistant" => prefixed(prefix, date, cap_excerpt(&t.content)),
                    _ => prefixed(prefix, date, &t.content),
                };
                // The server caps the whole prompt field, prefix included.
                format!("[{}] {}", t.role, cap_bytes(&text, USER_PROMPT_MAX_BYTES))
            }
            _ => format!("[{}] {}", t.role, t.content),
        })
        .collect();
    // Captured text only carries a session-level date when the replay puts
    // one on `session-start`; per-turn prefixes are already in the turns.
    let header = !(mode == IngestMode::CapturedSession
        && matches!(prefix, DatePrefix::Turn | DatePrefix::None));
    if header {
        format!("[Source session date: {date}]\n{}", rendered.join("\n\n"))
    } else {
        rendered.join("\n\n")
    }
}

#[derive(serde::Serialize)]
struct BatchItem {
    url: String,
    body: serde_json::Value,
}

/// Build the full event replay for one question's haystack.
fn build_items(q: &Question, prefix: DatePrefix) -> Vec<BatchItem> {
    let project = project_for(q);
    let cwd = format!("/lme/{project}");
    let mut items = Vec::new();
    let mut push = |event: &str, session_id: &str, extra: serde_json::Value| {
        let mut url = format!(
            "/hook?event={event}&agent=claude-code&workspace={EVAL_WORKSPACE}&project={project}&session_id={session_id}"
        );
        if event == "stop" {
            url.push_str("&capture_assistant=1");
        }
        let mut body = json!({
            "session_id": session_id,
            "cwd": cwd,
        });
        if let serde_json::Value::Object(extra) = extra {
            body.as_object_mut()
                .expect("body is an object")
                .extend(extra);
        }
        items.push(BatchItem { url, body });
    };

    for (idx, session) in q.haystack_sessions.iter().enumerate() {
        let sid = &stored_session_uuid(q, &q.haystack_session_ids[idx]).to_string();
        let date = &q.haystack_dates[idx];
        let mut start = json!({ "hook_event_name": "SessionStart", "source": "startup" });
        if prefix == DatePrefix::SessionStart {
            // The session-start title lands in the page's raw-observation
            // list, so the date stays indexed once per session.
            start["title"] = json!(format!("session date: {date}"));
        }
        push("session-start", sid, start);
        for turn in session {
            match turn.role.as_str() {
                "user" => push(
                    "user-prompt-submit",
                    sid,
                    json!({
                        "hook_event_name": "UserPromptSubmit",
                        "prompt": prefixed(prefix, date, &turn.content),
                    }),
                ),
                "assistant" => push(
                    "stop",
                    sid,
                    json!({
                        "hook_event_name": "Stop",
                        "_ai_memory_assistant": {
                            "version": 1,
                            "excerpt": prefixed(prefix, date, cap_excerpt(&turn.content)),
                        },
                    }),
                ),
                other => {
                    tracing::warn!(role = other, "unknown role in dataset; skipping turn");
                }
            }
        }
        push(
            "session-end",
            sid,
            json!({ "hook_event_name": "SessionEnd", "reason": "exit" }),
        );
    }
    items
}

/// Replay one question's haystack. Retries items the server's per-source
/// rate limiting skipped, until everything is accepted — silently missing
/// haystack data would corrupt the benchmark.
pub async fn ingest_question(
    client: &reqwest::Client,
    base_url: &str,
    q: &Question,
    prefix: DatePrefix,
) -> Result<IngestStats> {
    let started = Instant::now();
    let mut request_ms = Vec::new();
    let items = build_items(q, prefix);
    let total = items.len();
    let mut pending: Vec<&BatchItem> = items.iter().collect();
    let mut attempts = 0usize;
    while !pending.is_empty() {
        attempts += 1;
        if attempts > 60 {
            bail!(
                "question {}: {} hook items still unaccepted after {} passes",
                q.question_id,
                pending.len(),
                attempts - 1
            );
        }
        let mut next = Vec::new();
        for chunk in pending.chunks(MAX_BATCH) {
            let sent = Instant::now();
            let resp = client
                .post(format!("{base_url}/hook/batch"))
                .bearer_auth(EVAL_AUTH_TOKEN)
                .json(&chunk)
                .send()
                .await
                .context("posting hook batch")?
                .error_for_status()
                .context("hook batch rejected")?;
            let ack: serde_json::Value = resp.json().await?;
            request_ms.push(sent.elapsed().as_millis());
            let accepted: Vec<usize> = match ack.get("accepted_indices") {
                Some(serde_json::Value::Array(idx)) => idx
                    .iter()
                    .filter_map(|v| v.as_u64().map(|v| v as usize))
                    .collect(),
                _ => {
                    let prefix = ack.get("accepted").and_then(|v| v.as_u64()).unwrap_or(0) as usize;
                    (0..prefix).collect()
                }
            };
            let accepted_set: std::collections::HashSet<usize> = accepted.into_iter().collect();
            for (i, item) in chunk.iter().enumerate() {
                if !accepted_set.contains(&i) {
                    next.push(*item);
                }
            }
        }
        if !next.is_empty() {
            tokio::time::sleep(Duration::from_millis(500)).await;
        }
        pending = next;
    }
    Ok(IngestStats {
        items: total,
        elapsed: started.elapsed(),
        request_ms,
        docs_over_16kib: 0,
    })
}

/// Page title for a session written by the page ingest modes. Neutral on
/// purpose: it carries the date (as the hook pages' titles do) and nothing
/// that could favour the evidence session.
fn session_page_title(date: &str) -> String {
    format!("Chat session {date}")
}

/// Write each haystack session of `q` as one page under
/// `sessions/<uuid>.md` through `POST /admin/write-page`, sequentially
/// (questions still run concurrently). A repeated session id is written
/// once, at its first occurrence.
pub async fn write_session_pages(
    client: &reqwest::Client,
    base_url: &str,
    q: &Question,
    mode: IngestMode,
    prefix: DatePrefix,
) -> Result<IngestStats> {
    let started = Instant::now();
    let project = project_for(q);
    let mut seen = std::collections::HashSet::new();
    let mut stats = IngestStats::default();
    for ((sid, date), turns) in q
        .haystack_session_ids
        .iter()
        .zip(&q.haystack_dates)
        .zip(&q.haystack_sessions)
    {
        if !seen.insert(sid.as_str()) {
            continue;
        }
        let body = session_document(mode, prefix, date, turns);
        if body.len() > USER_PROMPT_MAX_BYTES {
            stats.docs_over_16kib += 1;
        }
        let request = json!({
            "workspace": EVAL_WORKSPACE,
            "project": project,
            "path": format!("sessions/{}.md", stored_session_uuid(q, sid)),
            "title": session_page_title(date),
            "body": body,
            // Hook-synthesised session pages are episodic too.
            "tier": "episodic",
        });
        let sent = Instant::now();
        client
            .post(format!("{base_url}/admin/write-page"))
            .bearer_auth(EVAL_AUTH_TOKEN)
            .json(&request)
            .send()
            .await
            .context("posting write-page")?
            .error_for_status()
            .with_context(|| format!("write-page rejected for {sid}"))?;
        stats.request_ms.push(sent.elapsed().as_millis());
        stats.items += 1;
    }
    stats.elapsed = started.elapsed();
    Ok(stats)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::retrieval::dataset::Turn;

    fn q() -> Question {
        serde_json::from_value(serde_json::json!({
            "question_id": "qX",
            "question_type": "single-session-user",
            "question": "?",
            "answer": "a",
            "question_date": "d",
            "haystack_dates": ["2023/05/20 (Sat) 02:21"],
            "haystack_session_ids": ["sharegpt_abc_0"],
            "haystack_sessions": [[
                {"role": "user", "content": "hello"},
                {"role": "assistant", "content": "hi"}
            ]],
            "answer_session_ids": ["sharegpt_abc_0"]
        }))
        .unwrap()
    }

    #[test]
    fn replay_matches_the_production_hook_cadence() {
        let items = build_items(&q(), DatePrefix::Turn);
        let events: Vec<&str> = items
            .iter()
            .map(|i| {
                i.url
                    .split("event=")
                    .nth(1)
                    .unwrap()
                    .split('&')
                    .next()
                    .unwrap()
            })
            .collect();
        assert_eq!(
            events,
            ["session-start", "user-prompt-submit", "stop", "session-end"]
        );
        // stop carries the opt-in marker and the capture flag
        let stop = &items[2];
        assert!(stop.url.contains("capture_assistant=1"));
        assert_eq!(stop.body["_ai_memory_assistant"]["version"], 1);
        // every item scopes explicitly
        for item in &items {
            assert!(item.url.contains("workspace=longmemeval"));
            assert!(item.url.contains("project=lme-qX"));
        }
    }

    #[test]
    fn session_dates_are_visible_to_the_retriever() {
        let items = build_items(&q(), DatePrefix::Turn);
        let prompt = items[1].body["prompt"].as_str().unwrap();
        assert!(prompt.starts_with("[session date: 2023/05/20"));
    }

    #[test]
    fn excerpts_are_capped_on_char_boundaries() {
        let long = format!("{}é", "x".repeat(EXCERPT_MAX_BYTES - 1));
        let capped = cap_excerpt(&long);
        assert!(capped.len() <= EXCERPT_MAX_BYTES);
        assert!(std::str::from_utf8(capped.as_bytes()).is_ok());
    }

    #[test]
    fn replayed_session_ids_are_per_question_uuids() {
        let first = q();
        let mut second = q();
        second.question_id = "qY".into();
        let a = stored_session_uuid(&first, "sharegpt_abc_0");
        let b = stored_session_uuid(&second, "sharegpt_abc_0");
        // A session shared by two haystacks must not share a stored id.
        assert_ne!(a, b);
        assert_eq!(a, stored_session_uuid(&first, "sharegpt_abc_0"));
        // The replay sends the UUID itself, which the store keeps verbatim.
        for item in build_items(&first, DatePrefix::Turn) {
            assert_eq!(
                item.body["session_id"].as_str(),
                Some(a.to_string().as_str())
            );
            assert!(item.url.contains(&format!("session_id={a}")));
        }
    }

    #[test]
    fn unknown_roles_are_skipped_not_mislabelled() {
        let mut question = q();
        question.haystack_sessions[0].push(Turn {
            role: "system".into(),
            content: "x".into(),
            has_answer: None,
        });
        let items = build_items(&question, DatePrefix::Turn);
        // still start, prompt, stop, end — the system turn produced nothing
        assert_eq!(items.len(), 4);
    }

    #[test]
    fn session_start_prefix_moves_the_date_off_the_turns() {
        let items = build_items(&q(), DatePrefix::SessionStart);
        assert_eq!(
            items[0].body["title"].as_str(),
            Some("session date: 2023/05/20 (Sat) 02:21")
        );
        assert_eq!(items[1].body["prompt"].as_str(), Some("hello"));
        assert_eq!(
            items[2].body["_ai_memory_assistant"]["excerpt"].as_str(),
            Some("hi")
        );
    }

    #[test]
    fn no_date_prefix_leaves_turns_and_session_start_bare() {
        let items = build_items(&q(), DatePrefix::None);
        assert!(items[0].body.get("title").is_none());
        assert_eq!(items[1].body["prompt"].as_str(), Some("hello"));
    }

    #[test]
    fn whole_session_documents_keep_every_byte_of_every_turn() {
        let long = "y".repeat(EXCERPT_MAX_BYTES * 3);
        let turns = vec![
            Turn {
                role: "user".into(),
                content: "hello".into(),
                has_answer: None,
            },
            Turn {
                role: "assistant".into(),
                content: long.clone(),
                has_answer: None,
            },
        ];
        let doc = session_document(IngestMode::WholeSession, DatePrefix::Turn, "D", &turns);
        assert_eq!(
            doc,
            format!("[Source session date: D]\n[user] hello\n\n[assistant] {long}")
        );
    }

    #[test]
    fn captured_session_documents_apply_the_capture_caps_and_prefix() {
        let turns = vec![
            Turn {
                role: "user".into(),
                content: "u".repeat(USER_PROMPT_MAX_BYTES + 10),
                has_answer: None,
            },
            Turn {
                role: "assistant".into(),
                content: "a".repeat(EXCERPT_MAX_BYTES + 10),
                has_answer: None,
            },
        ];
        let doc = session_document(IngestMode::CapturedSession, DatePrefix::Turn, "D", &turns);
        let parts: Vec<&str> = doc.split("\n\n").collect();
        let user = parts[0].strip_prefix("[user] ").unwrap();
        assert!(user.starts_with("[session date: D] u"));
        assert_eq!(user.len(), USER_PROMPT_MAX_BYTES);
        let assistant = parts[1]
            .strip_prefix("[assistant] [session date: D] ")
            .unwrap();
        assert_eq!(assistant.len(), EXCERPT_MAX_BYTES);

        let bare = session_document(IngestMode::CapturedSession, DatePrefix::None, "D", &turns);
        assert!(bare.starts_with("[user] uuu"));
        let once = session_document(
            IngestMode::CapturedSession,
            DatePrefix::SessionStart,
            "D",
            &turns,
        );
        assert!(once.starts_with("[Source session date: D]\n[user] uuu"));
    }
}