Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
M13: end-to-end handoff + recall smoke test + richer rule-based handoff
AkitaOnRailsf5a74d6c9c1d
diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rsindex 4c2cd210..7a0767d3 100644--- a/crates/ai-memory-hooks/src/router.rs+++ b/crates/ai-memory-hooks/src/router.rs@@ -299,35 +299,60 @@ fn build_auto_handoff(cwd: Option<String>,observations: &[ai_memory_core::Observation],) -> NewHandoff {- let mut prompts: Vec<&str> = Vec::new();+ // Prefer obs.body (the full prompt) over obs.title (first-line ++ // truncated to 80 chars for log/list display). When body is+ // empty fall back to title so we never produce an empty entry.+ fn pick_text(obs: &ai_memory_core::Observation) -> &str {+ if !obs.body.is_empty() {+ obs.body.as_str()+ } else {+ obs.title.as_str()+ }+ }+ /// Cap so a single 10-page prompt doesn't blow up the handoff.+ /// The body is already scrubbed at insert time; this is just a+ /// length budget. 1500 chars ≈ 250 words ≈ a paragraph.+ fn cap(s: &str) -> String {+ const MAX: usize = 1500;+ if s.chars().count() <= MAX {+ s.to_string()+ } else {+ let truncated: String = s.chars().take(MAX).collect();+ format!("{truncated}…")+ }+ }+ let mut prompts: Vec<String> = Vec::new();let mut tools: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();for obs in observations {match obs.kind {- ObservationKind::UserPrompt if !obs.title.is_empty() => prompts.push(&obs.title),+ ObservationKind::UserPrompt => {+ let text = pick_text(obs);+ if !text.is_empty() {+ prompts.push(text.to_string());+ }+ }ObservationKind::PostToolUse | ObservationKind::PreToolUse if !obs.title.is_empty() => {- tools.insert(&obs.title);+ tools.insert(obs.title.as_str());}_ => {}}}- let last_prompt = prompts.last().copied();- let summary = match (prompts.first().copied(), last_prompt) {- (Some(first), Some(last)) if first == last => format!("Session focused on: {first}"),- (Some(first), Some(last)) => format!("Started: {first}. Last: {last}."),- (Some(first), None) => format!("Started: {first}."),+ let first_prompt = prompts.first().cloned();+ let last_prompt = prompts.last().cloned();+ let summary = match (&first_prompt, &last_prompt) {+ (Some(first), Some(last)) if first == last => format!("Session focused on: {}", cap(first)),+ (Some(first), Some(last)) => format!("Started: {}\n\nLast: {}", cap(first), cap(last),),+ (Some(first), None) => format!("Started: {}", cap(first)),_ => format!("Session ended; {} observations recorded.",observations.len()),};- let open_questions = if prompts.is_empty() {- Vec::new()- } else {+ let open_questions = if let Some(last) = last_prompt {// Heuristic: last user prompt often *is* the open question.- vec![format!(- "Continue from: {}",- last_prompt.unwrap_or_default()- )]+ vec![format!("Continue from: {}", cap(&last))]+ } else {+ Vec::new()};let next_steps = if tools.is_empty() {Vec::new()diff --git a/tests/e2e/handoff_smoke.sh b/tests/e2e/handoff_smoke.shnew file mode 100755index 00000000..2667c504--- /dev/null+++ b/tests/e2e/handoff_smoke.sh@@ -0,0 +1,346 @@+#!/usr/bin/env bash+# End-to-end handoff + recall smoke test for ai-memory.+#+# What this validates:+#+# 1. Two real LLM calls against Gemini free-tier (different model+# variants per session so any recall must come from ai-memory, not+# a per-process cache or the same model "remembering" itself).+# 2. ai-memory's /hook ingress — observations land in the store.+# 3. ai-memory's auto-handoff creation at SessionEnd.+# 4. ai-memory's GET /handoff endpoint — the path SessionStart hooks+# use to surface prior context to the next agent CLI.+# 5. Sanitisation end-to-end — a planted "sk-canary-LEAK_ME_PLEASE_…"+# secret in session 1's prompt must NOT appear in any persisted+# state (observations, wiki pages, handoff body, on-disk markdown).+#+# Why direct Gemini REST and not opencode/codex/claude-code:+# - opencode's `run` mode pays a ~3 min bootstrap (mise/node + sqlite+# migration) per fresh data dir, and didn't reliably exit after the+# LLM responded in our smoke tests. Not viable for an automated+# test. The /hook endpoint is the same regardless of which agent+# CLI POSTs to it, so the test injects hook events directly while+# driving the LLM with the simplest possible client. The agent-CLI+# integrations are still documented + shipped (see hooks/) and+# exercised by the unit tests in ai-memory-hooks.+# - Gemini's free tier has per-user quotas (vs OpenRouter's shared+# pool which was returning 429s when this test was written) and+# responds in under 5 s for the prompts here.+#+# Required env: GEMINI_API_KEY. Get one free at+# https://aistudio.google.com/app/apikey (no credit card needed).+#+# Isolation: ai-memory's data dir + the on-the-fly opencode-style+# config live under a tempdir and are removed on exit. Re-runnable on+# any machine that has cargo + curl + jq.++set -euo pipefail++# --------------------------------------------------------------------+# Setup+# --------------------------------------------------------------------++REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+TEST_DIR="$(mktemp -d -t ai-memory-e2e-XXXXXX)"+LOG_FILE="$TEST_DIR/test.log"+SERVER_PID=""++cleanup() {+ local rc=$?+ if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then+ kill "$SERVER_PID" 2>/dev/null || true+ wait "$SERVER_PID" 2>/dev/null || true+ fi+ if [[ $rc -ne 0 ]]; then+ echo+ echo "==== TEST FAILED (rc=$rc) ===="+ echo "Preserved for inspection: $TEST_DIR"+ echo "Log: $LOG_FILE"+ else+ rm -rf "$TEST_DIR"+ fi+ exit $rc+}+trap cleanup EXIT INT TERM++step() { printf '\n=== %s ===\n' "$*"; }++require_cmd() {+ command -v "$1" >/dev/null 2>&1 || {+ echo "FATAL: required command '$1' not found in PATH" >&2+ exit 64+ }+}+require_cmd curl+require_cmd jq+require_cmd cargo++if [[ -z "${GEMINI_API_KEY:-}" ]]; then+ echo "FATAL: GEMINI_API_KEY not set." >&2+ echo " Get one (free) at https://aistudio.google.com/app/apikey" >&2+ exit 64+fi++# Free port for ai-memory.+PORT="$(python3 -c "import socket; s=socket.socket(); s.bind(('127.0.0.1',0)); print(s.getsockname()[1]); s.close()")"+SERVER_URL="http://127.0.0.1:$PORT"++# Two deliberately different Gemini variants. Both are free-tier;+# overridable via env when the defaults rotate out.+MODEL_A="${MODEL_A:-gemini-2.5-flash}"+MODEL_B="${MODEL_B:-gemini-2.5-flash-lite}"++# Isolate ai-memory's data dir; leave $HOME alone so cargo's target+# cache + the user's git config etc. stay accessible.+export AI_MEMORY_DATA_DIR="$TEST_DIR/ai-memory-data"+mkdir -p "$AI_MEMORY_DATA_DIR" "$TEST_DIR/blog"++# Mini blog project — non-coding topic so models don't "know" the+# answer absent the handoff.+cat >"$TEST_DIR/blog/TOPIC.md" <<'EOF'+# Blog draft: Rust Borrow Checker by Example+Plan (session 1):+- Three examples: closure captures + move, &mut aliasing, lifetime elision.+- ~1500 words, friendly tone.+EOF++# Canary that MUST be redacted end-to-end.+CANARY_KEY="sk-canary-LEAK_ME_PLEASE_e2e_smoketest_xxxxxxxxxxxx"++# --------------------------------------------------------------------+# Gemini helper+# --------------------------------------------------------------------++# Usage: gemini_call <model> <prompt-string>+# Echoes the model's text response to stdout. thinkingBudget=0+# disables reasoning tokens so maxOutputTokens is spent on the actual+# answer (gemini-2.5-flash is a reasoning model by default).+gemini_call() {+ local model="$1" prompt="$2"+ local body+ body="$(jq -n --arg p "$prompt" '{+ contents: [{ parts: [{ text: $p }] }],+ generationConfig: {+ maxOutputTokens: 800,+ thinkingConfig: { thinkingBudget: 0 }+ }+ }')"+ curl -sS --max-time 90 \+ "https://generativelanguage.googleapis.com/v1beta/models/$model:generateContent?key=$GEMINI_API_KEY" \+ -H "Content-Type: application/json" \+ -d "$body" \+ | jq -r '.candidates[0].content.parts[0].text // (.error.message | tostring | "GEMINI_ERROR: " + .)'+}++# --------------------------------------------------------------------+# Build + start ai-memory+# --------------------------------------------------------------------++step "Building ai-memory release binary"+cd "$REPO_ROOT"+cargo build --release --bin ai-memory --quiet 2>&1 | tee -a "$LOG_FILE"+AI_MEMORY="$REPO_ROOT/target/release/ai-memory"++step "Initialising ai-memory data dir at $AI_MEMORY_DATA_DIR"+"$AI_MEMORY" init >>"$LOG_FILE" 2>&1++step "Starting ai-memory server on $SERVER_URL"+"$AI_MEMORY" serve \+ --transport http --bind "127.0.0.1:$PORT" \+ --workspace e2e-test --project blog \+ >>"$LOG_FILE" 2>&1 &+SERVER_PID=$!+sleep 2+if ! kill -0 "$SERVER_PID" 2>/dev/null; then+ echo "FATAL: ai-memory server died on startup" >&2+ tail -30 "$LOG_FILE" >&2+ exit 1+fi+# Verify it's actually listening before continuing.+for _ in 1 2 3 4 5; do+ curl -sf -o /dev/null --max-time 2 "$SERVER_URL/mcp" && break+ sleep 1+done++# --------------------------------------------------------------------+# Session 1 — model A. Plant the plan + the canary.+# --------------------------------------------------------------------++SESSION_ID_1="$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid)"++step "Session 1 ($MODEL_A): session-start hook"+echo "{\"session_id\":\"$SESSION_ID_1\",\"cwd\":\"$TEST_DIR/blog\",\"model\":\"$MODEL_A\"}" \+ | curl -sS --max-time 3 \+ -X POST "$SERVER_URL/hook?event=session-start&agent=open-code" \+ -H "Content-Type: application/json" \+ --data-binary @- >>"$LOG_FILE" 2>&1++step "Session 1: invoke model A with the plan + canary"+S1_PROMPT="I'm planning a blog post titled 'Rust Borrow Checker by Example'.+Three concrete examples — (1) closure captures and the move keyword,+(2) double-mutable references and the &mut aliasing rule,+(3) lifetime elision pitfalls at function boundaries.+Length: about 1500 words. Tone: friendly and conversational.++For the ops dashboard I'm also storing this admin token:+$CANARY_KEY (do not share).++Acknowledge the plan in one short paragraph; no changes."+S1_RESPONSE="$(gemini_call "$MODEL_A" "$S1_PROMPT")"+echo "$S1_RESPONSE" >"$TEST_DIR/session1_response.txt"+echo "--- model A response (first 30 lines) ---"+echo "$S1_RESPONSE" | head -30+echo "--- end model A response ---"++# Tell ai-memory what the user prompt was so a meaningful handoff+# is built at SessionEnd.+step "Session 1: forward user-prompt + session-end hooks"+echo "{\"session_id\":\"$SESSION_ID_1\",\"prompt\":$(jq -Rs <<<"$S1_PROMPT")}" \+ | curl -sS --max-time 3 \+ -X POST "$SERVER_URL/hook?event=user-prompt&agent=open-code" \+ -H "Content-Type: application/json" \+ --data-binary @- >>"$LOG_FILE" 2>&1++# Synthetic "tool use" observation so the handoff's "next steps"+# section has tool names to surface.+for tool in "Read" "Edit" "Write"; do+ echo "{\"session_id\":\"$SESSION_ID_1\",\"tool\":\"$tool\"}" \+ | curl -sS --max-time 2 \+ -X POST "$SERVER_URL/hook?event=post-tool-use&agent=open-code" \+ -H "Content-Type: application/json" \+ --data-binary @- >>"$LOG_FILE" 2>&1+done++echo "{\"session_id\":\"$SESSION_ID_1\",\"cwd\":\"$TEST_DIR/blog\"}" \+ | curl -sS --max-time 10 \+ -X POST "$SERVER_URL/hook?event=session-end&agent=open-code" \+ -H "Content-Type: application/json" \+ --data-binary @- >>"$LOG_FILE" 2>&1++# Server work is async (writer actor + auto-commit). Give it a beat.+sleep 2++# --------------------------------------------------------------------+# Probe: handoff exists; canary already scrubbed at this point+# --------------------------------------------------------------------++step "Probe: fetch handoff via GET /handoff"+HANDOFF_MD="$(curl -sS --max-time 5 "$SERVER_URL/handoff?agent=open-code")"+echo "$HANDOFF_MD" >"$TEST_DIR/handoff.md"+echo "--- handoff body ---"+echo "$HANDOFF_MD"+echo "--- end handoff body ---"+[[ -n "$HANDOFF_MD" ]] || { echo "FAIL: /handoff returned empty" >&2; exit 1; }++# --------------------------------------------------------------------+# Session 2 — model B. The handoff is the only bridge.+# --------------------------------------------------------------------++SESSION_ID_2="$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid)"++step "Session 2 ($MODEL_B): session-start hook"+echo "{\"session_id\":\"$SESSION_ID_2\",\"cwd\":\"$TEST_DIR/blog\",\"model\":\"$MODEL_B\"}" \+ | curl -sS --max-time 3 \+ -X POST "$SERVER_URL/hook?event=session-start&agent=open-code" \+ -H "Content-Type: application/json" \+ --data-binary @- >>"$LOG_FILE" 2>&1++step "Session 2: invoke model B, with the handoff prepended"+S2_PROMPT="$(cat <<EOF+$HANDOFF_MD++---++The user has resumed work on the blog draft. Using only the handoff+above, summarise what was planned in the previous session: the title,+the three example topics, the length target, and the tone. Be+concrete — name each example. Then ask whether they want to start+drafting example 1.+EOF+)"+S2_RESPONSE="$(gemini_call "$MODEL_B" "$S2_PROMPT")"+echo "$S2_RESPONSE" >"$TEST_DIR/session2_response.txt"+echo "--- model B response ---"+echo "$S2_RESPONSE"+echo "--- end model B response ---"++# --------------------------------------------------------------------+# Assertions+# --------------------------------------------------------------------++PASS=0+FAIL=0+check_contains() {+ local label="$1" source="$2"+ shift 2+ local found=0+ for needle in "$@"; do+ if echo "$source" | grep -iqF "$needle"; then+ found=1+ break+ fi+ done+ if [[ $found -eq 1 ]]; then+ echo " PASS: $label"+ PASS=$((PASS+1))+ else+ echo " FAIL: $label (tried: $*)"+ FAIL=$((FAIL+1))+ fi+}++step "Recall assertions: model B output reflects session 1's plan"+check_contains "title (borrow checker)" "$S2_RESPONSE" "borrow checker" "borrowck" "borrow-checker"+check_contains "example 1 (closure / move)" "$S2_RESPONSE" "closure" "move"+check_contains "example 2 (mut aliasing)" "$S2_RESPONSE" "mutable" "&mut" "aliasing"+check_contains "example 3 (lifetime elision)" "$S2_RESPONSE" "lifetime" "elision"+check_contains "length target" "$S2_RESPONSE" "1500" "1,500" "fifteen hundred"+check_contains "tone descriptor" "$S2_RESPONSE" "friendly" "conversational" "casual"++step "Sanitisation assertions: canary NEVER in persisted state"+LEAKED=0+for path in "$AI_MEMORY_DATA_DIR/wiki" "$AI_MEMORY_DATA_DIR/db" "$TEST_DIR/handoff.md"; do+ if [[ -e "$path" ]] && grep -rqF "LEAK_ME_PLEASE" "$path" 2>/dev/null; then+ echo " FAIL: canary 'LEAK_ME_PLEASE' found under $path:"+ grep -rnF "LEAK_ME_PLEASE" "$path" 2>/dev/null | head -5+ LEAKED=1+ fi+done+if [[ $LEAKED -eq 0 ]]; then+ echo " PASS: canary scrubbed from wiki, db, and handoff body"+ PASS=$((PASS+1))+else+ FAIL=$((FAIL+1))+fi++# Also assert the canary text is in the model-B response only if the+# handoff didn't accidentally re-introduce it (it shouldn't, since+# build_auto_handoff extracts from already-scrubbed observations).+if echo "$S2_RESPONSE" | grep -qF "LEAK_ME_PLEASE"; then+ echo " FAIL: canary surfaced into model B's context via handoff"+ FAIL=$((FAIL+1))+else+ echo " PASS: canary not in model B's input context"+ PASS=$((PASS+1))+fi++step "Handoff content assertion"+if echo "$HANDOFF_MD" | grep -iqE "borrow|blog|rust|borrowck"; then+ echo " PASS: handoff body references the session-1 topic"+ PASS=$((PASS+1))+else+ echo " FAIL: handoff body looks unrelated to the session-1 topic"+ FAIL=$((FAIL+1))+fi++# --------------------------------------------------------------------+# Summary+# --------------------------------------------------------------------++echo+echo "================================="+echo " PASS: $PASS"+echo " FAIL: $FAIL"+echo "================================="+[[ $FAIL -gt 0 ]] && exit 1+exit 0