Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

M13: end-to-end handoff + recall smoke test + richer rule-based handoff

AkitaOnRailsf5a74d6c9c1d
diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rs
index 4c2cd210..7a0767d3 100644
--- a/crates/ai-memory-hooks/src/router.rs
+++ b/crates/ai-memory-hooks/src/router.rs
@@ -299,35 +299,60 @@ fn build_auto_handoff(
cwd: Option<String>,
observations: &[ai_memory_core::Observation],
) -> NewHandoff {
- let mut prompts: Vec<&str> = Vec::new();
+ // Prefer obs.body (the full prompt) over obs.title (first-line +
+ // truncated to 80 chars for log/list display). When body is
+ // empty fall back to title so we never produce an empty entry.
+ fn pick_text(obs: &ai_memory_core::Observation) -> &str {
+ if !obs.body.is_empty() {
+ obs.body.as_str()
+ } else {
+ obs.title.as_str()
+ }
+ }
+ /// Cap so a single 10-page prompt doesn't blow up the handoff.
+ /// The body is already scrubbed at insert time; this is just a
+ /// length budget. 1500 chars ≈ 250 words ≈ a paragraph.
+ fn cap(s: &str) -> String {
+ const MAX: usize = 1500;
+ if s.chars().count() <= MAX {
+ s.to_string()
+ } else {
+ let truncated: String = s.chars().take(MAX).collect();
+ format!("{truncated}…")
+ }
+ }
+ let mut prompts: Vec<String> = Vec::new();
let mut tools: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();
for obs in observations {
match obs.kind {
- ObservationKind::UserPrompt if !obs.title.is_empty() => prompts.push(&obs.title),
+ ObservationKind::UserPrompt => {
+ let text = pick_text(obs);
+ if !text.is_empty() {
+ prompts.push(text.to_string());
+ }
+ }
ObservationKind::PostToolUse | ObservationKind::PreToolUse if !obs.title.is_empty() => {
- tools.insert(&obs.title);
+ tools.insert(obs.title.as_str());
}
_ => {}
}
}
- let last_prompt = prompts.last().copied();
- let summary = match (prompts.first().copied(), last_prompt) {
- (Some(first), Some(last)) if first == last => format!("Session focused on: {first}"),
- (Some(first), Some(last)) => format!("Started: {first}. Last: {last}."),
- (Some(first), None) => format!("Started: {first}."),
+ let first_prompt = prompts.first().cloned();
+ let last_prompt = prompts.last().cloned();
+ let summary = match (&first_prompt, &last_prompt) {
+ (Some(first), Some(last)) if first == last => format!("Session focused on: {}", cap(first)),
+ (Some(first), Some(last)) => format!("Started: {}\n\nLast: {}", cap(first), cap(last),),
+ (Some(first), None) => format!("Started: {}", cap(first)),
_ => format!(
"Session ended; {} observations recorded.",
observations.len()
),
};
- let open_questions = if prompts.is_empty() {
- Vec::new()
- } else {
+ let open_questions = if let Some(last) = last_prompt {
// Heuristic: last user prompt often *is* the open question.
- vec![format!(
- "Continue from: {}",
- last_prompt.unwrap_or_default()
- )]
+ vec![format!("Continue from: {}", cap(&last))]
+ } else {
+ Vec::new()
};
let next_steps = if tools.is_empty() {
Vec::new()
diff --git a/tests/e2e/handoff_smoke.sh b/tests/e2e/handoff_smoke.sh
new file mode 100755
index 00000000..2667c504
--- /dev/null
+++ b/tests/e2e/handoff_smoke.sh
@@ -0,0 +1,346 @@
+#!/usr/bin/env bash
+# End-to-end handoff + recall smoke test for ai-memory.
+#
+# What this validates:
+#
+# 1. Two real LLM calls against Gemini free-tier (different model
+# variants per session so any recall must come from ai-memory, not
+# a per-process cache or the same model "remembering" itself).
+# 2. ai-memory's /hook ingress — observations land in the store.
+# 3. ai-memory's auto-handoff creation at SessionEnd.
+# 4. ai-memory's GET /handoff endpoint — the path SessionStart hooks
+# use to surface prior context to the next agent CLI.
+# 5. Sanitisation end-to-end — a planted "sk-canary-LEAK_ME_PLEASE_…"
+# secret in session 1's prompt must NOT appear in any persisted
+# state (observations, wiki pages, handoff body, on-disk markdown).
+#
+# Why direct Gemini REST and not opencode/codex/claude-code:
+# - opencode's `run` mode pays a ~3 min bootstrap (mise/node + sqlite
+# migration) per fresh data dir, and didn't reliably exit after the
+# LLM responded in our smoke tests. Not viable for an automated
+# test. The /hook endpoint is the same regardless of which agent
+# CLI POSTs to it, so the test injects hook events directly while
+# driving the LLM with the simplest possible client. The agent-CLI
+# integrations are still documented + shipped (see hooks/) and
+# exercised by the unit tests in ai-memory-hooks.
+# - Gemini's free tier has per-user quotas (vs OpenRouter's shared
+# pool which was returning 429s when this test was written) and
+# responds in under 5 s for the prompts here.
+#
+# Required env: GEMINI_API_KEY. Get one free at
+# https://aistudio.google.com/app/apikey (no credit card needed).
+#
+# Isolation: ai-memory's data dir + the on-the-fly opencode-style
+# config live under a tempdir and are removed on exit. Re-runnable on
+# any machine that has cargo + curl + jq.
+
+set -euo pipefail
+
+# --------------------------------------------------------------------
+# Setup
+# --------------------------------------------------------------------
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+TEST_DIR="$(mktemp -d -t ai-memory-e2e-XXXXXX)"
+LOG_FILE="$TEST_DIR/test.log"
+SERVER_PID=""
+
+cleanup() {
+ local rc=$?
+ if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then
+ kill "$SERVER_PID" 2>/dev/null || true
+ wait "$SERVER_PID" 2>/dev/null || true
+ fi
+ if [[ $rc -ne 0 ]]; then
+ echo
+ echo "==== TEST FAILED (rc=$rc) ===="
+ echo "Preserved for inspection: $TEST_DIR"
+ echo "Log: $LOG_FILE"
+ else
+ rm -rf "$TEST_DIR"
+ fi
+ exit $rc
+}
+trap cleanup EXIT INT TERM
+
+step() { printf '\n=== %s ===\n' "$*"; }
+
+require_cmd() {
+ command -v "$1" >/dev/null 2>&1 || {
+ echo "FATAL: required command '$1' not found in PATH" >&2
+ exit 64
+ }
+}
+require_cmd curl
+require_cmd jq
+require_cmd cargo
+
+if [[ -z "${GEMINI_API_KEY:-}" ]]; then
+ echo "FATAL: GEMINI_API_KEY not set." >&2
+ echo " Get one (free) at https://aistudio.google.com/app/apikey" >&2
+ exit 64
+fi
+
+# Free port for ai-memory.
+PORT="$(python3 -c "import socket; s=socket.socket(); s.bind(('127.0.0.1',0)); print(s.getsockname()[1]); s.close()")"
+SERVER_URL="http://127.0.0.1:$PORT"
+
+# Two deliberately different Gemini variants. Both are free-tier;
+# overridable via env when the defaults rotate out.
+MODEL_A="${MODEL_A:-gemini-2.5-flash}"
+MODEL_B="${MODEL_B:-gemini-2.5-flash-lite}"
+
+# Isolate ai-memory's data dir; leave $HOME alone so cargo's target
+# cache + the user's git config etc. stay accessible.
+export AI_MEMORY_DATA_DIR="$TEST_DIR/ai-memory-data"
+mkdir -p "$AI_MEMORY_DATA_DIR" "$TEST_DIR/blog"
+
+# Mini blog project — non-coding topic so models don't "know" the
+# answer absent the handoff.
+cat >"$TEST_DIR/blog/TOPIC.md" <<'EOF'
+# Blog draft: Rust Borrow Checker by Example
+Plan (session 1):
+- Three examples: closure captures + move, &mut aliasing, lifetime elision.
+- ~1500 words, friendly tone.
+EOF
+
+# Canary that MUST be redacted end-to-end.
+CANARY_KEY="sk-canary-LEAK_ME_PLEASE_e2e_smoketest_xxxxxxxxxxxx"
+
+# --------------------------------------------------------------------
+# Gemini helper
+# --------------------------------------------------------------------
+
+# Usage: gemini_call <model> <prompt-string>
+# Echoes the model's text response to stdout. thinkingBudget=0
+# disables reasoning tokens so maxOutputTokens is spent on the actual
+# answer (gemini-2.5-flash is a reasoning model by default).
+gemini_call() {
+ local model="$1" prompt="$2"
+ local body
+ body="$(jq -n --arg p "$prompt" '{
+ contents: [{ parts: [{ text: $p }] }],
+ generationConfig: {
+ maxOutputTokens: 800,
+ thinkingConfig: { thinkingBudget: 0 }
+ }
+ }')"
+ curl -sS --max-time 90 \
+ "https://generativelanguage.googleapis.com/v1beta/models/$model:generateContent?key=$GEMINI_API_KEY" \
+ -H "Content-Type: application/json" \
+ -d "$body" \
+ | jq -r '.candidates[0].content.parts[0].text // (.error.message | tostring | "GEMINI_ERROR: " + .)'
+}
+
+# --------------------------------------------------------------------
+# Build + start ai-memory
+# --------------------------------------------------------------------
+
+step "Building ai-memory release binary"
+cd "$REPO_ROOT"
+cargo build --release --bin ai-memory --quiet 2>&1 | tee -a "$LOG_FILE"
+AI_MEMORY="$REPO_ROOT/target/release/ai-memory"
+
+step "Initialising ai-memory data dir at $AI_MEMORY_DATA_DIR"
+"$AI_MEMORY" init >>"$LOG_FILE" 2>&1
+
+step "Starting ai-memory server on $SERVER_URL"
+"$AI_MEMORY" serve \
+ --transport http --bind "127.0.0.1:$PORT" \
+ --workspace e2e-test --project blog \
+ >>"$LOG_FILE" 2>&1 &
+SERVER_PID=$!
+sleep 2
+if ! kill -0 "$SERVER_PID" 2>/dev/null; then
+ echo "FATAL: ai-memory server died on startup" >&2
+ tail -30 "$LOG_FILE" >&2
+ exit 1
+fi
+# Verify it's actually listening before continuing.
+for _ in 1 2 3 4 5; do
+ curl -sf -o /dev/null --max-time 2 "$SERVER_URL/mcp" && break
+ sleep 1
+done
+
+# --------------------------------------------------------------------
+# Session 1 — model A. Plant the plan + the canary.
+# --------------------------------------------------------------------
+
+SESSION_ID_1="$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid)"
+
+step "Session 1 ($MODEL_A): session-start hook"
+echo "{\"session_id\":\"$SESSION_ID_1\",\"cwd\":\"$TEST_DIR/blog\",\"model\":\"$MODEL_A\"}" \
+ | curl -sS --max-time 3 \
+ -X POST "$SERVER_URL/hook?event=session-start&agent=open-code" \
+ -H "Content-Type: application/json" \
+ --data-binary @- >>"$LOG_FILE" 2>&1
+
+step "Session 1: invoke model A with the plan + canary"
+S1_PROMPT="I'm planning a blog post titled 'Rust Borrow Checker by Example'.
+Three concrete examples — (1) closure captures and the move keyword,
+(2) double-mutable references and the &mut aliasing rule,
+(3) lifetime elision pitfalls at function boundaries.
+Length: about 1500 words. Tone: friendly and conversational.
+
+For the ops dashboard I'm also storing this admin token:
+$CANARY_KEY (do not share).
+
+Acknowledge the plan in one short paragraph; no changes."
+S1_RESPONSE="$(gemini_call "$MODEL_A" "$S1_PROMPT")"
+echo "$S1_RESPONSE" >"$TEST_DIR/session1_response.txt"
+echo "--- model A response (first 30 lines) ---"
+echo "$S1_RESPONSE" | head -30
+echo "--- end model A response ---"
+
+# Tell ai-memory what the user prompt was so a meaningful handoff
+# is built at SessionEnd.
+step "Session 1: forward user-prompt + session-end hooks"
+echo "{\"session_id\":\"$SESSION_ID_1\",\"prompt\":$(jq -Rs <<<"$S1_PROMPT")}" \
+ | curl -sS --max-time 3 \
+ -X POST "$SERVER_URL/hook?event=user-prompt&agent=open-code" \
+ -H "Content-Type: application/json" \
+ --data-binary @- >>"$LOG_FILE" 2>&1
+
+# Synthetic "tool use" observation so the handoff's "next steps"
+# section has tool names to surface.
+for tool in "Read" "Edit" "Write"; do
+ echo "{\"session_id\":\"$SESSION_ID_1\",\"tool\":\"$tool\"}" \
+ | curl -sS --max-time 2 \
+ -X POST "$SERVER_URL/hook?event=post-tool-use&agent=open-code" \
+ -H "Content-Type: application/json" \
+ --data-binary @- >>"$LOG_FILE" 2>&1
+done
+
+echo "{\"session_id\":\"$SESSION_ID_1\",\"cwd\":\"$TEST_DIR/blog\"}" \
+ | curl -sS --max-time 10 \
+ -X POST "$SERVER_URL/hook?event=session-end&agent=open-code" \
+ -H "Content-Type: application/json" \
+ --data-binary @- >>"$LOG_FILE" 2>&1
+
+# Server work is async (writer actor + auto-commit). Give it a beat.
+sleep 2
+
+# --------------------------------------------------------------------
+# Probe: handoff exists; canary already scrubbed at this point
+# --------------------------------------------------------------------
+
+step "Probe: fetch handoff via GET /handoff"
+HANDOFF_MD="$(curl -sS --max-time 5 "$SERVER_URL/handoff?agent=open-code")"
+echo "$HANDOFF_MD" >"$TEST_DIR/handoff.md"
+echo "--- handoff body ---"
+echo "$HANDOFF_MD"
+echo "--- end handoff body ---"
+[[ -n "$HANDOFF_MD" ]] || { echo "FAIL: /handoff returned empty" >&2; exit 1; }
+
+# --------------------------------------------------------------------
+# Session 2 — model B. The handoff is the only bridge.
+# --------------------------------------------------------------------
+
+SESSION_ID_2="$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid)"
+
+step "Session 2 ($MODEL_B): session-start hook"
+echo "{\"session_id\":\"$SESSION_ID_2\",\"cwd\":\"$TEST_DIR/blog\",\"model\":\"$MODEL_B\"}" \
+ | curl -sS --max-time 3 \
+ -X POST "$SERVER_URL/hook?event=session-start&agent=open-code" \
+ -H "Content-Type: application/json" \
+ --data-binary @- >>"$LOG_FILE" 2>&1
+
+step "Session 2: invoke model B, with the handoff prepended"
+S2_PROMPT="$(cat <<EOF
+$HANDOFF_MD
+
+---
+
+The user has resumed work on the blog draft. Using only the handoff
+above, summarise what was planned in the previous session: the title,
+the three example topics, the length target, and the tone. Be
+concrete — name each example. Then ask whether they want to start
+drafting example 1.
+EOF
+)"
+S2_RESPONSE="$(gemini_call "$MODEL_B" "$S2_PROMPT")"
+echo "$S2_RESPONSE" >"$TEST_DIR/session2_response.txt"
+echo "--- model B response ---"
+echo "$S2_RESPONSE"
+echo "--- end model B response ---"
+
+# --------------------------------------------------------------------
+# Assertions
+# --------------------------------------------------------------------
+
+PASS=0
+FAIL=0
+check_contains() {
+ local label="$1" source="$2"
+ shift 2
+ local found=0
+ for needle in "$@"; do
+ if echo "$source" | grep -iqF "$needle"; then
+ found=1
+ break
+ fi
+ done
+ if [[ $found -eq 1 ]]; then
+ echo " PASS: $label"
+ PASS=$((PASS+1))
+ else
+ echo " FAIL: $label (tried: $*)"
+ FAIL=$((FAIL+1))
+ fi
+}
+
+step "Recall assertions: model B output reflects session 1's plan"
+check_contains "title (borrow checker)" "$S2_RESPONSE" "borrow checker" "borrowck" "borrow-checker"
+check_contains "example 1 (closure / move)" "$S2_RESPONSE" "closure" "move"
+check_contains "example 2 (mut aliasing)" "$S2_RESPONSE" "mutable" "&mut" "aliasing"
+check_contains "example 3 (lifetime elision)" "$S2_RESPONSE" "lifetime" "elision"
+check_contains "length target" "$S2_RESPONSE" "1500" "1,500" "fifteen hundred"
+check_contains "tone descriptor" "$S2_RESPONSE" "friendly" "conversational" "casual"
+
+step "Sanitisation assertions: canary NEVER in persisted state"
+LEAKED=0
+for path in "$AI_MEMORY_DATA_DIR/wiki" "$AI_MEMORY_DATA_DIR/db" "$TEST_DIR/handoff.md"; do
+ if [[ -e "$path" ]] && grep -rqF "LEAK_ME_PLEASE" "$path" 2>/dev/null; then
+ echo " FAIL: canary 'LEAK_ME_PLEASE' found under $path:"
+ grep -rnF "LEAK_ME_PLEASE" "$path" 2>/dev/null | head -5
+ LEAKED=1
+ fi
+done
+if [[ $LEAKED -eq 0 ]]; then
+ echo " PASS: canary scrubbed from wiki, db, and handoff body"
+ PASS=$((PASS+1))
+else
+ FAIL=$((FAIL+1))
+fi
+
+# Also assert the canary text is in the model-B response only if the
+# handoff didn't accidentally re-introduce it (it shouldn't, since
+# build_auto_handoff extracts from already-scrubbed observations).
+if echo "$S2_RESPONSE" | grep -qF "LEAK_ME_PLEASE"; then
+ echo " FAIL: canary surfaced into model B's context via handoff"
+ FAIL=$((FAIL+1))
+else
+ echo " PASS: canary not in model B's input context"
+ PASS=$((PASS+1))
+fi
+
+step "Handoff content assertion"
+if echo "$HANDOFF_MD" | grep -iqE "borrow|blog|rust|borrowck"; then
+ echo " PASS: handoff body references the session-1 topic"
+ PASS=$((PASS+1))
+else
+ echo " FAIL: handoff body looks unrelated to the session-1 topic"
+ FAIL=$((FAIL+1))
+fi
+
+# --------------------------------------------------------------------
+# Summary
+# --------------------------------------------------------------------
+
+echo
+echo "================================="
+echo " PASS: $PASS"
+echo " FAIL: $FAIL"
+echo "================================="
+[[ $FAIL -gt 0 ]] && exit 1
+exit 0