Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
"""
Ingest LoCoMo conversations into Claude Code's auto-memory system.
Same flow as openclaw: for each sample, send each session's bundled conversation
to Claude Code via `claude -p`, letting its auto-memory system extract and persist
memories. Each session is a separate `claude -p` invocation (independent conversation).
All samples share one isolated project directory so Claude Code accumulates memories
from all sessions into a single memory store.
Usage:
python ingest.py
python ingest.py --sample 0
python ingest.py --sample conv-26
python ingest.py --api-url http://localhost:8000 --api-key sk-xxx
python ingest.py --force-ingest # re-ingest even if already done
"""
import argparse
import csv
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from typing import Optional
SCRIPT_DIR = Path(__file__).parent.resolve()
DEFAULT_DATA_PATH = str(SCRIPT_DIR / ".." / "locomo10.json")
DEFAULT_PROJECT_ROOT = "/tmp/locomo-eval"
DEFAULT_HOME = "/tmp/claude-eval-home"
DEFAULT_INGEST_RECORD = str(SCRIPT_DIR / "result" / ".ingest_record.json")
DEFAULT_SUCCESS_CSV = str(SCRIPT_DIR / "result" / "ingest_success.csv")
DEFAULT_ERROR_LOG = str(SCRIPT_DIR / "result" / "ingest_errors.log")
SUCCESS_CSV_FIELDS = [
"timestamp",
"sample_id",
"session",
"date_time",
"speakers",
"input_tokens",
"cache_creation_input_tokens",
"cache_read_input_tokens",
"output_tokens",
"reasoning_tokens",
"total_cost_usd",
"duration_ms",
"response_preview",
]
# ---------------------------------------------------------------------------
# LoCoMo data loading
# ---------------------------------------------------------------------------
def load_locomo_data(path: str, sample_id: Optional[str] = None) -> list[dict]:
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
if sample_id is not None:
try:
idx = int(sample_id)
if idx < 0 or idx >= len(data):
raise ValueError(f"Sample index {idx} out of range (0-{len(data) - 1})")
return [data[idx]]
except ValueError:
pass
matched = [s for s in data if s.get("sample_id") == sample_id]
if not matched:
raise ValueError(f"sample_id '{sample_id}' not found")
return matched
return data
def format_locomo_message(msg: dict) -> str:
speaker = msg.get("speaker", "unknown")
text = msg.get("text", "")
line = f"{speaker}: {text}"
img_urls = msg.get("img_url", [])
if isinstance(img_urls, str) and img_urls:
img_urls = [img_urls]
for _url in img_urls or []:
caption = msg.get("blip_caption", "")
if caption:
line += f"\n (image: {caption})"
return line
def build_session_messages(item: dict) -> list[dict]:
"""Build bundled session messages for one LoCoMo sample.
Same format as openclaw: each session becomes a single string with all
messages concatenated, prefixed with date/time header.
"""
conv = item["conversation"]
speaker_a = conv["speaker_a"]
speaker_b = conv["speaker_b"]
session_keys = sorted(
[k for k in conv if k.startswith("session_") and not k.endswith("_date_time")],
key=lambda k: int(k.split("_")[1]),
)
sessions = []
for sk in session_keys:
sess_num = int(sk.split("_")[1])
raw_messages = conv[sk]
if not isinstance(raw_messages, list) or not raw_messages:
continue
dt_key = f"{sk}_date_time"
date_time = conv.get(dt_key, "")
parts = [f"[group chat conversation: {date_time}]"]
for msg in raw_messages:
parts.append(format_locomo_message(msg))
combined = "\n\n".join(parts)
sessions.append(
{
"message": combined,
"meta": {
"sample_id": item["sample_id"],
"session_key": sk,
"session_num": sess_num,
"date_time": date_time,
"speakers": f"{speaker_a} & {speaker_b}",
},
}
)
return sessions
# ---------------------------------------------------------------------------
# Ingest record (deduplication)
# ---------------------------------------------------------------------------
def load_ingest_record(path: str) -> dict:
try:
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
except (FileNotFoundError, json.JSONDecodeError):
return {}
def save_ingest_record(record: dict, path: str) -> None:
Path(path).parent.mkdir(parents=True, exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(record, f, indent=2, ensure_ascii=False)
def is_already_ingested(sample_id: str, session_key: str, record: dict) -> bool:
key = f"claudecode:{sample_id}:{session_key}"
return key in record and record[key].get("success", False)
def mark_ingested(sample_id: str, session_key: str, record: dict, meta: dict) -> None:
key = f"claudecode:{sample_id}:{session_key}"
record[key] = {"success": True, "timestamp": int(time.time()), "meta": meta}
def write_success_csv(record: dict, csv_path: str) -> None:
Path(csv_path).parent.mkdir(parents=True, exist_ok=True)
file_exists = os.path.exists(csv_path)
with open(csv_path, "a", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=SUCCESS_CSV_FIELDS)
if not file_exists:
writer.writeheader()
writer.writerow(record)
def write_error_log(path: str, sample_id: str, session_key: str, error: str) -> None:
Path(path).parent.mkdir(parents=True, exist_ok=True)
ts = time.strftime("%Y-%m-%d %H:%M:%S")
with open(path, "a", encoding="utf-8") as f:
f.write(f"[{ts}] ERROR [{sample_id}/{session_key}]: {error}\n")
# ---------------------------------------------------------------------------
# Claude Code invocation
# ---------------------------------------------------------------------------
def _run_claude_once(
prompt: str,
project_dir: str,
env: dict,
model: Optional[str] = None,
timeout_sec: int = 600,
extra_flags: Optional[list] = None,
) -> dict:
"""Single claude -p invocation. Returns parsed result dict."""
cmd = [
"claude",
"-p",
prompt,
"--output-format",
"json",
"--dangerously-skip-permissions",
"--setting-sources",
"",
"--disable-slash-commands",
"--strict-mcp-config",
]
if extra_flags:
cmd.extend(extra_flags)
if model:
cmd.extend(["--model", model])
try:
result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=timeout_sec,
cwd=project_dir,
env=env,
)
stdout = result.stdout.strip()
if not stdout:
return {
"response": f"[ERROR] empty stdout: {result.stderr[:200]}",
"usage": {},
"duration_ms": 0,
"cost": 0,
}
try:
body = json.loads(stdout)
except json.JSONDecodeError:
return {
"response": f"[ERROR] JSON parse: {stdout[:200]}",
"usage": {},
"duration_ms": 0,
"cost": 0,
}
if body.get("is_error"):
return {
"response": f"[ERROR] {body.get('result', 'unknown')}",
"usage": body.get("usage", {}),
"duration_ms": body.get("duration_ms", 0),
"cost": body.get("total_cost_usd", 0),
}
return {
"response": body.get("result", ""),
"usage": body.get("usage", {}),
"duration_ms": body.get("duration_ms", 0),
"num_turns": body.get("num_turns", 0),
"cost": body.get("total_cost_usd", 0),
}
except subprocess.TimeoutExpired:
return {"response": "[TIMEOUT]", "usage": {}, "duration_ms": timeout_sec * 1000, "cost": 0}
except Exception as e:
return {"response": f"[ERROR] {e}", "usage": {}, "duration_ms": 0, "cost": 0}
def _build_env(
home_dir: str, api_key: Optional[str], auth_token: Optional[str], api_url: Optional[str]
) -> dict:
env = os.environ.copy()
env["HOME"] = home_dir
if api_key:
env["ANTHROPIC_API_KEY"] = api_key
if auth_token:
env["ANTHROPIC_AUTH_TOKEN"] = auth_token
if api_url:
env["ANTHROPIC_BASE_URL"] = api_url
return env
def _run_with_retry(
prompt: str,
project_dir: str,
env: dict,
model: Optional[str] = None,
timeout_sec: int = 600,
retries: int = 2,
extra_flags: Optional[list] = None,
) -> dict:
"""Run claude -p with retries on TIMEOUT/ERROR."""
for attempt in range(retries + 1):
result = _run_claude_once(prompt, project_dir, env, model, timeout_sec, extra_flags)
resp = result["response"]
if not resp.startswith("[TIMEOUT]") and not resp.startswith("[ERROR]"):
return result
if attempt < retries:
print(f" [retry {attempt + 1}/{retries}] {resp[:80]}", file=sys.stderr)
return result
def run_claude_ingest(
message: str,
project_dir: str,
home_dir: str,
api_url: Optional[str] = None,
api_key: Optional[str] = None,
auth_token: Optional[str] = None,
model: Optional[str] = None,
timeout_sec: int = 600,
retries: int = 2,
) -> dict:
"""Send a session's conversation to Claude Code and let auto-memory process it."""
env = _build_env(home_dir, api_key, auth_token, api_url)
return _run_with_retry(message, project_dir, env, model, timeout_sec, retries)
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
parser = argparse.ArgumentParser(
description="Ingest LoCoMo conversations into Claude Code auto-memory"
)
parser.add_argument(
"--input",
default=DEFAULT_DATA_PATH,
help=f"Path to locomo JSON (default: {DEFAULT_DATA_PATH})",
)
parser.add_argument(
"--sample",
default=None,
help="Sample index (0-based) or sample_id. Default: all.",
)
parser.add_argument(
"--project-root",
default=DEFAULT_PROJECT_ROOT,
help=f"Root for sample project dirs (default: {DEFAULT_PROJECT_ROOT})",
)
parser.add_argument(
"--home",
default=DEFAULT_HOME,
help=f"Isolated HOME directory (default: {DEFAULT_HOME})",
)
parser.add_argument(
"--api-url",
default=None,
help="Custom API base URL (ANTHROPIC_BASE_URL)",
)
parser.add_argument(
"--api-key",
default=None,
help="API key (ANTHROPIC_API_KEY)",
)
parser.add_argument(
"--auth-token",
default=None,
help="Auth token (ANTHROPIC_AUTH_TOKEN), alternative to --api-key",
)
parser.add_argument(
"--model",
default=None,
help="Model name/alias (e.g. sonnet, opus)",
)
parser.add_argument(
"--force-ingest",
action="store_true",
help="Re-ingest even if already recorded as done",
)
parser.add_argument(
"--clear-ingest-record",
action="store_true",
help="Clear all ingest records before running",
)
parser.add_argument(
"--record",
default=DEFAULT_INGEST_RECORD,
help=f"Ingest record file (default: {DEFAULT_INGEST_RECORD})",
)
parser.add_argument(
"--success-csv",
default=DEFAULT_SUCCESS_CSV,
help=f"Success CSV path (default: {DEFAULT_SUCCESS_CSV})",
)
parser.add_argument(
"--error-log",
default=DEFAULT_ERROR_LOG,
help=f"Error log path (default: {DEFAULT_ERROR_LOG})",
)
parser.add_argument(
"--timeout",
type=int,
default=600,
help="Timeout per session in seconds (default: 600)",
)
parser.add_argument(
"--prompt-prefix",
default="",
help="Optional prefix prepended before each session's conversation text "
"(e.g. to nudge auto-memory). Default: empty (bare conversation).",
)
args = parser.parse_args()
api_key = args.api_key or os.environ.get("ANTHROPIC_API_KEY", "")
auth_token = args.auth_token or os.environ.get("ANTHROPIC_AUTH_TOKEN", "")
api_url = args.api_url or os.environ.get("ANTHROPIC_BASE_URL")
if not api_key and not auth_token:
print(
"Error: API key required (--api-key/ANTHROPIC_API_KEY or --auth-token/ANTHROPIC_AUTH_TOKEN)",
file=sys.stderr,
)
sys.exit(1)
if not os.path.exists(args.input):
print(f"Error: input file not found: {args.input}", file=sys.stderr)
sys.exit(1)
# Ingest record
if args.clear_ingest_record:
ingest_record: dict = {}
save_ingest_record(ingest_record, args.record)
print("[INFO] Cleared ingest records", file=sys.stderr)
else:
ingest_record = load_ingest_record(args.record)
samples = load_locomo_data(args.input, args.sample)
print(f"[INFO] Loaded {len(samples)} sample(s)", file=sys.stderr)
print(f"[INFO] Project root: {args.project_root}", file=sys.stderr)
print(f"[INFO] HOME: {args.home}", file=sys.stderr)
# Ensure isolated HOME has .claude directory
os.makedirs(os.path.join(args.home, ".claude"), exist_ok=True)
total_sessions = 0
success_count = 0
skip_count = 0
error_count = 0
for item in samples:
sample_id = item["sample_id"]
sessions = build_session_messages(item)
# All samples share one project dir so memories accumulate together
project_dir = os.path.join(args.project_root, sample_id)
os.makedirs(project_dir, exist_ok=True)
print(f"\n=== Sample {sample_id} ({len(sessions)} sessions) ===", file=sys.stderr)
for sess in sessions:
meta = sess["meta"]
msg = sess["message"]
session_key = meta["session_key"]
label = f"{session_key} ({meta['date_time']})"
total_sessions += 1
if not args.force_ingest and is_already_ingested(sample_id, session_key, ingest_record):
print(f" [{label}] SKIP (already ingested)", file=sys.stderr)
skip_count += 1
continue
preview = msg.replace("\n", " | ")[:80]
print(f" [{label}] {preview}...", file=sys.stderr)
send_msg = f"{args.prompt_prefix}\n\n{msg}" if args.prompt_prefix else msg
t0 = time.time()
result = run_claude_ingest(
send_msg,
project_dir,
args.home,
api_url=api_url,
api_key=api_key,
auth_token=auth_token,
model=args.model,
timeout_sec=args.timeout,
)
elapsed = time.time() - t0
response = result["response"]
usage = result.get("usage", {})
if response.startswith("[ERROR]") or response.startswith("[TIMEOUT]"):
print(f" -> {response[:80]} ({elapsed:.1f}s)", file=sys.stderr)
write_error_log(args.error_log, sample_id, session_key, response)
error_count += 1
else:
print(
f" -> {response[:80]}{'...' if len(response) > 80 else ''}"
f" ({elapsed:.1f}s)",
file=sys.stderr,
)
mark_ingested(
sample_id,
session_key,
ingest_record,
{
"date_time": meta["date_time"],
"duration_ms": result.get("duration_ms", 0),
},
)
save_ingest_record(ingest_record, args.record)
write_success_csv(
{
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
"sample_id": sample_id,
"session": session_key,
"date_time": meta["date_time"],
"speakers": meta["speakers"],
"input_tokens": usage.get("input_tokens", 0),
"cache_creation_input_tokens": usage.get("cache_creation_input_tokens", 0),
"cache_read_input_tokens": usage.get("cache_read_input_tokens", 0),
"output_tokens": usage.get("output_tokens", 0),
"reasoning_tokens": usage.get("reasoning_tokens", 0),
"total_cost_usd": result.get("cost", 0),
"duration_ms": result.get("duration_ms", 0),
"response_preview": response[:100],
},
args.success_csv,
)
success_count += 1
print("\n=== Ingest summary ===", file=sys.stderr)
print(f" Total sessions: {total_sessions}", file=sys.stderr)
print(f" Succeeded: {success_count}", file=sys.stderr)
print(f" Skipped: {skip_count}", file=sys.stderr)
print(f" Failed: {error_count}", file=sys.stderr)
# Write sample mapping for eval.py
mapping_path = os.path.join(args.project_root, "sample_mapping.json")
mapping = {}
for item in samples:
sid = item["sample_id"]
mapping[sid] = {
"sample_id": sid,
"project_dir": os.path.join(args.project_root, sid),
}
with open(mapping_path, "w", encoding="utf-8") as f:
json.dump(mapping, f, indent=2, ensure_ascii=False)
print(f" Mapping: {mapping_path}", file=sys.stderr)
if __name__ == "__main__":
main()