Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
"""
Ingest LoCoMo conversations into Supermemory.
Each sample gets an isolated Supermemory namespace keyed by containerTag = sample_id
(e.g. "conv-26"). Sessions are formatted as date-prefixed JSON content strings,
matching the memorybench supermemory provider convention.
Usage:
# Ingest all samples
python ingest.py
# Ingest a specific sample
python ingest.py --sample conv-26
# Ingest specific sessions
python ingest.py --sample conv-26 --sessions 1-4
# Force re-ingest even if already done
python ingest.py --force-ingest
# Set Supermemory API key via env or flag
SUPERMEMORY_API_KEY=xxx python ingest.py
python ingest.py --api-key xxx
"""
import argparse
import json
import os
import re
import sys
import time
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any, Optional
from dotenv import load_dotenv
load_dotenv(Path.home() / ".openviking_benchmark_env")
try:
from supermemory import Supermemory
except ImportError:
print("Error: supermemory package not installed. Run: pip install supermemory", file=sys.stderr)
sys.exit(1)
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
SCRIPT_DIR = Path(__file__).parent.resolve()
DEFAULT_DATA_PATH = str(SCRIPT_DIR / ".." / "data" / "locomo10.json")
DEFAULT_RECORD_PATH = str(SCRIPT_DIR / "result" / ".ingest_record.json")
DEFAULT_LOG_PATH = str(SCRIPT_DIR / "result" / "ingest_errors.log")
# ---------------------------------------------------------------------------
# Tag sanitization (must match openclaw-supermemory's sanitizeTag logic)
# ---------------------------------------------------------------------------
def sanitize_tag(raw: str) -> str:
"""Sanitize a tag string to match openclaw-supermemory convention.
Replaces non-alphanumeric/underscore chars with '_', collapses runs, strips edges.
e.g. 'conv-26' -> 'conv_26'
"""
tag = re.sub(r"[^a-zA-Z0-9_]", "_", raw)
tag = re.sub(r"_+", "_", tag)
tag = tag.strip("_")
return tag
# ---------------------------------------------------------------------------
# LoCoMo data loading
# ---------------------------------------------------------------------------
def load_locomo_data(path: str, sample_id: Optional[str] = None) -> list[dict]:
"""Load LoCoMo JSON and optionally filter to one sample by sample_id or numeric index."""
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
if sample_id is not None:
try:
idx = int(sample_id)
if idx < 0 or idx >= len(data):
raise ValueError(f"Sample index {idx} out of range (0-{len(data) - 1})")
return [data[idx]]
except ValueError:
pass
matched = [s for s in data if s.get("sample_id") == sample_id]
if not matched:
raise ValueError(f"sample_id '{sample_id}' not found")
return matched
return data
def parse_session_range(s: str) -> tuple[int, int]:
"""Parse '1-4' or '3' into (lo, hi) inclusive tuple."""
if "-" in s:
lo, hi = s.split("-", 1)
return int(lo), int(hi)
n = int(s)
return n, n
def parse_locomo_date(date_time: str) -> Optional[str]:
"""Parse a LoCoMo date_time string to ISO 8601 format.
e.g. '1:56 pm on 8 May, 2023' -> '2023-05-08T13:56:00.000Z'
"""
import re
from datetime import datetime, timezone
match = re.search(r"(\d+):(\d+)\s*(am|pm)\s*on\s*(\d+)\s*(\w+),?\s*(\d+)", date_time, re.IGNORECASE)
if not match:
return None
hour_str, minute, ampm, day, month_name, year = match.groups()
hour = int(hour_str)
if ampm.lower() == "pm" and hour != 12:
hour += 12
if ampm.lower() == "am" and hour == 12:
hour = 0
month_names = ["january","february","march","april","may","june",
"july","august","september","october","november","december"]
try:
month = next(i + 1 for i, n in enumerate(month_names) if n.startswith(month_name.lower()))
dt = datetime(int(year), month, int(day), hour, int(minute), tzinfo=timezone.utc)
return dt.strftime("%Y-%m-%dT%H:%M:%S.000Z")
except (StopIteration, ValueError):
return None
def build_session_content(
item: dict,
session_key: str,
date_time: str,
) -> str:
conv = item["conversation"]
raw_messages = conv[session_key]
lines = [f"[{msg.get('speaker', '').capitalize()}]: {msg.get('text', '')}" for msg in raw_messages]
session_str = "\n".join(lines)
if date_time:
return (
f"Here is the date the following session took place: {date_time}\n\n"
f"Here is the session as a stringified JSON:\n{session_str}"
)
else:
return f"Here is the session as a stringified JSON:\n{session_str}"
def build_sessions(
item: dict,
session_range: Optional[tuple[int, int]] = None,
) -> list[dict]:
"""
Extract sessions from a LoCoMo sample.
Returns list of dicts with keys:
- content: formatted string for supermemory
- meta: session metadata
"""
conv = item["conversation"]
session_keys = sorted(
[k for k in conv if k.startswith("session_") and not k.endswith("_date_time")],
key=lambda k: int(k.split("_")[1]),
)
sessions = []
for sk in session_keys:
sess_num = int(sk.split("_")[1])
if session_range:
lo, hi = session_range
if sess_num < lo or sess_num > hi:
continue
raw_messages = conv[sk]
if not isinstance(raw_messages, list) or not raw_messages:
continue
dt_key = f"{sk}_date_time"
date_time = conv.get(dt_key, "")
date_iso = parse_locomo_date(date_time) if date_time else None
content = build_session_content(item, sk, date_time)
sessions.append(
{
"content": content,
"meta": {
"sample_id": item["sample_id"],
"session_key": sk,
"date_time": date_time,
"date_iso": date_iso or "",
"speaker_a": conv["speaker_a"],
"speaker_b": conv["speaker_b"],
},
}
)
return sessions
# ---------------------------------------------------------------------------
# Ingest record (progress tracking)
# ---------------------------------------------------------------------------
def load_ingest_record(path: str) -> dict:
try:
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
except (FileNotFoundError, json.JSONDecodeError):
return {}
def save_ingest_record(record: dict, path: str) -> None:
Path(path).parent.mkdir(parents=True, exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
json.dump(record, f, indent=2, ensure_ascii=False)
def is_already_ingested(sample_id: str, session_key: str, record: dict) -> bool:
key = f"supermemory:{sample_id}:{session_key}"
return key in record and record[key].get("success", False)
def mark_ingested(
sample_id: str,
session_key: str,
record: dict,
doc_id: str,
meta: Optional[dict] = None,
) -> None:
key = f"supermemory:{sample_id}:{session_key}"
record[key] = {
"success": True,
"timestamp": int(time.time()),
"doc_id": doc_id,
"meta": meta or {},
}
def write_error_log(path: str, sample_id: str, session_key: str, error: str) -> None:
Path(path).parent.mkdir(parents=True, exist_ok=True)
ts = time.strftime("%Y-%m-%d %H:%M:%S")
with open(path, "a", encoding="utf-8") as f:
f.write(f"[{ts}] ERROR [{sample_id}/{session_key}]: {error}\n")
# ---------------------------------------------------------------------------
# Supermemory indexing poll
# ---------------------------------------------------------------------------
def poll_document(client: Supermemory, doc_id: str, timeout_sec: int = 600) -> str:
"""
Poll a single document until done/failed/TIMEOUT.
Returns final status string.
"""
backoff = 1.0
start = time.time()
while True:
if time.time() - start > timeout_sec:
return "TIMEOUT"
try:
doc = client.documents.get(doc_id)
doc_status = getattr(doc, "status", None) or (doc.get("status") if isinstance(doc, dict) else None)
if doc_status == "failed":
return "failed"
if doc_status == "done":
return "done"
except Exception as e:
print(f" [poll] Error checking doc {doc_id}: {e}", file=sys.stderr)
time.sleep(backoff)
backoff = min(backoff * 1.2, 5.0)
def poll_all_documents(
client: Supermemory,
pending: dict[str, dict], # doc_id -> {"session_key", "label", ...}
timeout_sec: int = 600,
threads: int = 8,
) -> tuple[dict[str, str], dict[str, str]]:
"""
Poll all pending doc_ids in parallel.
Returns (done_map, failed_map): doc_id -> label.
"""
done_map: dict[str, str] = {}
failed_map: dict[str, str] = {}
with ThreadPoolExecutor(max_workers=threads) as executor:
futures = {
executor.submit(poll_document, client, doc_id, timeout_sec): doc_id
for doc_id in pending
}
for fut in as_completed(futures):
doc_id = futures[fut]
label = pending[doc_id]["label"]
try:
status = fut.result()
except Exception as e:
status = f"ERROR: {e}"
if status == "done":
done_map[doc_id] = label
print(f" [{label}] indexed OK doc_id={doc_id}", file=sys.stderr)
else:
failed_map[doc_id] = label
print(f" [{label}] indexing {status} doc_id={doc_id}", file=sys.stderr)
return done_map, failed_map
# ---------------------------------------------------------------------------
# Core ingest logic
# ---------------------------------------------------------------------------
def upload_session(
client: Supermemory,
content: str,
container_tag: str,
meta: dict,
) -> str:
"""Upload one session to Supermemory (no indexing wait). Returns doc_id."""
response = client.add(
content=content,
container_tag=container_tag,
metadata={
"session_key": meta.get("session_key", ""),
"date_time": meta.get("date_time", ""),
"date": meta.get("date_iso", ""),
"speaker_a": meta.get("speaker_a", ""),
"speaker_b": meta.get("speaker_b", ""),
},
)
doc_id = getattr(response, "id", None) or (response.get("id") if isinstance(response, dict) else None)
if not doc_id:
raise RuntimeError(f"Supermemory add() returned no id: {response}")
return doc_id
def ingest_sample(
item: dict,
client: Supermemory,
ingest_record: dict,
record_lock: threading.Lock,
args: argparse.Namespace,
session_range: Optional[tuple[int, int]],
) -> tuple[int, int, int, int]:
"""Ingest one sample. Returns (total, success, skip, error) counts."""
sample_id: str = item["sample_id"]
container_tag = sanitize_tag(sample_id)
sessions = build_sessions(item, session_range)
print(f"\n=== Sample {sample_id} ({len(sessions)} sessions) [containerTag={container_tag}] ===", file=sys.stderr)
total = len(sessions)
success = skip = error = 0
# Phase 1: upload all sessions serially (within this sample)
to_poll: dict[str, dict] = {}
for sess in sessions:
meta = sess["meta"]
session_key = meta["session_key"]
label = f"{session_key} ({meta['date_time']})"
with record_lock:
already = not args.force_ingest and is_already_ingested(sample_id, session_key, ingest_record)
if already:
print(f" [{label}] SKIP (already ingested)", file=sys.stderr)
skip += 1
continue
try:
doc_id = upload_session(client, sess["content"], container_tag, meta)
print(f" [{label}] uploaded doc_id={doc_id}", file=sys.stderr)
to_poll[doc_id] = {"session_key": session_key, "label": label, "meta": meta}
except Exception as e:
print(f" [{label}] UPLOAD ERROR: {e}", file=sys.stderr)
write_error_log(args.error_log, sample_id, session_key, str(e))
error += 1
if not to_poll:
return total, success, skip, error
# Phase 2: poll all uploaded docs in parallel until all done
print(f" [INFO] Polling {len(to_poll)} docs for indexing completion...", file=sys.stderr)
t0 = time.time()
done_map, failed_map = poll_all_documents(client, to_poll, threads=args.threads)
elapsed = time.time() - t0
print(f" [INFO] Indexing done in {elapsed:.1f}s: {len(done_map)} OK, {len(failed_map)} failed", file=sys.stderr)
# Phase 3: save ingest records (thread-safe)
with record_lock:
for doc_id in done_map:
info = to_poll[doc_id]
mark_ingested(sample_id, info["session_key"], ingest_record, doc_id, info["meta"])
success += 1
save_ingest_record(ingest_record, args.record)
for doc_id in failed_map:
info = to_poll[doc_id]
write_error_log(args.error_log, sample_id, info["session_key"], f"indexing failed/timeout for doc {doc_id}")
error += 1
return total, success, skip, error
def run_ingest(args: argparse.Namespace) -> None:
api_key = args.api_key or os.environ.get("SUPERMEMORY_API_KEY", "")
if not api_key:
print("Error: Supermemory API key required (--api-key or SUPERMEMORY_API_KEY env var)", file=sys.stderr)
sys.exit(1)
client = Supermemory(api_key=api_key)
session_range = parse_session_range(args.sessions) if args.sessions else None
if args.clear_ingest_record:
ingest_record: dict = {}
save_ingest_record(ingest_record, args.record)
print("[INFO] Cleared existing ingest records", file=sys.stderr)
else:
ingest_record = load_ingest_record(args.record)
samples = load_locomo_data(args.input, args.sample)
if args.limit:
samples = samples[: args.limit]
print(f"[INFO] Loaded {len(samples)} sample(s) (sample_concurrency={args.sample_concurrency})", file=sys.stderr)
total_sessions = success_count = skip_count = error_count = 0
record_lock = threading.Lock()
with ThreadPoolExecutor(max_workers=args.sample_concurrency) as executor:
futures = {
executor.submit(
ingest_sample, item, client, ingest_record, record_lock, args, session_range
): item["sample_id"]
for item in samples
}
for fut in as_completed(futures):
sample_id = futures[fut]
try:
t, s, sk, e = fut.result()
total_sessions += t
success_count += s
skip_count += sk
error_count += e
except Exception as exc:
print(f"[ERROR] Sample {sample_id} failed: {exc}", file=sys.stderr)
error_count += 1
print(f"\n=== Ingest summary ===", file=sys.stderr)
print(f" Total sessions: {total_sessions}", file=sys.stderr)
print(f" Succeeded: {success_count}", file=sys.stderr)
print(f" Skipped: {skip_count}", file=sys.stderr)
print(f" Failed: {error_count}", file=sys.stderr)
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main() -> None:
parser = argparse.ArgumentParser(description="Ingest LoCoMo conversations into Supermemory")
parser.add_argument(
"--input",
default=DEFAULT_DATA_PATH,
help="Path to locomo10.json (default: ../data/locomo10.json)",
)
parser.add_argument(
"--api-key",
default=None,
help="Supermemory API key (or set SUPERMEMORY_API_KEY env var)",
)
parser.add_argument(
"--sample",
default=None,
help="Sample index (0-based int) or sample_id string (e.g. conv-26). Default: all.",
)
parser.add_argument(
"--limit",
type=int,
default=None,
help="Max number of samples to ingest. Default: all.",
)
parser.add_argument(
"--sessions",
default=None,
help="Session range, e.g. '1-4' or '3'. Default: all.",
)
parser.add_argument(
"--record",
default=DEFAULT_RECORD_PATH,
help=f"Path to ingest progress record (default: {DEFAULT_RECORD_PATH})",
)
parser.add_argument(
"--error-log",
default=DEFAULT_LOG_PATH,
help=f"Path to error log (default: {DEFAULT_LOG_PATH})",
)
parser.add_argument(
"--force-ingest",
action="store_true",
default=False,
help="Re-ingest even if already recorded as done",
)
parser.add_argument(
"--clear-ingest-record",
action="store_true",
default=False,
help="Clear all existing ingest records before running",
)
parser.add_argument(
"--threads",
type=int,
default=8,
help="Concurrent threads for indexing poll within a sample (default: 8)",
)
parser.add_argument(
"--sample-concurrency",
type=int,
default=3,
help="Number of samples to ingest concurrently (default: 3)",
)
args = parser.parse_args()
run_ingest(args)
if __name__ == "__main__":
main()