Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
add import (#965)
yeshion23333cd4fec25f062
diff --git a/bot/eval/locomo/import_to_ov.py b/bot/eval/locomo/import_to_ov.pynew file mode 100644index 000000000..efe9c3e05--- /dev/null+++ b/bot/eval/locomo/import_to_ov.py@@ -0,0 +1,444 @@+"""+OpenViking data import tool.++Import conversations from LoCoMo JSON or plain text files into OpenViking memory.++Usage:+ # Import LoCoMo JSON conversations+ uv run python import_to_ov.py locomo10.json --sample 0 --sessions 1-4++ # Import plain text conversations+ uv run python import_to_ov.py example.txt+"""++import argparse+import json+import subprocess+import sys+import time+from datetime import datetime+++def parse_test_file(path: str) -> list[dict]:+ """Parse txt test file into sessions.++ Each session is a dict with:+ - messages: list of user message strings+ """+ with open(path, "r", encoding="utf-8") as f:+ content = f.read()++ raw_sessions = content.split("---\n")+ sessions = []++ for raw in raw_sessions:+ lines = [line for line in raw.strip().splitlines() if line.strip()]+ if not lines:+ continue++ messages = []+ for line in lines:+ if not line.startswith("eval:"): # Skip eval lines+ messages.append(line)++ if messages:+ sessions.append({"messages": messages})++ return sessions+++def format_locomo_message(msg: dict) -> str:+ """Format a single LoCoMo message into a natural chat-style string.++ Output format:+ Speaker: text here+ image_url: caption+ """+ speaker = msg.get("speaker", "unknown")+ text = msg.get("text", "")+ line = f"{speaker}: {text}"++ img_urls = msg.get("img_url", [])+ if isinstance(img_urls, str):+ img_urls = [img_urls]+ blip = msg.get("blip_caption", "")++ if img_urls:+ for url in img_urls:+ caption = f": {blip}" if blip else ""+ line += f"\n{url}{caption}"+ elif blip:+ line += f"\n({blip})"++ return line+++def load_locomo_data(+ path: str,+ sample_index: int | None = None,+) -> list[dict]:+ """Load LoCoMo JSON and optionally filter to one sample."""+ with open(path, "r", encoding="utf-8") as f:+ data = json.load(f)++ if sample_index is not None:+ if sample_index < 0 or sample_index >= len(data):+ print(f"Error: sample index {sample_index} out of range (0-{len(data)-1})", file=sys.stderr)+ sys.exit(1)+ return [data[sample_index]]+ return data+++def build_session_messages(+ item: dict,+ session_range: tuple[int, int] | None = None,+) -> list[dict]:+ """Build bundled session messages for one LoCoMo sample.++ Returns list of dicts with keys: message, meta.+ """+ conv = item["conversation"]+ speakers = f"{conv['speaker_a']} & {conv['speaker_b']}"++ session_keys = sorted(+ [k for k in conv if k.startswith("session_") and not k.endswith("_date_time")],+ key=lambda k: int(k.split("_")[1]),+ )++ sessions = []+ for sk in session_keys:+ sess_num = int(sk.split("_")[1])+ if session_range:+ lo, hi = session_range+ if sess_num < lo or sess_num > hi:+ continue++ dt_key = f"{sk}_date_time"+ date_time = conv.get(dt_key, "")++ parts = [f"[group chat conversation: {date_time}]"]+ for msg in conv[sk]:+ parts.append(format_locomo_message(msg))+ combined = "\n\n".join(parts)++ sessions.append({+ "message": combined,+ "meta": {+ "sample_id": item["sample_id"],+ "session_key": sk,+ "date_time": date_time,+ "speakers": speakers,+ },+ })++ return sessions+++# ---------------------------------------------------------------------------+# Ingest record helpers (avoid duplicate ingestion)+# ---------------------------------------------------------------------------++def load_ingest_record(record_path: str = ".ingest_record.json") -> dict:+ """Load existing ingest record file, return empty dict if not exists."""+ try:+ with open(record_path, "r", encoding="utf-8") as f:+ return json.load(f)+ except (FileNotFoundError, json.JSONDecodeError):+ return {}+++def save_ingest_record(record: dict, record_path: str = ".ingest_record.json") -> None:+ """Save ingest record to file."""+ with open(record_path, "w", encoding="utf-8") as f:+ json.dump(record, f, indent=2, ensure_ascii=False)+++def is_already_ingested(+ sample_id: str | int,+ session_key: str,+ record: dict,+) -> bool:+ """Check if a specific session has already been successfully ingested."""+ key = f"viking:{sample_id}:{session_key}"+ return key in record and record[key].get("success", False)+++def mark_ingested(+ sample_id: str | int,+ session_key: str,+ record: dict,+ meta: dict | None = None,+) -> None:+ """Mark a session as successfully ingested."""+ key = f"viking:{sample_id}:{session_key}"+ record[key] = {+ "success": True,+ "timestamp": int(time.time()),+ "meta": meta or {},+ }+++# ---------------------------------------------------------------------------+# OpenViking import+# ---------------------------------------------------------------------------++def viking_ingest(msg: str) -> None:+ """Save a message to OpenViking via `ov add-memory`."""+ result = subprocess.run(+ ["ov", "add-memory", msg],+ capture_output=True,+ text=True,+ )+ if result.returncode != 0:+ raise RuntimeError(result.stderr.strip() or f"ov exited with code {result.returncode}")+++# ---------------------------------------------------------------------------+# Main import logic+# ---------------------------------------------------------------------------++def parse_session_range(s: str) -> tuple[int, int]:+ """Parse '1-4' or '3' into (lo, hi) inclusive tuple."""+ if "-" in s:+ lo, hi = s.split("-", 1)+ return int(lo), int(hi)+ n = int(s)+ return n, n+++def run_import(args: argparse.Namespace) -> None:+ session_range = parse_session_range(args.sessions) if args.sessions else None++ # Handle ingest record operations+ if args.clear_ingest_record:+ ingest_record = {}+ save_ingest_record(ingest_record)+ print(f"[INFO] All existing ingest records cleared", file=sys.stderr)+ else:+ ingest_record = load_ingest_record()++ # Open output files for incremental writing+ txt_output = open(args.output, "a", encoding="utf-8")+ jsonl_output = open(f"{args.output}.jsonl", "a", encoding="utf-8")++ # Write run header+ run_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")+ txt_output.write(f"\n=== Import run at {run_time} ===\n")+ txt_output.flush()++ skipped_count = 0+ success_count = 0+ error_count = 0++ if args.input.endswith(".json"):+ # LoCoMo JSON format+ samples = load_locomo_data(args.input, args.sample)++ for item in samples:+ sample_id = item["sample_id"]+ sessions = build_session_messages(item, session_range)++ print(f"\n=== Sample {sample_id} ===", file=sys.stderr)+ print(f" {len(sessions)} session(s) to import", file=sys.stderr)++ for sess in sessions:+ meta = sess["meta"]+ msg = sess["message"]+ label = f"{meta['session_key']} ({meta['date_time']})"++ # Skip already ingested sessions unless force-ingest is enabled+ if not args.force_ingest and is_already_ingested(sample_id, meta['session_key'], ingest_record):+ print(f" [{label}] [SKIP] already imported (use --force-ingest to reprocess)", file=sys.stderr)+ skipped_count += 1++ # Write skip record+ result = {+ "timestamp": run_time,+ "sample_id": sample_id,+ "session": meta["session_key"],+ "status": "skipped",+ "reason": "already imported"+ }+ txt_output.write(f"[{sample_id}/{meta['session_key']}] SKIPPED: already imported\n")+ jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")+ txt_output.flush()+ jsonl_output.flush()+ continue++ preview = msg.replace("\n", " | ")[:80]+ print(f" [{label}] {preview}...", file=sys.stderr)++ try:+ viking_ingest(msg)+ print(f" -> [SUCCESS] imported to OpenViking", file=sys.stderr)+ success_count += 1++ # Write success record+ result = {+ "timestamp": run_time,+ "sample_id": sample_id,+ "session": meta["session_key"],+ "status": "success",+ "meta": meta+ }+ txt_output.write(f"[{sample_id}/{meta['session_key']}] SUCCESS\n")+ jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")+ txt_output.flush()+ jsonl_output.flush()++ # Mark as successfully ingested+ mark_ingested(sample_id, meta['session_key'], ingest_record, {+ "date_time": meta['date_time'],+ "speakers": meta['speakers']+ })+ save_ingest_record(ingest_record) # Save immediately after success++ except Exception as e:+ print(f" -> [ERROR] {e}", file=sys.stderr)+ error_count += 1++ # Write error record+ result = {+ "timestamp": run_time,+ "sample_id": sample_id,+ "session": meta["session_key"],+ "status": "error",+ "error": str(e)+ }+ txt_output.write(f"[{sample_id}/{meta['session_key']}] ERROR: {str(e)}\n")+ jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")+ txt_output.flush()+ jsonl_output.flush()++ else:+ # Plain text format+ sessions = parse_test_file(args.input)+ print(f"Found {len(sessions)} session(s) in text file", file=sys.stderr)++ for idx, session in enumerate(sessions, start=1):+ session_key = f"txt-session-{idx}"+ print(f"\n=== Text Session {idx} ===", file=sys.stderr)++ # Skip already ingested sessions unless force-ingest is enabled+ if not args.force_ingest and is_already_ingested("txt", session_key, ingest_record):+ print(f" [SKIP] already imported (use --force-ingest to reprocess)", file=sys.stderr)+ skipped_count += 1++ # Write skip record+ result = {+ "timestamp": run_time,+ "sample_id": "txt",+ "session": session_key,+ "status": "skipped",+ "reason": "already imported"+ }+ txt_output.write(f"[txt/{session_key}] SKIPPED: already imported\n")+ jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")+ txt_output.flush()+ jsonl_output.flush()+ continue++ combined_msg = "\n\n".join(session["messages"])+ preview = combined_msg.replace("\n", " | ")[:80]+ print(f" {preview}...", file=sys.stderr)++ try:+ viking_ingest(combined_msg)+ print(f" -> [SUCCESS] imported to OpenViking", file=sys.stderr)+ success_count += 1++ # Write success record+ result = {+ "timestamp": run_time,+ "sample_id": "txt",+ "session": session_key,+ "status": "success",+ "session_index": idx+ }+ txt_output.write(f"[txt/{session_key}] SUCCESS\n")+ jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")+ txt_output.flush()+ jsonl_output.flush()++ mark_ingested("txt", session_key, ingest_record, {+ "session_index": idx+ })+ save_ingest_record(ingest_record) # Save immediately after success++ except Exception as e:+ print(f" -> [ERROR] {e}", file=sys.stderr)+ error_count += 1++ # Write error record+ result = {+ "timestamp": run_time,+ "sample_id": "txt",+ "session": session_key,+ "status": "error",+ "error": str(e)+ }+ txt_output.write(f"[txt/{session_key}] ERROR: {str(e)}\n")+ jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")+ txt_output.flush()+ jsonl_output.flush()++ # Close output files+ txt_output.close()+ jsonl_output.close()++ # Final summary+ total_processed = success_count + error_count + skipped_count+ print(f"\n=== Import summary ===", file=sys.stderr)+ print(f"Total sessions: {total_processed}", file=sys.stderr)+ print(f"Successfully imported: {success_count}", file=sys.stderr)+ print(f"Failed: {error_count}", file=sys.stderr)+ print(f"Skipped (already imported): {skipped_count}", file=sys.stderr)+ print(f"Results saved to: {args.output} (text) and {args.output}.jsonl (JSON Lines)", file=sys.stderr)+++# ---------------------------------------------------------------------------+# CLI+# ---------------------------------------------------------------------------++def main():+ parser = argparse.ArgumentParser(description="Import conversations into OpenViking")+ parser.add_argument(+ "--input",+ default="./test_data/locomo10.json",+ help="Path to input file (.txt or LoCoMo .json)"+ )+ parser.add_argument(+ "--output",+ default="./result/import_results",+ help="Path to output file (default: import_results)",+ )+ parser.add_argument(+ "--sample",+ type=int,+ default=None,+ help="LoCoMo JSON: sample index (0-based). Default: all samples.",+ )+ parser.add_argument(+ "--sessions",+ default=None,+ help="LoCoMo JSON: session range, e.g. '1-4' or '3'. Default: all sessions.",+ )+ parser.add_argument(+ "--force-ingest",+ action="store_true",+ default=False,+ help="Force re-import even if already recorded as completed",+ )+ parser.add_argument(+ "--clear-ingest-record",+ action="store_true",+ default=False,+ help="Clear all existing ingest records before running",+ )+ args = parser.parse_args()++ run_import(args)+++if __name__ == "__main__":+ main()