Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(bot): import ov eval script (#1108)
yeshion233338628f2719554
diff --git a/bot/eval/locomo/import_to_ov.py b/bot/eval/locomo/import_to_ov.pyindex c44923362..7a531d04a 100644--- a/bot/eval/locomo/import_to_ov.py+++ b/bot/eval/locomo/import_to_ov.py@@ -12,15 +12,25 @@ Usage:"""import argparse+import asyncio+import csvimport jsonimport sysimport timefrom datetime import datetime+from pathlib import Path+from typing import List, Dict, Any, Tuple, Optionalimport openviking as ov+from openviking.message.part import TextPart-def parse_test_file(path: str) -> list[dict]:+def _get_session_number(session_key: str) -> int:+ """Extract session number from session key."""+ return int(session_key.split("_")[1])+++def parse_test_file(path: str) -> List[Dict[str, Any]]:"""Parse txt test file into sessions.Each session is a dict with:@@ -48,55 +58,66 @@ def parse_test_file(path: str) -> list[dict]:return sessions-def format_locomo_message(msg: dict, index: int | None = None) -> str:- """Format a single LoCoMo message into chat-style string.+def format_locomo_message(msg: Dict[str, Any]) -> str:+ """Format a single LoCoMo message into a natural chat-style string.Output format:- [index][Speaker]: text here+ Speaker: text here+ image_url: caption"""speaker = msg.get("speaker", "unknown")text = msg.get("text", "")- if index is not None:- return f"[{index}][{speaker}]: {text}"- return f"[{speaker}]: {text}"+ line = f"{speaker}: {text}"++ img_urls = msg.get("img_url", [])+ if isinstance(img_urls, str):+ img_urls = [img_urls]+ blip = msg.get("blip_caption", "")++ if img_urls:+ for url in img_urls:+ caption = f": {blip}" if blip else ""+ line += f"\n{url}{caption}"+ elif blip:+ line += f"\n({blip})"++ return linedef load_locomo_data(path: str,- sample_index: int | None = None,-) -> list[dict]:+ sample_index: Optional[int] = None,+) -> List[Dict[str, Any]]:"""Load LoCoMo JSON and optionally filter to one sample."""with open(path, "r", encoding="utf-8") as f:data = json.load(f)if sample_index is not None:if sample_index < 0 or sample_index >= len(data):- print(f"Error: sample index {sample_index} out of range (0-{len(data)-1})", file=sys.stderr)- sys.exit(1)+ raise ValueError(f"Sample index {sample_index} out of range (0-{len(data)-1})")return [data[sample_index]]return datadef build_session_messages(- item: dict,- session_range: tuple[int, int] | None = None,-) -> list[dict]:- """Build session messages for one LoCoMo sample.+ item: Dict[str, Any],+ session_range: Optional[Tuple[int, int]] = None,+) -> List[Dict[str, Any]]:+ """Build bundled session messages for one LoCoMo sample.- Returns list of dicts with keys: messages, meta.- Each dict represents a session with multiple messages (user/assistant role).+ Returns list of dicts with keys: message, meta."""conv = item["conversation"]speakers = f"{conv['speaker_a']} & {conv['speaker_b']}"session_keys = sorted([k for k in conv if k.startswith("session_") and not k.endswith("_date_time")],- key=lambda k: int(k.split("_")[1]),+ key=_get_session_number,)sessions = []for sk in session_keys:- sess_num = int(sk.split("_")[1])+ sess_num = _get_session_number(sk)if session_range:lo, hi = session_rangeif sess_num < lo or sess_num > hi:@@ -105,20 +126,13 @@ def build_session_messages(dt_key = f"{sk}_date_time"date_time = conv.get(dt_key, "")- # Extract messages with all as user role, including speaker in content- messages = []- for idx, msg in enumerate(conv[sk]):- speaker = msg.get("speaker", "unknown")- text = msg.get("text", "")- messages.append({- "role": "user",- "text": f"[{speaker}]: {text}",- "speaker": speaker,- "index": idx- })+ parts = [f"[group chat conversation: {date_time}]"]+ for msg in conv[sk]:+ parts.append(format_locomo_message(msg))+ combined = "\n\n".join(parts)sessions.append({- "messages": messages,+ "message": combined,"meta": {"sample_id": item["sample_id"],"session_key": sk,@@ -134,8 +148,55 @@ def build_session_messages(# Ingest record helpers (avoid duplicate ingestion)# ---------------------------------------------------------------------------+def load_success_csv(csv_path: str = "import_success.csv") -> set:+ """加载成功导入的CSV记录,返回已成功的键集合"""+ success_keys = set()+ if Path(csv_path).exists():+ with open(csv_path, "r", encoding="utf-8") as f:+ reader = csv.DictReader(f)+ for row in reader:+ key = f"viking:{row['sample_id']}:{row['session']}"+ success_keys.add(key)+ return success_keys+++def write_success_record(record: Dict[str, Any], csv_path: str = "import_success.csv") -> None:+ """写入成功记录到CSV文件"""+ file_exists = Path(csv_path).exists()+ fieldnames = ["timestamp", "sample_id", "session", "date_time", "speakers",+ "embedding_tokens", "vlm_tokens", "llm_input_tokens",+ "llm_output_tokens", "total_tokens"]++ with open(csv_path, "a", encoding="utf-8", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ if not file_exists:+ writer.writeheader()++ writer.writerow({+ "timestamp": record["timestamp"],+ "sample_id": record["sample_id"],+ "session": record["session"],+ "date_time": record.get("meta", {}).get("date_time", ""),+ "speakers": record.get("meta", {}).get("speakers", ""),+ "embedding_tokens": record["token_usage"].get("embedding", 0),+ "vlm_tokens": record["token_usage"].get("vlm", 0),+ "llm_input_tokens": record["token_usage"].get("llm_input", 0),+ "llm_output_tokens": record["token_usage"].get("llm_output", 0),+ "total_tokens": record["token_usage"].get("total", 0)+ })+++def write_error_record(record: Dict[str, Any], error_path: str = "import_errors.log") -> None:+ """写入错误记录到日志文件"""+ with open(error_path, "a", encoding="utf-8") as f:+ timestamp = record["timestamp"]+ sample_id = record["sample_id"]+ session = record["session"]+ error = record["error"]+ f.write(f"[{timestamp}] ERROR [{sample_id}/{session}]: {error}\n")-def load_ingest_record(record_path: str = "result/ingest_record.json") -> dict:++def load_ingest_record(record_path: str = "./result/.ingest_record.json") -> Dict[str, Any]:"""Load existing ingest record file, return empty dict if not exists."""try:with open(record_path, "r", encoding="utf-8") as f:@@ -144,7 +205,7 @@ def load_ingest_record(record_path: str = "result/ingest_record.json") -> dict:return {}-def save_ingest_record(record: dict, record_path: str = "result/ingest_record.json") -> None:+def save_ingest_record(record: Dict[str, Any], record_path: str = "./result/.ingest_record.json") -> None:"""Save ingest record to file."""with open(record_path, "w", encoding="utf-8") as f:json.dump(record, f, indent=2, ensure_ascii=False)@@ -153,18 +214,21 @@ def save_ingest_record(record: dict, record_path: str = "result/ingest_record.jsdef is_already_ingested(sample_id: str | int,session_key: str,- record: dict,+ record: Dict[str, Any],+ success_keys: Optional[set] = None,) -> bool:"""Check if a specific session has already been successfully ingested."""key = f"viking:{sample_id}:{session_key}"+ if success_keys is not None and key in success_keys:+ return Truereturn key in record and record[key].get("success", False)def mark_ingested(sample_id: str | int,session_key: str,- record: dict,- meta: dict | None = None,+ record: Dict[str, Any],+ meta: Optional[Dict[str, Any]] = None,) -> None:"""Mark a session as successfully ingested."""key = f"viking:{sample_id}:{session_key}"@@ -178,61 +242,82 @@ def mark_ingested(# ---------------------------------------------------------------------------# OpenViking import# ---------------------------------------------------------------------------+def _parse_token_usage(token_data: Dict[str, Any]) -> Dict[str, int]:+ """解析Token使用数据(仅支持新版token_usage格式)"""+ usage = token_data["token_usage"]+ return {+ "embedding": usage["embedding"]["total_tokens"],+ "vlm": usage["llm"]["total_tokens"],+ "llm_input": usage["llm"]["prompt_tokens"],+ "llm_output": usage["llm"]["completion_tokens"],+ "total": usage["total"]["total_tokens"]+ }-def viking_ingest(messages: list[dict], session_time: str = None) -> None:- """Save messages to OpenViking via SyncHTTPClient (add messages + commit session).-- Args:- messages: List of message dicts with role and text- session_time: Session time string (e.g., "9:36 am on 2 April, 2023")+async def viking_ingest(msg: str, openviking_url: str, semaphore: asyncio.Semaphore) -> Dict[str, int]:+ """Save a message to OpenViking via OpenViking SDK client.+ Returns token usage dict with embedding and vlm token counts."""- from datetime import datetime+ # 使用信号量控制并发+ async with semaphore:+ # Create client+ client = ov.AsyncHTTPClient(url=openviking_url)+ await client.initialize()- # 解析 session_time- created_at = None- if session_time:try:- dt = datetime.strptime(session_time, "%I:%M %p on %d %B, %Y")- created_at = dt.isoformat()- except ValueError:- print(f"Warning: Failed to parse session_time: {session_time}", file=sys.stderr)+ # Create session+ create_res = await client.create_session()+ session_id = create_res["session_id"]+ session = client.session(session_id)- client = ov.SyncHTTPClient()- client.initialize()+ # Add message+ await session.add_message(+ role="user",+ parts=[TextPart(text=msg)]+ )- # Create new session- session_result = client.create_session()- session_id = session_result.get('session_id')+ # Commit+ result = await session.commit(telemetry=True)- # Add messages one by one- for msg in messages:- client.add_message(session_id, role=msg["role"], content=msg["text"], created_at=created_at)+ if not (result.get("status") == "accepted" and result.get("task_id")):+ raise RuntimeError(f"Commit failed: {result}")- # Commit session to trigger memory extraction- commit_result = client.commit_session(session_id)- task_id = commit_result.get("task_id")+ # 轮询等待异步任务完成+ task_id = result["task_id"]+ max_wait = 1200 # 最多等待20分钟+ waited = 0- # Wait for commit task to complete- if task_id:- now = time.time()- while True:- task = client.get_task(task_id)- if not task or task.get("status") in ("completed", "failed"):- break- time.sleep(1)- elapsed = time.time() - now- status = task.get("status", "unknown") if task else "not found"+ while waited < max_wait:+ task = await client.get_task(task_id)+ if task["status"] == "completed":+ token_usage = _parse_token_usage(task["result"])+ break+ elif task["status"] == "failed":+ raise RuntimeError(f"Commit failed: {task.get('error', 'Unknown error')}")- client.close()+ # 指数退避策略,避免频繁请求+ await asyncio.sleep(min(1 << (waited // 10), 60))+ waited += 1+ else:+ raise RuntimeError(f"Commit timed out after {max_wait} seconds")++ return token_usage++ finally:+ await client.close()+++def sync_viking_ingest(msg: str, openviking_url: str) -> Dict[str, int]:+ """Synchronous wrapper for viking_ingest to maintain existing API."""+ semaphore = asyncio.Semaphore(1) # 同步调用时使用信号量为1+ return asyncio.run(viking_ingest(msg, openviking_url, semaphore))# ---------------------------------------------------------------------------# Main import logic# -----------------------------------------------------------------------------def parse_session_range(s: str) -> tuple[int, int]:+def parse_session_range(s: str) -> Tuple[int, int]:"""Parse '1-4' or '3' into (lo, hi) inclusive tuple."""if "-" in s:lo, hi = s.split("-", 1)@@ -241,7 +326,69 @@ def parse_session_range(s: str) -> tuple[int, int]:return n, n-def run_import(args: argparse.Namespace) -> None:+async def process_single_session(+ msg: str,+ sample_id: str | int,+ session_key: str,+ meta: Dict[str, Any],+ run_time: str,+ ingest_record: Dict[str, Any],+ args: argparse.Namespace,+ semaphore: asyncio.Semaphore+) -> Dict[str, Any]:+ """处理单个会话的导入任务"""+ try:+ token_usage = await viking_ingest(msg, args.openviking_url, semaphore)+ print(f" -> [SUCCESS] [{sample_id}/{session_key}] imported to OpenViking", file=sys.stderr)++ # Extract token counts+ embedding_tokens = token_usage.get("embedding", 0)+ vlm_tokens = token_usage.get("vlm", 0)+ print(f" -> [USAGE] [{sample_id}/{session_key}] Embedding tokens: {embedding_tokens}, VLM tokens: {vlm_tokens}", file=sys.stderr)++ # Write success record+ result = {+ "timestamp": run_time,+ "sample_id": sample_id,+ "session": session_key,+ "status": "success",+ "meta": meta,+ "token_usage": token_usage,+ "embedding_tokens": embedding_tokens,+ "vlm_tokens": vlm_tokens+ }++ # 写入成功CSV+ write_success_record(result, args.success_csv)++ # Mark as successfully ingested+ mark_ingested(sample_id, session_key, ingest_record, meta)+ save_ingest_record(ingest_record) # Save immediately after success++ return result++ except Exception as e:+ print(f" -> [ERROR] [{sample_id}/{session_key}] {e}", file=sys.stderr)++ # Write error record+ result = {+ "timestamp": run_time,+ "sample_id": sample_id,+ "session": session_key,+ "status": "error",+ "error": str(e)+ }++ # 写入错误日志+ write_error_record(result, args.error_log)++ return result+++async def run_import(args: argparse.Namespace) -> None:+ # 初始化信号量控制并发+ semaphore = asyncio.Semaphore(args.parallel)+session_range = parse_session_range(args.sessions) if args.sessions else None# Handle ingest record operations@@ -252,18 +399,21 @@ def run_import(args: argparse.Namespace) -> None:else:ingest_record = load_ingest_record()- # Open output files for incremental writing- txt_output = open(args.output, "a", encoding="utf-8")- jsonl_output = open(f"{args.output}.jsonl", "a", encoding="utf-8")+ # 加载成功CSV记录用于去重+ success_keys = set()+ if not args.force_ingest:+ success_keys = load_success_csv(args.success_csv)+ print(f"[INFO] Loaded {len(success_keys)} existing success records from {args.success_csv}", file=sys.stderr)# Write run headerrun_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")- txt_output.write(f"\n=== Import run at {run_time} ===\n")- txt_output.flush()skipped_count = 0success_count = 0error_count = 0+ total_embedding_tokens = 0+ total_vlm_tokens = 0+ tasks: List[asyncio.Task] = []if args.input.endswith(".json"):# LoCoMo JSON format@@ -278,73 +428,33 @@ def run_import(args: argparse.Namespace) -> None:for sess in sessions:meta = sess["meta"]- messages = sess["messages"]- label = f"{meta['session_key']} ({meta['date_time']})"+ msg = sess["message"]+ session_key = meta["session_key"]+ label = f"{session_key} ({meta['date_time']})"# Skip already ingested sessions unless force-ingest is enabled- if not args.force_ingest and is_already_ingested(sample_id, meta['session_key'], ingest_record):+ if not args.force_ingest and is_already_ingested(sample_id, session_key, ingest_record, success_keys):print(f" [{label}] [SKIP] already imported (use --force-ingest to reprocess)", file=sys.stderr)skipped_count += 1-- # Write skip record- result = {- "timestamp": run_time,- "sample_id": sample_id,- "session": meta["session_key"],- "status": "skipped",- "reason": "already imported"- }- txt_output.write(f"[{sample_id}/{meta['session_key']}] SKIPPED: already imported\n")- jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")- txt_output.flush()- jsonl_output.flush()continue- # Preview messages- preview = " | ".join([f"{msg['role']}: {msg['text'][:30]}..." for msg in messages[:3]])- print(f" [{label}] {preview}", file=sys.stderr)-- try:- viking_ingest(messages, session_time=meta.get("date_time"))- print(f" -> [SUCCESS] imported to OpenViking", file=sys.stderr)- success_count += 1-- # Write success record- result = {- "timestamp": run_time,- "sample_id": sample_id,- "session": meta["session_key"],- "status": "success",- "meta": meta- }- txt_output.write(f"[{sample_id}/{meta['session_key']}] SUCCESS\n")- jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")- txt_output.flush()- jsonl_output.flush()-- # Mark as successfully ingested- mark_ingested(sample_id, meta['session_key'], ingest_record, {- "date_time": meta['date_time'],- "speakers": meta['speakers']- })- save_ingest_record(ingest_record) # Save immediately after success-- except Exception as e:- print(f" -> [ERROR] {e}", file=sys.stderr)- error_count += 1-- # Write error record- result = {- "timestamp": run_time,- "sample_id": sample_id,- "session": meta["session_key"],- "status": "error",- "error": str(e)- }- txt_output.write(f"[{sample_id}/{meta['session_key']}] ERROR: {str(e)}\n")- jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")- txt_output.flush()- jsonl_output.flush()+ preview = msg.replace("\n", " | ")[:80]+ print(f" [{label}] {preview}...", file=sys.stderr)++ # 创建异步任务+ task = asyncio.create_task(+ process_single_session(+ msg=msg,+ sample_id=sample_id,+ session_key=session_key,+ meta=meta,+ run_time=run_time,+ ingest_record=ingest_record,+ args=args,+ semaphore=semaphore+ )+ )+ tasks.append(task)else:# Plain text format@@ -356,80 +466,47 @@ def run_import(args: argparse.Namespace) -> None:print(f"\n=== Text Session {idx} ===", file=sys.stderr)# Skip already ingested sessions unless force-ingest is enabled- if not args.force_ingest and is_already_ingested("txt", session_key, ingest_record):+ if not args.force_ingest and is_already_ingested("txt", session_key, ingest_record, success_keys):print(f" [SKIP] already imported (use --force-ingest to reprocess)", file=sys.stderr)skipped_count += 1-- # Write skip record- result = {- "timestamp": run_time,- "sample_id": "txt",- "session": session_key,- "status": "skipped",- "reason": "already imported"- }- txt_output.write(f"[txt/{session_key}] SKIPPED: already imported\n")- jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")- txt_output.flush()- jsonl_output.flush()continue- # For plain text, all messages as user role- messages = []- for i, text in enumerate(session["messages"]):- messages.append({- "role": "user",- "text": text.strip(),- "speaker": "user",- "index": i- })-- preview = " | ".join([f"{msg['role']}: {msg['text'][:30]}..." for msg in messages[:3]])- print(f" {preview}", file=sys.stderr)-- try:- viking_ingest(messages)- print(f" -> [SUCCESS] imported to OpenViking", file=sys.stderr)- success_count += 1-- # Write success record- result = {- "timestamp": run_time,- "sample_id": "txt",- "session": session_key,- "status": "success",- "session_index": idx- }- txt_output.write(f"[txt/{session_key}] SUCCESS\n")- jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")- txt_output.flush()- jsonl_output.flush()-- mark_ingested("txt", session_key, ingest_record, {- "session_index": idx- })- save_ingest_record(ingest_record) # Save immediately after success-- except Exception as e:- print(f" -> [ERROR] {e}", file=sys.stderr)- error_count += 1-- # Write error record- result = {- "timestamp": run_time,- "sample_id": "txt",- "session": session_key,- "status": "error",- "error": str(e)- }- txt_output.write(f"[txt/{session_key}] ERROR: {str(e)}\n")- jsonl_output.write(json.dumps(result, ensure_ascii=False) + "\n")- txt_output.flush()- jsonl_output.flush()-- # Close output files- txt_output.close()- jsonl_output.close()+ combined_msg = "\n\n".join(session["messages"])+ preview = combined_msg.replace("\n", " | ")[:80]+ print(f" {preview}...", file=sys.stderr)++ # 创建异步任务+ task = asyncio.create_task(+ process_single_session(+ msg=combined_msg,+ sample_id="txt",+ session_key=session_key,+ meta={"session_index": idx},+ run_time=run_time,+ ingest_record=ingest_record,+ args=args,+ semaphore=semaphore+ )+ )+ tasks.append(task)++ # 等待所有任务完成+ print(f"\n[INFO] Starting import with {args.parallel} concurrent workers, {len(tasks)} tasks to process", file=sys.stderr)+ results = await asyncio.gather(*tasks, return_exceptions=True)++ # 统计结果+ for result in results:+ if isinstance(result, Exception):+ error_count += 1+ print(f"[UNEXPECTED ERROR] Task failed with exception: {result}", file=sys.stderr)+ continue++ if result["status"] == "success":+ success_count += 1+ total_embedding_tokens += result["embedding_tokens"]+ total_vlm_tokens += result["vlm_tokens"]+ elif result["status"] == "error":+ error_count += 1# Final summarytotal_processed = success_count + error_count + skipped_count@@ -438,14 +515,21 @@ def run_import(args: argparse.Namespace) -> None:print(f"Successfully imported: {success_count}", file=sys.stderr)print(f"Failed: {error_count}", file=sys.stderr)print(f"Skipped (already imported): {skipped_count}", file=sys.stderr)- print(f"Results saved to: {args.output} (text) and {args.output}.jsonl (JSON Lines)", file=sys.stderr)+ print(f"\n=== Token usage summary ===", file=sys.stderr)+ print(f"Total Embedding tokens: {total_embedding_tokens}", file=sys.stderr)+ print(f"Total VLM tokens: {total_vlm_tokens}", file=sys.stderr)+ if success_count > 0:+ print(f"Average Embedding per session: {total_embedding_tokens // success_count}", file=sys.stderr)+ print(f"Average VLM per session: {total_vlm_tokens // success_count}", file=sys.stderr)+ print(f"\nResults saved to:", file=sys.stderr)+ print(f" - Success records: {args.success_csv}", file=sys.stderr)+ print(f" - Error logs: {args.error_log}", file=sys.stderr)# ---------------------------------------------------------------------------# CLI# ----------------------------------------------------------------------------def main():parser = argparse.ArgumentParser(description="Import conversations into OpenViking")parser.add_argument(@@ -454,9 +538,25 @@ def main():help="Path to input file (.txt or LoCoMo .json)")parser.add_argument(- "--output",- default="./result/import_results",- help="Path to output file (default: import_results)",+ "--success-csv",+ default="./result/import_success.csv",+ help="Path to success records CSV file (default: import_success.csv)",+ )+ parser.add_argument(+ "--error-log",+ default="./result/import_errors.log",+ help="Path to error log file (default: import_errors.log)",+ )+ parser.add_argument(+ "--openviking-url",+ default="http://localhost:1933",+ help="OpenViking service URL (default: http://localhost:1933)",+ )+ parser.add_argument(+ "--parallel",+ type=int,+ default=5,+ help="Number of concurrent import workers (default: 5)",)parser.add_argument("--sample",@@ -483,8 +583,16 @@ def main():)args = parser.parse_args()- run_import(args)+ # 确保输出目录存在+ Path(args.success_csv).parent.mkdir(parents=True, exist_ok=True)+ Path(args.error_log).parent.mkdir(parents=True, exist_ok=True)++ try:+ asyncio.run(run_import(args))+ except ValueError as e:+ print(f"Error: {e}", file=sys.stderr)+ sys.exit(1)if __name__ == "__main__":- main()+ main()\ No newline at end of file