Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(benchmark): add Hermes OpenViking LoCoMo scripts (#1985)
Hao Zhe665b7d980fe2
diff --git a/benchmark/locomo/README.md b/benchmark/locomo/README.mdindex fe5b8268c..86d870fa6 100644--- a/benchmark/locomo/README.md+++ b/benchmark/locomo/README.md@@ -25,7 +25,8 @@ benchmark/locomo/│ └── result/ # 评测结果目录├── mem0/ # mem0 评测脚本(详见 mem0/README.md)├── supermemory/ # Supermemory 评测脚本(详见 supermemory/README.md)-└── claudecode/ # Claude Code 评测脚本(详见 claudecode/README.md)+├── claudecode/ # Claude Code 评测脚本(详见 claudecode/README.md)+└── hermes/ # Hermes Agent 评测脚本(详见 hermes/README.md)```---diff --git a/benchmark/locomo/hermes/README.md b/benchmark/locomo/hermes/README.mdnew file mode 100644index 000000000..708678611--- /dev/null+++ b/benchmark/locomo/hermes/README.md@@ -0,0 +1,56 @@+# Hermes LoCoMo Benchmark++This directory runs LoCoMo QA through Hermes Agent with three memory paths:++- `native`: imports transcripts into Hermes native memory, then evaluates through Hermes.+- `e2e`: imports transcripts through Hermes with the OpenViking memory plugin enabled, commits the resulting OpenViking sessions, then evaluates through Hermes.+- `preingest`: imports transcripts directly into OpenViking, then evaluates through Hermes against the preloaded OpenViking state.++Use `run_full_eval.sh` for normal runs:++```bash+cd benchmark/locomo/hermes+./run_full_eval.sh --suite native+./run_full_eval.sh --suite e2e+./run_full_eval.sh --suite preingest+```++## Required Services++- Hermes gateway running at `HERMES_URL` (default: `http://127.0.0.1:8642`).+- OpenViking server running at `OPENVIKING_URL` (default: `http://127.0.0.1:1933`) for `e2e` and `preingest`.+- LoCoMo dataset at `../data/locomo10.json`, or set `LOCOMO_JSON=/path/to/locomo10.json`.+- Judge model credentials through `JUDGE_TOKEN` or `ARK_API_KEY`.++The default OpenViking benchmark setup is local and does not require an API key. For non-default namespaces or authenticated servers, set `OPENVIKING_ACCOUNT`, `OPENVIKING_USER`, `OPENVIKING_AGENT`, and `OPENVIKING_API_KEY` consistently for import and eval.++## Fresh Runs++For OpenViking suites, start from a fresh OpenViking workspace when using `--force-ingest`:++```bash+rm -rf ~/.openviking/data/*+./run_full_eval.sh --suite e2e --force-ingest --force-eval+```++`--force-ingest` resets benchmark CSV/token files. It does not delete OpenViking server state. If old OpenViking archives remain, deterministic benchmark session IDs can reuse stale extracted memories.++Use `-cp` or `--checkpoint` on `e2e` and `preingest` runs to copy the OpenViking state after import:++```bash+./run_full_eval.sh --suite e2e -cp+```++## Results And Resume Behavior++Each run writes to a timestamped `result_*` directory unless `RESULT_DIR` or `--result-dir` is provided. Important files:++- `import_success.csv`: successful ingest sessions and Hermes ingest usage where applicable.+- `qa_results.csv`: answers, Hermes QA usage, tool calls, judge result, and reasoning.+- `import_true_tokens.csv`: OpenViking model-token delta observed after import.+- `eval_true_tokens.csv`: OpenViking model-token delta observed after QA.+- `stats.log`: final score and token summary.++Interrupted runs can be resumed in the same result directory. OpenViking token delta CSVs append one row per completed import/eval pass, so final stats sum all rows rather than reading only the last row.++Hermes token accounting prefers `state.db` when available through `HERMES_STATE_DB` or `HERMES_HOME/state.db`; otherwise stats fall back to the benchmark CSV usage fields.diff --git a/benchmark/locomo/hermes/eval.py b/benchmark/locomo/hermes/eval.pynew file mode 100644index 000000000..d59122ddb--- /dev/null+++ b/benchmark/locomo/hermes/eval.py@@ -0,0 +1,829 @@+"""+Shared Hermes LoCoMo QA evaluator.+"""++from __future__ import annotations++import argparse+import asyncio+import csv+import json+import os+import sys+import time+from concurrent.futures import ThreadPoolExecutor, as_completed+from datetime import datetime+from pathlib import Path+from threading import Lock++import httpx+import requests++try:+ from dotenv import load_dotenv+except ImportError:++ def load_dotenv(*_args, **_kwargs):+ return False+++env_file = Path.home() / ".openviking_benchmark_env"+load_dotenv(env_file)++DEFAULT_BASE_URL = os.getenv("HERMES_GATEWAY_BASE_URL", "http://127.0.0.1:8642")+DEFAULT_MODEL = os.getenv("HERMES_GATEWAY_MODEL", "hermes-agent")+DEFAULT_TOKEN = os.getenv("HERMES_GATEWAY_TOKEN") or os.getenv("API_SERVER_KEY") or ""+DEFAULT_OPENVIKING_URL = os.getenv("OPENVIKING_ENDPOINT", "http://127.0.0.1:1933")+DEFAULT_DATASET_LOCATION = "benchmark/locomo/data/locomo10.json"+MAX_HTTP_ATTEMPTS = 2+DEFAULT_ERROR_RETRIES = int(os.getenv("QA_ERROR_RETRIES", "2"))+QA_INSTRUCTIONS = "Answer to the best of your ability and reasonable inference."++csv_lock = Lock()+++def default_locomo_input(script_dir: Path) -> str:+ return os.getenv("LOCOMO_JSON") or str(script_dir.parent / "data" / "locomo10.json")+++def dataset_missing_message(path: str) -> str:+ return (+ f"LoCoMo dataset not found: {path}\n"+ f"Set LOCOMO_JSON=/path/to/locomo10.json, pass a script input path, "+ f"or place it at {DEFAULT_DATASET_LOCATION}."+ )+++def openviking_headers() -> dict[str, str]:+ headers = {+ "X-OpenViking-Account": os.getenv("OPENVIKING_ACCOUNT", "default"),+ "X-OpenViking-User": os.getenv("OPENVIKING_USER", "default"),+ "X-OpenViking-Agent": os.getenv("OPENVIKING_AGENT", "hermes"),+ }+ api_key = os.getenv("OPENVIKING_API_KEY", "")+ if api_key:+ headers["X-API-Key"] = api_key+ return headers+++def require_suite(value: str) -> str:+ if value not in {"baseline", "e2e", "preingest"}:+ raise argparse.ArgumentTypeError("suite must be one of: baseline, e2e, preingest")+ return value+++def result_dir_name(suite: str) -> str:+ if suite == "baseline":+ return "result_baseline"+ if suite == "e2e":+ return "result_e2e"+ return "result_preingest"+++def conversation_prefix(suite: str) -> str:+ if suite == "baseline":+ return "locomo-native-qa-"+ if suite == "e2e":+ return "locomo-e2e-qa-"+ return "locomo-ovpreingest-qa-"+++def load_locomo_data(path: str, sample_index: int | None = None) -> list[dict]:+ try:+ with open(path, "r", encoding="utf-8") as f:+ data = json.load(f)+ except FileNotFoundError:+ print(f"Error: {dataset_missing_message(path)}", file=sys.stderr)+ sys.exit(1)+ except json.JSONDecodeError as e:+ print(f"Error: invalid JSON in {path}: {e}", file=sys.stderr)+ sys.exit(1)++ if sample_index is None:+ return data+ if sample_index < 0 or sample_index >= len(data):+ print(+ f"Error: sample index {sample_index} out of range (0-{len(data) - 1})", file=sys.stderr+ )+ sys.exit(1)+ return [data[sample_index]]+++def parse_locomo_datetime(date_str: str) -> datetime | None:+ try:+ if " on " in date_str:+ return datetime.strptime(date_str.split(" on ")[-1].strip(), "%d %B, %Y")+ except ValueError:+ return None+ return None+++def get_sample_question_time(sample: dict) -> str | None:+ conversation = sample.get("conversation", {})+ session_keys = [+ key for key in conversation.keys() if key.startswith("session_") and "date_time" not in key+ ]+ if not session_keys:+ return None++ def session_num(key: str) -> int:+ try:+ return int(key.replace("session_", ""))+ except ValueError:+ return 0++ session_keys.sort(key=session_num, reverse=True)+ for key in session_keys:+ if not conversation.get(key):+ continue+ date_str = conversation.get(f"session_{session_num(key)}_date_time")+ if not date_str:+ continue+ parsed = parse_locomo_datetime(date_str)+ if parsed:+ return parsed.strftime("%Y-%m-%d")+ return None+++def normalize_usage(raw: dict | None) -> dict:+ raw = raw or {}+ input_tokens = raw.get("input_tokens") or raw.get("prompt_tokens", 0)+ output_tokens = raw.get("output_tokens") or raw.get("completion_tokens", 0)+ cache_read_tokens = raw.get("cache_read_tokens") or raw.get("cacheRead", 0)+ cache_write_tokens = raw.get("cache_write_tokens") or raw.get("cacheWrite", 0)+ total_tokens = raw.get("total_tokens") or raw.get("totalTokens")+ if total_tokens is None:+ total_tokens = int(input_tokens or 0) + int(output_tokens or 0)++ return {+ "input_tokens": int(input_tokens or 0),+ "output_tokens": int(output_tokens or 0),+ "cache_read_tokens": int(cache_read_tokens or 0),+ "cache_write_tokens": int(cache_write_tokens or 0),+ "total_tokens": int(total_tokens or 0),+ }+++def extract_response_text(body: dict) -> str:+ for item in body.get("output", []):+ if item.get("type") != "message":+ continue+ for content in item.get("content", []):+ if content.get("type") == "output_text":+ return content.get("text", "")+ if "text" in content:+ return content["text"]+ return ""+++def extract_tool_names(body: dict) -> list[str]:+ return [item["requested_name"] for item in extract_tool_trace(body)]+++def extract_tool_trace(body: dict) -> list[dict]:+ trace = []+ for item in body.get("output", []):+ if item.get("type") != "function_call":+ continue+ requested_name = item.get("requested_name") or item.get("name")+ if not isinstance(requested_name, str) or not requested_name:+ continue+ executed_name = item.get("executed_name")+ if not isinstance(executed_name, str):+ executed_name = ""+ if not executed_name and not item.get("was_blocked"):+ executed_name = requested_name+ trace.append(+ {+ "requested_name": requested_name,+ "executed_name": executed_name,+ "was_rewritten": bool(item.get("was_rewritten")),+ "was_blocked": bool(item.get("was_blocked")),+ }+ )+ return trace+++def send_gateway_request(+ base_url: str,+ token: str,+ model: str,+ conversation: str,+ input_text: str,+ instructions: str = QA_INSTRUCTIONS,+ timeout: int = 600,+) -> tuple[dict, float, str]:+ url = f"{base_url.rstrip('/')}/v1/responses"+ headers = {+ "Authorization": f"Bearer {token}",+ "Content-Type": "application/json",+ }+ payload = {+ "model": model,+ "input": input_text,+ "conversation": conversation,+ "session_id": conversation,+ "store": False,+ }+ if instructions:+ payload["instructions"] = instructions++ last_error: requests.RequestException | None = None+ total_elapsed = 0.0+ for attempt in range(1, MAX_HTTP_ATTEMPTS + 1):+ started_at = time.perf_counter()+ try:+ resp = requests.post(url, headers=headers, json=payload, timeout=timeout)+ elapsed = time.perf_counter() - started_at+ total_elapsed += elapsed+ resp.raise_for_status()+ return resp.json(), total_elapsed, resp.headers.get("X-Hermes-Session-Id", "")+ except requests.HTTPError as exc:+ elapsed = time.perf_counter() - started_at+ total_elapsed += elapsed+ if attempt >= MAX_HTTP_ATTEMPTS:+ raise+ last_error = exc+ except (requests.ConnectionError, requests.Timeout) as exc:+ elapsed = time.perf_counter() - started_at+ total_elapsed += elapsed+ if attempt >= MAX_HTTP_ATTEMPTS:+ raise+ last_error = exc+ time.sleep(float(2 ** (attempt - 1)))+ if last_error is not None:+ raise last_error+ raise RuntimeError("unreachable")+++def build_question_prompt(+ suite: str,+ question: str,+ question_time: str | None,+ sample_id: str,+) -> str:+ if question_time:+ return f"Current date: {question_time}. {question}"+ return question+++def process_single_question(+ item: dict,+ sample_idx: int,+ question_index: int,+ qa: dict,+ args: argparse.Namespace,+ csv_path: str,+) -> dict:+ sample_id = item["sample_id"]+ question = qa["question"]+ expected = str(qa["answer"])+ category = str(qa.get("category", ""))+ evidence = qa.get("evidence", [])+ question_time = get_sample_question_time(item)+ conversation = f"{args.conversation_prefix}{sample_id}-q{question_index}"++ if args.suite == "baseline":+ mode = "native DB, session_search"+ elif args.suite == "e2e":+ mode = "mixed memory E2E"+ else:+ mode = "ov-preingest, no replay"+ print(f"[{sample_id}] Q{question_index}: asking ({mode})", file=sys.stderr)++ try:+ body, qa_latency, hermes_session_id = send_gateway_request(+ base_url=args.base_url,+ token=args.token,+ model=args.model,+ conversation=conversation,+ input_text=build_question_prompt(+ args.suite,+ question,+ question_time,+ sample_id,+ ),+ timeout=args.timeout,+ )+ qa_usage = normalize_usage(body.get("usage"))+ response = extract_response_text(body)+ tool_trace = extract_tool_trace(body)+ tool_names = [item["requested_name"] for item in tool_trace]+ except requests.HTTPError as e:+ response = f"[ERROR] HTTP {e.response.status_code}: {e.response.text}"+ qa_usage = normalize_usage({})+ qa_latency = 0.0+ tool_trace = []+ tool_names = []+ hermes_session_id = ""+ except Exception as e:+ response = f"[ERROR] {e}"+ qa_usage = normalize_usage({})+ qa_latency = 0.0+ tool_trace = []+ tool_names = []+ hermes_session_id = ""++ record = {+ "sample_id": sample_id,+ "sample_idx": sample_idx,+ "qi": question_index,+ "question": question,+ "expected": expected,+ "response": response,+ "category": category,+ "evidence": json.dumps(evidence, ensure_ascii=False),+ "question_time": question_time or "",+ "conversation": conversation,+ "hermes_session_id": hermes_session_id,+ "qa_input_tokens": qa_usage["input_tokens"],+ "qa_output_tokens": qa_usage["output_tokens"],+ "qa_cache_read_tokens": qa_usage["cache_read_tokens"],+ "qa_cache_write_tokens": qa_usage["cache_write_tokens"],+ "qa_total_tokens": qa_usage["total_tokens"],+ "qa_latency_sec": f"{qa_latency:.4f}",+ "tool_call_count": len(tool_names),+ "tool_names": json.dumps(tool_names, ensure_ascii=False),+ "executed_tool_call_count": len([item for item in tool_trace if item["executed_name"]]),+ "executed_tool_names": json.dumps(+ [item["executed_name"] for item in tool_trace if item["executed_name"]],+ ensure_ascii=False,+ ),+ "rewritten_tool_call_count": sum(1 for item in tool_trace if item["was_rewritten"]),+ "blocked_tool_call_count": sum(1 for item in tool_trace if item["was_blocked"]),+ "result": "",+ "reasoning": "",+ }++ with csv_lock:+ save_record_to_csv(csv_path, record)++ print(f"[{sample_id}] Q{question_index}: done ({response[:60]}...)", file=sys.stderr)+ return record+++def is_error_record(row: dict) -> bool:+ return str(row.get("response", "")).lstrip().startswith("[ERROR]")+++def count_error_records(csv_path: str) -> int:+ if not os.path.exists(csv_path):+ return 0++ count = 0+ with open(csv_path, "r", encoding="utf-8", newline="") as f:+ for row in csv.DictReader(f):+ if is_error_record(row):+ count += 1+ return count+++def prune_error_records(csv_path: str) -> int:+ if not os.path.exists(csv_path):+ return 0++ with open(csv_path, "r", encoding="utf-8", newline="") as f:+ reader = csv.DictReader(f)+ fieldnames = reader.fieldnames or []+ rows = list(reader)++ kept = [row for row in rows if not is_error_record(row)]+ removed = len(rows) - len(kept)+ if removed <= 0:+ return 0++ with open(csv_path, "w", encoding="utf-8", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ writer.writeheader()+ writer.writerows(kept)+ f.flush()+ return removed+++def load_executed_records(csv_path: str) -> set[tuple[str, int]]:+ executed = set()+ if not os.path.exists(csv_path):+ return executed++ with open(csv_path, "r", encoding="utf-8", newline="") as f:+ for row in csv.DictReader(f):+ if is_error_record(row):+ continue+ try:+ executed.add((row["sample_id"], int(row["qi"])))+ except Exception:+ continue+ return executed+++def save_record_to_csv(csv_path: str, record: dict) -> None:+ os.makedirs(os.path.dirname(csv_path), exist_ok=True)+ fieldnames = [+ "sample_id",+ "sample_idx",+ "qi",+ "question",+ "expected",+ "response",+ "category",+ "evidence",+ "question_time",+ "conversation",+ "hermes_session_id",+ "qa_input_tokens",+ "qa_output_tokens",+ "qa_cache_read_tokens",+ "qa_cache_write_tokens",+ "qa_total_tokens",+ "qa_latency_sec",+ "tool_call_count",+ "tool_names",+ "executed_tool_call_count",+ "executed_tool_names",+ "rewritten_tool_call_count",+ "blocked_tool_call_count",+ "result",+ "reasoning",+ ]+ expected_header = ",".join(fieldnames)+ should_write_header = not os.path.exists(csv_path)++ if not should_write_header:+ with open(csv_path, "r", encoding="utf-8", newline="") as f:+ current_header = f.readline().strip()+ if current_header != expected_header:+ with open(csv_path, "r", encoding="utf-8", newline="") as f:+ existing_rows = list(csv.DictReader(f))+ with open(csv_path, "w", encoding="utf-8", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ writer.writeheader()+ for row in existing_rows:+ writer.writerow({key: row.get(key, "") for key in fieldnames})++ with open(csv_path, "a", encoding="utf-8", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ if should_write_header:+ writer.writeheader()+ writer.writerow(record)+ f.flush()+++def reset_generated_file(path: str | Path, label: str) -> None:+ target = Path(path)+ if target.exists():+ target.unlink()+ print(f"Removed existing {label}: {target}", file=sys.stderr)+++def parse_observer_model_totals(status_text: str) -> tuple[int, int, int, int]:+ embedding_prompt = 0+ embedding_completion = 0+ vlm_prompt = 0+ vlm_completion = 0+ current_section = ""++ for raw_line in status_text.splitlines():+ line = raw_line.strip()+ if not line:+ continue+ if line.endswith("Models:"):+ current_section = line.split(" ", 1)[0].lower()+ continue+ if not line.startswith("|"):+ continue++ cols = [col.strip() for col in line.split("|") if col.strip()]+ if len(cols) < 7 or cols[0] == "Model":+ continue++ prompt = int(cols[3]) if cols[3].isdigit() else 0+ completion = int(cols[4]) if cols[4].isdigit() else 0+ if current_section == "embedding":+ embedding_prompt += prompt+ embedding_completion += completion+ elif current_section == "vlm":+ vlm_prompt += prompt+ vlm_completion += completion++ return embedding_prompt, embedding_completion, vlm_prompt, vlm_completion+++async def _read_model_totals(+ client: httpx.AsyncClient, openviking_url: str+) -> tuple[int, int, int, int] | None:+ try:+ resp = await client.get(f"{openviking_url}/api/v1/observer/models")+ if resp.status_code != 200:+ return None+ status_text = resp.json().get("result", {}).get("status", "")+ except Exception:+ return None++ if "No model usage data available" in status_text:+ return 0, 0, 0, 0+ return parse_observer_model_totals(status_text)+++async def _read_queue_totals(+ client: httpx.AsyncClient, openviking_url: str+) -> tuple[int, int, int] | None:+ try:+ resp = await client.get(f"{openviking_url}/api/v1/observer/queue")+ if resp.status_code != 200:+ return None+ status_text = resp.json().get("result", {}).get("status", "")+ except Exception:+ return None++ for raw_line in status_text.splitlines():+ line = raw_line.strip()+ if not line.startswith("|") or "TOTAL" not in line:+ continue+ cols = [col.strip() for col in line.split("|") if col.strip()]+ if len(cols) < 4 or cols[0] != "TOTAL":+ continue+ try:+ return int(cols[1]), int(cols[2]), int(cols[3])+ except ValueError:+ return None+ return None+++def _append_true_token_record(output_file: str, record: dict) -> None:+ fieldnames = [+ "timestamp",+ "embedding_input_tokens",+ "embedding_output_tokens",+ "vlm_llm_input_tokens",+ "vlm_llm_output_tokens",+ ]+ expected_header = ",".join(fieldnames)+ mode = "a"+ should_write_header = not os.path.exists(output_file)++ if not should_write_header:+ with open(output_file, "r", encoding="utf-8", newline="") as f:+ current_header = f.readline().strip()+ if current_header != expected_header:+ mode = "w"+ should_write_header = True++ with open(output_file, mode, encoding="utf-8", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ if should_write_header:+ writer.writeheader()+ writer.writerow(record)+++async def wait_for_queues_and_record_totals(+ openviking_url: str,+ output_file: str,+ baseline_processed: int = 0,+ baseline_model_totals: tuple[int, int, int, int] | None = None,+ max_wait_sec: int = 1800,+ settle_checks: int = 3,+ poll_interval: float = 2.0,+) -> None:+ print(+ f"\n[INFO] Waiting for OpenViking background queues to drain "+ f"(baseline_processed={baseline_processed}, settle={settle_checks})...",+ file=sys.stderr,+ )++ async with httpx.AsyncClient(headers=openviking_headers()) as client:+ started = time.perf_counter()+ idle_streak = 0+ last_totals: tuple[int, int, int] | None = None++ while True:+ elapsed = time.perf_counter() - started+ if elapsed > max_wait_sec:+ print(+ f"[WARN] Queue drain timeout after {elapsed:.0f}s (last totals={last_totals})",+ file=sys.stderr,+ )+ break++ totals = await _read_queue_totals(client, openviking_url)+ if totals is None:+ await asyncio.sleep(poll_interval)+ continue++ pending, in_progress, processed = totals+ last_totals = totals+ queues_idle = pending == 0 and in_progress == 0++ if queues_idle:+ idle_streak += 1+ if idle_streak >= settle_checks:+ print(+ f"[INFO] Background queues drained. Processed {processed - baseline_processed} item(s) since baseline.",+ file=sys.stderr,+ )+ break+ else:+ idle_streak = 0+ print(+ f" .. queue state pending={pending} in_progress={in_progress} processed={processed}",+ file=sys.stderr,+ )++ await asyncio.sleep(poll_interval)++ print("[INFO] Fetching final true token usage from observer...", file=sys.stderr)+ try:+ final_totals = await _read_model_totals(client, openviking_url)+ if final_totals is None:+ print("[INFO] No model usage recorded by OpenViking server.", file=sys.stderr)+ return++ (+ baseline_embedding_prompt,+ baseline_embedding_completion,+ baseline_vlm_prompt,+ baseline_vlm_completion,+ ) = baseline_model_totals or (0, 0, 0, 0)+ (+ final_embedding_prompt,+ final_embedding_completion,+ final_vlm_prompt,+ final_vlm_completion,+ ) = final_totals++ delta_embedding_prompt = max(final_embedding_prompt - baseline_embedding_prompt, 0)+ delta_embedding_completion = max(+ final_embedding_completion - baseline_embedding_completion, 0+ )+ delta_vlm_prompt = max(final_vlm_prompt - baseline_vlm_prompt, 0)+ delta_vlm_completion = max(final_vlm_completion - baseline_vlm_completion, 0)++ print("\n=== TRUE OpenViking Token Delta For This Run ===", file=sys.stderr)+ print(f"Embedding Input (Prompt) Delta: {delta_embedding_prompt}", file=sys.stderr)+ print(+ f"Embedding Output (Completion) Delta: {delta_embedding_completion}",+ file=sys.stderr,+ )+ print(f"VLM/LLM Input (Prompt) Delta: {delta_vlm_prompt}", file=sys.stderr)+ print(f"VLM/LLM Output (Completion) Delta: {delta_vlm_completion}", file=sys.stderr)+ print(+ f"[INFO] Final cumulative totals: embedding_in={final_embedding_prompt}, "+ f"embedding_out={final_embedding_completion}, vlm_in={final_vlm_prompt}, "+ f"vlm_out={final_vlm_completion}",+ file=sys.stderr,+ )++ _append_true_token_record(+ output_file,+ {+ "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),+ "embedding_input_tokens": delta_embedding_prompt,+ "embedding_output_tokens": delta_embedding_completion,+ "vlm_llm_input_tokens": delta_vlm_prompt,+ "vlm_llm_output_tokens": delta_vlm_completion,+ },+ )+ print(f"True token totals saved to: {output_file}", file=sys.stderr)+ except Exception as e:+ print(f"[ERROR] Failed to fetch final token usage: {e}", file=sys.stderr)+++def run_eval_pass(+ samples: list[dict], args: argparse.Namespace, executed_records: set[tuple[str, int]]+) -> None:+ for sample_idx, item in enumerate(samples, start=1):+ sample_id = item["sample_id"]+ qas = [qa for qa in item.get("qa", []) if str(qa.get("category", "")) != "5"]+ if args.count is not None:+ qas = qas[: args.count]++ tasks = []+ for question_index, qa in enumerate(qas, start=1):+ if (sample_id, question_index) in executed_records:+ print(f"[{sample_id}] skip Q{question_index}: already recorded", file=sys.stderr)+ continue+ tasks.append((question_index, qa))++ if not tasks:+ continue++ print(+ f"[{sample_id}] running {len(tasks)} question(s) with {args.parallel} workers",+ file=sys.stderr,+ )+ with ThreadPoolExecutor(max_workers=max(1, args.parallel)) as executor:+ futures = [+ executor.submit(+ process_single_question, item, sample_idx, question_index, qa, args, args.output+ )+ for question_index, qa in tasks+ ]+ for future in as_completed(futures):+ future.result()+++def run_eval(args: argparse.Namespace) -> None:+ if not args.token:+ print("Error: HERMES_GATEWAY_TOKEN or --token is required", file=sys.stderr)+ sys.exit(1)++ samples = load_locomo_data(args.input, args.sample)++ if args.force:+ reset_generated_file(args.output, "QA CSV")+ reset_generated_file(+ Path(args.output).parent / "eval_true_tokens.csv", "eval true-token CSV"+ )+ else:+ removed = prune_error_records(args.output)+ if removed:+ print(f"Pruned {removed} previous error row(s) for retry", file=sys.stderr)++ baseline_processed = 0+ baseline_model_totals = None+ if args.suite in {"e2e", "preingest"}:++ async def snapshot_processed() -> int:+ async with httpx.AsyncClient(headers=openviking_headers()) as client:+ totals = await _read_queue_totals(client, args.openviking_url)+ return totals[2] if totals else 0++ async def snapshot_models() -> tuple[int, int, int, int] | None:+ async with httpx.AsyncClient(headers=openviking_headers()) as client:+ return await _read_model_totals(client, args.openviking_url)++ baseline_processed = asyncio.run(snapshot_processed())+ baseline_model_totals = asyncio.run(snapshot_models())+ print(f"[INFO] Observer baseline processed={baseline_processed}", file=sys.stderr)++ max_rounds = max(0, args.error_retries) + 1+ for round_index in range(max_rounds):+ executed_records = (+ set() if args.force and round_index == 0 else load_executed_records(args.output)+ )+ print(+ f"Loaded {len(executed_records)} executed records from {args.output}", file=sys.stderr+ )+ if round_index:+ print(f"Retry round {round_index}/{max_rounds - 1}", file=sys.stderr)++ run_eval_pass(samples, args, executed_records)++ error_count = count_error_records(args.output)+ if error_count == 0 or round_index >= max_rounds - 1:+ break++ removed = prune_error_records(args.output)+ print(f"Retrying {removed} error row(s)", file=sys.stderr)++ print(f"Done. Results written to {args.output}", file=sys.stderr)++ if args.suite in {"e2e", "preingest"}:+ true_token_file = Path(args.output).parent / "eval_true_tokens.csv"+ asyncio.run(+ wait_for_queues_and_record_totals(+ args.openviking_url,+ str(true_token_file),+ baseline_processed=baseline_processed,+ baseline_model_totals=baseline_model_totals,+ )+ )+++def main() -> None:+ script_dir = Path(__file__).parent.resolve()+ parser = argparse.ArgumentParser(description="Shared Hermes LoCoMo QA evaluator")+ parser.add_argument(+ "input", nargs="?", default=default_locomo_input(script_dir), help="Path to LoCoMo JSON"+ )+ parser.add_argument("--suite", type=require_suite, required=True, help="Benchmark suite")+ parser.add_argument("--output", default=None, help="Path to output CSV")+ parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="Hermes gateway base URL")+ parser.add_argument("--token", default=DEFAULT_TOKEN, help="Hermes gateway token")+ parser.add_argument("--model", default=DEFAULT_MODEL, help="Hermes gateway model name")+ parser.add_argument("--sample", type=int, default=None, help="Sample index (0-based)")+ parser.add_argument("--count", type=int, default=None, help="Questions per sample")+ parser.add_argument("--parallel", type=int, default=4, help="Parallel question workers")+ parser.add_argument("--timeout", type=int, default=600, help="Gateway request timeout seconds")+ parser.add_argument(+ "--force", action="store_true", help="Ignore existing CSV and rerun questions"+ )+ parser.add_argument(+ "--error-retries",+ type=int,+ default=DEFAULT_ERROR_RETRIES,+ help="Retry rows whose response starts with [ERROR] this many times",+ )+ parser.add_argument(+ "--openviking-url", default=DEFAULT_OPENVIKING_URL, help="OpenViking service URL"+ )+ args = parser.parse_args()++ if args.output is None:+ args.output = str(script_dir / result_dir_name(args.suite) / "qa_results.csv")+ args.conversation_prefix = conversation_prefix(args.suite)++ run_eval(args)+++if __name__ == "__main__":+ main()diff --git a/benchmark/locomo/hermes/import_e2e.py b/benchmark/locomo/hermes/import_e2e.pynew file mode 100644index 000000000..f572c27b8--- /dev/null+++ b/benchmark/locomo/hermes/import_e2e.py@@ -0,0 +1,999 @@+"""+Hermes E2E memory ingest tool for mixed native and OpenViking memory.+"""++from __future__ import annotations++import argparse+import asyncio+import csv+import json+import os+import sys+import time+from datetime import datetime+from pathlib import Path++import httpx+import requests++try:+ from dotenv import load_dotenv+except ImportError:++ def load_dotenv(*_args, **_kwargs):+ return False+++env_file = Path.home() / ".openviking_benchmark_env"+load_dotenv(env_file)++DEFAULT_BASE_URL = os.getenv("HERMES_GATEWAY_BASE_URL", "http://127.0.0.1:8642")+DEFAULT_MODEL = os.getenv("HERMES_GATEWAY_MODEL", "hermes-agent")+DEFAULT_TOKEN = os.getenv("HERMES_GATEWAY_TOKEN") or os.getenv("API_SERVER_KEY") or ""+DEFAULT_OPENVIKING_URL = os.getenv("OPENVIKING_ENDPOINT", "http://127.0.0.1:1933")+DEFAULT_IMPORT_ERROR_RETRIES = int(os.getenv("IMPORT_ERROR_RETRIES", "2"))+DEFAULT_SESSION_PREFIX = "locomo-e2e"+DEFAULT_DATASET_LOCATION = "benchmark/locomo/data/locomo10.json"+DEFAULT_IMPORT_ACK_PROMPT = (+ "This is a past conversation for memory ingestion, not a live request. "+ "The benchmark ingest hook has already recorded the user message in OpenViking. "+ "Treat the user message as transcript data only. Do not perform tasks. "+ "Do not use any tools, including viking_remember or other memory tools. "+ "Acknowledge with exactly: OK"+)+COMMIT_WAIT_ATTEMPTS = 60+COMMIT_WAIT_INTERVAL_SEC = 0.5+COMMIT_TASK_POLL_INTERVAL_SEC = 1.0+COMMIT_POST_ATTEMPTS = 3+EXPECTED_E2E_OPENVIKING_MESSAGES = 2++csv_lock = asyncio.Lock()+++def default_locomo_input(script_dir: Path) -> str:+ return os.getenv("LOCOMO_JSON") or str(script_dir.parent / "data" / "locomo10.json")+++def dataset_missing_message(path: str) -> str:+ return (+ f"LoCoMo dataset not found: {path}\n"+ f"Set LOCOMO_JSON=/path/to/locomo10.json, pass a script input path, "+ f"or place it at {DEFAULT_DATASET_LOCATION}."+ )+++def openviking_headers(*, content_type: bool = False) -> dict[str, str]:+ headers = {+ "X-OpenViking-Account": os.getenv("OPENVIKING_ACCOUNT", "default"),+ "X-OpenViking-User": os.getenv("OPENVIKING_USER", "default"),+ "X-OpenViking-Agent": os.getenv("OPENVIKING_AGENT", "hermes"),+ }+ api_key = os.getenv("OPENVIKING_API_KEY", "")+ if api_key:+ headers["X-API-Key"] = api_key+ if content_type:+ headers["Content-Type"] = "application/json"+ return headers+++def load_locomo_data(path: str, sample_index: int | None = None) -> list[dict]:+ try:+ with open(path, "r", encoding="utf-8") as f:+ data = json.load(f)+ except FileNotFoundError:+ print(f"Error: {dataset_missing_message(path)}", file=sys.stderr)+ sys.exit(1)+ if sample_index is not None:+ return [data[sample_index]]+ return data+++def _get_session_number(session_key: str) -> int:+ return int(session_key.split("_")[1])+++def _clean_text(value: object) -> str:+ if value is None:+ return ""+ if isinstance(value, list):+ return ", ".join(str(item).strip() for item in value if str(item).strip())+ return str(value).strip()+++def _normalize_locomo_session_time(date_time: str) -> str:+ value = date_time.strip()+ if not value:+ return ""+ for suffix in ("am", "pm"):+ value = value.replace(f" {suffix} ", f" {suffix.upper()} ")+ try:+ parsed = datetime.strptime(value, "%I:%M %p on %d %B, %Y")+ except ValueError:+ return ""+ return parsed.strftime("%Y-%m-%d %H:%M")+++def _visual_metadata_line(speaker: str, msg: dict) -> str:+ caption = _clean_text(msg.get("blip_caption"))+ query = _clean_text(msg.get("query"))+ details = []+ if caption:+ details.append(f"caption: {caption}")+ if query:+ details.append(f"query: {query}")+ if not details:+ return ""+ return f"[{speaker} visual]: " + "; ".join(details)+++def _session_header_lines(conv: dict, session_key: str, date_time: str) -> list[str]:+ speaker_a = conv.get("speaker_a", "A")+ speaker_b = conv.get("speaker_b", "B")+ lines = [+ f"The following is a conversation transcript between {speaker_a} and {speaker_b}.",+ "---",+ ]+ if date_time:+ lines.append(f"\n[Session: {date_time}]")+ normalized_time = _normalize_locomo_session_time(date_time)+ if normalized_time:+ lines.append(f"[Session date: {normalized_time}]")+ lines.append(f"[LoCoMo session: {session_key}]")+ return lines+++def _message_lines(msg: dict) -> list[str]:+ speaker = msg.get("speaker", "unknown")+ text = _clean_text(msg.get("text"))+ lines = [f"[{speaker}]: {text}"]+ visual_line = _visual_metadata_line(speaker, msg)+ if visual_line:+ lines.append(visual_line)+ return lines+++def build_session_transcripts(+ sample: dict,+) -> list[dict]:+ conv = sample.get("conversation", {})+ session_keys = sorted(+ [k for k in conv if k.startswith("session_") and not k.endswith("_date_time")],+ key=_get_session_number,+ )++ transcripts = []++ for session_key in session_keys:+ date_time = conv.get(f"{session_key}_date_time", "")+ lines = _session_header_lines(conv, session_key, date_time)++ for msg in conv[session_key]:+ lines.extend(_message_lines(msg))++ lines.append("---")+ transcripts.append(+ {+ "sample_id": sample["sample_id"],+ "session": session_key,+ "date_time": date_time,+ "transcript": "\n".join(lines),+ }+ )+ return transcripts+++def format_transcript(sample: dict) -> str:+ sample_with_id = dict(sample)+ sample_with_id.setdefault("sample_id", "")+ return "\n\n".join(+ item["transcript"]+ for item in build_session_transcripts(+ sample_with_id,+ )+ )+++def _build_session_id(sample_id: str, session_key: str) -> str:+ return f"{DEFAULT_SESSION_PREFIX}-{sample_id}-{session_key}"+++def ingest_session(+ sample_id: str,+ session_key: str,+ transcript: str,+ args: argparse.Namespace,+) -> dict:+ url = f"{args.base_url.rstrip('/')}/v1/chat/completions"+ session_id = _build_session_id(sample_id, session_key)+ headers = {+ "Authorization": f"Bearer {args.token}",+ "Content-Type": "application/json",+ "X-Hermes-Session-Id": session_id,+ }+ messages = [+ {"role": "system", "content": DEFAULT_IMPORT_ACK_PROMPT},+ {"role": "user", "content": transcript},+ ]+ payload = {+ "model": args.model,+ "messages": messages,+ }++ started_at = time.perf_counter()+ resp = requests.post(url, headers=headers, json=payload, timeout=args.timeout)+ elapsed = time.perf_counter() - started_at+ resp.raise_for_status()+ body = resp.json()+ resolved_session_id = resp.headers.get("X-Hermes-Session-Id", session_id)++ usage = body.get("usage", {})+ total_tokens = usage.get("total_tokens", 0)+ input_tokens = usage.get("input_tokens", usage.get("prompt_tokens", 0))+ output_tokens = usage.get("output_tokens", usage.get("completion_tokens", 0))+ cache_read = usage.get("cache_read_tokens", usage.get("cacheRead", 0))+ cache_write = usage.get("cache_write_tokens", usage.get("cacheWrite", 0))++ print(f" -> [{sample_id}/{session_key}] Ingested in {elapsed:.2f}s (Tokens: {total_tokens})")++ return {+ "sample_id": sample_id,+ "session": session_key,+ "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),+ "conversation": resolved_session_id,+ "total_tokens": total_tokens,+ "input_tokens": input_tokens,+ "output_tokens": output_tokens,+ "cache_read": cache_read,+ "cache_write": cache_write,+ "session_id": resolved_session_id,+ "status": "success",+ }+++def load_success_keys(csv_path: str) -> set[str]:+ keys: set[str] = set()+ if not os.path.exists(csv_path):+ return keys+ with open(csv_path, "r", encoding="utf-8") as f:+ for row in csv.DictReader(f):+ sample_id = row.get("sample_id", "")+ session_key = row.get("session", "")+ if not sample_id:+ continue+ if session_key:+ keys.add(f"{sample_id}:{session_key}")+ else:+ keys.add(f"{sample_id}:*")+ return keys+++def is_already_ingested(sample_id: str, session_key: str, success_keys: set[str]) -> bool:+ return f"{sample_id}:{session_key}" in success_keys or f"{sample_id}:*" in success_keys+++def _unwrap_openviking_dict(body: object) -> dict:+ if not isinstance(body, dict):+ return {}+ result = body.get("result")+ if isinstance(result, dict):+ return result+ return body+++def _read_openviking_session(base_url: str, session_id: str) -> dict | None:+ resp = requests.get(+ f"{base_url}/api/v1/sessions/{session_id}",+ headers=openviking_headers(),+ timeout=10,+ )+ if resp.status_code != 200:+ return None+ try:+ body = resp.json()+ except Exception:+ return None+ if isinstance(body, dict) and body.get("status") not in {None, "ok"}:+ return None+ session = _unwrap_openviking_dict(body)+ return session if session else None+++def _resolve_openviking_workspace() -> Path | None:+ source = os.getenv("OPENVIKING_STATE_SOURCE", "").strip()+ if source:+ return Path(source).expanduser().resolve()++ config_candidates: list[Path] = []+ if os.getenv("OPENVIKING_CONFIG_FILE"):+ config_candidates.append(Path(os.environ["OPENVIKING_CONFIG_FILE"]).expanduser())+ config_candidates.append(Path.home() / ".openviking" / "ov.conf")+ config_candidates.append(Path("/etc/openviking/ov.conf"))++ for config_path in config_candidates:+ if not config_path.exists():+ continue+ try:+ raw = os.path.expandvars(config_path.read_text(encoding="utf-8-sig"))+ data = json.loads(raw)+ except Exception:+ continue+ storage = data.get("storage", {})+ if isinstance(storage, dict) and storage.get("workspace"):+ return Path(storage["workspace"]).expanduser().resolve()++ return (Path.home() / ".openviking" / "data").expanduser().resolve()+++def _find_done_marker(session_id: str) -> Path | None:+ workspace = _resolve_openviking_workspace()+ if workspace is None or not workspace.exists():+ return None++ account = os.getenv("OPENVIKING_ACCOUNT", "default")+ direct_history = workspace / "viking" / account / "session" / session_id / "history"+ try:+ if direct_history.exists():+ for path in sorted(direct_history.glob("*/.done")):+ return path+ except OSError:+ return None+ return None+++def _can_check_done_marker() -> bool:+ workspace = _resolve_openviking_workspace()+ return bool(workspace and workspace.exists())+++def _wait_for_done_marker(session_id: str, timeout_sec: float) -> bool | None:+ if not _can_check_done_marker():+ return None++ deadline = time.monotonic() + max(0.0, timeout_sec)+ while True:+ if _find_done_marker(session_id):+ return True+ if time.monotonic() >= deadline:+ return False+ time.sleep(COMMIT_TASK_POLL_INTERVAL_SEC)+++def _wait_for_session_write_barrier(base_url: str, session_id: str) -> tuple[bool, int, int]:+ last_pending_tokens = 0+ last_message_count = 0++ for _ in range(COMMIT_WAIT_ATTEMPTS):+ if _wait_for_done_marker(session_id, 0) is True:+ return True, last_pending_tokens, last_message_count++ session = _read_openviking_session(base_url, session_id)+ if session:+ last_pending_tokens = int(session.get("pending_tokens") or 0)+ last_message_count = int(session.get("message_count") or 0)+ if last_message_count >= EXPECTED_E2E_OPENVIKING_MESSAGES:+ print(+ f" -> [{session_id}] OpenViking write barrier passed "+ f"(pending_tokens={last_pending_tokens}, message_count={last_message_count})"+ )+ return True, last_pending_tokens, last_message_count++ time.sleep(COMMIT_WAIT_INTERVAL_SEC)++ return False, last_pending_tokens, last_message_count+++def _read_latest_commit_task(base_url: str, session_id: str) -> dict | None:+ try:+ resp = requests.get(+ f"{base_url}/api/v1/tasks",+ headers=openviking_headers(),+ params={+ "task_type": "session_commit",+ "resource_id": session_id,+ "limit": 1,+ },+ timeout=10,+ )+ resp.raise_for_status()+ body = resp.json()+ except Exception:+ return None++ result = body.get("result") if isinstance(body, dict) else None+ if isinstance(result, list) and result:+ task = result[0]+ return task if isinstance(task, dict) else None+ return None+++def _wait_for_done_after_task_completion(session_id: str) -> bool:+ done = _wait_for_done_marker(session_id, 30)+ if done is not True:+ raise RuntimeError(f"task completed but .done marker is not visible for {session_id}")+ print(f" -> [{session_id}] OpenViking memory extraction completed")+ return True+++def _wait_for_task_completion(+ base_url: str, session_id: str, task_id: str, deadline: float+) -> bool:+ task_url = f"{base_url}/api/v1/tasks/{task_id}"+ last_status = "unknown"++ while time.monotonic() < deadline:+ if _wait_for_done_marker(session_id, 0) is True:+ print(f" -> [{session_id}] OpenViking memory extraction completed (.done)")+ return True++ task_resp = requests.get(task_url, headers=openviking_headers(), timeout=10)+ task_resp.raise_for_status()+ task_body = task_resp.json()+ task_result = _unwrap_openviking_dict(task_body)+ last_status = str(task_result.get("status") or "unknown")+ if last_status == "completed":+ return _wait_for_done_after_task_completion(session_id)+ if last_status in {"failed", "cancelled"}:+ raise RuntimeError(f"Task {task_id} {last_status}: {task_result}")+ time.sleep(COMMIT_TASK_POLL_INTERVAL_SEC)++ raise RuntimeError(f"Task {task_id} timed out after commit wait (last_status={last_status})")+++def _await_existing_or_ambiguous_commit(base_url: str, session_id: str, deadline: float) -> bool:+ while time.monotonic() < deadline:+ if _wait_for_done_marker(session_id, 0) is True:+ print(f" -> [{session_id}] OpenViking memory extraction completed (.done)")+ return True++ task = _read_latest_commit_task(base_url, session_id)+ if task:+ status = str(task.get("status") or "unknown")+ if status == "completed":+ return _wait_for_done_after_task_completion(session_id)+ if status in {"failed", "cancelled"}:+ raise RuntimeError(f"Task {task.get('task_id')} {status}: {task}")+ time.sleep(COMMIT_TASK_POLL_INTERVAL_SEC)+ continue++ session = _read_openviking_session(base_url, session_id)+ if session:+ pending_tokens = int(session.get("pending_tokens") or 0)+ message_count = int(session.get("message_count") or 0)+ if message_count >= EXPECTED_E2E_OPENVIKING_MESSAGES:+ return False+ if pending_tokens == 0 and message_count == 0:+ time.sleep(COMMIT_TASK_POLL_INTERVAL_SEC)+ continue++ time.sleep(COMMIT_TASK_POLL_INTERVAL_SEC)++ raise RuntimeError(f"Timed out waiting for ambiguous commit completion for {session_id}")+++def commit_openviking_session(+ openviking_url: str,+ session_id: str,+ task_timeout_sec: int | None = None,+) -> bool:+ try:+ if task_timeout_sec is None:+ task_timeout_sec = int(os.getenv("QUEUE_MAX_WAIT_SEC", "1800"))++ base_url = openviking_url.rstrip("/")+ url = f"{base_url}/api/v1/sessions/{session_id}/commit"+ headers = openviking_headers(content_type=True)+ deadline = time.monotonic() + max(1, task_timeout_sec)++ if _wait_for_done_marker(session_id, 0) is True:+ print(f" -> [{session_id}] OpenViking memory extraction already completed (.done)")+ return True++ ready, last_pending_tokens, last_message_count = _wait_for_session_write_barrier(+ base_url,+ session_id,+ )+ if _wait_for_done_marker(session_id, 0) is True:+ print(f" -> [{session_id}] OpenViking memory extraction already completed (.done)")+ return True+ if not ready:+ raise RuntimeError(+ f"session {session_id} did not reach the OpenViking write barrier before "+ f"commit (expected_message_count={EXPECTED_E2E_OPENVIKING_MESSAGES}, "+ f"pending_tokens={last_pending_tokens}, message_count={last_message_count})"+ )++ for attempt in range(1, COMMIT_POST_ATTEMPTS + 1):+ if _wait_for_done_marker(session_id, 0) is True:+ print(f" -> [{session_id}] OpenViking memory extraction already completed (.done)")+ return True++ try:+ resp = requests.post(url, headers=headers, timeout=30)+ resp.raise_for_status()+ try:+ body = resp.json()+ except Exception:+ body = {}+ result = _unwrap_openviking_dict(body)+ task_id = result.get("task_id")+ print(+ f" -> [{session_id}] Triggered OpenViking session commit (task_id={task_id})"+ )++ if task_id:+ return _wait_for_task_completion(base_url, session_id, task_id, deadline)++ done = _wait_for_done_marker(session_id, max(0, deadline - time.monotonic()))+ if done is not True:+ raise RuntimeError(f"session {session_id} committed but .done was not written")+ return True+ except Exception as e:+ print(+ f" -> [{session_id}] OpenViking commit attempt "+ f"{attempt}/{COMMIT_POST_ATTEMPTS} did not finish cleanly: {e}"+ )+ if _await_existing_or_ambiguous_commit(base_url, session_id, deadline):+ return True+ if attempt >= COMMIT_POST_ATTEMPTS:+ raise+ time.sleep(COMMIT_TASK_POLL_INTERVAL_SEC)++ return False+ except Exception as e:+ print(f" -> [{session_id}] Failed to trigger OpenViking commit: {e}")+ return False+++async def write_success_record(record: dict, csv_path: str) -> None:+ os.makedirs(os.path.dirname(csv_path), exist_ok=True)+ fieldnames = [+ "timestamp",+ "sample_id",+ "session",+ "conversation",+ "total_tokens",+ "input_tokens",+ "output_tokens",+ "cache_read",+ "cache_write",+ "status",+ ]++ async with csv_lock:+ file_exists = os.path.exists(csv_path)+ with open(csv_path, "a", encoding="utf-8", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ if not file_exists:+ writer.writeheader()+ writer.writerow({key: record.get(key, "") for key in fieldnames})+++def reset_generated_file(path: str | Path, label: str) -> None:+ target = Path(path)+ if target.exists():+ target.unlink()+ print(f"Removed existing {label}: {target}", file=sys.stderr)+++def parse_observer_model_totals(status_text: str) -> tuple[int, int, int, int]:+ embedding_prompt = 0+ embedding_completion = 0+ vlm_prompt = 0+ vlm_completion = 0+ current_section = ""++ for raw_line in status_text.splitlines():+ line = raw_line.strip()+ if not line:+ continue+ if line.endswith("Models:"):+ current_section = line.split(" ", 1)[0].lower()+ continue+ if not line.startswith("|"):+ continue++ cols = [col.strip() for col in line.split("|") if col.strip()]+ if len(cols) < 7 or cols[0] == "Model":+ continue++ prompt = int(cols[3]) if cols[3].isdigit() else 0+ completion = int(cols[4]) if cols[4].isdigit() else 0++ if current_section == "embedding":+ embedding_prompt += prompt+ embedding_completion += completion+ elif current_section == "vlm":+ vlm_prompt += prompt+ vlm_completion += completion++ return embedding_prompt, embedding_completion, vlm_prompt, vlm_completion+++async def _read_model_totals(+ client: httpx.AsyncClient, openviking_url: str+) -> tuple[int, int, int, int] | None:+ try:+ resp = await client.get(+ f"{openviking_url}/api/v1/observer/models", headers=openviking_headers()+ )+ if resp.status_code != 200:+ return None+ status_text = resp.json().get("result", {}).get("status", "")+ except Exception:+ return None++ if "No model usage data available" in status_text:+ return 0, 0, 0, 0+ return parse_observer_model_totals(status_text)+++async def _read_queue_totals(+ client: httpx.AsyncClient, openviking_url: str+) -> tuple[int, int, int] | None:+ try:+ resp = await client.get(+ f"{openviking_url}/api/v1/observer/queue", headers=openviking_headers()+ )+ if resp.status_code != 200:+ return None+ status_text = resp.json().get("result", {}).get("status", "")+ except Exception:+ return None++ for raw_line in status_text.splitlines():+ line = raw_line.strip()+ if not line.startswith("|") or "TOTAL" not in line:+ continue+ cols = [col.strip() for col in line.split("|") if col.strip()]+ if len(cols) < 4 or cols[0] != "TOTAL":+ continue+ try:+ return int(cols[1]), int(cols[2]), int(cols[3])+ except ValueError:+ return None+ return None+++def _append_true_token_record(output_file: str, record: dict) -> None:+ fieldnames = [+ "timestamp",+ "embedding_input_tokens",+ "embedding_output_tokens",+ "vlm_llm_input_tokens",+ "vlm_llm_output_tokens",+ ]+ expected_header = ",".join(fieldnames)+ mode = "a"+ should_write_header = not os.path.exists(output_file)++ if not should_write_header:+ with open(output_file, "r", encoding="utf-8", newline="") as f:+ current_header = f.readline().strip()+ if current_header != expected_header:+ mode = "w"+ should_write_header = True++ with open(output_file, mode, encoding="utf-8", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ if should_write_header:+ writer.writeheader()+ writer.writerow(record)+++async def wait_for_queues_and_record_totals(+ openviking_url: str,+ output_file: str,+ baseline_processed: int = 0,+ baseline_model_totals: tuple[int, int, int, int] | None = None,+ expected_processed_delta: int = 0,+ max_wait_sec: int = 1800,+ settle_checks: int = 3,+ poll_interval: float = 2.0,+) -> None:+ has_processed_target = expected_processed_delta > 0+ target_processed = baseline_processed + max(0, expected_processed_delta)+ target_note = f", target_processed={target_processed}" if has_processed_target else ""+ print(+ f"\n[INFO] Waiting for OpenViking background queues to drain "+ f"(baseline_processed={baseline_processed}{target_note}, settle={settle_checks})...",+ file=sys.stderr,+ )++ async with httpx.AsyncClient() as client:+ started = time.perf_counter()+ idle_streak = 0+ last_totals: tuple[int, int, int] | None = None+ timed_out = False++ while True:+ elapsed = time.perf_counter() - started+ if elapsed > max_wait_sec:+ print(+ f"[ERROR] Queue drain timeout after {elapsed:.0f}s "+ f"(last totals={last_totals}, idle_streak={idle_streak})",+ file=sys.stderr,+ )+ timed_out = True+ break++ totals = await _read_queue_totals(client, openviking_url)+ if totals is None:+ await asyncio.sleep(poll_interval)+ continue++ pending, in_progress, processed = totals+ last_totals = totals+ expected_work_done = True if not has_processed_target else processed >= target_processed+ queues_idle = pending == 0 and in_progress == 0++ if expected_work_done and queues_idle:+ idle_streak += 1+ if idle_streak >= settle_checks:+ print(+ f"[INFO] Background queues drained. Processed "+ f"{processed - baseline_processed} item(s) since baseline.",+ file=sys.stderr,+ )+ break+ else:+ idle_streak = 0+ if not expected_work_done and queues_idle:+ print(+ f" .. waiting for expected import work "+ f"(processed={processed}, target={target_processed})",+ file=sys.stderr,+ )+ else:+ print(+ f" .. queue state pending={pending} in_progress={in_progress} "+ f"processed={processed}",+ file=sys.stderr,+ )++ await asyncio.sleep(poll_interval)++ if timed_out:+ raise TimeoutError(+ "OpenViking background queues did not finish before checkpoint "+ f"(target_processed={target_processed if has_processed_target else 'none'}, "+ f"last_totals={last_totals})"+ )++ print("[INFO] Fetching final true token usage from observer...", file=sys.stderr)+ try:+ final_totals = await _read_model_totals(client, openviking_url)+ if final_totals is None:+ print("[INFO] No model usage recorded by OpenViking server.", file=sys.stderr)+ return++ (+ baseline_embedding_prompt,+ baseline_embedding_completion,+ baseline_vlm_prompt,+ baseline_vlm_completion,+ ) = baseline_model_totals or (0, 0, 0, 0)+ (+ final_embedding_prompt,+ final_embedding_completion,+ final_vlm_prompt,+ final_vlm_completion,+ ) = final_totals++ delta_embedding_prompt = max(final_embedding_prompt - baseline_embedding_prompt, 0)+ delta_embedding_completion = max(+ final_embedding_completion - baseline_embedding_completion, 0+ )+ delta_vlm_prompt = max(final_vlm_prompt - baseline_vlm_prompt, 0)+ delta_vlm_completion = max(final_vlm_completion - baseline_vlm_completion, 0)++ print("\n=== TRUE OpenViking Token Delta For This Run ===", file=sys.stderr)+ print(f"Embedding Input (Prompt) Delta: {delta_embedding_prompt}", file=sys.stderr)+ print(+ f"Embedding Output (Completion) Delta: {delta_embedding_completion}",+ file=sys.stderr,+ )+ print(f"VLM/LLM Input (Prompt) Delta: {delta_vlm_prompt}", file=sys.stderr)+ print(f"VLM/LLM Output (Completion) Delta: {delta_vlm_completion}", file=sys.stderr)+ print(+ f"[INFO] Final cumulative totals: embedding_in={final_embedding_prompt}, "+ f"embedding_out={final_embedding_completion}, vlm_in={final_vlm_prompt}, "+ f"vlm_out={final_vlm_completion}",+ file=sys.stderr,+ )++ _append_true_token_record(+ output_file,+ {+ "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),+ "embedding_input_tokens": delta_embedding_prompt,+ "embedding_output_tokens": delta_embedding_completion,+ "vlm_llm_input_tokens": delta_vlm_prompt,+ "vlm_llm_output_tokens": delta_vlm_completion,+ },+ )+ print(f"True token totals saved to: {output_file}", file=sys.stderr)+ except Exception as e:+ print(f"[ERROR] Failed to fetch final token usage: {e}", file=sys.stderr)+++async def process_session(+ sample_id: str,+ session_key: str,+ transcript: str,+ args: argparse.Namespace,+) -> dict | None:+ max_attempts = max(0, getattr(args, "error_retries", DEFAULT_IMPORT_ERROR_RETRIES)) + 1+ retry_delay = float(getattr(args, "retry_delay_sec", 2.0))+ loop = asyncio.get_running_loop()+ record: dict | None = None++ for attempt in range(1, max_attempts + 1):+ try:+ record = await loop.run_in_executor(+ None,+ ingest_session,+ sample_id,+ session_key,+ transcript,+ args,+ )+ break+ except Exception as e:+ if attempt >= max_attempts:+ print(f" -> [{sample_id}/{session_key}] ERROR after {attempt} attempt(s): {e}")+ return None+ print(+ f" -> [{sample_id}/{session_key}] ERROR attempt {attempt}/{max_attempts}: {e}; retrying"+ )+ await asyncio.sleep(retry_delay)++ try:+ session_id = record.get("session_id")+ if not session_id:+ raise RuntimeError("Hermes response did not include a session id")++ committed = False+ for attempt in range(1, max_attempts + 1):+ committed = await loop.run_in_executor(+ None,+ commit_openviking_session,+ args.openviking_url,+ session_id,+ )+ if committed:+ break+ if attempt < max_attempts:+ print(+ f" -> [{session_id}] Commit failed attempt {attempt}/{max_attempts}; retrying"+ )+ await asyncio.sleep(retry_delay)+ if not committed:+ raise RuntimeError("OpenViking session commit failed")++ await write_success_record(record, args.success_csv)+ return record+ except Exception as e:+ print(f" -> [{sample_id}/{session_key}] ERROR: {e}")+ return None+++async def main() -> None:+ script_dir = Path(__file__).parent.resolve()+ default_input = default_locomo_input(script_dir)+ default_success_csv = str(script_dir / "result_e2e" / "import_success.csv")++ parser = argparse.ArgumentParser(+ description="Ingest LoCoMo transcripts into Hermes OpenViking E2E"+ )+ parser.add_argument("--input", default=default_input, help="Path to LoCoMo JSON")+ parser.add_argument("--success-csv", default=default_success_csv, help="Success records CSV")+ parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="Hermes gateway URL")+ parser.add_argument("--token", default=DEFAULT_TOKEN, help="Hermes gateway token")+ parser.add_argument("--model", default=DEFAULT_MODEL, help="Hermes model name")+ parser.add_argument("--sample", type=int, default=None, help="Sample index to process")+ parser.add_argument("--timeout", type=int, default=1200, help="Request timeout")+ parser.add_argument(+ "--force-ingest", action="store_true", help="Ignore existing records and re-ingest"+ )+ parser.add_argument(+ "--openviking-url", default=DEFAULT_OPENVIKING_URL, help="OpenViking service URL"+ )+ parser.add_argument("--parallel", type=int, default=4, help="Parallel ingest workers")+ parser.add_argument(+ "--queue-max-wait-sec",+ type=int,+ default=1800,+ help="Maximum seconds to wait for OpenViking observer queue deltas",+ )+ parser.add_argument(+ "--error-retries",+ type=int,+ default=DEFAULT_IMPORT_ERROR_RETRIES,+ help="Retry failed import sessions/commits this many times",+ )+ parser.add_argument(+ "--retry-delay-sec",+ type=float,+ default=2.0,+ help="Seconds to wait between import retry attempts",+ )+ args = parser.parse_args()+ os.environ["QUEUE_MAX_WAIT_SEC"] = str(args.queue_max_wait_sec)++ if not args.token:+ print("Error: API token required", file=sys.stderr)+ sys.exit(1)++ if args.force_ingest:+ reset_generated_file(args.success_csv, "import success CSV")+ reset_generated_file(+ Path(args.success_csv).parent / "import_true_tokens.csv", "import true-token CSV"+ )++ existing_success_keys = set()+ if not args.force_ingest:+ existing_success_keys = load_success_keys(args.success_csv)++ samples = load_locomo_data(args.input, args.sample)++ print(+ f"Starting E2E ingest for {len(samples)} samples with {args.parallel} workers "+ "(commit=locomo-session, session_ids=locomo-session)..."+ )++ async with httpx.AsyncClient() as snap_client:+ baseline_totals = await _read_queue_totals(snap_client, args.openviking_url)+ baseline_model_totals = await _read_model_totals(snap_client, args.openviking_url)+ baseline_processed = baseline_totals[2] if baseline_totals else 0+ print(f"[INFO] Observer baseline processed={baseline_processed}", file=sys.stderr)++ semaphore = asyncio.Semaphore(max(1, args.parallel))+ failed_sessions: list[str] = []++ async def run_limited(sample_id: str, session_key: str, payload: dict) -> None:+ async with semaphore:+ record = await process_session(sample_id, session_key, payload["transcript"], args)+ if not record:+ failed_sessions.append(f"{sample_id}/{session_key}")+ await asyncio.sleep(1)++ tasks = []+ for item in samples:+ sample_id = item["sample_id"]+ for session_payload in build_session_transcripts(item):+ session_key = session_payload["session"]+ if not args.force_ingest and is_already_ingested(+ sample_id, session_key, existing_success_keys+ ):+ print(f" -> [{sample_id}/{session_key}] SKIP (already ingested)")+ continue+ tasks.append(run_limited(sample_id, session_key, session_payload))++ if tasks:+ await asyncio.gather(*tasks)++ print("Ingest complete.")++ if failed_sessions:+ print(+ f"Import failed for {len(failed_sessions)} session(s) after retries: "+ + ", ".join(failed_sessions[:20]),+ file=sys.stderr,+ )+ if len(failed_sessions) > 20:+ print(f"... and {len(failed_sessions) - 20} more", file=sys.stderr)+ sys.exit(1)++ true_token_file = Path(args.success_csv).parent / "import_true_tokens.csv"+ await wait_for_queues_and_record_totals(+ args.openviking_url,+ str(true_token_file),+ baseline_processed=baseline_processed,+ baseline_model_totals=baseline_model_totals,+ max_wait_sec=args.queue_max_wait_sec,+ )+++if __name__ == "__main__":+ asyncio.run(main())diff --git a/benchmark/locomo/hermes/import_to_native.py b/benchmark/locomo/hermes/import_to_native.pynew file mode 100644index 000000000..5beb1b780--- /dev/null+++ b/benchmark/locomo/hermes/import_to_native.py@@ -0,0 +1,376 @@+"""+Hermes built-in memory ingest tool.++Sends LoCoMo conversation transcripts to the Hermes API server.+The server persists them into `~/.hermes/state.db` so they can be+retrieved later through Hermes native memory tools.+"""++from __future__ import annotations++import argparse+import asyncio+import csv+import json+import os+import sys+import time+from datetime import datetime+from pathlib import Path++import requests++try:+ from dotenv import load_dotenv+except ImportError:++ def load_dotenv(*_args, **_kwargs):+ return False+++env_file = Path.home() / ".openviking_benchmark_env"+load_dotenv(env_file)++DEFAULT_BASE_URL = os.getenv("HERMES_GATEWAY_BASE_URL", "http://127.0.0.1:8642")+DEFAULT_MODEL = os.getenv("HERMES_GATEWAY_MODEL", "hermes-agent")+DEFAULT_TOKEN = os.getenv("HERMES_GATEWAY_TOKEN") or os.getenv("API_SERVER_KEY") or ""+DEFAULT_IMPORT_ERROR_RETRIES = int(os.getenv("IMPORT_ERROR_RETRIES", "2"))+DEFAULT_DATASET_LOCATION = "benchmark/locomo/data/locomo10.json"+++def default_locomo_input(script_dir: Path) -> str:+ return os.getenv("LOCOMO_JSON") or str(script_dir.parent / "data" / "locomo10.json")+++def dataset_missing_message(path: str) -> str:+ return (+ f"LoCoMo dataset not found: {path}\n"+ f"Set LOCOMO_JSON=/path/to/locomo10.json, pass a script input path, "+ f"or place it at {DEFAULT_DATASET_LOCATION}."+ )+++def load_locomo_data(path: str, sample_index: int | None = None) -> list[dict]:+ try:+ with open(path, "r", encoding="utf-8") as f:+ data = json.load(f)+ except FileNotFoundError:+ print(f"Error: {dataset_missing_message(path)}", file=sys.stderr)+ sys.exit(1)+ if sample_index is not None:+ return [data[sample_index]]+ return data+++def _get_session_number(session_key: str) -> int:+ return int(session_key.split("_")[1])+++def _session_keys(conv: dict) -> list[str]:+ return sorted(+ [k for k in conv if k.startswith("session_") and not k.endswith("_date_time")],+ key=_get_session_number,+ )+++def _clean_text(value: object) -> str:+ if value is None:+ return ""+ if isinstance(value, list):+ return ", ".join(str(item).strip() for item in value if str(item).strip())+ return str(value).strip()+++def _normalize_locomo_session_time(date_time: str) -> str:+ value = date_time.strip()+ if not value:+ return ""+ for suffix in ("am", "pm"):+ value = value.replace(f" {suffix} ", f" {suffix.upper()} ")+ try:+ parsed = datetime.strptime(value, "%I:%M %p on %d %B, %Y")+ except ValueError:+ return ""+ return parsed.strftime("%Y-%m-%d %H:%M")+++def _visual_metadata_line(speaker: str, msg: dict) -> str:+ caption = _clean_text(msg.get("blip_caption"))+ query = _clean_text(msg.get("query"))+ details = []+ if caption:+ details.append(f"caption: {caption}")+ if query:+ details.append(f"query: {query}")+ if not details:+ return ""+ return f"[{speaker} visual]: " + "; ".join(details)+++def _message_line(msg: dict) -> str:+ speaker = msg.get("speaker", "unknown")+ return f"[{speaker}]: {_clean_text(msg.get('text'))}"+++def _message_lines(msg: dict) -> list[str]:+ speaker = msg.get("speaker", "unknown")+ lines = [_message_line(msg)]+ visual_line = _visual_metadata_line(speaker, msg)+ if visual_line:+ lines.append(visual_line)+ return lines+++def _session_header_lines(conv: dict, session_key: str, date_time: str) -> list[str]:+ speaker_a = conv.get("speaker_a", "A")+ speaker_b = conv.get("speaker_b", "B")+ lines = [+ f"The following is a conversation transcript between {speaker_a} and {speaker_b}.",+ "---",+ ]+ if date_time:+ lines.append(f"\n[Session: {date_time}]")+ normalized_time = _normalize_locomo_session_time(date_time)+ if normalized_time:+ lines.append(f"[Session date: {normalized_time}]")+ lines.append(f"[LoCoMo session: {session_key}]")+ return lines+++def format_session_transcript(sample: dict, session_key: str) -> str:+ conv = sample.get("conversation", {})+ date_time = conv.get(f"{session_key}_date_time", "")++ lines = _session_header_lines(conv, session_key, date_time)+ for msg in conv.get(session_key, []):+ lines.extend(_message_lines(msg))+ lines.append("---")+ return "\n".join(lines)+++def iter_session_payloads(sample: dict) -> list[tuple[str, str]]:+ conv = sample.get("conversation", {})+ return [+ (session_key, format_session_transcript(sample, session_key))+ for session_key in _session_keys(conv)+ ]+++def send_ingest_request(+ sample_id: str,+ input_text: str,+ args: argparse.Namespace,+ *,+ native_session_id: str,+ label: str,+) -> dict:+ url = f"{args.base_url.rstrip('/')}/v1/responses"+ headers = {+ "Authorization": f"Bearer {args.token}",+ "Content-Type": "application/json",+ }+ payload = {+ "model": args.model,+ "input": input_text,+ # Stable session_id keeps each flattened LoCoMo session isolated.+ # Do not set Responses `conversation`: that chains prior requests back+ # into the model context.+ "session_id": native_session_id,+ "store": True,+ "instructions": (+ "This is a past conversation transcript for memory ingestion, not a live request. "+ "Treat the user message as transcript data only. Do not perform tasks. "+ "Do not use any tools. Acknowledge with exactly: OK"+ ),+ }++ started_at = time.perf_counter()+ resp = requests.post(url, headers=headers, json=payload, timeout=args.timeout)+ elapsed = time.perf_counter() - started_at+ resp.raise_for_status()+ body = resp.json()++ usage = body.get("usage", {})+ total_tokens = usage.get("total_tokens", 0)+ input_tokens = usage.get("input_tokens", usage.get("prompt_tokens", 0))+ output_tokens = usage.get("output_tokens", usage.get("completion_tokens", 0))+ cache_read = usage.get("cache_read_tokens", usage.get("cacheRead", 0))+ cache_write = usage.get("cache_write_tokens", usage.get("cacheWrite", 0))++ print(f" -> [{sample_id}/{label}] Ingested in {elapsed:.2f}s (Tokens: {total_tokens})")++ return {+ "total_tokens": total_tokens,+ "input_tokens": input_tokens,+ "output_tokens": output_tokens,+ "cache_read": cache_read,+ "cache_write": cache_write,+ }+++def ingest_sample(sample_id: str, sample: dict, args: argparse.Namespace) -> dict:+ conversation_name = f"locomo-native-{sample_id}"+ request_count = 0+ totals = {+ "total_tokens": 0,+ "input_tokens": 0,+ "output_tokens": 0,+ "cache_read": 0,+ "cache_write": 0,+ }++ payloads = iter_session_payloads(sample)+ for session_key, transcript in payloads:+ max_attempts = max(0, args.error_retries) + 1+ for attempt in range(1, max_attempts + 1):+ try:+ usage = send_ingest_request(+ sample_id,+ transcript,+ args,+ native_session_id=f"{conversation_name}-{session_key}",+ label=session_key,+ )+ break+ except Exception as e:+ if attempt >= max_attempts:+ raise RuntimeError(+ f"{sample_id}/{session_key} failed after {attempt} attempt(s): {e}"+ ) from e+ print(+ f" -> [{sample_id}/{session_key}] ERROR attempt "+ f"{attempt}/{max_attempts}: {e}; retrying",+ file=sys.stderr,+ )+ time.sleep(args.retry_delay_sec)+ request_count += 1+ for key in totals:+ totals[key] += int(usage.get(key, 0))++ return {+ "sample_id": sample_id,+ "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),+ "conversation": conversation_name,+ "ingest_granularity": "session",+ "request_count": request_count,+ **totals,+ "status": "success",+ }+++def write_success_record(record: dict, csv_path: str) -> None:+ os.makedirs(os.path.dirname(csv_path), exist_ok=True)+ file_exists = os.path.exists(csv_path)+ legacy_fieldnames = ["timestamp", "sample_id", "conversation", "total_tokens", "status"]+ fieldnames = [+ "timestamp",+ "sample_id",+ "conversation",+ "ingest_granularity",+ "request_count",+ "input_tokens",+ "output_tokens",+ "cache_read",+ "cache_write",+ "total_tokens",+ "status",+ ]+ if file_exists and os.path.getsize(csv_path) > 0:+ with open(csv_path, "r", encoding="utf-8", newline="") as f:+ existing_header = f.readline().strip().split(",")+ if existing_header == legacy_fieldnames:+ fieldnames = legacy_fieldnames+ elif existing_header != fieldnames:+ raise RuntimeError(+ f"Unexpected import CSV header in {csv_path}; use a fresh result directory"+ )++ with open(csv_path, "a", encoding="utf-8", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ if not file_exists:+ writer.writeheader()+ writer.writerow({key: record.get(key, "") for key in fieldnames})+++def reset_generated_file(path: str | Path, label: str) -> None:+ target = Path(path)+ if target.exists():+ target.unlink()+ print(f"Removed existing {label}: {target}", file=sys.stderr)+++async def main() -> None:+ script_dir = Path(__file__).parent.resolve()+ default_input = default_locomo_input(script_dir)+ default_success_csv = str(script_dir / "result_baseline" / "import_success.csv")++ parser = argparse.ArgumentParser(description="Ingest LoCoMo transcripts into Hermes state.db")+ parser.add_argument("--input", default=default_input, help="Path to LoCoMo JSON")+ parser.add_argument("--success-csv", default=default_success_csv, help="Success records CSV")+ parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="Hermes gateway URL")+ parser.add_argument("--token", default=DEFAULT_TOKEN, help="Hermes gateway token")+ parser.add_argument("--model", default=DEFAULT_MODEL, help="Hermes model name")+ parser.add_argument("--sample", type=int, default=None, help="Sample index to process")+ parser.add_argument("--timeout", type=int, default=600, help="Request timeout")+ parser.add_argument(+ "--force-ingest", action="store_true", help="Ignore existing records and re-ingest"+ )+ parser.add_argument(+ "--error-retries",+ type=int,+ default=DEFAULT_IMPORT_ERROR_RETRIES,+ help="Retry failed session imports this many times",+ )+ parser.add_argument(+ "--retry-delay-sec",+ type=float,+ default=2.0,+ help="Seconds to wait between import retry attempts",+ )+ args = parser.parse_args()++ if not args.token:+ print("Error: API token required", file=sys.stderr)+ sys.exit(1)++ if args.force_ingest:+ reset_generated_file(args.success_csv, "import success CSV")++ existing_samples = set()+ if not args.force_ingest and os.path.exists(args.success_csv):+ with open(args.success_csv, "r", encoding="utf-8") as f:+ for row in csv.DictReader(f):+ existing_samples.add(row["sample_id"])++ samples = load_locomo_data(args.input, args.sample)++ print(f"Starting native session ingest for {len(samples)} samples...")+ failed_samples: list[str] = []+ for item in samples:+ sample_id = item["sample_id"]+ if sample_id in existing_samples:+ print(f" -> [{sample_id}] SKIP (already ingested)")+ continue++ try:+ record = ingest_sample(sample_id, item, args)+ write_success_record(record, args.success_csv)+ except Exception as e:+ print(f" -> [{sample_id}] ERROR: {e}", file=sys.stderr)+ failed_samples.append(sample_id)++ if failed_samples:+ print(+ "Native import failed for "+ f"{len(failed_samples)} sample(s): {', '.join(failed_samples[:20])}",+ file=sys.stderr,+ )+ if len(failed_samples) > 20:+ print(f"... and {len(failed_samples) - 20} more", file=sys.stderr)+ sys.exit(1)++ print("Ingest complete.")+++if __name__ == "__main__":+ asyncio.run(main())diff --git a/benchmark/locomo/hermes/import_to_ov.py b/benchmark/locomo/hermes/import_to_ov.pynew file mode 100644index 000000000..e30533b84--- /dev/null+++ b/benchmark/locomo/hermes/import_to_ov.py@@ -0,0 +1,862 @@+"""+OpenViking data import tool for LoCoMo benchmark.+"""++from __future__ import annotations++import argparse+import asyncio+import csv+import json+import os+import sys+import traceback+from datetime import datetime, timedelta+from pathlib import Path+from typing import Any, Dict, List, Optional++import httpx++import openviking as ov++try:+ from dotenv import load_dotenv+except ImportError:++ def load_dotenv(*_args, **_kwargs):+ return False+++env_file = Path.home() / ".openviking_benchmark_env"+load_dotenv(env_file)++DEFAULT_OPENVIKING_URL = os.getenv("OPENVIKING_ENDPOINT", "http://127.0.0.1:1933")+DEFAULT_IMPORT_ERROR_RETRIES = int(os.getenv("IMPORT_ERROR_RETRIES", "2"))+DEFAULT_SESSION_PREFIX = "locomo-ovpreingest"+DEFAULT_OPENVIKING_ACCOUNT = os.getenv("OPENVIKING_ACCOUNT", "default")+DEFAULT_OPENVIKING_USER = os.getenv("OPENVIKING_USER", "default")+DEFAULT_OPENVIKING_AGENT = os.getenv("OPENVIKING_AGENT", "hermes")+DEFAULT_OPENVIKING_API_KEY = os.getenv("OPENVIKING_API_KEY", "")+DEFAULT_DATASET_LOCATION = "benchmark/locomo/data/locomo10.json"+++def default_locomo_input(script_dir: Path) -> str:+ return os.getenv("LOCOMO_JSON") or str(script_dir.parent / "data" / "locomo10.json")+++def dataset_missing_message(path: str) -> str:+ return (+ f"LoCoMo dataset not found: {path}\n"+ f"Set LOCOMO_JSON=/path/to/locomo10.json, pass a script input path, "+ f"or place it at {DEFAULT_DATASET_LOCATION}."+ )+++def _get_session_number(session_key: str) -> int:+ return int(session_key.split("_")[1])+++def _build_session_id(sample_id: str, session_key: str) -> str:+ return f"{DEFAULT_SESSION_PREFIX}-{sample_id}-{session_key}"+++def _clean_text(value: object) -> str:+ if value is None:+ return ""+ if isinstance(value, list):+ return ", ".join(str(item).strip() for item in value if str(item).strip())+ return str(value).strip()+++def _parse_locomo_session_datetime(date_time: str) -> Optional[datetime]:+ value = _clean_text(date_time)+ if not value:+ return None++ normalized = (+ value.replace(" a.m.", " AM")+ .replace(" p.m.", " PM")+ .replace(" am ", " AM ")+ .replace(" pm ", " PM ")+ )+ formats = (+ "%I:%M %p on %d %B, %Y",+ "%I %p on %d %B, %Y",+ "%I:%M %p on %B %d, %Y",+ "%I %p on %B %d, %Y",+ "%d %B, %Y",+ "%B %d, %Y",+ )+ for fmt in formats:+ try:+ return datetime.strptime(normalized, fmt)+ except ValueError:+ continue+ return None+++def _normalize_locomo_session_time(date_time: str) -> str:+ parsed = _parse_locomo_session_datetime(date_time)+ if parsed is None:+ return ""+ return parsed.strftime("%Y-%m-%d %H:%M")+++def _visual_metadata_line(speaker: str, msg: dict) -> str:+ caption = _clean_text(msg.get("blip_caption"))+ query = _clean_text(msg.get("query"))+ details = []+ if caption:+ details.append(f"caption: {caption}")+ if query:+ details.append(f"query: {query}")+ if not details:+ return ""+ return f"[{speaker} visual]: " + "; ".join(details)+++def _build_session_transcript(+ conv: Dict[str, Any],+ session_key: str,+ date_time: str,+) -> str:+ speaker_a = conv.get("speaker_a", "A")+ speaker_b = conv.get("speaker_b", "B")+ lines = [+ f"The following is a conversation transcript between {speaker_a} and {speaker_b}.",+ "---",+ ]+ if date_time:+ lines.append(f"[Session: {date_time}]")+ normalized_time = _normalize_locomo_session_time(date_time)+ if normalized_time:+ lines.append(f"[Session date: {normalized_time}]")+ lines.append(f"[LoCoMo session: {session_key}]")++ for msg in conv[session_key]:+ speaker = msg.get("speaker", "unknown")+ text = _clean_text(msg.get("text"))+ lines.append(f"[{speaker}]: {text}")+ visual_line = _visual_metadata_line(speaker, msg)+ if visual_line:+ lines.append(visual_line)++ lines.append("---")+ return "\n".join(lines)+++def load_locomo_data(path: str, sample_index: Optional[int] = None) -> List[Dict[str, Any]]:+ try:+ with open(path, "r", encoding="utf-8") as f:+ data = json.load(f)+ except FileNotFoundError:+ print(f"Error: {dataset_missing_message(path)}", file=sys.stderr)+ sys.exit(1)++ if sample_index is not None:+ if sample_index < 0 or sample_index >= len(data):+ raise ValueError(f"Sample index {sample_index} out of range (0-{len(data) - 1})")+ return [data[sample_index]]+ return data+++def build_session_messages(item: Dict[str, Any]) -> List[Dict[str, Any]]:+ conv = item["conversation"]+ speakers = f"{conv['speaker_a']} & {conv['speaker_b']}"++ session_keys = sorted(+ [key for key in conv if key.startswith("session_") and not key.endswith("_date_time")],+ key=_get_session_number,+ )++ sessions = []+ for session_key in session_keys:+ date_time = conv.get(f"{session_key}_date_time", "")+ utterance_count = len(conv[session_key])+ transcript = _build_session_transcript(conv, session_key, date_time)+ messages = [+ {+ "role": "user",+ "text": transcript,+ "index": 0,+ "utterance_count": utterance_count,+ }+ ]++ sessions.append(+ {+ "messages": messages,+ "meta": {+ "sample_id": item["sample_id"],+ "session_key": session_key,+ "date_time": date_time,+ "speakers": speakers,+ },+ }+ )++ return sessions+++def load_success_csv(csv_path: str) -> set:+ success_keys = set()+ if Path(csv_path).exists():+ with open(csv_path, "r", encoding="utf-8") as f:+ reader = csv.DictReader(f)+ for row in reader:+ success_keys.add(f"viking:{row['sample_id']}:{row['session']}")+ return success_keys+++def write_success_record(record: Dict[str, Any], csv_path: str) -> None:+ file_exists = Path(csv_path).exists()+ fieldnames = [+ "timestamp",+ "sample_id",+ "session",+ "date_time",+ "speakers",+ "embedding_tokens",+ "llm_total_tokens",+ "llm_input_tokens",+ "llm_output_tokens",+ "llm_cache_read",+ "llm_cache_write",+ "total_tokens",+ ]++ with open(csv_path, "a", encoding="utf-8", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ if not file_exists:+ writer.writeheader()+ writer.writerow(+ {+ "timestamp": record["timestamp"],+ "sample_id": record["sample_id"],+ "session": record["session"],+ "date_time": record.get("meta", {}).get("date_time", ""),+ "speakers": record.get("meta", {}).get("speakers", ""),+ "embedding_tokens": record["token_usage"].get("embedding", 0),+ "llm_total_tokens": record["token_usage"].get("llm_total", 0),+ "llm_input_tokens": record["token_usage"].get("llm_input", 0),+ "llm_output_tokens": record["token_usage"].get("llm_output", 0),+ "llm_cache_read": record["token_usage"].get("llm_cache_read", 0),+ "llm_cache_write": record["token_usage"].get("llm_cache_write", 0),+ "total_tokens": record["token_usage"].get("total", 0),+ }+ )+++def write_error_record(record: Dict[str, Any], error_path: str) -> None:+ with open(error_path, "a", encoding="utf-8") as f:+ f.write(+ f"[{record['timestamp']}] ERROR [{record['sample_id']}/{record['session']}]: {record['error']}\n"+ )+++def is_already_ingested(+ sample_id: str, session_key: str, success_keys: Optional[set] = None+) -> bool:+ return success_keys is not None and f"viking:{sample_id}:{session_key}" in success_keys+++def _parse_token_usage(commit_result: Dict[str, Any]) -> Dict[str, int]:+ if "result" in commit_result:+ result = commit_result["result"]+ if "token_usage" in result:+ token_usage = result["token_usage"]+ embedding = token_usage.get("embedding", {})+ llm = token_usage.get("llm", {})+ embed_total = embedding.get("total_tokens", embedding.get("total", 0))+ llm_total = llm.get("total_tokens", llm.get("total", 0))+ llm_input = llm.get("prompt_tokens", llm.get("input", 0))+ llm_output = llm.get("completion_tokens", llm.get("output", 0))+ return {+ "embedding": embed_total,+ "llm_total": llm_total,+ "llm_input": llm_input,+ "llm_output": llm_output,+ "llm_cache_read": llm.get("cache_read", 0),+ "llm_cache_write": llm.get("cache_write", 0),+ "total": token_usage.get("total", {}).get("total_tokens", embed_total + llm_total),+ }++ telemetry = commit_result.get("telemetry", {}).get("summary", {})+ tokens = telemetry.get("tokens", {})+ return {+ "embedding": tokens.get("embedding", {}).get("total", 0),+ "llm_total": tokens.get("llm", {}).get("total", 0),+ "llm_input": tokens.get("llm", {}).get("input", 0),+ "llm_output": tokens.get("llm", {}).get("output", 0),+ "llm_cache_read": tokens.get("llm", {}).get("cache_read", 0),+ "llm_cache_write": tokens.get("llm", {}).get("cache_write", 0),+ "total": tokens.get("total", 0),+ }+++async def viking_ingest(+ messages: List[Dict[str, Any]],+ openviking_url: str,+ session_time: Optional[str] = None,+ session_id: Optional[str] = None,+ replace_session: bool = False,+ account: str = DEFAULT_OPENVIKING_ACCOUNT,+ user: str = DEFAULT_OPENVIKING_USER,+ agent: str = DEFAULT_OPENVIKING_AGENT,+ api_key: str = DEFAULT_OPENVIKING_API_KEY,+) -> Dict[str, Any]:+ base_datetime = _parse_locomo_session_datetime(session_time or "")+ if session_time and base_datetime is None:+ print(f"Warning: Failed to parse session_time: {session_time}", file=sys.stderr)++ client = ov.AsyncHTTPClient(+ url=openviking_url,+ api_key=api_key or None,+ account=account,+ user=user,+ agent_id=agent,+ )+ await client.initialize()++ try:+ if session_id and replace_session:+ try:+ await client.delete_session(session_id)+ except Exception as e:+ message = str(e).lower()+ if "not found" not in message and "not_found" not in message:+ raise++ create_res = await client.create_session(session_id)+ resolved_session_id = create_res["session_id"]++ for idx, msg in enumerate(messages):+ msg_created_at = None+ if base_datetime:+ msg_created_at = (base_datetime + timedelta(seconds=idx)).isoformat()+ await client.add_message(+ session_id=resolved_session_id,+ role=msg["role"],+ parts=[{"type": "text", "text": msg["text"]}],+ created_at=msg_created_at,+ role_id=msg.get("role_id"),+ )++ result = await client.commit_session(resolved_session_id, telemetry=True)+ if result.get("status") not in ("committed", "accepted"):+ raise RuntimeError(f"Commit failed: {result}")++ task_id = result.get("task_id")+ if task_id:+ max_attempts = 3600+ for _ in range(max_attempts):+ task = await client.get_task(task_id)+ status = task.get("status") if task else "unknown"+ if status == "completed":+ token_usage = _parse_token_usage(task)+ break+ if status in ("failed", "cancelled", "unknown"):+ raise RuntimeError(f"Task {task_id} {status}: {task}")+ await asyncio.sleep(1)+ else:+ raise RuntimeError(f"Task {task_id} timed out after {max_attempts} attempts")+ else:+ token_usage = {+ "embedding": 0,+ "llm_total": 0,+ "llm_input": 0,+ "llm_output": 0,+ "llm_cache_read": 0,+ "llm_cache_write": 0,+ "total": 0,+ }++ return {+ "token_usage": token_usage,+ "task_id": task_id,+ "trace_id": result.get("trace_id", ""),+ "session_id": resolved_session_id,+ }+ finally:+ await client.close()+++def parse_observer_model_totals(status_text: str) -> tuple[int, int, int, int]:+ embedding_prompt = 0+ embedding_completion = 0+ vlm_prompt = 0+ vlm_completion = 0+ current_section = ""++ for raw_line in status_text.splitlines():+ line = raw_line.strip()+ if not line:+ continue+ if line.endswith("Models:"):+ current_section = line.split(" ", 1)[0].lower()+ continue+ if not line.startswith("|"):+ continue+ cols = [col.strip() for col in line.split("|") if col.strip()]+ if len(cols) < 7 or cols[0] == "Model":+ continue++ prompt = int(cols[3]) if cols[3].isdigit() else 0+ completion = int(cols[4]) if cols[4].isdigit() else 0+ if current_section == "embedding":+ embedding_prompt += prompt+ embedding_completion += completion+ elif current_section == "vlm":+ vlm_prompt += prompt+ vlm_completion += completion++ return embedding_prompt, embedding_completion, vlm_prompt, vlm_completion+++async def _read_model_totals(+ client: httpx.AsyncClient, openviking_url: str+) -> tuple[int, int, int, int] | None:+ try:+ resp = await client.get(f"{openviking_url}/api/v1/observer/models")+ if resp.status_code != 200:+ return None+ status_text = resp.json().get("result", {}).get("status", "")+ except Exception:+ return None++ if "No model usage data available" in status_text:+ return 0, 0, 0, 0+ return parse_observer_model_totals(status_text)+++async def _read_queue_totals(+ client: httpx.AsyncClient, openviking_url: str+) -> tuple[int, int, int] | None:+ try:+ resp = await client.get(f"{openviking_url}/api/v1/observer/queue")+ if resp.status_code != 200:+ return None+ status_text = resp.json().get("result", {}).get("status", "")+ except Exception:+ return None++ for raw_line in status_text.splitlines():+ line = raw_line.strip()+ if not line.startswith("|") or "TOTAL" not in line:+ continue+ cols = [col.strip() for col in line.split("|") if col.strip()]+ if len(cols) < 4 or cols[0] != "TOTAL":+ continue+ try:+ return int(cols[1]), int(cols[2]), int(cols[3])+ except ValueError:+ return None+ return None+++def append_true_token_record(output_file: str, record: Dict[str, Any]) -> None:+ fieldnames = [+ "timestamp",+ "embedding_input_tokens",+ "embedding_output_tokens",+ "vlm_llm_input_tokens",+ "vlm_llm_output_tokens",+ ]+ expected_header = ",".join(fieldnames)+ mode = "a"+ should_write_header = not Path(output_file).exists()++ if not should_write_header:+ with open(output_file, "r", encoding="utf-8", newline="") as f:+ current_header = f.readline().strip()+ if current_header != expected_header:+ mode = "w"+ should_write_header = True++ with open(output_file, mode, encoding="utf-8", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ if should_write_header:+ writer.writeheader()+ writer.writerow(record)+++async def process_single_session(+ messages: List[Dict[str, Any]],+ sample_id: str,+ session_key: str,+ meta: Dict[str, Any],+ run_time: str,+ args: argparse.Namespace,+) -> Dict[str, Any]:+ session_id = _build_session_id(sample_id, session_key)+ max_attempts = max(0, getattr(args, "error_retries", DEFAULT_IMPORT_ERROR_RETRIES)) + 1+ retry_delay = float(getattr(args, "retry_delay_sec", 2.0))+ last_error: Exception | None = None++ for attempt in range(1, max_attempts + 1):+ try:+ replace_session = args.force_ingest+ if attempt > 1:+ replace_session = True+ print(+ f" -> [RETRY] [{sample_id}/{session_key}] "+ f"attempt {attempt}/{max_attempts}; replacing partial session {session_id}",+ file=sys.stderr,+ )++ result = await viking_ingest(+ messages,+ args.openviking_url,+ meta.get("date_time"),+ session_id=session_id,+ replace_session=replace_session,+ account=args.account,+ user=args.user,+ agent=args.agent,+ api_key=args.api_key,+ )+ token_usage = result["token_usage"]+ task_id = result.get("task_id")+ trace_id = result.get("trace_id", "")+ resolved_session_id = result.get("session_id", session_id)+ print(+ f" -> [COMPLETED] [{sample_id}/{session_key}] "+ f"embed={token_usage.get('embedding', 0)}, llm={token_usage.get('llm_total', 0)}, "+ f"session_id={resolved_session_id}, task_id={task_id}, trace_id={trace_id}",+ file=sys.stderr,+ )++ record = {+ "timestamp": run_time,+ "sample_id": sample_id,+ "session": session_key,+ "status": "success",+ "meta": meta,+ "token_usage": token_usage,+ }+ write_success_record(record, args.success_csv)+ return record+ except Exception as e:+ last_error = e+ if attempt < max_attempts:+ print(+ f" -> [ERROR] [{sample_id}/{session_key}] "+ f"attempt {attempt}/{max_attempts}: {e}; retrying",+ file=sys.stderr,+ )+ await asyncio.sleep(retry_delay)+ continue+ print(f" -> [ERROR] [{sample_id}/{session_key}] {e}", file=sys.stderr)+ traceback.print_exc(file=sys.stderr)++ record = {+ "timestamp": run_time,+ "sample_id": sample_id,+ "session": session_key,+ "status": "error",+ "error": str(last_error) if last_error else "unknown import error",+ }+ write_error_record(record, args.error_log)+ return record+++async def wait_for_queues_and_record_totals(+ openviking_url: str,+ output_file: str,+ baseline_processed: int = 0,+ baseline_model_totals: tuple[int, int, int, int] | None = None,+ expected_processed_delta: int = 0,+ max_wait_sec: int = 1800,+ settle_checks: int = 3,+ poll_interval: float = 2.0,+) -> None:+ has_processed_target = expected_processed_delta > 0+ target_processed = baseline_processed + max(0, expected_processed_delta)+ target_note = f", target_processed={target_processed}" if has_processed_target else ""+ print(+ f"\n[INFO] Waiting for OpenViking background queues to drain "+ f"(baseline_processed={baseline_processed}{target_note}, settle={settle_checks})...",+ file=sys.stderr,+ )++ async with httpx.AsyncClient() as client:+ started = asyncio.get_running_loop().time()+ idle_streak = 0+ last_totals: tuple[int, int, int] | None = None+ timed_out = False++ while True:+ elapsed = asyncio.get_running_loop().time() - started+ if elapsed > max_wait_sec:+ print(+ f"[ERROR] Queue drain timeout after {elapsed:.0f}s "+ f"(last totals={last_totals}, idle_streak={idle_streak})",+ file=sys.stderr,+ )+ timed_out = True+ break++ totals = await _read_queue_totals(client, openviking_url)+ if totals is None:+ await asyncio.sleep(poll_interval)+ continue++ pending, in_progress, processed = totals+ last_totals = totals+ expected_work_done = True if not has_processed_target else processed >= target_processed+ queues_idle = pending == 0 and in_progress == 0++ if expected_work_done and queues_idle:+ idle_streak += 1+ if idle_streak >= settle_checks:+ print(+ f"[INFO] Background queues drained. Processed "+ f"{processed - baseline_processed} item(s) since baseline.",+ file=sys.stderr,+ )+ break+ else:+ idle_streak = 0+ if not expected_work_done and queues_idle:+ print(+ f" .. waiting for expected import work "+ f"(processed={processed}, target={target_processed})",+ file=sys.stderr,+ )+ else:+ print(+ f" .. queue state pending={pending} in_progress={in_progress} "+ f"processed={processed}",+ file=sys.stderr,+ )+ await asyncio.sleep(poll_interval)++ if timed_out:+ raise TimeoutError(+ "OpenViking background queues did not finish before checkpoint "+ f"(target_processed={target_processed if has_processed_target else 'none'}, "+ f"last_totals={last_totals})"+ )++ print("[INFO] Fetching final true token usage from observer...", file=sys.stderr)+ try:+ final_totals = await _read_model_totals(client, openviking_url)+ if final_totals is None:+ print("[INFO] No model usage recorded by OpenViking server.", file=sys.stderr)+ return++ (+ baseline_embedding_prompt,+ baseline_embedding_completion,+ baseline_vlm_prompt,+ baseline_vlm_completion,+ ) = baseline_model_totals or (0, 0, 0, 0)+ (+ final_embedding_prompt,+ final_embedding_completion,+ final_vlm_prompt,+ final_vlm_completion,+ ) = final_totals++ delta_embedding_prompt = max(final_embedding_prompt - baseline_embedding_prompt, 0)+ delta_embedding_completion = max(+ final_embedding_completion - baseline_embedding_completion, 0+ )+ delta_vlm_prompt = max(final_vlm_prompt - baseline_vlm_prompt, 0)+ delta_vlm_completion = max(final_vlm_completion - baseline_vlm_completion, 0)++ print("\n=== TRUE OpenViking Token Delta For This Run ===", file=sys.stderr)+ print(f"Embedding Input (Prompt) Delta: {delta_embedding_prompt}", file=sys.stderr)+ print(+ f"Embedding Output (Completion) Delta: {delta_embedding_completion}",+ file=sys.stderr,+ )+ print(f"VLM/LLM Input (Prompt) Delta: {delta_vlm_prompt}", file=sys.stderr)+ print(f"VLM/LLM Output (Completion) Delta: {delta_vlm_completion}", file=sys.stderr)+ print(+ f"[INFO] Final cumulative totals: embedding_in={final_embedding_prompt}, "+ f"embedding_out={final_embedding_completion}, vlm_in={final_vlm_prompt}, "+ f"vlm_out={final_vlm_completion}",+ file=sys.stderr,+ )++ append_true_token_record(+ output_file,+ {+ "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),+ "embedding_input_tokens": delta_embedding_prompt,+ "embedding_output_tokens": delta_embedding_completion,+ "vlm_llm_input_tokens": delta_vlm_prompt,+ "vlm_llm_output_tokens": delta_vlm_completion,+ },+ )+ print(f"True token totals saved to: {output_file}", file=sys.stderr)+ except Exception as e:+ print(f"[ERROR] Failed to fetch final token usage: {e}", file=sys.stderr)+++async def run_import(args: argparse.Namespace) -> None:+ success_keys = set()+ if not args.force_ingest:+ success_keys = load_success_csv(args.success_csv)+ print(+ f"[INFO] Loaded {len(success_keys)} existing success records from {args.success_csv}",+ file=sys.stderr,+ )++ run_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")+ skipped_count = 0+ samples = load_locomo_data(args.input, args.sample)+ tasks = []+ semaphore = asyncio.Semaphore(max(1, args.parallel))++ async def run_limited(coro):+ async with semaphore:+ return await coro++ async with httpx.AsyncClient() as snap_client:+ baseline_totals = await _read_queue_totals(snap_client, args.openviking_url)+ baseline_model_totals = await _read_model_totals(snap_client, args.openviking_url)+ baseline_processed = baseline_totals[2] if baseline_totals else 0+ print(f"[INFO] Observer baseline processed={baseline_processed}", file=sys.stderr)++ for item in samples:+ sample_id = item["sample_id"]+ sessions = build_session_messages(item)++ for sess in sessions:+ meta = sess["meta"]+ messages = sess["messages"]+ session_key = meta["session_key"]+ label = f"{session_key} ({meta['date_time']})"++ if not args.force_ingest and is_already_ingested(sample_id, session_key, success_keys):+ print(+ f" [{label}] [SKIP] already imported (use --force-ingest to reprocess)",+ file=sys.stderr,+ )+ skipped_count += 1+ continue++ tasks.append(+ run_limited(+ process_single_session(+ messages=messages,+ sample_id=sample_id,+ session_key=session_key,+ meta=meta,+ run_time=run_time,+ args=args,+ )+ )+ )++ if tasks:+ print(+ f"\n[INFO] Starting import with {len(tasks)} tasks to process (parallel={max(1, args.parallel)})",+ file=sys.stderr,+ )+ results = await asyncio.gather(*tasks, return_exceptions=True)+ else:+ results = []++ failed_sessions = []+ for result in results:+ if isinstance(result, Exception):+ failed_sessions.append(f"exception:{result}")+ elif isinstance(result, dict) and result.get("status") == "error":+ failed_sessions.append(f"{result.get('sample_id')}/{result.get('session')}")++ if failed_sessions:+ print(+ f"Import failed for {len(failed_sessions)} session(s) after retries: "+ + ", ".join(failed_sessions[:20]),+ file=sys.stderr,+ )+ if len(failed_sessions) > 20:+ print(f"... and {len(failed_sessions) - 20} more", file=sys.stderr)+ sys.exit(1)++ true_token_file = Path(args.success_csv).parent / "import_true_tokens.csv"+ await wait_for_queues_and_record_totals(+ args.openviking_url,+ str(true_token_file),+ baseline_processed=baseline_processed,+ baseline_model_totals=baseline_model_totals,+ max_wait_sec=args.queue_max_wait_sec,+ )+++def reset_generated_file(path: str | Path, label: str) -> None:+ target = Path(path)+ if target.exists():+ target.unlink()+ print(f"Removed existing {label}: {target}", file=sys.stderr)+++def main():+ script_dir = Path(__file__).parent.resolve()+ default_input = default_locomo_input(script_dir)+ default_success_csv = str(script_dir / "result_preingest" / "import_success.csv")+ default_error_log = str(script_dir / "result_preingest" / "import_errors.log")++ parser = argparse.ArgumentParser(description="Import LoCoMo conversations into OpenViking")+ parser.add_argument("--input", default=default_input, help="Path to LoCoMo .json")+ parser.add_argument(+ "--success-csv", default=default_success_csv, help="Success records CSV path"+ )+ parser.add_argument("--error-log", default=default_error_log, help="Error log path")+ parser.add_argument(+ "--openviking-url", default=DEFAULT_OPENVIKING_URL, help="OpenViking service URL"+ )+ parser.add_argument(+ "--account", default=DEFAULT_OPENVIKING_ACCOUNT, help="OpenViking account namespace"+ )+ parser.add_argument("--user", default=DEFAULT_OPENVIKING_USER, help="OpenViking user namespace")+ parser.add_argument(+ "--agent", default=DEFAULT_OPENVIKING_AGENT, help="OpenViking agent namespace"+ )+ parser.add_argument("--api-key", default=DEFAULT_OPENVIKING_API_KEY, help="OpenViking API key")+ parser.add_argument("--sample", type=int, default=None, help="Sample index (0-based)")+ parser.add_argument(+ "--force-ingest", action="store_true", default=False, help="Force re-import"+ )+ parser.add_argument("--parallel", type=int, default=4, help="Max concurrent session imports")+ parser.add_argument(+ "--queue-max-wait-sec",+ type=int,+ default=1800,+ help="Maximum seconds to wait for OpenViking observer queue deltas",+ )+ parser.add_argument(+ "--error-retries",+ type=int,+ default=DEFAULT_IMPORT_ERROR_RETRIES,+ help="Retry failed imports this many times",+ )+ parser.add_argument(+ "--retry-delay-sec",+ type=float,+ default=2.0,+ help="Seconds to wait between import retry attempts",+ )+ args = parser.parse_args()++ Path(args.success_csv).parent.mkdir(parents=True, exist_ok=True)+ Path(args.error_log).parent.mkdir(parents=True, exist_ok=True)+ if args.force_ingest:+ reset_generated_file(args.success_csv, "import success CSV")+ reset_generated_file(args.error_log, "import error log")+ reset_generated_file(+ Path(args.success_csv).parent / "import_true_tokens.csv", "import true-token CSV"+ )++ try:+ asyncio.run(run_import(args))+ except ValueError as e:+ print(f"Error: {e}", file=sys.stderr)+ sys.exit(1)+++if __name__ == "__main__":+ main()diff --git a/benchmark/locomo/hermes/judge.py b/benchmark/locomo/hermes/judge.pynew file mode 100644index 000000000..5de7bb708--- /dev/null+++ b/benchmark/locomo/hermes/judge.py@@ -0,0 +1,237 @@+import argparse+import asyncio+import csv+import json+import os+import sys+from pathlib import Path++from openai import AsyncOpenAI++try:+ from dotenv import load_dotenv+except ImportError:++ def load_dotenv(*_args, **_kwargs):+ return False+++env_file = Path.home() / ".openviking_benchmark_env"+load_dotenv(env_file)+DEFAULT_JUDGE_ERROR_RETRIES = int(os.getenv("JUDGE_ERROR_RETRIES", "2"))+++def is_retryable_judge_error(reasoning: str) -> bool:+ return reasoning.startswith("[API ERROR]") or reasoning.startswith("[PARSE ERROR]")+++def require_suite(value: str) -> str:+ if value not in {"baseline", "e2e", "preingest"}:+ raise argparse.ArgumentTypeError("suite must be one of: baseline, e2e, preingest")+ return value+++def default_input_path(script_dir: Path, suite: str) -> str:+ if suite == "baseline":+ return str(script_dir / "result_baseline" / "qa_results.csv")+ if suite == "e2e":+ return str(script_dir / "result_e2e" / "qa_results.csv")+ return str(script_dir / "result_preingest" / "qa_results.csv")+++async def grade_answer(+ llm_client, model: str, question: str, gold_answer: str, response: str+) -> tuple[bool, str]:+ system_prompt = """+ You are an expert grader that determines if answers to questions match a gold standard answer+ """++ accuracy_prompt = f"""+ Your task is to label an answer to a question as 'CORRECT' or 'WRONG'. You will be given the following data:+ (1) a question (posed by one user to another user),+ (2) a 'gold' (ground truth) answer,+ (3) a generated answer+ which you will score as CORRECT/WRONG.++ The point of the question is to ask about something one user should know about the other user based on their prior conversations.+ The gold answer will usually be a concise and short answer that includes the referenced topic, for example:+ Question: Do you remember what I got the last time I went to Hawaii?+ Gold answer: A shell necklace+ The generated answer might be much longer, but you should be generous with your grading - as long as it touches on the same topic as the gold answer, it should be counted as CORRECT.++ For time related questions, the gold answer will be a specific date, month, year, etc. The generated answer might be much longer or use relative time references (like "last Tuesday" or "next month"), but you should be generous with your grading - as long as it refers to the same date or time period as the gold answer, it should be counted as CORRECT. Even if the format differs (e.g., "May 7th" vs "7 May"), consider it CORRECT if it's the same date.++ Now it's time for the real question:+ Question: {question}+ Gold answer: {gold_answer}+ Generated answer: {response}++ First, provide a short (one sentence) explanation of your reasoning, then finish with CORRECT or WRONG.+ Do NOT include both CORRECT and WRONG in your response, or it will break the evaluation script.++ Respond with JSON only: {{"is_correct": "CORRECT" or "WRONG", "reasoning": "your explanation"}}+ """++ try:+ resp = await llm_client.chat.completions.create(+ model=model,+ messages=[+ {"role": "system", "content": system_prompt},+ {"role": "user", "content": accuracy_prompt},+ ],+ temperature=0,+ timeout=60,+ )+ content = resp.choices[0].message.content.strip()+ start_idx = content.find("{")+ end_idx = content.rfind("}")+ if start_idx != -1 and end_idx != -1:+ result = json.loads(content[start_idx : end_idx + 1].strip())+ is_correct = result.get("is_correct", "WRONG").strip().upper() == "CORRECT"+ reasoning = result.get("reasoning", "")+ return is_correct, reasoning+ return False, f"[PARSE ERROR] Invalid response: {content}"+ except Exception as e:+ return False, f"[API ERROR] {str(e)}"+++def load_answers(input_path: str) -> tuple[list[dict], list[str]]:+ if not os.path.exists(input_path):+ raise FileNotFoundError(f"Input file not found: {input_path}")++ with open(input_path, "r", encoding="utf-8", newline="") as f:+ reader = csv.DictReader(f)+ fieldnames = reader.fieldnames.copy()+ if "reasoning" not in fieldnames:+ fieldnames.append("reasoning")+ rows = list(reader)+ return rows, fieldnames+++async def main():+ script_dir = Path(__file__).parent.resolve()+ parser = argparse.ArgumentParser(description="Shared QA judge for Hermes LoCoMo benchmark")+ parser.add_argument("--suite", type=require_suite, default="baseline", help="Benchmark suite")+ parser.add_argument("--input", default=None, help="Path to QA result csv file")+ parser.add_argument(+ "--base-url", default="https://ark.cn-beijing.volces.com/api/v3", help="LLM API base URL"+ )+ parser.add_argument(+ "--token",+ default=os.getenv("ARK_API_KEY", os.getenv("OPENAI_API_KEY", "")),+ help="LLM API token",+ )+ parser.add_argument("--model", default="doubao-seed-2-0-pro-260215", help="Judge model name")+ parser.add_argument("--parallel", type=int, default=5, help="Parallel request count")+ parser.add_argument(+ "--error-retries",+ type=int,+ default=DEFAULT_JUDGE_ERROR_RETRIES,+ help="Retry judge API/parse failures this many times",+ )+ parser.add_argument(+ "--retry-delay-sec",+ type=float,+ default=2.0,+ help="Seconds to wait between judge retry attempts",+ )+ args = parser.parse_args()++ if args.input is None:+ args.input = default_input_path(script_dir, args.suite)++ if not args.token:+ print("Error: API token is required")+ print("\nSet API key via:")+ print(" 1. Create ~/.openviking_benchmark_env with: ARK_API_KEY=your_key")+ print(" 2. Or pass --token your_key")+ print(" 3. Or set env var: export ARK_API_KEY=your_key")+ raise SystemExit(1)++ rows, fieldnames = load_answers(args.input)++ valid_rows = []+ ungraded = []+ for idx, row in enumerate(rows):+ if row.get("category", "") == "5":+ continue+ valid_rows.append(idx)+ if not row.get("result"):+ ungraded.append(idx)++ total = len(rows)+ valid_total = len(valid_rows)+ print(+ f"Total answers: {total}, valid (category != 5): {valid_total}, ungraded: {len(ungraded)}"+ )++ if not ungraded:+ print("All valid answers already graded, exit")+ return++ client = AsyncOpenAI(base_url=args.base_url, api_key=args.token)+ semaphore = asyncio.Semaphore(args.parallel)+ file_lock = asyncio.Lock()++ async def save_results():+ async with file_lock:+ temp_file = f"{args.input}.tmp"+ with open(temp_file, "w", encoding="utf-8", newline="") as f:+ writer = csv.DictWriter(f, fieldnames=fieldnames)+ writer.writeheader()+ writer.writerows(rows)+ os.replace(temp_file, args.input)++ async def process_row(idx: int):+ async with semaphore:+ row = rows[idx]+ question = row["question"]+ gold = row.get("expected") or row.get("answer")+ response = row["response"]+ print(f"Grading {idx + 1}/{total}: {question[:60]}...")+ max_attempts = max(0, args.error_retries) + 1+ is_correct = False+ reasoning = ""+ for attempt in range(1, max_attempts + 1):+ is_correct, reasoning = await grade_answer(+ client, args.model, question, gold, response+ )+ if not is_retryable_judge_error(reasoning):+ row["result"] = "CORRECT" if is_correct else "WRONG"+ break+ if attempt < max_attempts:+ print(+ f"Judge retry {attempt}/{max_attempts} for row {idx + 1}: {reasoning}",+ file=sys.stderr,+ )+ await asyncio.sleep(args.retry_delay_sec)+ else:+ row["result"] = ""+ if is_retryable_judge_error(reasoning):+ row["result"] = ""+ row["reasoning"] = reasoning+ await save_results()+ print(f"Saved result for {idx + 1}/{total}: {row['result'] or 'UNJUDGED'}")+ return idx, row++ await asyncio.gather(*[process_row(idx) for idx in ungraded])++ correct = sum(+ 1 for row in rows if row.get("category", "") != "5" and row.get("result") == "CORRECT"+ )+ total_graded = sum(1 for row in rows if row.get("category", "") != "5" and row.get("result"))+ remaining_ungraded = sum(+ 1 for row in rows if row.get("category", "") != "5" and not row.get("result")+ )+ accuracy = correct / total_graded if total_graded > 0 else 0.0+ print(f"\nGrading completed: {correct}/{total_graded} correct, accuracy: {accuracy:.2%}")+ print(f"All results saved to {args.input}")+ if remaining_ungraded:+ print(+ f"Judge left {remaining_ungraded} valid row(s) ungraded after retries", file=sys.stderr+ )+ raise SystemExit(1)+++if __name__ == "__main__":+ asyncio.run(main())diff --git a/benchmark/locomo/hermes/run_full_eval.sh b/benchmark/locomo/hermes/run_full_eval.shnew file mode 100755index 000000000..6453932b0--- /dev/null+++ b/benchmark/locomo/hermes/run_full_eval.sh@@ -0,0 +1,909 @@+#!/usr/bin/env bash+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+cd "$SCRIPT_DIR"++if [[ -z "${PYTHON:-}" ]]; then+ if [[ -x "$SCRIPT_DIR/../../../.venv/bin/python" ]]; then+ PYTHON="$SCRIPT_DIR/../../../.venv/bin/python"+ elif command -v python >/dev/null 2>&1; then+ PYTHON="python"+ else+ PYTHON="python3"+ fi+fi++usage() {+ cat <<'USAGE'+Run a fixed LoCoMo benchmark suite.++Suites:+ e2e Import through Hermes/OpenViking, then eval/judge/stats.+ preingest Import directly into OpenViking, then eval/judge/stats.+ native Import into Hermes native memory, then eval/judge/stats. No OpenViking checkpoint.++Usage:+ ./run_full_eval.sh --suite e2e [-cp] [options]+ ./run_full_eval.sh --suite preingest [-cp] [options]+ ./run_full_eval.sh --suite native [options]++Options:+ --suite NAME Required: e2e, preingest, or native.+ -cp, --checkpoint Copy an OpenViking checkpoint after import (e2e/preingest only).+ --sample N Process one LoCoMo sample index.+ --count N Limit QA questions per sample during eval.+ --force-ingest Re-ingest even if import_success.csv exists.+ --force-eval Re-run QA even if qa_results.csv exists.+ --skip-import Run eval/judge/stats only; assumes benchmark state is already loaded.+ --import-csv PATH Import CSV used for stats when --skip-import is used.+ --result-dir PATH Result directory for CSVs and logs.+ --snapshot-root PATH Root directory for OpenViking checkpoints when -cp is used.+ --state-source PATH OpenViking workspace to copy when -cp is used.+ --run-id ID Stable id used in result/checkpoint names.+ -h, --help Show this help.++Environment:+ LOCOMO_JSON, HERMES_URL, HERMES_TOKEN, HERMES_MODEL, OPENVIKING_URL+ JUDGE_BASE_URL, JUDGE_TOKEN, JUDGE_MODEL+ IMPORT_PARALLEL, QA_PARALLEL, JUDGE_PARALLEL+ IMPORT_ERROR_RETRIES, QA_ERROR_RETRIES, JUDGE_ERROR_RETRIES, QUEUE_MAX_WAIT_SEC+ PYTHON, HERMES_STATE_DB, OPENVIKING_CONFIG_FILE, OPENVIKING_STATE_SOURCE, SNAPSHOT_ROOT, RESULT_DIR+ E2E_PREFLIGHT, SKIP_IMPORT, IMPORT_CSV_OVERRIDE+ OPENVIKING_ACCOUNT, OPENVIKING_USER, OPENVIKING_AGENT, OPENVIKING_API_KEY++Dataset:+ By default this expects ../data/locomo10.json from benchmark/locomo/hermes.+ Set LOCOMO_JSON=/path/to/locomo10.json to use a local copy elsewhere.+USAGE+}++SUITE="${SUITE:-}"+LOCOMO_JSON="${LOCOMO_JSON:-../data/locomo10.json}"+HERMES_URL="${HERMES_URL:-http://127.0.0.1:8642}"+HERMES_TOKEN="${HERMES_TOKEN:-${API_SERVER_KEY:-}}"+HERMES_MODEL="${HERMES_MODEL:-hermes-agent}"+OPENVIKING_URL="${OPENVIKING_URL:-http://127.0.0.1:1933}"+OPENVIKING_ACCOUNT="${OPENVIKING_ACCOUNT:-default}"+OPENVIKING_USER="${OPENVIKING_USER:-default}"+OPENVIKING_AGENT="${OPENVIKING_AGENT:-hermes}"+OPENVIKING_API_KEY="${OPENVIKING_API_KEY:-}"+JUDGE_BASE_URL="${JUDGE_BASE_URL:-https://ark.cn-beijing.volces.com/api/v3}"+JUDGE_TOKEN="${JUDGE_TOKEN:-${ARK_API_KEY:-}}"+JUDGE_MODEL="${JUDGE_MODEL:-doubao-seed-2-0-pro-260215}"+IMPORT_PARALLEL="${IMPORT_PARALLEL:-4}"+QA_PARALLEL="${QA_PARALLEL:-4}"+JUDGE_PARALLEL="${JUDGE_PARALLEL:-5}"+IMPORT_ERROR_RETRIES="${IMPORT_ERROR_RETRIES:-2}"+QA_ERROR_RETRIES="${QA_ERROR_RETRIES:-2}"+JUDGE_ERROR_RETRIES="${JUDGE_ERROR_RETRIES:-2}"+QUEUE_MAX_WAIT_SEC="${QUEUE_MAX_WAIT_SEC:-1800}"+E2E_PREFLIGHT="${E2E_PREFLIGHT:-1}"+RUN_ID="${RUN_ID:-}"+RESULT_DIR="${RESULT_DIR:-}"+SNAPSHOT_ROOT="${SNAPSHOT_ROOT:-}"+OPENVIKING_STATE_SOURCE="${OPENVIKING_STATE_SOURCE:-}"+SAMPLE="${SAMPLE:-}"+COUNT="${COUNT:-}"+FORCE_INGEST="${FORCE_INGEST:-}"+FORCE_EVAL="${FORCE_EVAL:-}"+CHECKPOINT_REQUESTED="${CHECKPOINT_REQUESTED:-0}"+SKIP_IMPORT="${SKIP_IMPORT:-}"+IMPORT_CSV_OVERRIDE="${IMPORT_CSV_OVERRIDE:-}"++while [[ $# -gt 0 ]]; do+ case "$1" in+ --suite)+ SUITE="${2:-}"+ shift 2+ ;;+ --sample)+ SAMPLE="${2:-}"+ shift 2+ ;;+ --count)+ COUNT="${2:-}"+ shift 2+ ;;+ --force-ingest)+ FORCE_INGEST=1+ shift+ ;;+ --force-eval)+ FORCE_EVAL=1+ shift+ ;;+ --skip-import)+ SKIP_IMPORT=1+ shift+ ;;+ --import-csv)+ IMPORT_CSV_OVERRIDE="${2:-}"+ shift 2+ ;;+ -cp|--checkpoint)+ CHECKPOINT_REQUESTED=1+ shift+ ;;+ --result-dir)+ RESULT_DIR="${2:-}"+ shift 2+ ;;+ --snapshot-root)+ SNAPSHOT_ROOT="${2:-}"+ shift 2+ ;;+ --state-source)+ OPENVIKING_STATE_SOURCE="${2:-}"+ shift 2+ ;;+ --run-id)+ RUN_ID="${2:-}"+ shift 2+ ;;+ -h|--help)+ usage+ exit 0+ ;;+ *)+ echo "Unknown argument: $1" >&2+ usage >&2+ exit 1+ ;;+ esac+done++if [[ -z "$SUITE" ]]; then+ echo "Error: --suite is required" >&2+ usage >&2+ exit 1+fi++CHECKPOINT_OPENVIKING=0+CAN_CHECKPOINT_OPENVIKING=0+case "$SUITE" in+ e2e)+ SUITE_LABEL="e2e"+ EVAL_SUITE="e2e"+ IMPORT_SCRIPT="import_e2e.py"+ BENCHMARK_TITLE="Hermes OpenViking E2E LoCoMo benchmark"+ RUN_ID="${RUN_ID:-locomo-e2e-session-no-pe-$(date +%Y%m%d-%H%M%S)}"+ CAN_CHECKPOINT_OPENVIKING=1+ if [[ -z "$RESULT_DIR" ]]; then+ if [[ "$CHECKPOINT_REQUESTED" == "1" ]]; then+ RESULT_DIR="$SCRIPT_DIR/result_e2e_checkpointed_${RUN_ID}"+ else+ RESULT_DIR="$SCRIPT_DIR/result_e2e_${RUN_ID}"+ fi+ fi+ ;;+ preingest)+ SUITE_LABEL="preingest"+ EVAL_SUITE="preingest"+ IMPORT_SCRIPT="import_to_ov.py"+ BENCHMARK_TITLE="Hermes + OpenViking pre-ingest LoCoMo benchmark"+ RUN_ID="${RUN_ID:-locomo-preingest-session-timestamps-visuals-no-pe-$(date +%Y%m%d-%H%M%S)}"+ RESULT_DIR="${RESULT_DIR:-$SCRIPT_DIR/result_preingest_${RUN_ID}}"+ CAN_CHECKPOINT_OPENVIKING=1+ ;;+ native|baseline)+ SUITE_LABEL="native"+ EVAL_SUITE="baseline"+ IMPORT_SCRIPT="import_to_native.py"+ BENCHMARK_TITLE="Hermes Native Memory LoCoMo benchmark"+ RUN_ID="${RUN_ID:-locomo-native-session-no-pe-$(date +%Y%m%d-%H%M%S)}"+ RESULT_DIR="${RESULT_DIR:-$SCRIPT_DIR/result_baseline_${RUN_ID}}"+ ;;+ *)+ echo "Error: --suite must be one of: e2e, preingest, native" >&2+ exit 1+ ;;+esac++if [[ "$CHECKPOINT_REQUESTED" == "1" ]]; then+ if [[ "$CAN_CHECKPOINT_OPENVIKING" == "1" ]]; then+ CHECKPOINT_OPENVIKING=1+ else+ echo "Note: -cp/--checkpoint is ignored for native suite; native has no OpenViking checkpoint." >&2+ fi+fi++if [[ "$SKIP_IMPORT" == "1" && "$CHECKPOINT_OPENVIKING" == "1" ]]; then+ echo "Note: -cp/--checkpoint is ignored with --skip-import; no import checkpoint will be produced." >&2+ CHECKPOINT_OPENVIKING=0+fi++SNAPSHOT_ROOT="${SNAPSHOT_ROOT:-$SCRIPT_DIR/openviking_state_snapshots}"+IMPORT_CSV="${RESULT_DIR}/import_success.csv"+STATS_IMPORT_CSV="${IMPORT_CSV_OVERRIDE:-$IMPORT_CSV}"+QA_CSV="${RESULT_DIR}/qa_results.csv"+LOG_DIR="${RESULT_DIR}/logs"+SNAPSHOT_RUN_DIR="${SNAPSHOT_ROOT}/${RUN_ID}"+SNAPSHOT_DIR="${SNAPSHOT_RUN_DIR}/openviking_workspace"+SNAPSHOT_MANIFEST="${SNAPSHOT_RUN_DIR}/manifest.txt"+RESULT_MANIFEST="${RESULT_DIR}/openviking_checkpoint_manifest.txt"+RESOLVED_OPENVIKING_CONFIG_FILE=""+export OPENVIKING_ACCOUNT OPENVIKING_USER OPENVIKING_AGENT OPENVIKING_API_KEY++if [[ ! -f "$LOCOMO_JSON" ]]; then+ echo "Error: LoCoMo dataset not found: $LOCOMO_JSON" >&2+ echo "Set LOCOMO_JSON=/path/to/locomo10.json or place it at benchmark/locomo/data/locomo10.json." >&2+ exit 1+fi++mkdir -p "$RESULT_DIR" "$LOG_DIR"+if [[ "$CHECKPOINT_OPENVIKING" == "1" ]]; then+ mkdir -p "$SNAPSHOT_ROOT"+fi++resolve_openviking_state_source() {+ "$PYTHON" - <<'PY'+import json+import os+from pathlib import Path++config_candidates = []+if os.environ.get("OPENVIKING_CONFIG_FILE"):+ config_candidates.append(Path(os.environ["OPENVIKING_CONFIG_FILE"]).expanduser())+config_candidates.append(Path.home() / ".openviking" / "ov.conf")+config_candidates.append(Path("/etc/openviking/ov.conf"))++config_path = next((path for path in config_candidates if path.exists()), None)+workspace = ""+if config_path is not None:+ raw = os.path.expandvars(config_path.read_text(encoding="utf-8-sig"))+ data = json.loads(raw)+ storage = data.get("storage", {})+ if isinstance(storage, dict):+ workspace = storage.get("workspace") or ""++if not workspace:+ workspace = str(Path.home() / ".openviking" / "data")++print(Path(workspace).expanduser().resolve())+print(config_path.expanduser().resolve() if config_path is not None else "")+PY+}++ensure_openviking_state_source() {+ if [[ -z "$OPENVIKING_STATE_SOURCE" ]]; then+ mapfile -t resolved_state < <(resolve_openviking_state_source)+ OPENVIKING_STATE_SOURCE="${resolved_state[0]:-}"+ RESOLVED_OPENVIKING_CONFIG_FILE="${resolved_state[1]:-}"+ else+ OPENVIKING_STATE_SOURCE="$("$PYTHON" -c 'import pathlib, sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$OPENVIKING_STATE_SOURCE")"+ RESOLVED_OPENVIKING_CONFIG_FILE="${OPENVIKING_CONFIG_FILE:-}"+ fi++ if [[ -z "$OPENVIKING_STATE_SOURCE" || ! -d "$OPENVIKING_STATE_SOURCE" ]]; then+ echo "OpenViking state source does not exist: ${OPENVIKING_STATE_SOURCE:-<empty>}" >&2+ echo "Pass --state-source PATH or set OPENVIKING_STATE_SOURCE." >&2+ exit 1+ fi+}++verify_openviking_archive_readiness() {+ if [[ "$SUITE_LABEL" != "e2e" && "$SUITE_LABEL" != "preingest" ]]; then+ echo ">>> Step 2: SKIPPED (OpenViking archive readiness is not used for native suite)."+ return+ fi++ ensure_openviking_state_source++ echo ""+ echo ">>> Step 2: Waiting for OpenViking archive readiness..."+ OPENVIKING_STATE_SOURCE="$OPENVIKING_STATE_SOURCE" \+ LOCOMO_JSON="$LOCOMO_JSON" \+ SAMPLE="$SAMPLE" \+ IMPORT_CSV="$IMPORT_CSV" \+ RESULT_DIR="$RESULT_DIR" \+ SUITE_LABEL="$SUITE_LABEL" \+ QUEUE_MAX_WAIT_SEC="$QUEUE_MAX_WAIT_SEC" \+ "$PYTHON" - <<'PY'+import csv+import json+import os+import sys+import time+from pathlib import Path++source = Path(os.environ["OPENVIKING_STATE_SOURCE"]).expanduser().resolve()+locomo_json = Path(os.environ["LOCOMO_JSON"]).expanduser().resolve()+import_csv = Path(os.environ["IMPORT_CSV"]).expanduser().resolve()+result_dir = Path(os.environ["RESULT_DIR"]).expanduser().resolve()+suite = os.environ["SUITE_LABEL"]+sample_value = (os.environ.get("SAMPLE") or "").strip()+max_wait_sec = max(1, int(os.environ.get("QUEUE_MAX_WAIT_SEC") or "1800"))+account = os.environ.get("OPENVIKING_ACCOUNT", "default")+poll_interval = 5++def session_number(session_key: str) -> int:+ try:+ return int(session_key.split("_", 1)[1])+ except Exception:+ return 0++with locomo_json.open("r", encoding="utf-8") as f:+ data = json.load(f)++if sample_value:+ sample_index = int(sample_value)+ if sample_index < 0 or sample_index >= len(data):+ print(+ f"Sample index {sample_index} out of range for {locomo_json}",+ file=sys.stderr,+ )+ sys.exit(1)+ data = [data[sample_index]]++prefix = "locomo-e2e" if suite == "e2e" else "locomo-ovpreingest"+expected_ids = []+expected_pairs = set()+for item in data:+ sample_id = item.get("sample_id", "")+ conv = item.get("conversation", {})+ session_keys = sorted(+ [+ key for key in conv+ if key.startswith("session_") and not key.endswith("_date_time")+ ],+ key=session_number,+ )+ for session_key in session_keys:+ expected_pairs.add((sample_id, session_key))+ expected_ids.append(f"{prefix}-{sample_id}-{session_key}")++expected_ids = sorted(set(expected_ids))+if not expected_ids:+ print(f"No expected sessions found in {locomo_json}", file=sys.stderr)+ sys.exit(1)++def read_import_pairs() -> set[tuple[str, str]]:+ pairs = set()+ if not import_csv.exists():+ return pairs+ with import_csv.open("r", encoding="utf-8", newline="") as f:+ for row in csv.DictReader(f):+ sample_id = (row.get("sample_id") or "").strip()+ session_key = (row.get("session") or "").strip()+ if sample_id and session_key:+ pairs.add((sample_id, session_key))+ return pairs++def scan_expected_sessions():+ done_session_ids = set()+ done_files = 0+ memory_diff_files = 0+ failed_files = []++ for session_id in expected_ids:+ history_dir = source / "viking" / account / "session" / session_id / "history"+ try:+ if not history_dir.exists():+ continue+ for archive_dir in history_dir.iterdir():+ if not archive_dir.is_dir():+ continue++ done_marker = archive_dir / ".done"+ failed_marker = archive_dir / ".failed.json"+ memory_diff = archive_dir / "memory_diff.json"++ if done_marker.exists():+ done_files += 1+ done_session_ids.add(session_id)+ if memory_diff.exists():+ memory_diff_files += 1+ if failed_marker.exists():+ failed_files.append(failed_marker)+ except OSError:+ continue++ return done_session_ids, done_files, memory_diff_files, failed_files++deadline = time.monotonic() + max_wait_sec+last_status = ""++while True:+ done_session_ids, done_files, memory_diff_files, failed_files = scan_expected_sessions()+ expected_done = done_session_ids & set(expected_ids)+ missing = [session_id for session_id in expected_ids if session_id not in done_session_ids]+ status = (+ f"expected_sessions={len(expected_ids)}, "+ f"done_sessions={len(expected_done)}, "+ f"done_files={done_files}, memory_diff_files={memory_diff_files}, "+ f"failed_files={len(failed_files)}"+ )++ if failed_files:+ print("OpenViking failed archive markers exist:", file=sys.stderr)+ for path in failed_files[:20]:+ print(f" {path}", file=sys.stderr)+ if len(failed_files) > 20:+ print(f" ... and {len(failed_files) - 20} more", file=sys.stderr)+ sys.exit(1)++ if not missing:+ print(f"Archive readiness: {status}")+ import_pairs = read_import_pairs()+ missing_csv_pairs = sorted(expected_pairs - import_pairs)+ report = {+ "suite": suite,+ "expected_sessions": len(expected_ids),+ "done_sessions": len(expected_done),+ "import_csv_rows": len(import_pairs & expected_pairs),+ "missing_import_csv_rows": [+ {"sample_id": sample_id, "session": session_key}+ for sample_id, session_key in missing_csv_pairs+ ],+ }+ report_path = result_dir / "logs" / "archive_readiness_report.json"+ report_path.parent.mkdir(parents=True, exist_ok=True)+ report_path.write_text(json.dumps(report, indent=2), encoding="utf-8")+ if missing_csv_pairs:+ print(+ "Archive readiness passed, but import_success.csv is "+ f"missing {len(missing_csv_pairs)} row(s); see {report_path}",+ file=sys.stderr,+ )+ sys.exit(1)+ else:+ print(f"Archive readiness report: {report_path}")+ break++ if time.monotonic() >= deadline:+ print(+ "OpenViking archive readiness timed out: "+ f"{len(missing)} expected session(s) have no .done marker.",+ file=sys.stderr,+ )+ print(f"Last status: {status}", file=sys.stderr)+ for session_id in missing[:20]:+ print(f" {session_id}", file=sys.stderr)+ if len(missing) > 20:+ print(f" ... and {len(missing) - 20} more", file=sys.stderr)+ sys.exit(1)++ if status != last_status:+ print(f" .. archive readiness pending: {status}", file=sys.stderr)+ last_status = status+ time.sleep(poll_interval)+PY+}++copy_openviking_state() {+ if [[ "$CHECKPOINT_OPENVIKING" != "1" ]]; then+ echo ">>> Step 2: SKIPPED (OpenViking checkpoint disabled)."+ return+ fi++ ensure_openviking_state_source++ if [[ -e "$SNAPSHOT_DIR" ]]; then+ echo "Checkpoint destination already exists: $SNAPSHOT_DIR" >&2+ echo "Use a different --run-id or --snapshot-root." >&2+ exit 1+ fi++ local snapshot_abs+ snapshot_abs="$("$PYTHON" -c 'import pathlib, sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$SNAPSHOT_DIR")"+ case "${snapshot_abs}/" in+ "${OPENVIKING_STATE_SOURCE}/"*)+ echo "Refusing to place checkpoint inside the OpenViking source workspace." >&2+ echo "Source: $OPENVIKING_STATE_SOURCE" >&2+ echo "Destination: $snapshot_abs" >&2+ exit 1+ ;;+ esac++ mkdir -p "$SNAPSHOT_RUN_DIR"++ echo ""+ echo ">>> Step 2: Copying OpenViking state checkpoint..."+ echo "Source: $OPENVIKING_STATE_SOURCE"+ echo "Checkpoint: $SNAPSHOT_DIR"++ if command -v rsync >/dev/null 2>&1; then+ rsync -a --delete "${OPENVIKING_STATE_SOURCE}/" "${SNAPSHOT_DIR}/"+ else+ mkdir -p "$SNAPSHOT_DIR"+ cp -a "${OPENVIKING_STATE_SOURCE}/." "${SNAPSHOT_DIR}/"+ fi++ if [[ -n "$RESOLVED_OPENVIKING_CONFIG_FILE" && -f "$RESOLVED_OPENVIKING_CONFIG_FILE" ]]; then+ cp "$RESOLVED_OPENVIKING_CONFIG_FILE" "${SNAPSHOT_RUN_DIR}/ov.conf"+ fi++ {+ echo "OpenViking ${SUITE_LABEL} checkpoint"+ echo "created_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)"+ echo "run_id=$RUN_ID"+ echo "suite=$SUITE_LABEL"+ echo "prompting=no_pe"+ echo "import_shape=flattened_locomo_session"+ echo "result_dir=$RESULT_DIR"+ echo "openviking_url=$OPENVIKING_URL"+ echo "source_workspace=$OPENVIKING_STATE_SOURCE"+ echo "checkpoint_workspace=$SNAPSHOT_DIR"+ echo "checkpoint_manifest=$SNAPSHOT_MANIFEST"+ echo "config_file=${RESOLVED_OPENVIKING_CONFIG_FILE:-}"+ echo "import_csv=$IMPORT_CSV"+ echo "qa_csv=$QA_CSV"+ } > "$SNAPSHOT_MANIFEST"++ cp "$SNAPSHOT_MANIFEST" "$RESULT_MANIFEST"++ echo "Checkpoint manifest: $SNAPSHOT_MANIFEST"+ echo "Result manifest: $RESULT_MANIFEST"+ echo ">>> Step 2 done."+}++verify_hermes_openviking_target() {+ if [[ "$SUITE_LABEL" != "e2e" ]]; then+ return+ fi++ local preflight_value+ preflight_value="$(printf '%s' "$E2E_PREFLIGHT" | tr '[:upper:]' '[:lower:]')"+ case "$preflight_value" in+ 0|false|no|off)+ echo ">>> Preflight skipped (E2E_PREFLIGHT=$E2E_PREFLIGHT)."+ return+ ;;+ esac++ local probe_session_id="openviking-e2e-preflight-${RUN_ID}-$$"+ echo ""+ echo ">>> Preflight: verifying Hermes writes to the configured OpenViking target..."+ echo "Probe session: $probe_session_id"++ HERMES_URL="$HERMES_URL" \+ HERMES_TOKEN="$HERMES_TOKEN" \+ HERMES_MODEL="$HERMES_MODEL" \+ OPENVIKING_URL="$OPENVIKING_URL" \+ PROBE_SESSION_ID="$probe_session_id" \+ "$PYTHON" - <<'PY'+import os+import sys+import time++import requests+++def fail(message: str) -> None:+ print(f"Preflight failed: {message}", file=sys.stderr)+ sys.exit(1)+++hermes_url = os.environ["HERMES_URL"].rstrip("/")+openviking_url = os.environ["OPENVIKING_URL"].rstrip("/")+token = os.environ.get("HERMES_TOKEN", "")+model = os.environ["HERMES_MODEL"]+session_id = os.environ["PROBE_SESSION_ID"]++headers = {+ "Content-Type": "application/json",+ "X-Hermes-Session-Id": session_id,+}+if token:+ headers["Authorization"] = f"Bearer {token}"++payload = {+ "model": model,+ "messages": [+ {+ "role": "user",+ "content": f"OpenViking benchmark preflight probe {session_id}. Acknowledge with OK.",+ }+ ],+}++try:+ response = requests.post(+ f"{hermes_url}/v1/chat/completions",+ headers=headers,+ json=payload,+ timeout=300,+ )+ response.raise_for_status()+except Exception as exc:+ fail(f"Hermes request did not succeed: {exc}")++resolved_session_id = response.headers.get("X-Hermes-Session-Id") or session_id++ov_headers = {+ "X-OpenViking-Account": os.environ.get("OPENVIKING_ACCOUNT", "default"),+ "X-OpenViking-User": os.environ.get("OPENVIKING_USER", "default"),+ "X-OpenViking-Agent": os.environ.get("OPENVIKING_AGENT", "hermes"),+}+api_key = os.environ.get("OPENVIKING_API_KEY", "")+if api_key:+ ov_headers["X-API-Key"] = api_key++last_observed = "session not found"+deadline = time.monotonic() + 30.0+while time.monotonic() < deadline:+ try:+ session_response = requests.get(+ f"{openviking_url}/api/v1/sessions/{resolved_session_id}",+ headers=ov_headers,+ timeout=10,+ )+ except Exception as exc:+ last_observed = str(exc)+ time.sleep(0.5)+ continue++ if session_response.status_code == 200:+ try:+ body = session_response.json()+ except Exception:+ body = {}+ result = body.get("result") if isinstance(body, dict) else None+ session = result if isinstance(result, dict) else body+ pending_tokens = int(session.get("pending_tokens") or 0)+ message_count = int(session.get("message_count") or 0)+ last_observed = f"pending_tokens={pending_tokens}, message_count={message_count}"+ if pending_tokens > 0 or message_count > 0:+ print(+ "Preflight OK: expected OpenViking target received "+ f"{resolved_session_id} ({last_observed})."+ )+ try:+ requests.delete(+ f"{openviking_url}/api/v1/sessions/{resolved_session_id}",+ headers=ov_headers,+ timeout=10,+ )+ except Exception as exc:+ print(f"Warning: failed to delete preflight session: {exc}", file=sys.stderr)+ sys.exit(0)++ time.sleep(0.5)++fail(+ "Hermes completed the probe, but the configured OpenViking target did not receive it. "+ f"Expected OPENVIKING_URL={openviking_url}, session={resolved_session_id}, "+ f"namespace={ov_headers.get('X-OpenViking-Account')}/"+ f"{ov_headers.get('X-OpenViking-User')}/{ov_headers.get('X-OpenViking-Agent')}; "+ f"last observed: {last_observed}"+)+PY+}++warm_hermes_memory_provider() {+ if [[ "$SUITE_LABEL" != "e2e" ]]; then+ return+ fi++ echo ""+ echo ">>> Warmup: initializing Hermes memory provider before parallel QA..."++ HERMES_URL="$HERMES_URL" \+ HERMES_TOKEN="$HERMES_TOKEN" \+ HERMES_MODEL="$HERMES_MODEL" \+ "$PYTHON" - <<'PY'+import os+import sys+import uuid++import requests+++hermes_url = os.environ["HERMES_URL"].rstrip("/")+token = os.environ.get("HERMES_TOKEN", "")+model = os.environ["HERMES_MODEL"]+session_id = f"openviking-e2e-warmup-{uuid.uuid4().hex}"++headers = {"Content-Type": "application/json"}+if token:+ headers["Authorization"] = f"Bearer {token}"++payload = {+ "model": model,+ "input": "OpenViking memory provider warmup. Acknowledge with OK.",+ "conversation": session_id,+ "session_id": session_id,+ "store": False,+}++try:+ response = requests.post(+ f"{hermes_url}/v1/responses",+ headers=headers,+ json=payload,+ timeout=180,+ )+ response.raise_for_status()+except Exception as exc:+ print(f"Warmup failed: {exc}", file=sys.stderr)+ sys.exit(1)++print(f"Warmup OK: {session_id}")+PY+}++write_run_command() {+ {+ echo "#!/usr/bin/env bash"+ printf 'cd %q\n' "$SCRIPT_DIR"+ printf 'PYTHON=%q RUN_ID=%q RESULT_DIR=%q SNAPSHOT_ROOT=%q OPENVIKING_STATE_SOURCE=%q ' \+ "$PYTHON" "$RUN_ID" "$RESULT_DIR" "$SNAPSHOT_ROOT" "$OPENVIKING_STATE_SOURCE"+ printf 'IMPORT_PARALLEL=%q QA_PARALLEL=%q JUDGE_PARALLEL=%q ' \+ "$IMPORT_PARALLEL" "$QA_PARALLEL" "$JUDGE_PARALLEL"+ printf 'IMPORT_ERROR_RETRIES=%q QA_ERROR_RETRIES=%q JUDGE_ERROR_RETRIES=%q QUEUE_MAX_WAIT_SEC=%q ' \+ "$IMPORT_ERROR_RETRIES" "$QA_ERROR_RETRIES" "$JUDGE_ERROR_RETRIES" "$QUEUE_MAX_WAIT_SEC"+ printf 'OPENVIKING_ACCOUNT=%q OPENVIKING_USER=%q OPENVIKING_AGENT=%q E2E_PREFLIGHT=%q ' \+ "$OPENVIKING_ACCOUNT" "$OPENVIKING_USER" "$OPENVIKING_AGENT" "$E2E_PREFLIGHT"+ [[ -n "${HERMES_STATE_DB:-}" ]] && printf 'HERMES_STATE_DB=%q ' "$HERMES_STATE_DB"+ printf './run_full_eval.sh --suite %q' "$SUITE_LABEL"+ [[ "$CHECKPOINT_OPENVIKING" == "1" ]] && printf ' -cp'+ [[ -n "$SAMPLE" ]] && printf ' --sample %q' "$SAMPLE"+ [[ -n "$COUNT" ]] && printf ' --count %q' "$COUNT"+ [[ -n "$FORCE_INGEST" ]] && printf ' --force-ingest'+ [[ -n "$FORCE_EVAL" ]] && printf ' --force-eval'+ [[ "$SKIP_IMPORT" == "1" ]] && printf ' --skip-import'+ [[ -n "$IMPORT_CSV_OVERRIDE" ]] && printf ' --import-csv %q' "$IMPORT_CSV_OVERRIDE"+ printf '\n'+ } > "${RESULT_DIR}/run_command.sh"+ chmod +x "${RESULT_DIR}/run_command.sh"+}++echo "================================================================="+echo " $BENCHMARK_TITLE"+echo "================================================================="+echo "Suite: $SUITE_LABEL"+echo "Run id: $RUN_ID"+echo "LoCoMo data: $LOCOMO_JSON"+echo "Hermes URL: $HERMES_URL"+echo "OpenViking URL: $OPENVIKING_URL"+echo "Result dir: $RESULT_DIR"+echo "Python: $PYTHON"+echo "Checkpoint: $([[ "$CHECKPOINT_OPENVIKING" == "1" ]] && echo enabled || echo disabled)"+if [[ "$CHECKPOINT_OPENVIKING" == "1" ]]; then+ echo "Snapshot root: $SNAPSHOT_ROOT"+fi+echo "Import shape: flattened LoCoMo session"+echo "Import: $([[ "$SKIP_IMPORT" == "1" ]] && echo skipped || echo enabled)"+if [[ "$SKIP_IMPORT" != "1" ]]; then+ echo "Import workers: $IMPORT_PARALLEL"+fi+echo "QA workers: $QA_PARALLEL"+echo "Judge workers: $JUDGE_PARALLEL"+echo "Prompting: no PE"+echo "================================================================="++if [[ "$SKIP_IMPORT" != "1" && ( "$SUITE_LABEL" == "e2e" || "$SUITE_LABEL" == "preingest" ) ]]; then+ ensure_openviking_state_source+ export OPENVIKING_STATE_SOURCE+fi++if [[ "$SKIP_IMPORT" == "1" ]]; then+ echo ">>> Preflight: SKIPPED (--skip-import)."+else+ verify_hermes_openviking_target+fi+write_run_command+warm_hermes_memory_provider++echo ""+if [[ "$SKIP_IMPORT" == "1" ]]; then+ echo ">>> Step 1: SKIPPED (--skip-import)."+ IMPORT_RC=0+else+ echo ">>> Step 1: Running import..."+ IMPORT_ARGS=(+ --input "$LOCOMO_JSON"+ --success-csv "$IMPORT_CSV"+ )++ if [[ "$SUITE_LABEL" == "e2e" || "$SUITE_LABEL" == "native" ]]; then+ IMPORT_ARGS+=(--base-url "$HERMES_URL" --model "$HERMES_MODEL")+ [[ -n "$HERMES_TOKEN" ]] && IMPORT_ARGS+=(--token "$HERMES_TOKEN")+ fi++ if [[ "$SUITE_LABEL" == "e2e" ]]; then+ IMPORT_ARGS+=(+ --openviking-url "$OPENVIKING_URL"+ --parallel "$IMPORT_PARALLEL"+ --error-retries "$IMPORT_ERROR_RETRIES"+ --queue-max-wait-sec "$QUEUE_MAX_WAIT_SEC"+ )+ elif [[ "$SUITE_LABEL" == "preingest" ]]; then+ IMPORT_ARGS+=(+ --error-log "${RESULT_DIR}/import_errors.log"+ --openviking-url "$OPENVIKING_URL"+ --account "$OPENVIKING_ACCOUNT"+ --user "$OPENVIKING_USER"+ --agent "$OPENVIKING_AGENT"+ --parallel "$IMPORT_PARALLEL"+ --error-retries "$IMPORT_ERROR_RETRIES"+ --queue-max-wait-sec "$QUEUE_MAX_WAIT_SEC"+ )+ [[ -n "$OPENVIKING_API_KEY" ]] && IMPORT_ARGS+=(--api-key "$OPENVIKING_API_KEY")+ elif [[ "$SUITE_LABEL" == "native" ]]; then+ IMPORT_ARGS+=(--error-retries "$IMPORT_ERROR_RETRIES")+ fi++ [[ -n "$SAMPLE" ]] && IMPORT_ARGS+=(--sample "$SAMPLE")+ [[ -n "$FORCE_INGEST" ]] && IMPORT_ARGS+=(--force-ingest)+ export QUEUE_MAX_WAIT_SEC+ set +e+ "$PYTHON" "$IMPORT_SCRIPT" "${IMPORT_ARGS[@]}" 2>&1 | tee "${LOG_DIR}/import.log"+ IMPORT_RC=${PIPESTATUS[0]}+ set -e+ if [[ "$IMPORT_RC" -ne 0 ]]; then+ if [[ "$SUITE_LABEL" != "e2e" && "$SUITE_LABEL" != "preingest" ]]; then+ echo ">>> Step 1 import command failed with status $IMPORT_RC." >&2+ exit "$IMPORT_RC"+ fi+ echo ">>> Step 1 import command exited with status $IMPORT_RC; checking OpenViking archive readiness before deciding whether to continue." >&2+ fi+ echo ">>> Step 1 done."++ verify_openviking_archive_readiness+ if [[ "$IMPORT_RC" -ne 0 ]]; then+ echo ">>> Continuing because OpenViking archive readiness passed." >&2+ fi++ copy_openviking_state+fi++echo ""+echo ">>> Step 3: Running QA evaluation..."+EVAL_ARGS=(+ "$LOCOMO_JSON"+ --suite "$EVAL_SUITE"+ --output "$QA_CSV"+ --base-url "$HERMES_URL"+ --model "$HERMES_MODEL"+ --parallel "$QA_PARALLEL"+ --error-retries "$QA_ERROR_RETRIES"+)+if [[ "$SUITE_LABEL" == "e2e" || "$SUITE_LABEL" == "preingest" ]]; then+ EVAL_ARGS+=(--openviking-url "$OPENVIKING_URL")+fi+[[ -n "$HERMES_TOKEN" ]] && EVAL_ARGS+=(--token "$HERMES_TOKEN")+[[ -n "$SAMPLE" ]] && EVAL_ARGS+=(--sample "$SAMPLE")+[[ -n "$COUNT" ]] && EVAL_ARGS+=(--count "$COUNT")+[[ -n "$FORCE_EVAL" ]] && EVAL_ARGS+=(--force)+"$PYTHON" eval.py "${EVAL_ARGS[@]}" 2>&1 | tee "${LOG_DIR}/eval.log"+echo ">>> Step 3 done."++echo ""+echo ">>> Step 4: Judging answers with LLM..."+JUDGE_ARGS=(+ --suite "$EVAL_SUITE"+ --input "$QA_CSV"+ --base-url "$JUDGE_BASE_URL"+ --model "$JUDGE_MODEL"+ --parallel "$JUDGE_PARALLEL"+ --error-retries "$JUDGE_ERROR_RETRIES"+)+[[ -n "$JUDGE_TOKEN" ]] && JUDGE_ARGS+=(--token "$JUDGE_TOKEN")+"$PYTHON" judge.py "${JUDGE_ARGS[@]}" 2>&1 | tee "${LOG_DIR}/judge.log"+echo ">>> Step 4 done."++echo ""+echo ">>> Step 5: Final statistics"+STATS_ARGS=(+ --suite "$EVAL_SUITE"+ --input "$QA_CSV"+ --import-csv "$STATS_IMPORT_CSV"+)+if [[ -n "${HERMES_STATE_DB:-}" ]]; then+ STATS_ARGS+=(--hermes-state-db "$HERMES_STATE_DB")+elif [[ -n "${HERMES_HOME:-}" && -f "${HERMES_HOME}/state.db" ]]; then+ STATS_ARGS+=(--hermes-state-db "${HERMES_HOME}/state.db")+fi+"$PYTHON" stat_judge_result.py "${STATS_ARGS[@]}" 2>&1 | tee "${LOG_DIR}/stats.log"++echo ""+echo "================================================================="+echo " Benchmark complete"+echo "================================================================="+echo "Result dir: $RESULT_DIR"+if [[ "$CHECKPOINT_OPENVIKING" == "1" ]]; then+ echo "OpenViking checkpoint: $SNAPSHOT_DIR"+ echo "Checkpoint manifest: $SNAPSHOT_MANIFEST"+fidiff --git a/benchmark/locomo/hermes/stat_judge_result.py b/benchmark/locomo/hermes/stat_judge_result.pynew file mode 100644index 000000000..7e766231c--- /dev/null+++ b/benchmark/locomo/hermes/stat_judge_result.py@@ -0,0 +1,583 @@+from __future__ import annotations++import argparse+import csv+import os+import sqlite3+from collections import defaultdict+from dataclasses import dataclass+from pathlib import Path++HERMES_USAGE_KEYS = [+ "input_tokens",+ "output_tokens",+ "cache_read_tokens",+ "cache_write_tokens",+ "reasoning_tokens",+ "api_call_count",+ "tool_call_count",+]+++def require_suite(value: str) -> str:+ if value not in {"baseline", "e2e", "preingest"}:+ raise argparse.ArgumentTypeError("suite must be one of: baseline, e2e, preingest")+ return value+++def result_dir_name(suite: str) -> str:+ if suite == "baseline":+ return "result_baseline"+ if suite == "e2e":+ return "result_e2e"+ return "result_preingest"+++def summary_title(suite: str) -> str:+ if suite == "baseline":+ return "Hermes Baseline"+ if suite == "e2e":+ return "Hermes OpenViking E2E"+ return "Hermes + OpenViking (pre-ingest)"+++def format_optional_int(value: int | None) -> str:+ return "unavailable" if value is None else f"{value:,}"+++def format_optional_float(value: float | None) -> str:+ return "unavailable" if value is None else f"{value:,.2f}"+++def read_int(row: dict, key: str) -> int:+ try:+ return int(float(row.get(key, 0) or 0))+ except (ValueError, TypeError):+ return 0+++def read_float(row: dict, key: str) -> float:+ try:+ return float(row.get(key, 0) or 0)+ except (ValueError, TypeError):+ return 0.0+++def read_first_int(row: dict, keys: tuple[str, ...]) -> int:+ for key in keys:+ if key in row and row.get(key) not in {None, ""}:+ return read_int(row, key)+ return 0+++@dataclass+class HermesUsage:+ input_tokens: int = 0+ output_tokens: int = 0+ cache_read_tokens: int = 0+ cache_write_tokens: int = 0+ reasoning_tokens: int = 0+ api_call_count: int = 0+ tool_call_count: int = 0++ @property+ def total_tokens(self) -> int:+ return (+ self.input_tokens+ + self.output_tokens+ + self.cache_read_tokens+ + self.cache_write_tokens+ )++ @property+ def cache_tokens(self) -> int:+ return self.cache_read_tokens + self.cache_write_tokens++ @property+ def no_cache_tokens(self) -> int:+ return self.input_tokens + self.output_tokens++ @classmethod+ def from_mapping(cls, values: dict[str, int]) -> "HermesUsage":+ return cls(**{key: int(values.get(key, 0) or 0) for key in HERMES_USAGE_KEYS})+++@dataclass+class HermesUsageSummary:+ usage: HermesUsage+ source: str+ matched_sessions: int+ expected_sessions: int+ authoritative: bool+++@dataclass+class OpenVikingUsage:+ embedding_input_tokens: int = 0+ embedding_output_tokens: int = 0+ llm_input_tokens: int = 0+ llm_output_tokens: int = 0++ @property+ def embedding_tokens(self) -> int:+ return self.embedding_input_tokens + self.embedding_output_tokens++ @property+ def has_tokens(self) -> bool:+ return any(+ [+ self.embedding_input_tokens,+ self.embedding_output_tokens,+ self.llm_input_tokens,+ self.llm_output_tokens,+ ]+ )+++def read_true_token_csv(csv_path: Path) -> tuple[int, int, int, int]:+ if not csv_path.exists():+ return 0, 0, 0, 0+ totals = [0, 0, 0, 0]+ with open(csv_path, "r", encoding="utf-8", newline="") as f:+ for row in csv.DictReader(f):+ totals[0] += read_int(row, "embedding_input_tokens")+ totals[1] += read_int(row, "embedding_output_tokens")+ totals[2] += read_int(row, "vlm_llm_input_tokens")+ totals[3] += read_int(row, "vlm_llm_output_tokens")+ return totals[0], totals[1], totals[2], totals[3]+++def resolve_hermes_state_db(value: str | None) -> Path | None:+ if value:+ path = Path(value).expanduser()+ return path if path.exists() else None+ hermes_home = os.getenv("HERMES_HOME", "").strip()+ if hermes_home:+ path = Path(hermes_home).expanduser() / "state.db"+ return path if path.exists() else None+ return None+++def read_hermes_sessions(state_db: Path | None) -> dict[str, dict[str, int]]:+ if state_db is None:+ return {}+ try:+ conn = sqlite3.connect(state_db)+ except sqlite3.Error:+ return {}+ try:+ available = {row[1] for row in conn.execute("PRAGMA table_info(sessions)")}+ if "id" not in available:+ return {}+ columns = ["id"] + [key for key in HERMES_USAGE_KEYS if key in available]+ rows = conn.execute(f"SELECT {', '.join(columns)} FROM sessions").fetchall()+ except sqlite3.Error:+ return {}+ finally:+ conn.close()++ sessions: dict[str, dict[str, int]] = {}+ for row in rows:+ session_id = str(row[0])+ usage = dict.fromkeys(HERMES_USAGE_KEYS, 0)+ for idx, key in enumerate(columns[1:], start=1):+ try:+ usage[key] = int(row[idx] or 0)+ except (TypeError, ValueError):+ usage[key] = 0+ sessions[session_id] = usage+ return sessions+++def sum_hermes_usage(+ state_sessions: dict[str, dict[str, int]], session_ids: list[str]+) -> dict[str, int]:+ totals = dict.fromkeys(HERMES_USAGE_KEYS, 0)+ seen = set()+ for session_id in session_ids:+ if session_id in seen:+ continue+ seen.add(session_id)+ usage = state_sessions.get(session_id)+ if not usage:+ continue+ for key in HERMES_USAGE_KEYS:+ totals[key] += int(usage.get(key, 0) or 0)+ return totals+++def hermes_total_tokens(usage: dict[str, int]) -> int:+ return (+ usage.get("input_tokens", 0)+ + usage.get("output_tokens", 0)+ + usage.get("cache_read_tokens", 0)+ + usage.get("cache_write_tokens", 0)+ )+++def read_csv_rows(csv_path: str | Path) -> list[dict]:+ path = Path(csv_path)+ if not path.exists():+ return []+ with open(path, "r", encoding="utf-8", newline="") as f:+ return list(csv.DictReader(f))+++def valid_qa_rows(input_path: str | Path) -> list[dict]:+ return [row for row in read_csv_rows(input_path) if str(row.get("category", "")) != "5"]+++def qa_state_session_ids(rows: list[dict], state_sessions: dict[str, dict[str, int]]) -> list[str]:+ matched = []+ for row in rows:+ for key in ("hermes_session_id", "conversation"):+ session_id = row.get(key, "")+ if session_id in state_sessions:+ matched.append(session_id)+ break+ return matched+++def summarize_score(rows: list[dict]) -> dict:+ correct = 0+ wrong = 0+ category_rows = defaultdict(int)+ category_graded = defaultdict(int)+ category_correct = defaultdict(int)+ total_qa_latency = 0.0++ for row in rows:+ category = str(row.get("category", ""))+ category_rows[category] += 1+ if str(row.get("result", "")).upper() == "CORRECT":+ correct += 1+ category_correct[category] += 1+ category_graded[category] += 1+ elif str(row.get("result", "")).upper() == "WRONG":+ wrong += 1+ category_graded[category] += 1+ total_qa_latency += read_float(row, "qa_latency_sec")++ graded = correct + wrong+ category_stats = {}+ for category, row_count in category_rows.items():+ cat_graded = category_graded[category]+ cat_correct = category_correct[category]+ category_stats[category] = {+ "rows": row_count,+ "graded": cat_graded,+ "correct": cat_correct,+ "accuracy": cat_correct / cat_graded if cat_graded else 0.0,+ }+ return {+ "rows": len(rows),+ "graded": graded,+ "correct": correct,+ "wrong": wrong,+ "ungraded": len(rows) - graded,+ "accuracy": correct / graded if graded else 0.0,+ "avg_qa_latency": total_qa_latency / len(rows) if rows else 0.0,+ "categories": category_stats,+ }+++def sum_qa_csv_usage(rows: list[dict]) -> HermesUsage:+ return HermesUsage(+ input_tokens=sum(read_int(row, "qa_input_tokens") for row in rows),+ output_tokens=sum(read_int(row, "qa_output_tokens") for row in rows),+ cache_read_tokens=sum(read_int(row, "qa_cache_read_tokens") for row in rows),+ cache_write_tokens=sum(read_int(row, "qa_cache_write_tokens") for row in rows),+ )+++def summarize_qa_hermes_usage(+ rows: list[dict], state_sessions: dict[str, dict[str, int]]+) -> HermesUsageSummary:+ matched_ids = qa_state_session_ids(rows, state_sessions)+ matched_count = len(set(matched_ids))+ if rows and matched_count == len(rows):+ return HermesUsageSummary(+ usage=HermesUsage.from_mapping(sum_hermes_usage(state_sessions, matched_ids)),+ source="state.db",+ matched_sessions=matched_count,+ expected_sessions=len(rows),+ authoritative=True,+ )+ return HermesUsageSummary(+ usage=sum_qa_csv_usage(rows),+ source="qa_results.csv fallback",+ matched_sessions=matched_count,+ expected_sessions=len(rows),+ authoritative=False,+ )+++def import_state_session_ids(+ rows: list[dict],+ suite: str,+ state_sessions: dict[str, dict[str, int]],+) -> list[str]:+ if suite == "e2e":+ row_ids = []+ for row in rows:+ for key in ("session_id", "conversation"):+ session_id = row.get(key, "")+ if session_id in state_sessions:+ row_ids.append(session_id)+ break+ if row_ids:+ return row_ids+ ids = [+ session_id+ for session_id in state_sessions+ if session_id.startswith("locomo-e2e-") and not session_id.startswith("locomo-e2e-qa-")+ ]+ if ids:+ return sorted(ids)+ if suite == "baseline":+ ids = [+ session_id+ for session_id in state_sessions+ if session_id.startswith("locomo-native-")+ and not session_id.startswith("locomo-native-qa-")+ ]+ if ids:+ return sorted(ids)+ return [+ row.get("conversation", "") for row in rows if row.get("conversation", "") in state_sessions+ ]+++def sum_import_csv_usage(rows: list[dict], suite: str) -> HermesUsage:+ if suite == "baseline":+ total_tokens = sum(read_int(row, "total_tokens") for row in rows)+ input_tokens = sum(read_first_int(row, ("input_tokens",)) for row in rows)+ output_tokens = sum(read_first_int(row, ("output_tokens",)) for row in rows)+ cache_read = sum(read_first_int(row, ("cache_read", "cache_read_tokens")) for row in rows)+ cache_write = sum(+ read_first_int(row, ("cache_write", "cache_write_tokens")) for row in rows+ )+ if input_tokens or output_tokens or cache_read or cache_write:+ return HermesUsage(+ input_tokens=input_tokens,+ output_tokens=output_tokens,+ cache_read_tokens=cache_read,+ cache_write_tokens=cache_write,+ )+ return HermesUsage(input_tokens=total_tokens)++ if suite == "e2e":+ return HermesUsage(+ input_tokens=sum(read_int(row, "input_tokens") for row in rows),+ output_tokens=sum(read_int(row, "output_tokens") for row in rows),+ cache_read_tokens=sum(+ read_first_int(row, ("cache_read", "cache_read_tokens")) for row in rows+ ),+ cache_write_tokens=sum(+ read_first_int(row, ("cache_write", "cache_write_tokens")) for row in rows+ ),+ )++ return HermesUsage()+++def summarize_import_hermes_usage(+ import_csv: Path, suite: str, state_sessions: dict[str, dict[str, int]]+) -> HermesUsageSummary | None:+ if suite == "preingest" or not import_csv.exists():+ return None++ rows = read_csv_rows(import_csv)+ if not rows:+ return None++ matched_ids = import_state_session_ids(rows, suite, state_sessions)+ matched_count = len(set(matched_ids))+ state_is_complete = bool(matched_ids) and (suite != "e2e" or matched_count == len(rows))+ if state_is_complete:+ expected_count = len(rows) if suite == "e2e" else matched_count+ return HermesUsageSummary(+ usage=HermesUsage.from_mapping(sum_hermes_usage(state_sessions, matched_ids)),+ source="state.db",+ matched_sessions=matched_count,+ expected_sessions=expected_count,+ authoritative=True,+ )++ return HermesUsageSummary(+ usage=sum_import_csv_usage(rows, suite),+ source="import_success.csv fallback",+ matched_sessions=0,+ expected_sessions=len(rows),+ authoritative=False,+ )+++def read_openviking_usage(result_dir: Path) -> tuple[OpenVikingUsage, OpenVikingUsage]:+ import_emb_in, import_emb_out, import_llm_in, import_llm_out = read_true_token_csv(+ result_dir / "import_true_tokens.csv"+ )+ eval_emb_in, eval_emb_out, eval_llm_in, eval_llm_out = read_true_token_csv(+ result_dir / "eval_true_tokens.csv"+ )+ return (+ OpenVikingUsage(import_emb_in, import_emb_out, import_llm_in, import_llm_out),+ OpenVikingUsage(eval_emb_in, eval_emb_out, eval_llm_in, eval_llm_out),+ )+++def format_usage_pair(label: str, qa_value: int, import_value: int | None) -> str:+ if import_value is None:+ return f" {label}: {qa_value:,} (QA)"+ return f" {label}: {qa_value:,} (QA), {import_value:,} (Ingest)"+++def format_source(summary: HermesUsageSummary) -> str:+ if summary.source == "state.db":+ return (+ f"{summary.source}, matched {summary.matched_sessions:,}/"+ f"{summary.expected_sessions:,} sessions"+ )+ if summary.matched_sessions:+ return (+ f"{summary.source}, state.db matched {summary.matched_sessions:,}/"+ f"{summary.expected_sessions:,} sessions"+ )+ return summary.source+++def format_score_lines(score: dict) -> list[str]:+ lines = [+ "Score",+ (f" overall_accuracy: {score['accuracy']:.2%} ({score['correct']:,}/{score['graded']:,})"),+ ]+ if score["ungraded"]:+ lines.append(f" ungraded_rows: {score['ungraded']:,}")++ lines.append(" category_accuracy:")+ for category in sorted(+ score["categories"].keys(),+ key=lambda value: int(value) if value.isdigit() else value,+ ):+ item = score["categories"][category]+ lines.append(+ f" category {category}: {item['correct']:,}/{item['graded']:,} "+ f"({item['accuracy']:.2%})"+ )+ ungraded = item["rows"] - item["graded"]+ if ungraded:+ lines.append(f" ungraded_rows: {ungraded:,}")+ return lines+++def format_summary(+ suite: str,+ score: dict,+ qa_usage: HermesUsageSummary,+ import_usage: HermesUsageSummary | None,+ ov_import: OpenVikingUsage,+ ov_eval: OpenVikingUsage,+) -> list[str]:+ import_hermes = import_usage.usage if import_usage is not None else None+ llm_calls = qa_usage.usage.api_call_count if qa_usage.authoritative else None+ llm_calls_per_qa = (+ llm_calls / qa_usage.expected_sessions+ if llm_calls is not None and qa_usage.expected_sessions+ else None+ )++ lines = [+ f"=== {summary_title(suite)} Summary ===",+ "",+ *format_score_lines(score),+ "",+ "Hermes Usage",+ f" qa_source: {format_source(qa_usage)}",+ ]+ if import_usage is not None:+ lines.append(f" ingest_source: {format_source(import_usage)}")+ lines.extend(+ [+ format_usage_pair(+ "total_tokens",+ qa_usage.usage.total_tokens,+ import_hermes.total_tokens if import_hermes is not None else None,+ ),+ format_usage_pair(+ "cache_tokens",+ qa_usage.usage.cache_tokens,+ import_hermes.cache_tokens if import_hermes is not None else None,+ ),+ format_usage_pair(+ "no_cache_tokens",+ qa_usage.usage.no_cache_tokens,+ import_hermes.no_cache_tokens if import_hermes is not None else None,+ ),+ "",+ "OpenViking Usage",+ ]+ )++ if ov_import.has_tokens or ov_eval.has_tokens:+ lines.extend(+ [+ f" embedding: {ov_import.embedding_tokens:,} (Ingest), {ov_eval.embedding_tokens:,} (QA)",+ f" llm_input: {ov_import.llm_input_tokens:,} (Ingest), {ov_eval.llm_input_tokens:,} (QA)",+ f" llm_output: {ov_import.llm_output_tokens:,} (Ingest), {ov_eval.llm_output_tokens:,} (QA)",+ ]+ )+ else:+ lines.append(" unavailable")++ lines.extend(+ [+ "",+ "Runtime",+ f" avg_seconds_per_qa: {score['avg_qa_latency']:.4f}",+ f" hermes_qa_llm_calls: {format_optional_int(llm_calls)} total",+ f" hermes_qa_llm_calls_per_qa: {format_optional_float(llm_calls_per_qa)}",+ ]+ )+ return lines+++def main() -> None:+ script_dir = Path(__file__).parent.resolve()+ parser = argparse.ArgumentParser(description="Summarize shared Hermes LoCoMo judge results")+ parser.add_argument("--suite", type=require_suite, default="baseline", help="Benchmark suite")+ parser.add_argument("--input", default=None, help="Path to graded CSV")+ parser.add_argument("--import-csv", default=None, help="Path to import_success.csv")+ parser.add_argument(+ "--hermes-state-db",+ default=None,+ help="Path to Hermes state.db for authoritative token/cache accounting",+ )+ args = parser.parse_args()++ default_result_dir = script_dir / result_dir_name(args.suite)+ if args.input is None:+ args.input = str(default_result_dir / "qa_results.csv")+ result_dir = Path(args.input).expanduser().resolve().parent+ if args.import_csv is None:+ args.import_csv = str(result_dir / "import_success.csv")++ state_db = resolve_hermes_state_db(args.hermes_state_db)+ state_sessions = read_hermes_sessions(state_db)++ if not os.path.exists(args.input):+ print(f"Warning: QA result file not found: {args.input}")+ return++ rows = valid_qa_rows(args.input)+ score = summarize_score(rows)+ qa_usage = summarize_qa_hermes_usage(rows, state_sessions)+ import_usage = summarize_import_hermes_usage(Path(args.import_csv), args.suite, state_sessions)+ ov_import, ov_eval = read_openviking_usage(result_dir)+ output_lines = format_summary(args.suite, score, qa_usage, import_usage, ov_import, ov_eval)++ for line in output_lines:+ print(line)+++if __name__ == "__main__":+ main()