Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
"""
Shared Hermes LoCoMo QA evaluator.
"""
from __future__ import annotations
import argparse
import asyncio
import csv
import json
import os
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from pathlib import Path
from threading import Lock
import httpx
import requests
try:
from dotenv import load_dotenv
except ImportError:
def load_dotenv(*_args, **_kwargs):
return False
env_file = Path.home() / ".openviking_benchmark_env"
load_dotenv(env_file)
DEFAULT_BASE_URL = os.getenv("HERMES_GATEWAY_BASE_URL", "http://127.0.0.1:8642")
DEFAULT_MODEL = os.getenv("HERMES_GATEWAY_MODEL", "hermes-agent")
DEFAULT_TOKEN = os.getenv("HERMES_GATEWAY_TOKEN") or os.getenv("API_SERVER_KEY") or ""
DEFAULT_OPENVIKING_URL = os.getenv("OPENVIKING_ENDPOINT", "http://127.0.0.1:1933")
DEFAULT_DATASET_LOCATION = "benchmark/locomo/data/locomo10.json"
MAX_HTTP_ATTEMPTS = 2
DEFAULT_ERROR_RETRIES = int(os.getenv("QA_ERROR_RETRIES", "2"))
QA_INSTRUCTIONS = "Answer to the best of your ability and reasonable inference."
csv_lock = Lock()
def default_locomo_input(script_dir: Path) -> str:
return os.getenv("LOCOMO_JSON") or str(script_dir.parent / "data" / "locomo10.json")
def dataset_missing_message(path: str) -> str:
return (
f"LoCoMo dataset not found: {path}\n"
f"Set LOCOMO_JSON=/path/to/locomo10.json, pass a script input path, "
f"or place it at {DEFAULT_DATASET_LOCATION}."
)
def openviking_headers() -> dict[str, str]:
headers = {
"X-OpenViking-Account": os.getenv("OPENVIKING_ACCOUNT", "default"),
"X-OpenViking-User": os.getenv("OPENVIKING_USER", "default"),
}
api_key = os.getenv("OPENVIKING_API_KEY", "")
if api_key:
headers["X-API-Key"] = api_key
return headers
def require_suite(value: str) -> str:
if value not in {"baseline", "e2e", "preingest"}:
raise argparse.ArgumentTypeError("suite must be one of: baseline, e2e, preingest")
return value
def result_dir_name(suite: str) -> str:
if suite == "baseline":
return "result_baseline"
if suite == "e2e":
return "result_e2e"
return "result_preingest"
def conversation_prefix(suite: str) -> str:
if suite == "baseline":
return "locomo-native-qa-"
if suite == "e2e":
return "locomo-e2e-qa-"
return "locomo-ovpreingest-qa-"
def load_locomo_data(path: str, sample_index: int | None = None) -> list[dict]:
try:
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
except FileNotFoundError:
print(f"Error: {dataset_missing_message(path)}", file=sys.stderr)
sys.exit(1)
except json.JSONDecodeError as e:
print(f"Error: invalid JSON in {path}: {e}", file=sys.stderr)
sys.exit(1)
if sample_index is None:
return data
if sample_index < 0 or sample_index >= len(data):
print(
f"Error: sample index {sample_index} out of range (0-{len(data) - 1})", file=sys.stderr
)
sys.exit(1)
return [data[sample_index]]
def parse_locomo_datetime(date_str: str) -> datetime | None:
try:
if " on " in date_str:
return datetime.strptime(date_str.split(" on ")[-1].strip(), "%d %B, %Y")
except ValueError:
return None
return None
def get_sample_question_time(sample: dict) -> str | None:
conversation = sample.get("conversation", {})
session_keys = [
key for key in conversation.keys() if key.startswith("session_") and "date_time" not in key
]
if not session_keys:
return None
def session_num(key: str) -> int:
try:
return int(key.replace("session_", ""))
except ValueError:
return 0
session_keys.sort(key=session_num, reverse=True)
for key in session_keys:
if not conversation.get(key):
continue
date_str = conversation.get(f"session_{session_num(key)}_date_time")
if not date_str:
continue
parsed = parse_locomo_datetime(date_str)
if parsed:
return parsed.strftime("%Y-%m-%d")
return None
def normalize_usage(raw: dict | None) -> dict:
raw = raw or {}
input_tokens = raw.get("input_tokens") or raw.get("prompt_tokens", 0)
output_tokens = raw.get("output_tokens") or raw.get("completion_tokens", 0)
cache_read_tokens = raw.get("cache_read_tokens") or raw.get("cacheRead", 0)
cache_write_tokens = raw.get("cache_write_tokens") or raw.get("cacheWrite", 0)
total_tokens = raw.get("total_tokens") or raw.get("totalTokens")
if total_tokens is None:
total_tokens = int(input_tokens or 0) + int(output_tokens or 0)
return {
"input_tokens": int(input_tokens or 0),
"output_tokens": int(output_tokens or 0),
"cache_read_tokens": int(cache_read_tokens or 0),
"cache_write_tokens": int(cache_write_tokens or 0),
"total_tokens": int(total_tokens or 0),
}
def extract_response_text(body: dict) -> str:
for item in body.get("output", []):
if item.get("type") != "message":
continue
for content in item.get("content", []):
if content.get("type") == "output_text":
return content.get("text", "")
if "text" in content:
return content["text"]
return ""
def extract_tool_names(body: dict) -> list[str]:
return [item["requested_name"] for item in extract_tool_trace(body)]
def extract_tool_trace(body: dict) -> list[dict]:
trace = []
for item in body.get("output", []):
if item.get("type") != "function_call":
continue
requested_name = item.get("requested_name") or item.get("name")
if not isinstance(requested_name, str) or not requested_name:
continue
executed_name = item.get("executed_name")
if not isinstance(executed_name, str):
executed_name = ""
if not executed_name and not item.get("was_blocked"):
executed_name = requested_name
trace.append(
{
"requested_name": requested_name,
"executed_name": executed_name,
"was_rewritten": bool(item.get("was_rewritten")),
"was_blocked": bool(item.get("was_blocked")),
}
)
return trace
def send_gateway_request(
base_url: str,
token: str,
model: str,
conversation: str,
input_text: str,
instructions: str = QA_INSTRUCTIONS,
timeout: int = 600,
) -> tuple[dict, float, str]:
url = f"{base_url.rstrip('/')}/v1/responses"
headers = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
}
payload = {
"model": model,
"input": input_text,
"conversation": conversation,
"session_id": conversation,
"store": False,
}
if instructions:
payload["instructions"] = instructions
last_error: requests.RequestException | None = None
total_elapsed = 0.0
for attempt in range(1, MAX_HTTP_ATTEMPTS + 1):
started_at = time.perf_counter()
try:
resp = requests.post(url, headers=headers, json=payload, timeout=timeout)
elapsed = time.perf_counter() - started_at
total_elapsed += elapsed
resp.raise_for_status()
return resp.json(), total_elapsed, resp.headers.get("X-Hermes-Session-Id", "")
except requests.HTTPError as exc:
elapsed = time.perf_counter() - started_at
total_elapsed += elapsed
if attempt >= MAX_HTTP_ATTEMPTS:
raise
last_error = exc
except (requests.ConnectionError, requests.Timeout) as exc:
elapsed = time.perf_counter() - started_at
total_elapsed += elapsed
if attempt >= MAX_HTTP_ATTEMPTS:
raise
last_error = exc
time.sleep(float(2 ** (attempt - 1)))
if last_error is not None:
raise last_error
raise RuntimeError("unreachable")
def build_question_prompt(
suite: str,
question: str,
question_time: str | None,
sample_id: str,
) -> str:
if question_time:
return f"Current date: {question_time}. {question}"
return question
def process_single_question(
item: dict,
sample_idx: int,
question_index: int,
qa: dict,
args: argparse.Namespace,
csv_path: str,
) -> dict:
sample_id = item["sample_id"]
question = qa["question"]
expected = str(qa["answer"])
category = str(qa.get("category", ""))
evidence = qa.get("evidence", [])
question_time = get_sample_question_time(item)
conversation = f"{args.conversation_prefix}{sample_id}-q{question_index}"
if args.suite == "baseline":
mode = "native DB, session_search"
elif args.suite == "e2e":
mode = "mixed memory E2E"
else:
mode = "ov-preingest, no replay"
print(f"[{sample_id}] Q{question_index}: asking ({mode})", file=sys.stderr)
try:
body, qa_latency, hermes_session_id = send_gateway_request(
base_url=args.base_url,
token=args.token,
model=args.model,
conversation=conversation,
input_text=build_question_prompt(
args.suite,
question,
question_time,
sample_id,
),
timeout=args.timeout,
)
qa_usage = normalize_usage(body.get("usage"))
response = extract_response_text(body)
tool_trace = extract_tool_trace(body)
tool_names = [item["requested_name"] for item in tool_trace]
except requests.HTTPError as e:
response = f"[ERROR] HTTP {e.response.status_code}: {e.response.text}"
qa_usage = normalize_usage({})
qa_latency = 0.0
tool_trace = []
tool_names = []
hermes_session_id = ""
except Exception as e:
response = f"[ERROR] {e}"
qa_usage = normalize_usage({})
qa_latency = 0.0
tool_trace = []
tool_names = []
hermes_session_id = ""
record = {
"sample_id": sample_id,
"sample_idx": sample_idx,
"qi": question_index,
"question": question,
"expected": expected,
"response": response,
"category": category,
"evidence": json.dumps(evidence, ensure_ascii=False),
"question_time": question_time or "",
"conversation": conversation,
"hermes_session_id": hermes_session_id,
"qa_input_tokens": qa_usage["input_tokens"],
"qa_output_tokens": qa_usage["output_tokens"],
"qa_cache_read_tokens": qa_usage["cache_read_tokens"],
"qa_cache_write_tokens": qa_usage["cache_write_tokens"],
"qa_total_tokens": qa_usage["total_tokens"],
"qa_latency_sec": f"{qa_latency:.4f}",
"tool_call_count": len(tool_names),
"tool_names": json.dumps(tool_names, ensure_ascii=False),
"executed_tool_call_count": len([item for item in tool_trace if item["executed_name"]]),
"executed_tool_names": json.dumps(
[item["executed_name"] for item in tool_trace if item["executed_name"]],
ensure_ascii=False,
),
"rewritten_tool_call_count": sum(1 for item in tool_trace if item["was_rewritten"]),
"blocked_tool_call_count": sum(1 for item in tool_trace if item["was_blocked"]),
"result": "",
"reasoning": "",
}
with csv_lock:
save_record_to_csv(csv_path, record)
print(f"[{sample_id}] Q{question_index}: done ({response[:60]}...)", file=sys.stderr)
return record
def is_error_record(row: dict) -> bool:
return str(row.get("response", "")).lstrip().startswith("[ERROR]")
def count_error_records(csv_path: str) -> int:
if not os.path.exists(csv_path):
return 0
count = 0
with open(csv_path, "r", encoding="utf-8", newline="") as f:
for row in csv.DictReader(f):
if is_error_record(row):
count += 1
return count
def prune_error_records(csv_path: str) -> int:
if not os.path.exists(csv_path):
return 0
with open(csv_path, "r", encoding="utf-8", newline="") as f:
reader = csv.DictReader(f)
fieldnames = reader.fieldnames or []
rows = list(reader)
kept = [row for row in rows if not is_error_record(row)]
removed = len(rows) - len(kept)
if removed <= 0:
return 0
with open(csv_path, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(kept)
f.flush()
return removed
def load_executed_records(csv_path: str) -> set[tuple[str, int]]:
executed = set()
if not os.path.exists(csv_path):
return executed
with open(csv_path, "r", encoding="utf-8", newline="") as f:
for row in csv.DictReader(f):
if is_error_record(row):
continue
try:
executed.add((row["sample_id"], int(row["qi"])))
except Exception:
continue
return executed
def save_record_to_csv(csv_path: str, record: dict) -> None:
os.makedirs(os.path.dirname(csv_path), exist_ok=True)
fieldnames = [
"sample_id",
"sample_idx",
"qi",
"question",
"expected",
"response",
"category",
"evidence",
"question_time",
"conversation",
"hermes_session_id",
"qa_input_tokens",
"qa_output_tokens",
"qa_cache_read_tokens",
"qa_cache_write_tokens",
"qa_total_tokens",
"qa_latency_sec",
"tool_call_count",
"tool_names",
"executed_tool_call_count",
"executed_tool_names",
"rewritten_tool_call_count",
"blocked_tool_call_count",
"result",
"reasoning",
]
expected_header = ",".join(fieldnames)
should_write_header = not os.path.exists(csv_path)
if not should_write_header:
with open(csv_path, "r", encoding="utf-8", newline="") as f:
current_header = f.readline().strip()
if current_header != expected_header:
with open(csv_path, "r", encoding="utf-8", newline="") as f:
existing_rows = list(csv.DictReader(f))
with open(csv_path, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
for row in existing_rows:
writer.writerow({key: row.get(key, "") for key in fieldnames})
with open(csv_path, "a", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
if should_write_header:
writer.writeheader()
writer.writerow(record)
f.flush()
def reset_generated_file(path: str | Path, label: str) -> None:
target = Path(path)
if target.exists():
target.unlink()
print(f"Removed existing {label}: {target}", file=sys.stderr)
def parse_observer_model_totals(status_text: str) -> tuple[int, int, int, int]:
embedding_prompt = 0
embedding_completion = 0
vlm_prompt = 0
vlm_completion = 0
current_section = ""
for raw_line in status_text.splitlines():
line = raw_line.strip()
if not line:
continue
if line.endswith("Models:"):
current_section = line.split(" ", 1)[0].lower()
continue
if not line.startswith("|"):
continue
cols = [col.strip() for col in line.split("|") if col.strip()]
if len(cols) < 7 or cols[0] == "Model":
continue
prompt = int(cols[3]) if cols[3].isdigit() else 0
completion = int(cols[4]) if cols[4].isdigit() else 0
if current_section == "embedding":
embedding_prompt += prompt
embedding_completion += completion
elif current_section == "vlm":
vlm_prompt += prompt
vlm_completion += completion
return embedding_prompt, embedding_completion, vlm_prompt, vlm_completion
async def _read_model_totals(
client: httpx.AsyncClient, openviking_url: str
) -> tuple[int, int, int, int] | None:
try:
resp = await client.get(f"{openviking_url}/api/v1/observer/models")
if resp.status_code != 200:
return None
status_text = resp.json().get("result", {}).get("status", "")
except Exception:
return None
if "No model usage data available" in status_text:
return 0, 0, 0, 0
return parse_observer_model_totals(status_text)
async def _read_queue_totals(
client: httpx.AsyncClient, openviking_url: str
) -> tuple[int, int, int] | None:
try:
resp = await client.get(f"{openviking_url}/api/v1/observer/queue")
if resp.status_code != 200:
return None
status_text = resp.json().get("result", {}).get("status", "")
except Exception:
return None
for raw_line in status_text.splitlines():
line = raw_line.strip()
if not line.startswith("|") or "TOTAL" not in line:
continue
cols = [col.strip() for col in line.split("|") if col.strip()]
if len(cols) < 4 or cols[0] != "TOTAL":
continue
try:
return int(cols[1]), int(cols[2]), int(cols[3])
except ValueError:
return None
return None
def _append_true_token_record(output_file: str, record: dict) -> None:
fieldnames = [
"timestamp",
"embedding_input_tokens",
"embedding_output_tokens",
"vlm_llm_input_tokens",
"vlm_llm_output_tokens",
]
expected_header = ",".join(fieldnames)
mode = "a"
should_write_header = not os.path.exists(output_file)
if not should_write_header:
with open(output_file, "r", encoding="utf-8", newline="") as f:
current_header = f.readline().strip()
if current_header != expected_header:
mode = "w"
should_write_header = True
with open(output_file, mode, encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
if should_write_header:
writer.writeheader()
writer.writerow(record)
async def wait_for_queues_and_record_totals(
openviking_url: str,
output_file: str,
baseline_processed: int = 0,
baseline_model_totals: tuple[int, int, int, int] | None = None,
max_wait_sec: int = 1800,
settle_checks: int = 3,
poll_interval: float = 2.0,
) -> None:
print(
f"\n[INFO] Waiting for OpenViking background queues to drain "
f"(baseline_processed={baseline_processed}, settle={settle_checks})...",
file=sys.stderr,
)
async with httpx.AsyncClient(headers=openviking_headers()) as client:
started = time.perf_counter()
idle_streak = 0
last_totals: tuple[int, int, int] | None = None
while True:
elapsed = time.perf_counter() - started
if elapsed > max_wait_sec:
print(
f"[WARN] Queue drain timeout after {elapsed:.0f}s (last totals={last_totals})",
file=sys.stderr,
)
break
totals = await _read_queue_totals(client, openviking_url)
if totals is None:
await asyncio.sleep(poll_interval)
continue
pending, in_progress, processed = totals
last_totals = totals
queues_idle = pending == 0 and in_progress == 0
if queues_idle:
idle_streak += 1
if idle_streak >= settle_checks:
print(
f"[INFO] Background queues drained. Processed {processed - baseline_processed} item(s) since baseline.",
file=sys.stderr,
)
break
else:
idle_streak = 0
print(
f" .. queue state pending={pending} in_progress={in_progress} processed={processed}",
file=sys.stderr,
)
await asyncio.sleep(poll_interval)
print("[INFO] Fetching final true token usage from observer...", file=sys.stderr)
try:
final_totals = await _read_model_totals(client, openviking_url)
if final_totals is None:
print("[INFO] No model usage recorded by OpenViking server.", file=sys.stderr)
return
(
baseline_embedding_prompt,
baseline_embedding_completion,
baseline_vlm_prompt,
baseline_vlm_completion,
) = baseline_model_totals or (0, 0, 0, 0)
(
final_embedding_prompt,
final_embedding_completion,
final_vlm_prompt,
final_vlm_completion,
) = final_totals
delta_embedding_prompt = max(final_embedding_prompt - baseline_embedding_prompt, 0)
delta_embedding_completion = max(
final_embedding_completion - baseline_embedding_completion, 0
)
delta_vlm_prompt = max(final_vlm_prompt - baseline_vlm_prompt, 0)
delta_vlm_completion = max(final_vlm_completion - baseline_vlm_completion, 0)
print("\n=== TRUE OpenViking Token Delta For This Run ===", file=sys.stderr)
print(f"Embedding Input (Prompt) Delta: {delta_embedding_prompt}", file=sys.stderr)
print(
f"Embedding Output (Completion) Delta: {delta_embedding_completion}",
file=sys.stderr,
)
print(f"VLM/LLM Input (Prompt) Delta: {delta_vlm_prompt}", file=sys.stderr)
print(f"VLM/LLM Output (Completion) Delta: {delta_vlm_completion}", file=sys.stderr)
print(
f"[INFO] Final cumulative totals: embedding_in={final_embedding_prompt}, "
f"embedding_out={final_embedding_completion}, vlm_in={final_vlm_prompt}, "
f"vlm_out={final_vlm_completion}",
file=sys.stderr,
)
_append_true_token_record(
output_file,
{
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"embedding_input_tokens": delta_embedding_prompt,
"embedding_output_tokens": delta_embedding_completion,
"vlm_llm_input_tokens": delta_vlm_prompt,
"vlm_llm_output_tokens": delta_vlm_completion,
},
)
print(f"True token totals saved to: {output_file}", file=sys.stderr)
except Exception as e:
print(f"[ERROR] Failed to fetch final token usage: {e}", file=sys.stderr)
def run_eval_pass(
samples: list[dict], args: argparse.Namespace, executed_records: set[tuple[str, int]]
) -> None:
for sample_idx, item in enumerate(samples, start=1):
sample_id = item["sample_id"]
qas = [qa for qa in item.get("qa", []) if str(qa.get("category", "")) != "5"]
if args.count is not None:
qas = qas[: args.count]
tasks = []
for question_index, qa in enumerate(qas, start=1):
if (sample_id, question_index) in executed_records:
print(f"[{sample_id}] skip Q{question_index}: already recorded", file=sys.stderr)
continue
tasks.append((question_index, qa))
if not tasks:
continue
print(
f"[{sample_id}] running {len(tasks)} question(s) with {args.parallel} workers",
file=sys.stderr,
)
with ThreadPoolExecutor(max_workers=max(1, args.parallel)) as executor:
futures = [
executor.submit(
process_single_question, item, sample_idx, question_index, qa, args, args.output
)
for question_index, qa in tasks
]
for future in as_completed(futures):
future.result()
def run_eval(args: argparse.Namespace) -> None:
if not args.token:
print("Error: HERMES_GATEWAY_TOKEN or --token is required", file=sys.stderr)
sys.exit(1)
samples = load_locomo_data(args.input, args.sample)
if args.force:
reset_generated_file(args.output, "QA CSV")
reset_generated_file(
Path(args.output).parent / "eval_true_tokens.csv", "eval true-token CSV"
)
else:
removed = prune_error_records(args.output)
if removed:
print(f"Pruned {removed} previous error row(s) for retry", file=sys.stderr)
baseline_processed = 0
baseline_model_totals = None
if args.suite in {"e2e", "preingest"}:
async def snapshot_processed() -> int:
async with httpx.AsyncClient(headers=openviking_headers()) as client:
totals = await _read_queue_totals(client, args.openviking_url)
return totals[2] if totals else 0
async def snapshot_models() -> tuple[int, int, int, int] | None:
async with httpx.AsyncClient(headers=openviking_headers()) as client:
return await _read_model_totals(client, args.openviking_url)
baseline_processed = asyncio.run(snapshot_processed())
baseline_model_totals = asyncio.run(snapshot_models())
print(f"[INFO] Observer baseline processed={baseline_processed}", file=sys.stderr)
max_rounds = max(0, args.error_retries) + 1
for round_index in range(max_rounds):
executed_records = (
set() if args.force and round_index == 0 else load_executed_records(args.output)
)
print(
f"Loaded {len(executed_records)} executed records from {args.output}", file=sys.stderr
)
if round_index:
print(f"Retry round {round_index}/{max_rounds - 1}", file=sys.stderr)
run_eval_pass(samples, args, executed_records)
error_count = count_error_records(args.output)
if error_count == 0 or round_index >= max_rounds - 1:
break
removed = prune_error_records(args.output)
print(f"Retrying {removed} error row(s)", file=sys.stderr)
print(f"Done. Results written to {args.output}", file=sys.stderr)
if args.suite in {"e2e", "preingest"}:
true_token_file = Path(args.output).parent / "eval_true_tokens.csv"
asyncio.run(
wait_for_queues_and_record_totals(
args.openviking_url,
str(true_token_file),
baseline_processed=baseline_processed,
baseline_model_totals=baseline_model_totals,
)
)
def main() -> None:
script_dir = Path(__file__).parent.resolve()
parser = argparse.ArgumentParser(description="Shared Hermes LoCoMo QA evaluator")
parser.add_argument(
"input", nargs="?", default=default_locomo_input(script_dir), help="Path to LoCoMo JSON"
)
parser.add_argument("--suite", type=require_suite, required=True, help="Benchmark suite")
parser.add_argument("--output", default=None, help="Path to output CSV")
parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="Hermes gateway base URL")
parser.add_argument("--token", default=DEFAULT_TOKEN, help="Hermes gateway token")
parser.add_argument("--model", default=DEFAULT_MODEL, help="Hermes gateway model name")
parser.add_argument("--sample", type=int, default=None, help="Sample index (0-based)")
parser.add_argument("--count", type=int, default=None, help="Questions per sample")
parser.add_argument("--parallel", type=int, default=4, help="Parallel question workers")
parser.add_argument("--timeout", type=int, default=600, help="Gateway request timeout seconds")
parser.add_argument(
"--force", action="store_true", help="Ignore existing CSV and rerun questions"
)
parser.add_argument(
"--error-retries",
type=int,
default=DEFAULT_ERROR_RETRIES,
help="Retry rows whose response starts with [ERROR] this many times",
)
parser.add_argument(
"--openviking-url", default=DEFAULT_OPENVIKING_URL, help="OpenViking service URL"
)
args = parser.parse_args()
if args.output is None:
args.output = str(script_dir / result_dir_name(args.suite) / "qa_results.csv")
args.conversation_prefix = conversation_prefix(args.suite)
run_eval(args)
if __name__ == "__main__":
main()