Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
"""
Evaluate LoCoMo QA via Supermemory + OpenClaw (agent mode).
Questions are sent to an OpenClaw agent which calls Supermemory internally.
Before each sample, eval.py automatically:
1. Updates ~/.openclaw/openclaw.json to set openclaw-supermemory.config.containerTag = sample_id
(and switches plugins.slots.memory to "openclaw-supermemory")
2. Restarts the OpenClaw gateway to pick up the new config
3. Runs all questions for that sample concurrently
Prerequisites:
- Conversations already ingested into Supermemory via ingest.py (containerTag = sanitize(sample_id))
- OpenClaw running locally with the openclaw-supermemory plugin installed
- ~/.openclaw/openclaw.json configured with openclaw-supermemory apiKey
Usage:
# Run QA + judge for all samples (6 concurrent threads)
python eval.py --threads 6 --judge
# Single sample
python eval.py --sample conv-26 --threads 6 --judge
# First 12 questions only
python eval.py --sample conv-26 --count 12 --threads 6 --judge
# Judge-only (grade existing responses in CSV)
python eval.py --judge-only
"""
import argparse
import csv
import json
import os
import re
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Optional
import requests
from dotenv import load_dotenv
load_dotenv(Path.home() / ".openviking_benchmark_env")
SCRIPT_DIR = Path(__file__).parent.resolve()
DEFAULT_DATA_PATH = str(SCRIPT_DIR / ".." / "data" / "locomo10.json")
DEFAULT_OUTPUT_PATH = str(SCRIPT_DIR / "result" / "qa_results.csv")
DEFAULT_OPENCLAW_URL = "http://127.0.0.1:18789"
DEFAULT_SESSION_KEY = "locomo-eval"
OPENCLAW_CONFIG_PATH = Path.home() / ".openclaw" / "openclaw.json"
# Serialize openclaw config updates across threads so each sample sees the right containerTag
_openclaw_config_lock = threading.Lock()
# ---------------------------------------------------------------------------
# Tag sanitization (must match openclaw-supermemory's sanitizeTag logic)
# ---------------------------------------------------------------------------
def sanitize_tag(raw: str) -> str:
"""Sanitize a tag string to match openclaw-supermemory convention.
e.g. 'conv-26' -> 'conv_26'
"""
tag = re.sub(r"[^a-zA-Z0-9_]", "_", raw)
tag = re.sub(r"_+", "_", tag)
tag = tag.strip("_")
return tag
# ---------------------------------------------------------------------------
# openclaw.json config helpers
# ---------------------------------------------------------------------------
def _update_openclaw_supermemory_container(sample_id: str) -> None:
"""
Rewrite ~/.openclaw/openclaw.json so that openclaw-supermemory uses
sanitize(sample_id) as containerTag. Also switches slots.memory to
'openclaw-supermemory' and disables autoCapture.
Must be called while holding _openclaw_config_lock.
"""
container_tag = sanitize_tag(sample_id)
with open(OPENCLAW_CONFIG_PATH, "r", encoding="utf-8") as f:
config = json.load(f)
plugins = config.setdefault("plugins", {})
entries = plugins.setdefault("entries", {})
sm_entry = entries.setdefault("openclaw-supermemory", {})
sm_entry["enabled"] = True
sm_entry.setdefault("config", {})["containerTag"] = container_tag
sm_entry["config"]["autoCapture"] = False
sm_entry["config"]["autoRecall"] = True
# Switch memory slot to supermemory
plugins.setdefault("slots", {})["memory"] = "openclaw-supermemory"
tmp = str(OPENCLAW_CONFIG_PATH) + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(config, f, indent=2, ensure_ascii=False)
os.replace(tmp, str(OPENCLAW_CONFIG_PATH))
def _restart_openclaw_gateway(base_url: str, startup_timeout: int = 30) -> None:
"""
Kill the running openclaw gateway process and restart it.
Waits until the gateway is ready to accept requests.
Must be called while holding _openclaw_config_lock.
"""
try:
subprocess.run(["pkill", "-f", "openclaw gateway"], capture_output=True)
except Exception as e:
print(f" [gateway] pkill failed: {e}", file=sys.stderr)
try:
subprocess.Popen(
["openclaw", "gateway"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
except Exception as e:
raise RuntimeError(f"Failed to start openclaw gateway: {e}")
# Wait for process to fully start
time.sleep(5)
# Wait until gateway is ready
health_url = f"{base_url.rstrip('/')}/health"
deadline = time.time() + startup_timeout
while time.time() < deadline:
try:
resp = requests.get(health_url, timeout=2)
if resp.status_code < 500:
return
except Exception:
pass
time.sleep(0.5)
raise RuntimeError(f"openclaw gateway did not become ready within {startup_timeout}s")
CATEGORY_NAMES = {
1: "single-hop",
2: "multi-hop",
3: "temporal",
4: "world-knowledge",
5: "adversarial",
}
# ---------------------------------------------------------------------------
# LoCoMo data loading
# ---------------------------------------------------------------------------
def load_locomo_data(path: str, sample_id: Optional[str] = None) -> list[dict]:
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
if sample_id is not None:
try:
idx = int(sample_id)
if idx < 0 or idx >= len(data):
raise ValueError(f"Sample index {idx} out of range (0-{len(data) - 1})")
return [data[idx]]
except ValueError:
pass
matched = [s for s in data if s.get("sample_id") == sample_id]
if not matched:
raise ValueError(f"sample_id '{sample_id}' not found")
return matched
return data
def get_sample_last_session_date(sample: dict) -> Optional[str]:
"""Return the date of the last session as YYYY-MM-DD, or None."""
conv = sample.get("conversation", {})
session_keys = [k for k in conv if k.startswith("session_") and "date_time" not in k]
if not session_keys:
return None
def sess_num(k: str) -> int:
try:
return int(k.split("_")[1])
except ValueError:
return 0
for sk in sorted(session_keys, key=sess_num, reverse=True):
if conv.get(sk):
dt_key = f"{sk}_date_time"
date_str = conv.get(dt_key, "")
if date_str and " on " in date_str:
try:
from datetime import datetime
date_part = date_str.split(" on ")[-1]
dt = datetime.strptime(date_part.strip(), "%d %B, %Y")
return dt.strftime("%Y-%m-%d")
except ValueError:
pass
return None
def load_qa_items(
data: list[dict],
skip_adversarial: bool = True,
question_index: Optional[int] = None,
count: Optional[int] = None,
) -> list[dict]:
items = []
for sample in data:
sample_id = sample.get("sample_id", "")
question_time = get_sample_last_session_date(sample)
for q_idx, qa in enumerate(sample.get("qa", [])):
if question_index is not None and q_idx != question_index:
continue
category = qa.get("category", 0)
if skip_adversarial and str(category) == "5":
continue
items.append(
{
"sample_id": sample_id,
"question_index": q_idx,
"question_id": f"{sample_id}_qa{q_idx}",
"question": qa["question"],
"answer": str(qa["answer"]),
"category": category,
"category_name": CATEGORY_NAMES.get(category, "unknown"),
"evidence": qa.get("evidence", []),
"question_time": question_time,
}
)
if count is not None:
items = items[:count]
return items
# ---------------------------------------------------------------------------
# CSV helpers
# ---------------------------------------------------------------------------
QA_FIELDNAMES = [
"sample_id",
"question_index",
"question_id",
"question",
"answer",
"category",
"category_name",
"question_time",
"evidence",
"response",
"input_tokens",
"output_tokens",
"total_tokens",
"time_cost",
"result",
"reasoning",
"timestamp",
]
def load_processed_ids(output_path: str) -> set[str]:
processed: set[str] = set()
if not os.path.exists(output_path):
return processed
try:
with open(output_path, "r", encoding="utf-8", newline="") as f:
for row in csv.DictReader(f):
if row.get("response"):
processed.add(row.get("question_id", ""))
except Exception as e:
print(f"[WARN] Error reading {output_path}: {e}", file=sys.stderr)
return processed
def save_row(output_path: str, row: dict, write_lock: threading.Lock) -> None:
with write_lock:
file_exists = os.path.exists(output_path)
with open(output_path, "a", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=QA_FIELDNAMES, extrasaction="ignore")
if not file_exists:
writer.writeheader()
writer.writerow(row)
f.flush()
# ---------------------------------------------------------------------------
# OpenClaw agent
# ---------------------------------------------------------------------------
def extract_openclaw_text(body: dict) -> str:
"""Extract assistant text from /v1/responses API response."""
try:
for item in body.get("output", []):
if item.get("type") == "message":
for content in item.get("content", []):
if content.get("type") == "output_text":
return content.get("text", "")
for item in body.get("output", []):
if "text" in item:
return item["text"]
for content in item.get("content", []):
if "text" in content:
return content["text"]
except Exception:
pass
return f"[ERROR: could not parse response: {body}]"
def get_openclaw_session_id(session_key: str) -> Optional[str]:
sessions_file = os.path.expanduser("~/.openclaw/agents/main/sessions/sessions.json")
try:
with open(sessions_file, "r") as f:
data = json.load(f)
return data.get(session_key, {}).get("sessionId")
except Exception:
return None
def parse_session_tokens(session_id: str, agent_id: str = "main") -> dict:
"""Sum up all LLM usage across all assistant messages in the session jsonl."""
sessions_dir = os.path.expanduser(f"~/.openclaw/agents/{agent_id}/sessions")
src = os.path.join(sessions_dir, f"{session_id}.jsonl")
total_input = total_output = total_cache_read = 0
try:
with open(src, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
obj = json.loads(line)
if obj.get("type") == "message" and obj.get("message", {}).get("role") == "assistant":
usage = obj["message"].get("usage", {})
total_input += usage.get("input", 0)
total_output += usage.get("output", 0)
total_cache_read += usage.get("cacheRead", 0)
except Exception:
pass
return {
"input_tokens": total_input,
"output_tokens": total_output,
"total_tokens": total_input + total_output + total_cache_read,
}
def send_to_openclaw(
question: str,
sample_id: str,
base_url: str,
token: str,
question_id: Optional[str] = None,
question_time: Optional[str] = None,
retries: int = 2,
) -> tuple[str, dict, float]:
"""
Send a question to an OpenClaw agent.
Returns (response_text, usage, time_cost).
"""
session_key = f"{DEFAULT_SESSION_KEY}-{question_id}" if question_id else DEFAULT_SESSION_KEY
url = f"{base_url.rstrip('/')}/v1/responses"
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {token}",
"X-OpenClaw-Session-Key": session_key,
}
payload = {
"model": "openclaw",
# Force explicit search: the openclaw-supermemory plugin skips semantic search
# on the first turn of a session, so autoRecall alone is not enough here.
"input": f"Use supermemory_search to find relevant memories, then Answer the question directly: {question}",
"stream": False,
"user": sample_id,
}
last_exc: Optional[Exception] = None
t0 = time.time()
for attempt in range(retries + 1):
try:
resp = requests.post(url, json=payload, headers=headers, timeout=300)
resp.raise_for_status()
body = resp.json()
response_text = extract_openclaw_text(body)
# Wait for openclaw to flush the session jsonl before parsing tokens
time.sleep(1)
session_id = get_openclaw_session_id(session_key)
if session_id:
usage = parse_session_tokens(session_id)
else:
usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
return response_text, usage, time.time() - t0
except Exception as e:
last_exc = e
if attempt < retries:
print(f" [retry {attempt + 1}/{retries}] {e}", file=sys.stderr)
raise RuntimeError(f"OpenClaw request failed after {retries + 1} attempts: {last_exc}")
# ---------------------------------------------------------------------------
# LLM judge
# ---------------------------------------------------------------------------
JUDGE_SYSTEM_PROMPT = "You are an expert grader that determines if answers to questions match a gold standard answer"
JUDGE_ACCURACY_PROMPT = """Your task is to label an answer to a question as 'CORRECT' or 'WRONG'. You will be given the following data:
(1) a question (posed by one user to another user),
(2) a 'gold' (ground truth) answer,
(3) a generated answer
which you will score as CORRECT/WRONG.
The point of the question is to ask about something one user should know about the other user based on their prior conversations.
The gold answer will usually be a concise and short answer that includes the referenced topic, for example:
Question: Do you remember what I got the last time I went to Hawaii?
Gold answer: A shell necklace
The generated answer might be much longer, but you should be generous with your grading - as long as it touches on the same topic as the gold answer, it should be counted as CORRECT.
For time related questions, the gold answer will be a specific date, month, year, etc. The generated answer might be much longer or use relative time references (like "last Tuesday" or "next month"), but you should be generous with your grading - as long as it refers to the same date or time period as the gold answer, it should be counted as CORRECT. Even if the format differs (e.g., "May 7th" vs "7 May"), consider it CORRECT if it's the same date.
Now it's time for the real question:
Question: {question}
Gold answer: {gold_answer}
Generated answer: {response}
First, provide a short (one sentence) explanation of your reasoning, then finish with CORRECT or WRONG.
Do NOT include both CORRECT and WRONG in your response, or it will break the evaluation script.
Respond with JSON only: {{"reasoning": "your explanation", "is_correct": "CORRECT" or "WRONG"}}"""
def judge_answer(
question: str,
gold_answer: str,
response: str,
judge_base_url: str,
judge_token: str,
judge_model: str,
) -> tuple[str, str]:
from openai import OpenAI
client = OpenAI(base_url=judge_base_url, api_key=judge_token)
prompt = JUDGE_ACCURACY_PROMPT.format(
question=question, gold_answer=gold_answer, response=response
)
try:
resp = client.chat.completions.create(
model=judge_model,
messages=[
{"role": "system", "content": JUDGE_SYSTEM_PROMPT},
{"role": "user", "content": prompt},
],
temperature=0,
timeout=60,
)
if not resp.choices:
raise ValueError("LLM returned empty or filtered response")
message = resp.choices[0].message
if message is None or message.content is None:
raise ValueError("LLM returned empty or filtered response")
content = message.content.strip()
start, end = content.find("{"), content.rfind("}")
if start != -1 and end != -1:
parsed = json.loads(content[start : end + 1])
label = "CORRECT" if parsed.get("is_correct", "WRONG").strip().upper() == "CORRECT" else "WRONG"
return label, parsed.get("reasoning", "")
return "WRONG", f"[PARSE ERROR] {content}"
except Exception as e:
return "WRONG", f"[API ERROR] {e}"
# ---------------------------------------------------------------------------
# Accuracy summary
# ---------------------------------------------------------------------------
def print_accuracy(rows: list[dict]) -> None:
graded = [r for r in rows if r.get("result") in ("CORRECT", "WRONG")]
if not graded:
print("\n[INFO] No graded results to summarize.", file=sys.stderr)
return
correct_total = sum(1 for r in graded if r["result"] == "CORRECT")
print("\n=== Accuracy Summary ===", file=sys.stderr)
print(f" Overall: {correct_total}/{len(graded)} = {correct_total/len(graded):.2%}", file=sys.stderr)
by_cat: dict[str, list[str]] = {}
for r in graded:
cat = r.get("category_name") or str(r.get("category", "?"))
by_cat.setdefault(cat, []).append(r["result"])
print(" By category:", file=sys.stderr)
for cat, results in sorted(by_cat.items()):
n = sum(1 for r in results if r == "CORRECT")
print(f" {cat:20s}: {n}/{len(results)} = {n/len(results):.2%}", file=sys.stderr)
# ---------------------------------------------------------------------------
# Main runners
# ---------------------------------------------------------------------------
def run_qa(args: argparse.Namespace) -> None:
openclaw_token = args.openclaw_token or os.environ.get("OPENCLAW_GATEWAY_TOKEN", "")
judge_token = args.judge_token or os.environ.get("ARK_API_KEY", os.environ.get("OPENAI_API_KEY", ""))
if args.judge and not judge_token:
print("Error: judge token required (--judge-token or ARK_API_KEY env var)", file=sys.stderr)
sys.exit(1)
data = load_locomo_data(args.input, args.sample)
qa_items = load_qa_items(
data,
skip_adversarial=args.skip_adversarial,
question_index=args.question_index,
count=args.count,
)
print(f"[INFO] {len(qa_items)} QA items loaded", file=sys.stderr)
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
# Remove ERROR rows from CSV before loading processed ids
if os.path.exists(args.output):
with open(args.output, "r", encoding="utf-8", newline="") as f:
reader = csv.DictReader(f)
fieldnames = reader.fieldnames or QA_FIELDNAMES
clean_rows = [r for r in reader if not r.get("response", "").startswith("[ERROR]")]
with open(args.output, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
writer.writeheader()
writer.writerows(clean_rows)
processed_ids = load_processed_ids(args.output)
remaining = [qa for qa in qa_items if qa["question_id"] not in processed_ids]
print(
f"[INFO] {len(processed_ids)} already done, {len(remaining)} remaining",
file=sys.stderr,
)
if not remaining:
print("[INFO] All questions already processed.", file=sys.stderr)
else:
write_lock = threading.Lock()
total = len(remaining)
# Group remaining questions by sample_id to minimize gateway restarts
from collections import defaultdict
by_sample: dict[str, list[tuple[int, dict]]] = defaultdict(list)
for i, qa in enumerate(remaining):
by_sample[qa["sample_id"]].append((i + 1, qa))
def run_one(qa: dict, idx: int) -> None:
print(
f" [{idx}/{total}] {qa['question_id']}: {qa['question'][:60]}...",
file=sys.stderr,
)
try:
response, usage, time_cost = send_to_openclaw(
qa["question"],
qa["sample_id"],
args.openclaw_url,
openclaw_token,
qa["question_id"],
qa.get("question_time"),
)
except Exception as e:
response = f"[ERROR] {e}"
usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
time_cost = 0.0
result_label, reasoning = "", ""
if args.judge and response and not response.startswith("[ERROR]"):
result_label, reasoning = judge_answer(
qa["question"],
qa["answer"],
response,
args.judge_base_url,
judge_token,
args.judge_model,
)
row = {
"sample_id": qa["sample_id"],
"question_index": qa["question_index"],
"question_id": qa["question_id"],
"question": qa["question"],
"answer": qa["answer"],
"category": qa["category"],
"category_name": qa["category_name"],
"question_time": qa.get("question_time", ""),
"evidence": json.dumps(qa.get("evidence", [])),
"response": response,
"input_tokens": usage.get("input_tokens", 0),
"output_tokens": usage.get("output_tokens", 0),
"total_tokens": usage.get("total_tokens", 0),
"time_cost": round(time_cost, 2),
"result": result_label,
"reasoning": reasoning,
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
}
save_row(args.output, row, write_lock)
label_str = f" → {result_label}" if result_label else ""
print(f" [{idx}/{total}] done {time_cost:.1f}s{label_str}", file=sys.stderr)
# Process sample by sample: restart gateway once per sample to pick up new containerTag
for sample_id, qa_list in by_sample.items():
container_tag = sanitize_tag(sample_id)
print(
f"\n[INFO] Switching to sample {sample_id} (containerTag={container_tag}), restarting openclaw gateway...",
file=sys.stderr,
)
with _openclaw_config_lock:
_update_openclaw_supermemory_container(sample_id)
_restart_openclaw_gateway(args.openclaw_url)
print(f"[INFO] Gateway ready, running {len(qa_list)} questions for {sample_id}", file=sys.stderr)
with ThreadPoolExecutor(max_workers=args.threads) as executor:
futures = {
executor.submit(run_one, qa, idx): qa
for idx, qa in qa_list
}
for fut in as_completed(futures):
try:
fut.result()
except Exception as e:
qa = futures[fut]
print(f" [ERROR] {qa['question_id']}: {e}", file=sys.stderr)
# Print token and latency summary
try:
with open(args.output, "r", encoding="utf-8", newline="") as f:
rows = list(csv.DictReader(f))
total_input = sum(int(r.get("input_tokens") or 0) for r in rows)
total_input_with_cache = sum(
int(r.get("total_tokens") or 0) - int(r.get("output_tokens") or 0) for r in rows
)
times = [float(r["time_cost"]) for r in rows if r.get("time_cost")]
avg_time = sum(times) / len(times) if times else 0.0
print(f"\n=== Token & Latency Summary ===", file=sys.stderr)
print(f" Total input tokens : {total_input}", file=sys.stderr)
print(f" Total input tokens (with cache): {total_input_with_cache}", file=sys.stderr)
print(f" Avg time per query : {avg_time:.1f}s", file=sys.stderr)
except Exception:
pass
if args.judge:
try:
with open(args.output, "r", encoding="utf-8", newline="") as f:
print_accuracy(list(csv.DictReader(f)))
except Exception:
pass
def run_judge_only(args: argparse.Namespace) -> None:
"""Grade responses in an existing CSV that lack a result label."""
if not os.path.exists(args.output):
print(f"Error: output file not found: {args.output}", file=sys.stderr)
sys.exit(1)
judge_token = args.judge_token or os.environ.get("ARK_API_KEY", os.environ.get("OPENAI_API_KEY", ""))
if not judge_token:
print("Error: judge token required (--judge-token or ARK_API_KEY env var)", file=sys.stderr)
sys.exit(1)
with open(args.output, "r", encoding="utf-8", newline="") as f:
reader = csv.DictReader(f)
fieldnames = list(reader.fieldnames or QA_FIELDNAMES)
rows = list(reader)
for extra in ("result", "reasoning"):
if extra not in fieldnames:
fieldnames.append(extra)
ungraded_indices = [i for i, row in enumerate(rows) if not row.get("result")]
print(f"[INFO] {len(rows)} rows total, {len(ungraded_indices)} ungraded", file=sys.stderr)
if not ungraded_indices:
print("[INFO] All rows already graded.", file=sys.stderr)
print_accuracy(rows)
return
file_lock = threading.Lock()
def grade_one(idx: int) -> None:
row = rows[idx]
label, reasoning = judge_answer(
row.get("question", ""),
row.get("answer", ""),
row.get("response", ""),
args.judge_base_url,
judge_token,
args.judge_model,
)
row["result"] = label
row["reasoning"] = reasoning
with file_lock:
tmp = args.output + ".tmp"
with open(tmp, "w", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
writer.writeheader()
writer.writerows(rows)
os.replace(tmp, args.output)
print(f" Graded {row.get('question_id','?')}: {label}", file=sys.stderr)
with ThreadPoolExecutor(max_workers=args.threads) as executor:
futures = [executor.submit(grade_one, idx) for idx in ungraded_indices]
for fut in as_completed(futures):
try:
fut.result()
except Exception as e:
print(f"[ERROR] grading failed: {e}", file=sys.stderr)
print_accuracy(rows)
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main() -> None:
parser = argparse.ArgumentParser(
description="Evaluate LoCoMo QA via OpenClaw agent (Supermemory-backed)",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
# Data selection
parser.add_argument("--input", default=DEFAULT_DATA_PATH, help="Path to locomo10.json")
parser.add_argument("--output", default=DEFAULT_OUTPUT_PATH, help="Path to output CSV")
parser.add_argument(
"--sample",
default=None,
help="Sample index (int) or sample_id (e.g. conv-26). Default: all.",
)
parser.add_argument(
"--question-index",
type=int,
default=None,
help="Single question index (0-based) within the sample.",
)
parser.add_argument("--count", type=int, default=None, help="Max QA items to process.")
parser.add_argument(
"--no-skip-adversarial",
dest="skip_adversarial",
action="store_false",
default=True,
help="Include category-5 adversarial questions (skipped by default).",
)
parser.add_argument("--threads", type=int, default=10, help="Concurrent threads per sample (default: 10)")
# OpenClaw
parser.add_argument(
"--openclaw-url",
default=DEFAULT_OPENCLAW_URL,
help=f"OpenClaw gateway URL (default: {DEFAULT_OPENCLAW_URL})",
)
parser.add_argument(
"--openclaw-token",
default=None,
help="OpenClaw auth token (or OPENCLAW_GATEWAY_TOKEN env var)",
)
# Judge
parser.add_argument(
"--judge",
action="store_true",
default=False,
help="Auto-judge each response right after answering.",
)
parser.add_argument(
"--judge-only",
action="store_true",
default=False,
help="Skip QA; only grade ungraded responses in the existing --output CSV.",
)
parser.add_argument(
"--judge-base-url",
default="https://ark.cn-beijing.volces.com/api/v3",
help="OpenAI-compatible API base URL for judge (default: Volcengine ARK)",
)
parser.add_argument(
"--judge-token",
default=None,
help="API token for judge (or ARK_API_KEY / OPENAI_API_KEY env var)",
)
parser.add_argument(
"--judge-model",
default="doubao-seed-2-0-pro-260215",
help="Judge model (default: doubao-seed-2-0-pro-260215)",
)
args = parser.parse_args()
if args.judge_only:
run_judge_only(args)
else:
run_qa(args)
if __name__ == "__main__":
main()