Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

29.6 KB
Raw Blame
"""
Evaluate LoCoMo QA via Supermemory + OpenClaw (agent mode).

Questions are sent to an OpenClaw agent which calls Supermemory internally.
Before each sample, eval.py automatically:
  1. Updates ~/.openclaw/openclaw.json to set openclaw-supermemory.config.containerTag = sample_id
     (and switches plugins.slots.memory to "openclaw-supermemory")
  2. Restarts the OpenClaw gateway to pick up the new config
  3. Runs all questions for that sample concurrently

Prerequisites:
  - Conversations already ingested into Supermemory via ingest.py (containerTag = sanitize(sample_id))
  - OpenClaw running locally with the openclaw-supermemory plugin installed
  - ~/.openclaw/openclaw.json configured with openclaw-supermemory apiKey

Usage:
    # Run QA + judge for all samples (6 concurrent threads)
    python eval.py --threads 6 --judge

    # Single sample
    python eval.py --sample conv-26 --threads 6 --judge

    # First 12 questions only
    python eval.py --sample conv-26 --count 12 --threads 6 --judge

    # Judge-only (grade existing responses in CSV)
    python eval.py --judge-only
"""

import argparse
import csv
import json
import os
import re
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Optional

import requests
from dotenv import load_dotenv

load_dotenv(Path.home() / ".openviking_benchmark_env")

SCRIPT_DIR = Path(__file__).parent.resolve()
DEFAULT_DATA_PATH = str(SCRIPT_DIR / ".." / "data" / "locomo10.json")
DEFAULT_OUTPUT_PATH = str(SCRIPT_DIR / "result" / "qa_results.csv")
DEFAULT_OPENCLAW_URL = "http://127.0.0.1:18789"
DEFAULT_SESSION_KEY = "locomo-eval"
OPENCLAW_CONFIG_PATH = Path.home() / ".openclaw" / "openclaw.json"

# Serialize openclaw config updates across threads so each sample sees the right containerTag
_openclaw_config_lock = threading.Lock()

# ---------------------------------------------------------------------------
# Tag sanitization (must match openclaw-supermemory's sanitizeTag logic)
# ---------------------------------------------------------------------------

def sanitize_tag(raw: str) -> str:
    """Sanitize a tag string to match openclaw-supermemory convention.
    e.g. 'conv-26' -> 'conv_26'
    """
    tag = re.sub(r"[^a-zA-Z0-9_]", "_", raw)
    tag = re.sub(r"_+", "_", tag)
    tag = tag.strip("_")
    return tag


# ---------------------------------------------------------------------------
# openclaw.json config helpers
# ---------------------------------------------------------------------------

def _update_openclaw_supermemory_container(sample_id: str) -> None:
    """
    Rewrite ~/.openclaw/openclaw.json so that openclaw-supermemory uses
    sanitize(sample_id) as containerTag. Also switches slots.memory to
    'openclaw-supermemory' and disables autoCapture.
    Must be called while holding _openclaw_config_lock.
    """
    container_tag = sanitize_tag(sample_id)

    with open(OPENCLAW_CONFIG_PATH, "r", encoding="utf-8") as f:
        config = json.load(f)

    plugins = config.setdefault("plugins", {})
    entries = plugins.setdefault("entries", {})

    sm_entry = entries.setdefault("openclaw-supermemory", {})
    sm_entry["enabled"] = True
    sm_entry.setdefault("config", {})["containerTag"] = container_tag
    sm_entry["config"]["autoCapture"] = False
    sm_entry["config"]["autoRecall"] = True

    # Switch memory slot to supermemory
    plugins.setdefault("slots", {})["memory"] = "openclaw-supermemory"

    tmp = str(OPENCLAW_CONFIG_PATH) + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        json.dump(config, f, indent=2, ensure_ascii=False)
    os.replace(tmp, str(OPENCLAW_CONFIG_PATH))


def _restart_openclaw_gateway(base_url: str, startup_timeout: int = 30) -> None:
    """
    Kill the running openclaw gateway process and restart it.
    Waits until the gateway is ready to accept requests.
    Must be called while holding _openclaw_config_lock.
    """
    try:
        subprocess.run(["pkill", "-f", "openclaw gateway"], capture_output=True)
    except Exception as e:
        print(f"    [gateway] pkill failed: {e}", file=sys.stderr)

    try:
        subprocess.Popen(
            ["openclaw", "gateway"],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
        )
    except Exception as e:
        raise RuntimeError(f"Failed to start openclaw gateway: {e}")

    # Wait for process to fully start
    time.sleep(5)

    # Wait until gateway is ready
    health_url = f"{base_url.rstrip('/')}/health"
    deadline = time.time() + startup_timeout
    while time.time() < deadline:
        try:
            resp = requests.get(health_url, timeout=2)
            if resp.status_code < 500:
                return
        except Exception:
            pass
        time.sleep(0.5)

    raise RuntimeError(f"openclaw gateway did not become ready within {startup_timeout}s")


CATEGORY_NAMES = {
    1: "single-hop",
    2: "multi-hop",
    3: "temporal",
    4: "world-knowledge",
    5: "adversarial",
}

# ---------------------------------------------------------------------------
# LoCoMo data loading
# ---------------------------------------------------------------------------

def load_locomo_data(path: str, sample_id: Optional[str] = None) -> list[dict]:
    with open(path, "r", encoding="utf-8") as f:
        data = json.load(f)

    if sample_id is not None:
        try:
            idx = int(sample_id)
            if idx < 0 or idx >= len(data):
                raise ValueError(f"Sample index {idx} out of range (0-{len(data) - 1})")
            return [data[idx]]
        except ValueError:
            pass
        matched = [s for s in data if s.get("sample_id") == sample_id]
        if not matched:
            raise ValueError(f"sample_id '{sample_id}' not found")
        return matched

    return data


def get_sample_last_session_date(sample: dict) -> Optional[str]:
    """Return the date of the last session as YYYY-MM-DD, or None."""
    conv = sample.get("conversation", {})
    session_keys = [k for k in conv if k.startswith("session_") and "date_time" not in k]
    if not session_keys:
        return None

    def sess_num(k: str) -> int:
        try:
            return int(k.split("_")[1])
        except ValueError:
            return 0

    for sk in sorted(session_keys, key=sess_num, reverse=True):
        if conv.get(sk):
            dt_key = f"{sk}_date_time"
            date_str = conv.get(dt_key, "")
            if date_str and " on " in date_str:
                try:
                    from datetime import datetime
                    date_part = date_str.split(" on ")[-1]
                    dt = datetime.strptime(date_part.strip(), "%d %B, %Y")
                    return dt.strftime("%Y-%m-%d")
                except ValueError:
                    pass
    return None


def load_qa_items(
    data: list[dict],
    skip_adversarial: bool = True,
    question_index: Optional[int] = None,
    count: Optional[int] = None,
) -> list[dict]:
    items = []
    for sample in data:
        sample_id = sample.get("sample_id", "")
        question_time = get_sample_last_session_date(sample)

        for q_idx, qa in enumerate(sample.get("qa", [])):
            if question_index is not None and q_idx != question_index:
                continue
            category = qa.get("category", 0)
            if skip_adversarial and str(category) == "5":
                continue
            items.append(
                {
                    "sample_id": sample_id,
                    "question_index": q_idx,
                    "question_id": f"{sample_id}_qa{q_idx}",
                    "question": qa["question"],
                    "answer": str(qa["answer"]),
                    "category": category,
                    "category_name": CATEGORY_NAMES.get(category, "unknown"),
                    "evidence": qa.get("evidence", []),
                    "question_time": question_time,
                }
            )

    if count is not None:
        items = items[:count]
    return items


# ---------------------------------------------------------------------------
# CSV helpers
# ---------------------------------------------------------------------------

QA_FIELDNAMES = [
    "sample_id",
    "question_index",
    "question_id",
    "question",
    "answer",
    "category",
    "category_name",
    "question_time",
    "evidence",
    "response",
    "input_tokens",
    "output_tokens",
    "total_tokens",
    "time_cost",
    "result",
    "reasoning",
    "timestamp",
]


def load_processed_ids(output_path: str) -> set[str]:
    processed: set[str] = set()
    if not os.path.exists(output_path):
        return processed
    try:
        with open(output_path, "r", encoding="utf-8", newline="") as f:
            for row in csv.DictReader(f):
                if row.get("response"):
                    processed.add(row.get("question_id", ""))
    except Exception as e:
        print(f"[WARN] Error reading {output_path}: {e}", file=sys.stderr)
    return processed


def save_row(output_path: str, row: dict, write_lock: threading.Lock) -> None:
    with write_lock:
        file_exists = os.path.exists(output_path)
        with open(output_path, "a", encoding="utf-8", newline="") as f:
            writer = csv.DictWriter(f, fieldnames=QA_FIELDNAMES, extrasaction="ignore")
            if not file_exists:
                writer.writeheader()
            writer.writerow(row)
            f.flush()


# ---------------------------------------------------------------------------
# OpenClaw agent
# ---------------------------------------------------------------------------

def extract_openclaw_text(body: dict) -> str:
    """Extract assistant text from /v1/responses API response."""
    try:
        for item in body.get("output", []):
            if item.get("type") == "message":
                for content in item.get("content", []):
                    if content.get("type") == "output_text":
                        return content.get("text", "")
        for item in body.get("output", []):
            if "text" in item:
                return item["text"]
            for content in item.get("content", []):
                if "text" in content:
                    return content["text"]
    except Exception:
        pass
    return f"[ERROR: could not parse response: {body}]"


def get_openclaw_session_id(session_key: str) -> Optional[str]:
    sessions_file = os.path.expanduser("~/.openclaw/agents/main/sessions/sessions.json")
    try:
        with open(sessions_file, "r") as f:
            data = json.load(f)
        return data.get(session_key, {}).get("sessionId")
    except Exception:
        return None


def parse_session_tokens(session_id: str, agent_id: str = "main") -> dict:
    """Sum up all LLM usage across all assistant messages in the session jsonl."""
    sessions_dir = os.path.expanduser(f"~/.openclaw/agents/{agent_id}/sessions")
    src = os.path.join(sessions_dir, f"{session_id}.jsonl")
    total_input = total_output = total_cache_read = 0
    try:
        with open(src, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                obj = json.loads(line)
                if obj.get("type") == "message" and obj.get("message", {}).get("role") == "assistant":
                    usage = obj["message"].get("usage", {})
                    total_input += usage.get("input", 0)
                    total_output += usage.get("output", 0)
                    total_cache_read += usage.get("cacheRead", 0)
    except Exception:
        pass
    return {
        "input_tokens": total_input,
        "output_tokens": total_output,
        "total_tokens": total_input + total_output + total_cache_read,
    }


def send_to_openclaw(
    question: str,
    sample_id: str,
    base_url: str,
    token: str,
    question_id: Optional[str] = None,
    question_time: Optional[str] = None,
    retries: int = 2,
) -> tuple[str, dict, float]:
    """
    Send a question to an OpenClaw agent.
    Returns (response_text, usage, time_cost).
    """
    session_key = f"{DEFAULT_SESSION_KEY}-{question_id}" if question_id else DEFAULT_SESSION_KEY

    url = f"{base_url.rstrip('/')}/v1/responses"
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {token}",
        "X-OpenClaw-Session-Key": session_key,
    }
    payload = {
        "model": "openclaw",
        # Force explicit search: the openclaw-supermemory plugin skips semantic search
        # on the first turn of a session, so autoRecall alone is not enough here.
        "input": f"Use supermemory_search to find relevant memories, then Answer the question directly: {question}",
        "stream": False,
        "user": sample_id,
    }

    last_exc: Optional[Exception] = None
    t0 = time.time()
    for attempt in range(retries + 1):
        try:
            resp = requests.post(url, json=payload, headers=headers, timeout=300)
            resp.raise_for_status()
            body = resp.json()
            response_text = extract_openclaw_text(body)

            # Wait for openclaw to flush the session jsonl before parsing tokens
            time.sleep(1)
            session_id = get_openclaw_session_id(session_key)
            if session_id:
                usage = parse_session_tokens(session_id)
            else:
                usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}

            return response_text, usage, time.time() - t0
        except Exception as e:
            last_exc = e
            if attempt < retries:
                print(f"    [retry {attempt + 1}/{retries}] {e}", file=sys.stderr)

    raise RuntimeError(f"OpenClaw request failed after {retries + 1} attempts: {last_exc}")


# ---------------------------------------------------------------------------
# LLM judge
# ---------------------------------------------------------------------------

JUDGE_SYSTEM_PROMPT = "You are an expert grader that determines if answers to questions match a gold standard answer"

JUDGE_ACCURACY_PROMPT = """Your task is to label an answer to a question as 'CORRECT' or 'WRONG'. You will be given the following data:
    (1) a question (posed by one user to another user),
    (2) a 'gold' (ground truth) answer,
    (3) a generated answer
which you will score as CORRECT/WRONG.

The point of the question is to ask about something one user should know about the other user based on their prior conversations.
The gold answer will usually be a concise and short answer that includes the referenced topic, for example:
Question: Do you remember what I got the last time I went to Hawaii?
Gold answer: A shell necklace
The generated answer might be much longer, but you should be generous with your grading - as long as it touches on the same topic as the gold answer, it should be counted as CORRECT.

For time related questions, the gold answer will be a specific date, month, year, etc. The generated answer might be much longer or use relative time references (like "last Tuesday" or "next month"), but you should be generous with your grading - as long as it refers to the same date or time period as the gold answer, it should be counted as CORRECT. Even if the format differs (e.g., "May 7th" vs "7 May"), consider it CORRECT if it's the same date.

Now it's time for the real question:
Question: {question}
Gold answer: {gold_answer}
Generated answer: {response}

First, provide a short (one sentence) explanation of your reasoning, then finish with CORRECT or WRONG.
Do NOT include both CORRECT and WRONG in your response, or it will break the evaluation script.

Respond with JSON only: {{"reasoning": "your explanation", "is_correct": "CORRECT" or "WRONG"}}"""


def judge_answer(
    question: str,
    gold_answer: str,
    response: str,
    judge_base_url: str,
    judge_token: str,
    judge_model: str,
) -> tuple[str, str]:
    from openai import OpenAI
    client = OpenAI(base_url=judge_base_url, api_key=judge_token)
    prompt = JUDGE_ACCURACY_PROMPT.format(
        question=question, gold_answer=gold_answer, response=response
    )
    try:
        resp = client.chat.completions.create(
            model=judge_model,
            messages=[
                {"role": "system", "content": JUDGE_SYSTEM_PROMPT},
                {"role": "user", "content": prompt},
            ],
            temperature=0,
            timeout=60,
        )
        if not resp.choices:
            raise ValueError("LLM returned empty or filtered response")
        message = resp.choices[0].message
        if message is None or message.content is None:
            raise ValueError("LLM returned empty or filtered response")
        content = message.content.strip()
        start, end = content.find("{"), content.rfind("}")
        if start != -1 and end != -1:
            parsed = json.loads(content[start : end + 1])
            label = "CORRECT" if parsed.get("is_correct", "WRONG").strip().upper() == "CORRECT" else "WRONG"
            return label, parsed.get("reasoning", "")
        return "WRONG", f"[PARSE ERROR] {content}"
    except Exception as e:
        return "WRONG", f"[API ERROR] {e}"


# ---------------------------------------------------------------------------
# Accuracy summary
# ---------------------------------------------------------------------------

def print_accuracy(rows: list[dict]) -> None:
    graded = [r for r in rows if r.get("result") in ("CORRECT", "WRONG")]
    if not graded:
        print("\n[INFO] No graded results to summarize.", file=sys.stderr)
        return

    correct_total = sum(1 for r in graded if r["result"] == "CORRECT")
    print("\n=== Accuracy Summary ===", file=sys.stderr)
    print(f"  Overall: {correct_total}/{len(graded)} = {correct_total/len(graded):.2%}", file=sys.stderr)

    by_cat: dict[str, list[str]] = {}
    for r in graded:
        cat = r.get("category_name") or str(r.get("category", "?"))
        by_cat.setdefault(cat, []).append(r["result"])

    print("  By category:", file=sys.stderr)
    for cat, results in sorted(by_cat.items()):
        n = sum(1 for r in results if r == "CORRECT")
        print(f"    {cat:20s}: {n}/{len(results)} = {n/len(results):.2%}", file=sys.stderr)


# ---------------------------------------------------------------------------
# Main runners
# ---------------------------------------------------------------------------

def run_qa(args: argparse.Namespace) -> None:
    openclaw_token = args.openclaw_token or os.environ.get("OPENCLAW_GATEWAY_TOKEN", "")

    judge_token = args.judge_token or os.environ.get("ARK_API_KEY", os.environ.get("OPENAI_API_KEY", ""))
    if args.judge and not judge_token:
        print("Error: judge token required (--judge-token or ARK_API_KEY env var)", file=sys.stderr)
        sys.exit(1)

    data = load_locomo_data(args.input, args.sample)
    qa_items = load_qa_items(
        data,
        skip_adversarial=args.skip_adversarial,
        question_index=args.question_index,
        count=args.count,
    )
    print(f"[INFO] {len(qa_items)} QA items loaded", file=sys.stderr)

    Path(args.output).parent.mkdir(parents=True, exist_ok=True)

    # Remove ERROR rows from CSV before loading processed ids
    if os.path.exists(args.output):
        with open(args.output, "r", encoding="utf-8", newline="") as f:
            reader = csv.DictReader(f)
            fieldnames = reader.fieldnames or QA_FIELDNAMES
            clean_rows = [r for r in reader if not r.get("response", "").startswith("[ERROR]")]
        with open(args.output, "w", encoding="utf-8", newline="") as f:
            writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
            writer.writeheader()
            writer.writerows(clean_rows)

    processed_ids = load_processed_ids(args.output)
    remaining = [qa for qa in qa_items if qa["question_id"] not in processed_ids]
    print(
        f"[INFO] {len(processed_ids)} already done, {len(remaining)} remaining",
        file=sys.stderr,
    )

    if not remaining:
        print("[INFO] All questions already processed.", file=sys.stderr)
    else:
        write_lock = threading.Lock()
        total = len(remaining)

        # Group remaining questions by sample_id to minimize gateway restarts
        from collections import defaultdict
        by_sample: dict[str, list[tuple[int, dict]]] = defaultdict(list)
        for i, qa in enumerate(remaining):
            by_sample[qa["sample_id"]].append((i + 1, qa))

        def run_one(qa: dict, idx: int) -> None:
            print(
                f"  [{idx}/{total}] {qa['question_id']}: {qa['question'][:60]}...",
                file=sys.stderr,
            )
            try:
                response, usage, time_cost = send_to_openclaw(
                    qa["question"],
                    qa["sample_id"],
                    args.openclaw_url,
                    openclaw_token,
                    qa["question_id"],
                    qa.get("question_time"),
                )
            except Exception as e:
                response = f"[ERROR] {e}"
                usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}
                time_cost = 0.0

            result_label, reasoning = "", ""
            if args.judge and response and not response.startswith("[ERROR]"):
                result_label, reasoning = judge_answer(
                    qa["question"],
                    qa["answer"],
                    response,
                    args.judge_base_url,
                    judge_token,
                    args.judge_model,
                )

            row = {
                "sample_id": qa["sample_id"],
                "question_index": qa["question_index"],
                "question_id": qa["question_id"],
                "question": qa["question"],
                "answer": qa["answer"],
                "category": qa["category"],
                "category_name": qa["category_name"],
                "question_time": qa.get("question_time", ""),
                "evidence": json.dumps(qa.get("evidence", [])),
                "response": response,
                "input_tokens": usage.get("input_tokens", 0),
                "output_tokens": usage.get("output_tokens", 0),
                "total_tokens": usage.get("total_tokens", 0),
                "time_cost": round(time_cost, 2),
                "result": result_label,
                "reasoning": reasoning,
                "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
            }
            save_row(args.output, row, write_lock)
            label_str = f"  → {result_label}" if result_label else ""
            print(f"  [{idx}/{total}] done {time_cost:.1f}s{label_str}", file=sys.stderr)

        # Process sample by sample: restart gateway once per sample to pick up new containerTag
        for sample_id, qa_list in by_sample.items():
            container_tag = sanitize_tag(sample_id)
            print(
                f"\n[INFO] Switching to sample {sample_id} (containerTag={container_tag}), restarting openclaw gateway...",
                file=sys.stderr,
            )
            with _openclaw_config_lock:
                _update_openclaw_supermemory_container(sample_id)
                _restart_openclaw_gateway(args.openclaw_url)
            print(f"[INFO] Gateway ready, running {len(qa_list)} questions for {sample_id}", file=sys.stderr)

            with ThreadPoolExecutor(max_workers=args.threads) as executor:
                futures = {
                    executor.submit(run_one, qa, idx): qa
                    for idx, qa in qa_list
                }
                for fut in as_completed(futures):
                    try:
                        fut.result()
                    except Exception as e:
                        qa = futures[fut]
                        print(f"  [ERROR] {qa['question_id']}: {e}", file=sys.stderr)

    # Print token and latency summary
    try:
        with open(args.output, "r", encoding="utf-8", newline="") as f:
            rows = list(csv.DictReader(f))
        total_input = sum(int(r.get("input_tokens") or 0) for r in rows)
        total_input_with_cache = sum(
            int(r.get("total_tokens") or 0) - int(r.get("output_tokens") or 0) for r in rows
        )
        times = [float(r["time_cost"]) for r in rows if r.get("time_cost")]
        avg_time = sum(times) / len(times) if times else 0.0
        print(f"\n=== Token & Latency Summary ===", file=sys.stderr)
        print(f"  Total input tokens             : {total_input}", file=sys.stderr)
        print(f"  Total input tokens (with cache): {total_input_with_cache}", file=sys.stderr)
        print(f"  Avg time per query             : {avg_time:.1f}s", file=sys.stderr)
    except Exception:
        pass

    if args.judge:
        try:
            with open(args.output, "r", encoding="utf-8", newline="") as f:
                print_accuracy(list(csv.DictReader(f)))
        except Exception:
            pass


def run_judge_only(args: argparse.Namespace) -> None:
    """Grade responses in an existing CSV that lack a result label."""
    if not os.path.exists(args.output):
        print(f"Error: output file not found: {args.output}", file=sys.stderr)
        sys.exit(1)

    judge_token = args.judge_token or os.environ.get("ARK_API_KEY", os.environ.get("OPENAI_API_KEY", ""))
    if not judge_token:
        print("Error: judge token required (--judge-token or ARK_API_KEY env var)", file=sys.stderr)
        sys.exit(1)

    with open(args.output, "r", encoding="utf-8", newline="") as f:
        reader = csv.DictReader(f)
        fieldnames = list(reader.fieldnames or QA_FIELDNAMES)
        rows = list(reader)

    for extra in ("result", "reasoning"):
        if extra not in fieldnames:
            fieldnames.append(extra)

    ungraded_indices = [i for i, row in enumerate(rows) if not row.get("result")]
    print(f"[INFO] {len(rows)} rows total, {len(ungraded_indices)} ungraded", file=sys.stderr)

    if not ungraded_indices:
        print("[INFO] All rows already graded.", file=sys.stderr)
        print_accuracy(rows)
        return

    file_lock = threading.Lock()

    def grade_one(idx: int) -> None:
        row = rows[idx]
        label, reasoning = judge_answer(
            row.get("question", ""),
            row.get("answer", ""),
            row.get("response", ""),
            args.judge_base_url,
            judge_token,
            args.judge_model,
        )
        row["result"] = label
        row["reasoning"] = reasoning
        with file_lock:
            tmp = args.output + ".tmp"
            with open(tmp, "w", encoding="utf-8", newline="") as f:
                writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
                writer.writeheader()
                writer.writerows(rows)
            os.replace(tmp, args.output)
        print(f"  Graded {row.get('question_id','?')}: {label}", file=sys.stderr)

    with ThreadPoolExecutor(max_workers=args.threads) as executor:
        futures = [executor.submit(grade_one, idx) for idx in ungraded_indices]
        for fut in as_completed(futures):
            try:
                fut.result()
            except Exception as e:
                print(f"[ERROR] grading failed: {e}", file=sys.stderr)

    print_accuracy(rows)


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------

def main() -> None:
    parser = argparse.ArgumentParser(
        description="Evaluate LoCoMo QA via OpenClaw agent (Supermemory-backed)",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__,
    )

    # Data selection
    parser.add_argument("--input", default=DEFAULT_DATA_PATH, help="Path to locomo10.json")
    parser.add_argument("--output", default=DEFAULT_OUTPUT_PATH, help="Path to output CSV")
    parser.add_argument(
        "--sample",
        default=None,
        help="Sample index (int) or sample_id (e.g. conv-26). Default: all.",
    )
    parser.add_argument(
        "--question-index",
        type=int,
        default=None,
        help="Single question index (0-based) within the sample.",
    )
    parser.add_argument("--count", type=int, default=None, help="Max QA items to process.")
    parser.add_argument(
        "--no-skip-adversarial",
        dest="skip_adversarial",
        action="store_false",
        default=True,
        help="Include category-5 adversarial questions (skipped by default).",
    )
    parser.add_argument("--threads", type=int, default=10, help="Concurrent threads per sample (default: 10)")

    # OpenClaw
    parser.add_argument(
        "--openclaw-url",
        default=DEFAULT_OPENCLAW_URL,
        help=f"OpenClaw gateway URL (default: {DEFAULT_OPENCLAW_URL})",
    )
    parser.add_argument(
        "--openclaw-token",
        default=None,
        help="OpenClaw auth token (or OPENCLAW_GATEWAY_TOKEN env var)",
    )

    # Judge
    parser.add_argument(
        "--judge",
        action="store_true",
        default=False,
        help="Auto-judge each response right after answering.",
    )
    parser.add_argument(
        "--judge-only",
        action="store_true",
        default=False,
        help="Skip QA; only grade ungraded responses in the existing --output CSV.",
    )
    parser.add_argument(
        "--judge-base-url",
        default="https://ark.cn-beijing.volces.com/api/v3",
        help="OpenAI-compatible API base URL for judge (default: Volcengine ARK)",
    )
    parser.add_argument(
        "--judge-token",
        default=None,
        help="API token for judge (or ARK_API_KEY / OPENAI_API_KEY env var)",
    )
    parser.add_argument(
        "--judge-model",
        default="doubao-seed-2-0-pro-260215",
        help="Judge model (default: doubao-seed-2-0-pro-260215)",
    )

    args = parser.parse_args()

    if args.judge_only:
        run_judge_only(args)
    else:
        run_qa(args)


if __name__ == "__main__":
    main()