Skip to content

Self-hosted AI workspace.

Python
35.1 KB
Raw Blame
#!/usr/bin/env python3
"""Multi-turn Odysseus tool-use eval for contextual follow-up behavior."""

from __future__ import annotations

import argparse
import contextlib
import json
import os
import re
import time
import uuid
from pathlib import Path
from typing import Any

import httpx

try:
    from scripts.eval_odysseus_tool_use import (
        _cookie,
        _raise_for_status_with_body,
        _sse_events,
        _tool_approval_from_event,
        _visible_event_text,
    )
except ModuleNotFoundError:
    from eval_odysseus_tool_use import (
        _cookie,
        _raise_for_status_with_body,
        _sse_events,
        _tool_approval_from_event,
        _visible_event_text,
    )


SCENARIOS: list[dict[str, Any]] = [
    {
        "scenario": "public_domain_art_links_followup",
        "turns": [
            {
                "message": "What are some good sites for public domain art?",
                "expected_tool": "no_tool",
                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
            },
            {
                "message": "send links",
                "expected_tool": "web_search",
                "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"],
            },
        ],
    },
    {
        "scenario": "notes_then_identity_boundary",
        "turns": [
            {
                "message": "what are my notes?",
                "expected_tool": "manage_notes",
                "expected_action": "list",
                "required_any": ["note", "[", "test scenario"],
            },
            {
                "message": "who are you?",
                "expected_tool": "no_tool",
                "required_any": ["odysseus", "assistant"],
            },
        ],
    },
    {
        "scenario": "email_followup_search",
        "turns": [
            {
                "message": "what is my latest email?",
                "expected_tool": "mcp__email__list_emails",
                "required_any": ["email", "from", "subject", "latest"],
            },
            {
                "message": "find emails from Runpod instead",
                "expected_tool": "mcp__email__search_emails",
                "required_any": ["runpod", "email", "no emails"],
            },
        ],
    },
    {
        "scenario": "calendar_then_general_fact",
        "turns": [
            {
                "message": "what is on my calendar?",
                "expected_tool": "manage_calendar",
                "expected_action": "list_events",
                "required_any": ["event", "calendar", "found", "no events"],
            },
            {
                "message": "what does VAT stand for?",
                "expected_tool": "no_tool",
                "required_any": ["value-added tax", "value added tax"],
            },
        ],
    },
    {
        "scenario": "public_domain_art_typo_links_followup",
        "turns": [
            {
                "message": "What are some good sites for public domain art?",
                "expected_tool": "no_tool",
                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
            },
            {
                "message": "sned links for those",
                "expected_tool": "web_search",
                "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"],
            },
        ],
    },
    {
        "scenario": "notes_then_calendar_switch",
        "turns": [
            {
                "message": "what are my notes?",
                "expected_tool": "manage_notes",
                "expected_action": "list",
                "required_any": ["note", "[", "test scenario"],
            },
            {
                "message": "what is on my calendar next week?",
                "expected_tool": "manage_calendar",
                "expected_action": "list_events",
                "required_any": ["event", "calendar", "found", "no events"],
            },
        ],
    },
    {
        "scenario": "email_then_ambiguous_links_clarify",
        "turns": [
            {
                "message": "what is my latest email?",
                "expected_tool": "mcp__email__list_emails",
                "required_any": ["email", "from", "subject", "latest"],
            },
            {
                "message": "send links",
                "expected_tool": "no_tool",
                "required_any": ["which links", "what links", "clarify", "what topic", "which topic"],
            },
        ],
    },
    {
        "scenario": "web_answer_then_calendar_boundary",
        "turns": [
            {
                "message": "What are some good sites for public domain art?",
                "expected_tool": "no_tool",
                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
            },
            {
                "message": "what is on my calendar?",
                "expected_tool": "manage_calendar",
                "expected_action": "list_events",
                "required_any": ["event", "calendar", "found", "no events"],
            },
        ],
    },
    {
        "scenario": "public_domain_art_sites_tail_followup",
        "turns": [
            {
                "message": "What are some good sites for public domain art?",
                "expected_tool": "no_tool",
                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
            },
            {
                "message": "send links for the sites",
                "expected_tool": "web_search",
                "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"],
            },
        ],
    },
    {
        "scenario": "public_domain_art_typo_bare_links_followup",
        "turns": [
            {
                "message": "What are some good sites for public domain art?",
                "expected_tool": "no_tool",
                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
            },
            {
                "message": "sned links",
                "expected_tool": "web_search",
                "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"],
            },
        ],
    },
    {
        "scenario": "public_domain_art_bare_websites_followup",
        "turns": [
            {
                "message": "What are some good sites for public domain art?",
                "expected_tool": "no_tool",
                "required_any": ["public domain", "met", "wikimedia", "rijksmuseum"],
            },
            {
                "message": "for the websites",
                "expected_tool": "web_search",
                "required_any": ["http", "wikimedia", "metmuseum", "rijksmuseum", "public domain"],
            },
        ],
    },
    {
        "scenario": "email_then_typo_links_tail_clarify",
        "turns": [
            {
                "message": "what is my latest email?",
                "expected_tool": "mcp__email__list_emails",
                "required_any": ["email", "from", "subject", "latest"],
            },
            {
                "message": "sned links for those",
                "expected_tool": "no_tool",
                "required_any": ["which links", "what links", "clarify", "what topic", "which topic", "topic"],
            },
        ],
    },
    {
        "scenario": "email_then_bare_websites_clarify",
        "turns": [
            {
                "message": "what is my latest email?",
                "expected_tool": "mcp__email__list_emails",
                "required_any": ["email", "from", "subject", "latest"],
            },
            {
                "message": "for the websites",
                "expected_tool": "no_tool",
                "required_any": ["which links", "what links", "clarify", "what topic", "which topic", "topic", "website"],
            },
        ],
    },
    {
        "scenario": "notes_then_typo_links_tail_clarify",
        "turns": [
            {
                "message": "what are my notes?",
                "expected_tool": "manage_notes",
                "expected_action": "list",
                "required_any": ["note", "[", "test scenario"],
            },
            {
                "message": "sned links for those",
                "expected_tool": "no_tool",
                "required_any": ["which links", "what links", "clarify", "what topic", "which topic", "topic"],
            },
        ],
    },
    {
        "scenario": "notes_crud_followthrough",
        "fixture_prefix": "ODY-EVAL-CRUD-NOTES-",
        "turns": [
            {
                "message": "Create a note titled ODY-EVAL-CRUD-NOTES-FLOW with content alpha checkpoint.",
                "expected_tool": "manage_notes",
                "expected_actions": ["add", "create"],
                "required_all": ["created", "ody-eval-crud-notes-flow"],
                "max_tool_count": 1,
            },
            {
                "message": "Update that note so its content says beta checkpoint.",
                "expected_tool": "manage_notes",
                "expected_action": "update",
                "required_all": ["updated", "note"],
                "max_tool_count": 1,
            },
            {
                "message": "Delete that note.",
                "expected_tool": "manage_notes",
                "expected_action": "delete",
                "required_all": ["deleted", "note"],
                "max_tool_count": 1,
            },
        ],
    },
    {
        "scenario": "calendar_crud_followthrough",
        "fixture_prefix": "ODY-EVAL-CRUD-CALENDAR-",
        "turns": [
            {
                "message": (
                    "Create a calendar event titled ODY-EVAL-CRUD-CALENDAR-FLOW "
                    "on 2026-08-25 from 10:00 to 10:30 at Test Lab."
                ),
                "expected_tool": "manage_calendar",
                "expected_actions": ["create_event", "create"],
                "required_all": ["created", "event", "ody-eval-crud-calendar-flow"],
                "max_tool_count": 1,
            },
            {
                "message": "Update that calendar event location to Blue Room.",
                "expected_tool": "manage_calendar",
                "expected_actions": ["update_event", "update"],
                "required_all": ["updated", "event"],
                "max_tool_count": 1,
            },
            {
                "message": "Delete that calendar event.",
                "expected_tool": "manage_calendar",
                "expected_actions": ["delete_event", "delete"],
                "required_all": ["deleted", "event"],
                "max_tool_count": 1,
            },
        ],
    },
    {
        "scenario": "memory_crud_followthrough",
        "fixture_prefix": "ODY-EVAL-CRUD-MEMORY-",
        "turns": [
            {
                "message": "Remember this temporary eval fact: ODY-EVAL-CRUD-MEMORY-FLOW alpha checkpoint.",
                "expected_tool": "manage_memory",
                "expected_action": "add",
                "required_all": ["memory", "added"],
                "max_tool_count": 1,
            },
            {
                "message": "Update that memory to say ODY-EVAL-CRUD-MEMORY-FLOW beta checkpoint.",
                "expected_tool": "manage_memory",
                "expected_action": "edit",
                "required_all": ["memory", "updated"],
                "max_tool_count": 1,
            },
            {
                "message": "Delete that memory.",
                "expected_tool": "manage_memory",
                "expected_action": "delete",
                "required_all": ["memory", "deleted"],
                "max_tool_count": 1,
            },
        ],
    },
    {
        "scenario": "memory_add_one_call_efficiency",
        "fixture_prefix": "ODY-EVAL-CRUD-MEMORY-",
        "turns": [
            {
                "message": "Remember this temporary eval fact: ODY-EVAL-CRUD-MEMORY-ONECALL alpha checkpoint.",
                "expected_tool": "manage_memory",
                "expected_action": "add",
                "required_all": ["memory", "added"],
                "max_tool_count": 1,
            },
            {
                "message": "Delete that memory.",
                "expected_tool": "manage_memory",
                "expected_action": "delete",
                "required_all": ["memory", "deleted"],
                "max_tool_count": 1,
            },
        ],
    },
    {
        "scenario": "memory_add_wording_variants_efficiency",
        "fixture_prefix": "ODY-EVAL-CRUD-MEMORY-",
        "turns": [
            {
                "message": "Save this as a memory: ODY-EVAL-CRUD-MEMORY-VAR-A alpha checkpoint.",
                "expected_tool": "manage_memory",
                "expected_action": "add",
                "required_all": ["memory", "added"],
                "max_tool_count": 1,
            },
            {
                "message": "Delete that memory.",
                "expected_tool": "manage_memory",
                "expected_action": "delete",
                "required_all": ["memory", "deleted"],
                "max_tool_count": 1,
            },
            {
                "message": "Add to memory that ODY-EVAL-CRUD-MEMORY-VAR-B beta checkpoint is temporary.",
                "expected_tool": "manage_memory",
                "expected_action": "add",
                "required_all": ["memory", "added"],
                "max_tool_count": 1,
            },
            {
                "message": "Remove that memory.",
                "expected_tool": "manage_memory",
                "expected_action": "delete",
                "required_all": ["memory", "deleted"],
                "max_tool_count": 1,
            },
            {
                "message": "Please remember: ODY-EVAL-CRUD-MEMORY-VAR-C gamma checkpoint.",
                "expected_tool": "manage_memory",
                "expected_action": "add",
                "required_all": ["memory", "added"],
                "max_tool_count": 1,
            },
            {
                "message": "Forget that memory.",
                "expected_tool": "manage_memory",
                "expected_action": "delete",
                "required_all": ["memory", "deleted"],
                "max_tool_count": 1,
            },
        ],
    },
    {
        "scenario": "memory_no_tool_boundary",
        "turns": [
            {
                "message": "do you remember what VAT stands for?",
                "expected_tool": "no_tool",
                "required_any": ["value-added tax", "value added tax"],
            },
            {
                "message": "what should I remember before buying public domain art?",
                "expected_tool": "no_tool",
                "required_any": ["license", "copyright", "public domain", "source"],
            },
            {
                "message": "remind me what Sweden is bordered by",
                "expected_tool": "no_tool",
                "required_any": ["norway", "finland"],
            },
            {
                "message": "what does it mean to remember something in a computer?",
                "expected_tool": "no_tool",
                "required_any": ["store", "storage", "memory", "data", "information"],
            },
        ],
    },
    {
        "scenario": "tasks_crud_followthrough",
        "fixture_prefix": "ODY-EVAL-CRUD-TASKS-",
        "turns": [
            {
                "message": (
                    "Create a scheduled task named ODY-EVAL-CRUD-TASKS-FLOW that runs daily at 09:00 UTC "
                    "and has prompt alpha checkpoint."
                ),
                "expected_tool": "manage_tasks",
                "expected_action": "create",
                "required_all": ["created", "task", "ody-eval-crud-tasks-flow"],
                "max_tool_count": 1,
            },
            {
                "message": "Update that task prompt to beta checkpoint.",
                "expected_tool": "manage_tasks",
                "expected_action": "edit",
                "required_all": ["updated", "task"],
                "max_tool_count": 1,
            },
            {
                "message": "Delete that task.",
                "expected_tool": "manage_tasks",
                "expected_action": "delete",
                "required_all": ["deleted", "task"],
                "max_tool_count": 1,
            },
        ],
    },
    {
        "scenario": "documents_create_delete_followthrough",
        "fixture_prefix": "ODY-EVAL-CRUD-DOCUMENTS-",
        "turns": [
            {
                "message": (
                    "Create an editor document titled ODY-EVAL-CRUD-DOCUMENTS-FLOW "
                    "with markdown content alpha checkpoint."
                ),
                "expected_tool": "create_document",
                "required_all": ["document", "ody-eval-crud-documents-flow"],
                "max_tool_count": 1,
            },
            {
                "message": "Delete that document.",
                "expected_tool": "manage_documents",
                "expected_action": "delete",
                "required_all": ["deleted", "document"],
                "max_tool_count": 1,
            },
        ],
    },
]


TOOL_ALIASES = {
    "mcp_email_list_emails": "mcp__email__list_emails",
    "mcp_email_search_emails": "mcp__email__search_emails",
    "list_emails": "mcp__email__list_emails",
    "search_emails": "mcp__email__search_emails",
}


def malformed_text_surface(response_text: str) -> bool:
    value = (response_text or "").lower()
    if any(
        marker in value
        for marker in (
            "<function",
            "<parameter",
            "function=",
            "parameter=",
            "</parameter",
            '"function"',
            "tool_call",
            "|end|",
            "web search results and fetched content",
            "search results summary:",
        )
    ):
        return True
    return any(
        re.search(pattern, response_text or "")
        for pattern in (
            r"\bIamOdysseus\b",
            r"\bSwedenisin\b",
            r"\bVATstands\b",
            r"\bpublic\s+domain\s+ar\b(?!t)",
            r"\bthe\s+me\s+open\s+access\b",
            r"\bar\s+institute\b(?!t)",
            r"\bpublicdomainar\b",
            r"\bTheMeOpenAccess\b",
            r"\bCanyouclarifywhich\b",
            r"\bwan me\s+link\b",
            r"\bwhat you wan me\b",
        )
    )


def canonical_tool(tool: str | None) -> str | None:
    if not tool:
        return tool
    return TOOL_ALIASES.get(tool, tool)


def parse_action(command: str | None) -> str:
    if not command:
        return ""
    try:
        parsed = json.loads(command)
    except json.JSONDecodeError:
        parsed = command
    if isinstance(parsed, dict):
        return str(parsed.get("action") or "")
    if isinstance(parsed, str):
        return parsed.strip().splitlines()[0] if parsed.strip() else ""
    return ""


def _fixture_owner() -> str:
    return os.environ.get("ODY_EVAL_OWNER", "pewds")


def _cleanup_crud_fixtures() -> None:
    """Remove only eval-owned CRUD artifacts created by this script."""
    owner = _fixture_owner()
    try:
        from core.database import (
            CalendarCal,
            CalendarEvent,
            Document,
            DocumentVersion,
            Note,
            ScheduledTask,
            SessionLocal,
        )
    except Exception as exc:
        print(json.dumps({"cleanup_warning": f"database import failed: {exc!r}"}), flush=True)
    else:
        db = SessionLocal()
        try:
            notes_q = db.query(Note).filter(Note.title.like("ODY-EVAL-CRUD-%"))
            if owner:
                notes_q = notes_q.filter(Note.owner == owner)
            for note in notes_q.all():
                db.delete(note)

            events_q = db.query(CalendarEvent).filter(CalendarEvent.summary.like("ODY-EVAL-CRUD-%"))
            if owner:
                events_q = events_q.join(CalendarCal, CalendarEvent.calendar_id == CalendarCal.id).filter(
                    CalendarCal.owner == owner
                )
            for event in events_q.all():
                db.delete(event)

            cals_q = db.query(CalendarCal).filter(CalendarCal.name.like("ODY-EVAL-CRUD-%"))
            if owner:
                cals_q = cals_q.filter(CalendarCal.owner == owner)
            for calendar in cals_q.all():
                db.delete(calendar)

            docs_q = db.query(Document).filter(Document.title.like("ODY-EVAL-CRUD-%"))
            if owner:
                docs_q = docs_q.filter(Document.owner == owner)
            for doc in docs_q.all():
                db.query(DocumentVersion).filter(DocumentVersion.document_id == doc.id).delete()
                db.delete(doc)

            tasks_q = db.query(ScheduledTask).filter(ScheduledTask.name.like("ODY-EVAL-CRUD-%"))
            if owner:
                tasks_q = tasks_q.filter(ScheduledTask.owner == owner)
            tasks_q.delete(synchronize_session=False)
            db.commit()
        except Exception as exc:
            db.rollback()
            print(json.dumps({"cleanup_warning": repr(exc)}), flush=True)
        finally:
            db.close()

    try:
        from src.constants import MEMORY_FILE
        memory_path = Path(MEMORY_FILE)
        if memory_path.exists():
            entries = json.loads(memory_path.read_text(encoding="utf-8"))
            if isinstance(entries, list):
                filtered = [
                    entry
                    for entry in entries
                    if not (
                        isinstance(entry, dict)
                        and "ODY-EVAL-CRUD-MEMORY-" in str(entry.get("text") or "")
                        and (not owner or entry.get("owner") == owner)
                    )
                ]
                if len(filtered) != len(entries):
                    memory_path.write_text(json.dumps(filtered, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
    except Exception as exc:
        print(json.dumps({"cleanup_warning": f"memory cleanup failed: {exc!r}"}), flush=True)


@contextlib.contextmanager
def _crud_fixture_cleanup(enabled: bool):
    if enabled:
        _cleanup_crud_fixtures()
    try:
        yield
    finally:
        if enabled:
            _cleanup_crud_fixtures()


def output_ok(event: dict[str, Any]) -> bool:
    if event.get("exit_code") not in (0, None):
        return False
    text = str(event.get("output") or "")
    return not text.lstrip().lower().startswith("error")


def event_action(event: dict[str, Any]) -> str:
    return parse_action(str(event.get("command") or ""))


def create_session(client: httpx.Client, args, name: str) -> str:
    response = client.post(
        args.base_url.rstrip("/") + "/api/session",
        data={
            "name": name,
            "endpoint_url": args.selected_endpoint_url or args.endpoint,
            "model": args.selected_model or args.model,
            "skip_validation": "true",
            "rag": "false",
            **({"endpoint_id": args.endpoint_id} if args.endpoint_id else {}),
        },
        timeout=30,
    )
    _raise_for_status_with_body(response)
    return response.json()["id"]


def run_turn(client: httpx.Client, args, session_id: str, spec: dict[str, Any]) -> dict[str, Any]:
    started = time.monotonic()
    events: list[dict[str, Any]] = []
    text: list[str] = []
    errors: list[dict[str, Any]] = []
    approval_turns = 0
    turn_data = {
        "message": spec["message"],
        "session": session_id,
        "mode": "agent",
        "agent_prompt_mode": args.prompt_mode,
        **({"selected_endpoint_id": args.endpoint_id} if args.endpoint_id else {}),
        **({"selected_endpoint_url": args.selected_endpoint_url} if args.selected_endpoint_url else {}),
        **({"selected_model": args.selected_model} if args.selected_model else {}),
    }
    try:
        while True:
            approval = None
            with client.stream(
                "POST",
                args.base_url.rstrip("/") + "/api/chat_stream",
                data=turn_data,
                headers={"Accept": "text/event-stream"},
                timeout=args.timeout,
            ) as response:
                _raise_for_status_with_body(response)
                for event in _sse_events(response):
                    events.append(event)
                    if event.get("type") == "error":
                        errors.append(event)
                    visible = _visible_event_text(event)
                    if visible:
                        if event.get("type") == "final_response":
                            text[:] = [visible]
                        else:
                            text.append(visible)
                    approval = approval or _tool_approval_from_event(event)
            if not args.auto_approve or not approval or approval_turns >= 3:
                break
            approval_turns += 1
            turn_data = {
                **turn_data,
                "tool_approval_id": approval["approval_id"],
                "tool_approval_decision": "approve",
            }
    except Exception as exc:
        errors.append({"type": "client_exception", "error": repr(exc)})

    starts = [event for event in events if event.get("type") == "tool_start"]
    outputs = [event for event in events if event.get("type") == "tool_output"]
    metrics = [
        event.get("data")
        for event in events
        if event.get("type") == "metrics" and isinstance(event.get("data"), dict)
    ]
    snapshots = [
        {
            key: event.get(key)
            for key in (
                "round",
                "model",
                "messages",
                "tools",
                "temperature",
                "max_tokens",
                "agent_prompt_mode",
            )
        }
        for event in events
        if event.get("type") == "model_request_snapshot"
    ]
    metric_tool_events = [
        tool_event
        for metric in metrics
        for tool_event in (metric.get("tool_events") or [])
        if isinstance(tool_event, dict)
    ]
    summarized_tool_events = [
        {
            "tool": canonical_tool(str(event.get("tool") or "")),
            "command": str(event.get("command") or ""),
            "exit_code": event.get("exit_code"),
            "output_preview": str(event.get("output") or "")[:500],
        }
        for event in [*outputs, *metric_tool_events]
        if isinstance(event, dict)
    ]
    observed_events = metric_tool_events or outputs or starts
    first = observed_events[0] if observed_events else {}
    first_tool = canonical_tool(first.get("tool"))
    first_action = parse_action(first.get("command"))
    response_text = "".join(text).strip()
    if not response_text and metrics:
        round_texts = metrics[-1].get("round_texts") or []
        response_text = next((str(item).strip() for item in reversed(round_texts) if str(item).strip()), "")

    expected_tool = spec["expected_tool"]
    expected_action = spec.get("expected_action") or ""
    expected_actions = [str(item) for item in (spec.get("expected_actions") or [])]
    max_tool_count = spec.get("max_tool_count")
    if expected_action and not expected_actions:
        expected_actions = [expected_action]
    if expected_tool == "no_tool":
        tool_ok = not observed_events
        execution_ok = bool(response_text) and not errors
    else:
        tool_ok = first_tool == expected_tool
        executed = [
            event
            for event in [*outputs, *metric_tool_events]
            if canonical_tool(str(event.get("tool") or "")) == expected_tool
            and (not expected_actions or event_action(event) in expected_actions)
            and output_ok(event)
        ]
        execution_ok = bool(executed) and not errors
    action_ok = not expected_actions or first_action in expected_actions
    lower_response = response_text.lower()
    required_any = [str(item).lower() for item in spec.get("required_any") or []]
    required_all = [str(item).lower() for item in spec.get("required_all") or []]
    response_quality_ok = bool(response_text) and (
        not required_any or any(item in lower_response for item in required_any)
    ) and all(item in lower_response for item in required_all)
    if malformed_text_surface(response_text):
        response_quality_ok = False
    tool_efficiency_ok = True
    if isinstance(max_tool_count, int):
        tool_efficiency_ok = len(observed_events) <= max_tool_count

    latest_metrics = metrics[-1] if metrics else {}
    usage_buckets = latest_metrics.get("usage_buckets") if isinstance(latest_metrics, dict) else None
    return {
        "message": spec["message"],
        "expected_tool": expected_tool,
        "expected_action": expected_action,
        "expected_actions": expected_actions,
        "first_tool": first_tool,
        "first_action": first_action,
        "tool_count": len(observed_events),
        "tool_ok": bool(tool_ok),
        "action_ok": bool(action_ok),
        "execution_ok": bool(execution_ok),
        "response_quality_ok": bool(response_quality_ok),
        "tool_efficiency_ok": bool(tool_efficiency_ok),
        "max_tool_count": max_tool_count,
        "stream_errors": errors,
        "response": response_text[:2000],
        "input_tokens": latest_metrics.get("input_tokens"),
        "output_tokens": latest_metrics.get("output_tokens"),
        "tokens_per_second": latest_metrics.get("tokens_per_second"),
        "request_context_tokens": latest_metrics.get("request_context_tokens"),
        "usage_buckets": usage_buckets if isinstance(usage_buckets, list) else [],
        "tool_events": summarized_tool_events,
        "elapsed_seconds": round(time.monotonic() - started, 3),
        "approval_turns": approval_turns,
        "model_request_snapshots": snapshots,
    }


def write_output(path: Path, records: list[dict[str, Any]], model: str) -> None:
    turns = [turn for record in records for turn in record["turns"]]
    summary = {
        "model": model,
        "scenarios": len(records),
        "turns": len(turns),
        "tool_success": sum(turn["tool_ok"] for turn in turns),
        "action_success": sum(turn["action_ok"] for turn in turns),
        "execution_success": sum(turn["execution_ok"] for turn in turns),
        "response_quality_success": sum(turn["response_quality_ok"] for turn in turns),
        "tool_efficiency_success": sum(turn.get("tool_efficiency_ok", True) for turn in turns),
        "stream_errors": sum(bool(turn["stream_errors"]) for turn in turns),
        "records": records,
    }
    tmp = path.with_name(path.name + ".tmp")
    tmp.write_text(json.dumps(summary, indent=2, ensure_ascii=True) + "\n")
    tmp.replace(path)


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--base-url", default="http://127.0.0.1:7011")
    parser.add_argument("--endpoint", default="http://host.docker.internal:18052/v1")
    parser.add_argument("--endpoint-id", default="v8c1000")
    parser.add_argument("--model", default="qwen35-9b-tool-router-v15-regular-chat-boundary-final")
    parser.add_argument("--selected-endpoint-url", default="")
    parser.add_argument("--selected-model", default="")
    parser.add_argument("--cookie-file", default="data/sessions.json")
    parser.add_argument("--output", required=True)
    parser.add_argument("--prompt-mode", default="compact")
    parser.add_argument("--timeout", type=float, default=180.0)
    parser.add_argument("--cases", default="")
    parser.add_argument("--no-auto-approve", dest="auto_approve", action="store_false")
    parser.add_argument("--keep-sessions", action="store_true")
    args = parser.parse_args()

    selected = {item.strip() for item in args.cases.split(",") if item.strip()}
    scenarios = [case for case in SCENARIOS if not selected or case["scenario"] in selected]
    unknown = selected - {case["scenario"] for case in SCENARIOS}
    if unknown:
        raise SystemExit(f"Unknown scenario(s): {', '.join(sorted(unknown))}")

    output = Path(args.output)
    output.parent.mkdir(parents=True, exist_ok=True)
    client = httpx.Client(
        cookies={"odysseus_session": _cookie(Path(args.cookie_file))},
        follow_redirects=False,
    )
    records: list[dict[str, Any]] = []
    try:
        needs_crud_cleanup = any(str(case.get("fixture_prefix") or "").startswith("ODY-EVAL-CRUD-") for case in scenarios)
        with _crud_fixture_cleanup(needs_crud_cleanup):
            for scenario in scenarios:
                session_id = create_session(
                    client,
                    args,
                    "[eval-context] "
                    + scenario["scenario"]
                    + " "
                    + time.strftime("%Y%m%d-%H%M%S")
                    + "-"
                    + uuid.uuid4().hex[:6],
                )
                turns = []
                try:
                    for spec in scenario["turns"]:
                        turn = run_turn(client, args, session_id, spec)
                        turns.append(turn)
                        print(
                            json.dumps(
                                {
                                    "scenario": scenario["scenario"],
                                    **{
                                        key: turn.get(key)
                                        for key in (
                                            "message",
                                            "expected_tool",
                                            "first_tool",
                                            "expected_action",
                                            "expected_actions",
                                            "first_action",
                                            "tool_ok",
                                            "action_ok",
                                            "execution_ok",
                                            "response_quality_ok",
                                            "tool_efficiency_ok",
                                            "max_tool_count",
                                            "tool_count",
                                            "input_tokens",
                                            "output_tokens",
                                            "elapsed_seconds",
                                            "stream_errors",
                                        )
                                    },
                                },
                                ensure_ascii=True,
                            ),
                            flush=True,
                        )
                finally:
                    if args.keep_sessions:
                        print(json.dumps({"kept_session": session_id, "scenario": scenario["scenario"]}), flush=True)
                    else:
                        try:
                            client.delete(args.base_url.rstrip("/") + f"/api/session/{session_id}", timeout=15)
                        except Exception:
                            pass
                records.append({"scenario": scenario["scenario"], "turns": turns})
                write_output(output, records, args.selected_model or args.model)
    finally:
        client.close()
    write_output(output, records, args.selected_model or args.model)
    summary = json.loads(output.read_text())
    print("SUMMARY", json.dumps({k: v for k, v in summary.items() if k != "records"}))


if __name__ == "__main__":
    main()