Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

43.5 KB
Raw Blame
#!/usr/bin/env python3
from __future__ import annotations

import argparse
import importlib.util
import json
import os
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any

from tau2_common import (
    assert_tau2_results_complete,
    domains,
    load_config,
    normalize_litellm_env,
    output_dir,
    resolve_path,
    run_id,
    simulator_policy_report,
    split_file,
    strategy_ids,
    tau2_context,
    tau2_repo,
    user_simulator_policy,
    write_json,
)

TRAIN_TRANSCRIPT_OPENVIKING_TEXT = "openviking_text"
DEFAULT_TRAIN_TOOL_OUTPUT_MAX_CHARS = 5000


def _reward(sim: dict[str, Any]) -> float:
    info = sim.get("reward_info") or {}
    value = info.get("reward", sim.get("reward", 0.0))
    try:
        return float(value)
    except (TypeError, ValueError):
        return 0.0


def _db_match(sim: dict[str, Any]) -> bool | None:
    info = sim.get("reward_info") or {}
    db = info.get("db_check") or {}
    if isinstance(db, dict):
        if "score" in db:
            return bool(db["score"])
        if "db_match" in db:
            return bool(db["db_match"])
    return sim.get("db_match")


def _strategy_int(
    config: dict[str, Any],
    strategy: dict[str, Any],
    key: str,
    *,
    fallback_key: str | None = None,
    default: int = 4,
) -> int:
    openviking = config.get("openviking", {})
    value = strategy.get(key)
    if value is None:
        value = openviking.get(key)
    if value is None and fallback_key:
        value = strategy.get(fallback_key)
    if value is None and fallback_key:
        value = openviking.get(fallback_key)
    if value is None:
        value = default
    return int(value)


def _strategy_optional_int(
    config: dict[str, Any],
    strategy: dict[str, Any],
    key: str,
    *,
    fallback_key: str | None = None,
) -> int | None:
    openviking = config.get("openviking", {})
    value = strategy.get(key)
    if value is None:
        value = openviking.get(key)
    if value is None and fallback_key:
        value = strategy.get(fallback_key)
    if value is None and fallback_key:
        value = openviking.get(fallback_key)
    if value is None:
        return None
    return int(value)


def _enabled(value: Any) -> bool:
    if isinstance(value, bool):
        return value
    if value is None:
        return False
    return str(value).strip().lower() in {"1", "true", "yes", "on"}


def _require_fixed_first_user(config: dict[str, Any]) -> bool:
    return _enabled(config.get("eval", {}).get("require_fixed_first_user"))


def _retrieval_budget(config: dict[str, Any], strategy: dict[str, Any]) -> dict[str, int | None]:
    retrieval_top_k = _strategy_int(config, strategy, "retrieval_top_k", default=4)
    first_user_retrieval_top_k = _strategy_int(
        config,
        strategy,
        "first_user_retrieval_top_k",
        fallback_key="retrieval_top_k",
        default=retrieval_top_k,
    )
    first_user_inject_top_k = _strategy_int(
        config,
        strategy,
        "first_user_inject_top_k",
        fallback_key="first_user_retrieval_top_k",
        default=first_user_retrieval_top_k,
    )
    prewrite_retrieval_top_k = _strategy_int(
        config,
        strategy,
        "prewrite_retrieval_top_k",
        fallback_key="retrieval_top_k",
        default=retrieval_top_k,
    )
    prewrite_inject_top_k = _strategy_int(
        config,
        strategy,
        "prewrite_inject_top_k",
        fallback_key="prewrite_retrieval_top_k",
        default=prewrite_retrieval_top_k,
    )
    memory_inject_max_chars = _strategy_optional_int(
        config,
        strategy,
        "memory_inject_max_chars",
    )
    first_user_memory_inject_max_chars = _strategy_optional_int(
        config,
        strategy,
        "first_user_memory_inject_max_chars",
        fallback_key="memory_inject_max_chars",
    )
    prewrite_memory_inject_max_chars = _strategy_optional_int(
        config,
        strategy,
        "prewrite_memory_inject_max_chars",
        fallback_key="memory_inject_max_chars",
    )
    for key, value in {
        "memory_inject_max_chars": memory_inject_max_chars,
        "first_user_memory_inject_max_chars": first_user_memory_inject_max_chars,
        "prewrite_memory_inject_max_chars": prewrite_memory_inject_max_chars,
    }.items():
        if value is not None and value < 0:
            raise ValueError(f"{strategy['id']} has negative {key}: {value}")
    return {
        "retrieval_top_k": retrieval_top_k,
        "first_user_retrieval_top_k": first_user_retrieval_top_k,
        "first_user_inject_top_k": first_user_inject_top_k,
        "prewrite_retrieval_top_k": prewrite_retrieval_top_k,
        "prewrite_inject_top_k": prewrite_inject_top_k,
        "memory_inject_max_chars": memory_inject_max_chars,
        "first_user_memory_inject_max_chars": first_user_memory_inject_max_chars,
        "prewrite_memory_inject_max_chars": prewrite_memory_inject_max_chars,
    }


def _memory_corpus_key_for(
    *,
    domain: str,
    strategy: dict[str, Any],
    train_num_tasks: int | None,
) -> str:
    corpus_id = str(strategy.get("corpus_id") or strategy["id"])
    raw_key = strategy.get("corpus_cache_key")
    if raw_key:
        key = str(raw_key).format(
            domain=domain,
            strategy_id=strategy["id"],
            corpus_id=corpus_id,
        )
    else:
        key = f"{domain}_{corpus_id}"
    if train_num_tasks is not None:
        key = f"{key}_train{train_num_tasks}"
    return key


def _memory_corpus_dir(config: dict[str, Any], configured_run_id: str, corpus_key: str) -> Path:
    raw = config.get("paths", {}).get("corpus_cache_dir")
    if raw:
        return resolve_path(str(raw)) / corpus_key
    return output_dir(config, configured_run_id) / "memory_corpora" / corpus_key


def _search_uri(search_memory_type: str) -> str:
    return f"viking://~/memories/{search_memory_type}"


def _train_transcript_format(strategy: dict[str, Any]) -> str:
    return str(strategy.get("train_transcript_format") or TRAIN_TRANSCRIPT_OPENVIKING_TEXT)


def _train_tool_output_max_chars(strategy: dict[str, Any]) -> int:
    raw = strategy.get("train_tool_output_max_chars")
    if raw is None:
        return DEFAULT_TRAIN_TOOL_OUTPUT_MAX_CHARS
    return int(raw)


def _train_skip_failed_sessions(strategy: dict[str, Any]) -> bool:
    return _enabled(strategy.get("train_skip_failed_sessions"))


def _manifest_openviking_identity(corpus_dir: Path) -> dict[str, str] | None:
    manifest_path = corpus_dir / "corpus_manifest.json"
    if not manifest_path.is_file():
        return None
    try:
        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
    except json.JSONDecodeError:
        return None
    openviking = manifest.get("openviking") or {}
    required = ("account", "user", "search_uri")
    if not all(openviking.get(key) for key in required):
        return None
    return {key: str(openviking[key]) for key in required}


def _metrics_from_tau2_results(results_path: Path) -> dict[str, Any]:
    data = json.loads(results_path.read_text(encoding="utf-8"))
    assert_tau2_results_complete(data, context=str(results_path))
    sims = data.get("simulations") or []
    rewards = [_reward(sim) for sim in sims]
    db_values = [_db_match(sim) for sim in sims]
    db_known = [value for value in db_values if value is not None]
    return {
        "simulation_count": len(sims),
        "avg_reward": sum(rewards) / len(rewards) if rewards else 0.0,
        "db_match_rate": (sum(1 for value in db_known if value) / len(db_known))
        if db_known
        else None,
    }


def _tau2_command(
    config: dict[str, Any],
    *,
    domain: str,
    strategy: dict[str, Any],
    configured_run_id: str,
    run_label: str,
    task_ids: list[str] | None,
    num_tasks: int | None,
    train_num_tasks: int | None,
    seed: int,
) -> list[str] | None:
    benchmark = config["benchmark"]
    model = config["model"]

    reasoning_effort = benchmark.get("reasoning_effort")
    agent_llm_args = '{"temperature":0.0}'
    user_llm_args = '{"temperature":0.0}'
    if reasoning_effort:
        agent_llm_args = f'{{"temperature":0.0,"reasoning_effort":"{reasoning_effort}"}}'
        user_llm_args = f'{{"temperature":0.0,"reasoning_effort":"{reasoning_effort}"}}'
    fixed_first_user_file = _fixed_first_user_file(config, domain)
    scope_prompt_file = _scope_prompt_file(config, strategy, domain)

    if (
        strategy.get("memory_backend") == "openviking"
        and strategy.get("train_memory_mode") == "experience_only"
    ):
        openviking = config["openviking"]
        corpus_id = str(strategy.get("corpus_id") or strategy["id"])
        resolved_train_num_tasks = (
            train_num_tasks if train_num_tasks is not None else strategy.get("train_num_tasks")
        )
        corpus_key = _memory_corpus_key_for(
            domain=domain,
            strategy=strategy,
            train_num_tasks=resolved_train_num_tasks,
        )
        corpus_dir = _memory_corpus_dir(config, configured_run_id, corpus_key)
        reuse_identity = _manifest_openviking_identity(corpus_dir)
        if reuse_identity is not None:
            account = reuse_identity["account"]
            user = reuse_identity["user"]
            search_uri = ""
        elif openviking.get("reuse_corpus_across_runs", False):
            account = f"{openviking['account']}-{corpus_key}"
            user = f"tau2-{domain}-{corpus_id}"
            search_uri = ""
        else:
            account = f"{openviking['account']}-{configured_run_id}-{domain}-{corpus_id}"
            user = f"tau2-{domain}-{corpus_id}"
            search_uri = ""
        search_memory_type = str(strategy.get("search_memory_type", "experiences"))
        if search_memory_type not in {"experiences", "trajectories"}:
            raise ValueError(
                f"Unsupported search_memory_type for {strategy['id']}: {search_memory_type}"
            )
        if not search_uri:
            search_uri = _search_uri(search_memory_type)
        budget = _retrieval_budget(config, strategy)
        command = [
            sys.executable,
            str(Path(__file__).with_name("run_memory_v2_eval.py")),
            "--tau2-repo",
            str(tau2_repo(config)),
            "--run-dir",
            str(output_dir(config, configured_run_id) / "memory_cells" / run_label),
            "--corpus-dir",
            str(corpus_dir),
            "--run-label",
            run_label,
            "--strategy-id",
            strategy["id"],
            "--domain",
            domain,
            "--train-split-name",
            str(benchmark.get("train_split_name", "train")),
            "--eval-split-name",
            str(benchmark.get("eval_split_name", "test")),
            "--max-steps",
            str(benchmark.get("max_steps", 200)),
            "--max-concurrency",
            str(benchmark.get("task_max_concurrency", 10)),
            "--agent-llm",
            str(model["agent_llm"]),
            "--user-llm",
            str(model["user_llm"]),
            "--agent-llm-args",
            agent_llm_args,
            "--user-llm-args",
            user_llm_args,
            "--openviking-url",
            str(openviking["url"]),
            "--openviking-account",
            account,
            "--openviking-user",
            user,
            "--search-uri",
            search_uri,
            "--retrieval-top-k",
            str(budget["retrieval_top_k"]),
            "--first-user-retrieval-top-k",
            str(budget["first_user_retrieval_top_k"]),
            "--first-user-inject-top-k",
            str(budget["first_user_inject_top_k"]),
            "--prewrite-retrieval-top-k",
            str(budget["prewrite_retrieval_top_k"]),
            "--prewrite-inject-top-k",
            str(budget["prewrite_inject_top_k"]),
            "--retrieval-mode",
            str(strategy.get("retrieval_mode", "first_user")),
            "--train-transcript-format",
            _train_transcript_format(strategy),
            "--train-tool-output-max-chars",
            str(_train_tool_output_max_chars(strategy)),
            "--seed",
            str(seed),
        ]
        if budget["memory_inject_max_chars"] is not None:
            command.extend(["--memory-inject-max-chars", str(budget["memory_inject_max_chars"])])
        if budget["first_user_memory_inject_max_chars"] is not None:
            command.extend(
                [
                    "--first-user-memory-inject-max-chars",
                    str(budget["first_user_memory_inject_max_chars"]),
                ]
            )
        if budget["prewrite_memory_inject_max_chars"] is not None:
            command.extend(
                [
                    "--prewrite-memory-inject-max-chars",
                    str(budget["prewrite_memory_inject_max_chars"]),
                ]
            )
        if _enabled(strategy.get("train_include_system_prompt")):
            command.append("--train-include-system-prompt")
        if _train_skip_failed_sessions(strategy):
            command.append("--train-skip-failed-sessions")
        if fixed_first_user_file is not None:
            command.extend(["--fixed-first-user-file", str(fixed_first_user_file)])
        if scope_prompt_file is not None:
            command.extend(["--scope-prompt-file", str(scope_prompt_file)])
        if task_ids:
            for task_id in task_ids:
                command.extend(["--task-id", task_id])
        elif num_tasks is not None:
            command.extend(["--num-tasks", str(num_tasks)])
        if resolved_train_num_tasks is not None:
            command.extend(["--train-num-tasks", str(resolved_train_num_tasks)])
        return command

    if strategy.get("memory_backend") != "none":
        return None

    command = [
        sys.executable,
        str(Path(__file__).with_name("run_memory_v2_eval.py")),
        "--tau2-repo",
        str(tau2_repo(config)),
        "--run-dir",
        str(output_dir(config, configured_run_id) / "memory_cells" / run_label),
        "--run-label",
        run_label,
        "--strategy-id",
        strategy["id"],
        "--domain",
        domain,
        "--eval-split-name",
        str(benchmark.get("eval_split_name", "test")),
        "--max-steps",
        str(benchmark.get("max_steps", 200)),
        "--max-concurrency",
        str(benchmark.get("task_max_concurrency", 10)),
        "--base-agent",
        str(benchmark.get("agent", "llm_agent")),
        "--user",
        str(benchmark.get("user", "user_simulator")),
        "--agent-llm",
        str(model["agent_llm"]),
        "--user-llm",
        str(model["user_llm"]),
        "--agent-llm-args",
        agent_llm_args,
        "--user-llm-args",
        user_llm_args,
        "--seed",
        str(seed),
        "--no-memory",
    ]
    if fixed_first_user_file is not None:
        command.extend(["--fixed-first-user-file", str(fixed_first_user_file)])
    if scope_prompt_file is not None:
        command.extend(["--scope-prompt-file", str(scope_prompt_file)])

    if task_ids:
        for task_id in task_ids:
            command.extend(["--task-id", task_id])
    elif num_tasks is not None:
        command.extend(["--num-tasks", str(num_tasks)])

    return command


def _fixed_first_user_file(config: dict[str, Any], domain: str) -> Path | None:
    raw = config.get("eval", {}).get("fixed_first_user_fixture")
    if raw is None:
        raw = config.get("eval", {}).get("fixed_first_user_fixtures")
    if isinstance(raw, dict):
        raw = raw.get(domain) or raw.get("default")
    if raw is None or str(raw).strip() == "":
        return None
    return resolve_path(str(raw))


def _scope_prompt_file(
    config: dict[str, Any], strategy: dict[str, Any], domain: str
) -> Path | None:
    raw = strategy.get("scope_prompt_file")
    if raw is None:
        raw = strategy.get("scope_prompt_files")
    if raw is None:
        raw = config.get("openviking", {}).get("scope_prompt_file")
    if raw is None:
        raw = config.get("openviking", {}).get("scope_prompt_files")
    if isinstance(raw, dict):
        raw = raw.get(domain) or raw.get("default")
    if raw is None or str(raw).strip() == "":
        return None
    return resolve_path(str(raw))


def _build_plan(
    config: dict[str, Any],
    configured_run_id: str,
    *,
    selected_domains: set[str] | None,
    selected_strategy_ids: set[str] | None,
    task_ids: list[str] | None,
    num_tasks: int | None,
    train_num_tasks: int | None,
    repeat_count_override: int | None,
    cell_concurrency_override: int | None,
    strategy_concurrency_override: int | None,
) -> dict[str, Any]:
    repeat_count = repeat_count_override or int(config["benchmark"].get("repeat_count", 8))
    base_seed = int(config["benchmark"].get("seed", 300))
    cell_timeout_seconds = int(config["benchmark"].get("cell_timeout_seconds", 0) or 0)
    strategy_concurrency = strategy_concurrency_override
    if strategy_concurrency is None:
        strategy_concurrency = cell_concurrency_override
    if strategy_concurrency is None:
        strategy_concurrency = config["benchmark"].get("strategy_concurrency")
    if strategy_concurrency is None:
        strategy_concurrency = config["benchmark"].get("cell_concurrency", 1)
    strategy_concurrency = max(1, int(strategy_concurrency or 1))
    policy_report = simulator_policy_report(config)
    require_fixed_first_user = _require_fixed_first_user(config)
    strategies = config.get("strategies") or []
    if selected_strategy_ids:
        unknown = selected_strategy_ids - set(strategy_ids(config))
        if unknown:
            raise ValueError(f"unknown strategy ids: {sorted(unknown)}")
        strategies = [
            strategy for strategy in strategies if strategy["id"] in selected_strategy_ids
        ]
    cells = []
    plan_domains = domains(config)
    if selected_domains:
        unknown_domains = selected_domains - set(plan_domains)
        if unknown_domains:
            raise ValueError(f"unknown domains: {sorted(unknown_domains)}")
        plan_domains = [domain for domain in plan_domains if domain in selected_domains]
    if require_fixed_first_user:
        missing_domains = [
            domain for domain in plan_domains if _fixed_first_user_file(config, domain) is None
        ]
        if missing_domains:
            protocol = config.get("eval", {}).get("protocol", "fixed_first_user_full8")
            raise ValueError(
                f"eval protocol {protocol!r} requires fixed-first-user fixtures for "
                f"{missing_domains}; set TAU2_RETAIL_FIXED_FIRST_USER_FILE and "
                "TAU2_AIRLINE_FIXED_FIRST_USER_FILE, or use a config that explicitly "
                "sets eval.require_fixed_first_user=false"
            )
    for domain in plan_domains:
        split_path = split_file(config, domain)
        for strategy in strategies:
            for repeat_index in range(repeat_count):
                seed = base_seed + repeat_index
                run_label = f"{configured_run_id}_{domain}_{strategy['id']}_r{repeat_index + 1}"
                command = _tau2_command(
                    config,
                    domain=domain,
                    strategy=strategy,
                    configured_run_id=configured_run_id,
                    run_label=run_label,
                    task_ids=task_ids,
                    num_tasks=num_tasks,
                    train_num_tasks=train_num_tasks,
                    seed=seed,
                )
                fixed_first_user_file = _fixed_first_user_file(config, domain)
                scope_prompt_file = _scope_prompt_file(config, strategy, domain)
                non_executable_reason = None
                if command is None:
                    non_executable_reason = (
                        "This OpenViking memory strategy is planned but not wired to "
                        "the TAU-2 adapter in this PR."
                    )
                cells.append(
                    {
                        "domain": domain,
                        "strategy_id": strategy["id"],
                        "strategy_label": strategy.get("label", strategy["id"]),
                        "repeat_index": repeat_index + 1,
                        "seed": seed,
                        "run_label": run_label,
                        "train_required": bool(strategy.get("train_required")),
                        "memory_backend": strategy.get("memory_backend"),
                        "corpus_id": strategy.get("corpus_id", strategy["id"]),
                        "corpus_key": _memory_corpus_key_for(
                            domain=domain,
                            strategy=strategy,
                            train_num_tasks=(
                                train_num_tasks
                                if train_num_tasks is not None
                                else strategy.get("train_num_tasks")
                            ),
                        ),
                        "corpus_dir": str(
                            _memory_corpus_dir(
                                config,
                                configured_run_id,
                                _memory_corpus_key_for(
                                    domain=domain,
                                    strategy=strategy,
                                    train_num_tasks=(
                                        train_num_tasks
                                        if train_num_tasks is not None
                                        else strategy.get("train_num_tasks")
                                    ),
                                ),
                            )
                        ),
                        "retrieval_mode": strategy.get("retrieval_mode"),
                        "train_transcript_format": _train_transcript_format(strategy),
                        "train_include_system_prompt": _enabled(
                            strategy.get("train_include_system_prompt")
                        ),
                        "train_skip_failed_sessions": _train_skip_failed_sessions(strategy),
                        "train_tool_output_max_chars": _train_tool_output_max_chars(strategy),
                        "retrieval_budget": _retrieval_budget(config, strategy),
                        "search_memory_type": strategy.get("search_memory_type", "experiences"),
                        "adapter_status": strategy.get("adapter_status", "ready"),
                        "executable": command is not None,
                        "user_simulator_policy": user_simulator_policy(config),
                        "user_simulator_policy_supported": policy_report["supported"],
                        "fixed_first_user_file": str(fixed_first_user_file)
                        if fixed_first_user_file
                        else None,
                        "scope_prompt_file": str(scope_prompt_file) if scope_prompt_file else None,
                        "split_file": str(split_path),
                        "command": command,
                        "non_executable_reason": non_executable_reason,
                    }
                )
    executable_cell_count = sum(1 for cell in cells if cell["executable"])
    return {
        "schema_version": "openviking.tau2.run_plan.v0",
        "run_id": configured_run_id,
        "status": "planned",
        "strategy_ids": strategy_ids(config),
        "domains": plan_domains,
        "tau2": tau2_context(config),
        "eval_protocol": config.get("eval", {}).get("protocol"),
        "require_fixed_first_user": require_fixed_first_user,
        "simulator_policy": policy_report,
        "cell_count": len(cells),
        "executable_cell_count": executable_cell_count,
        "pending_cell_count": len(cells) - executable_cell_count,
        "corpus_prepare_concurrency": int(config["benchmark"].get("corpus_prepare_concurrency", 1)),
        "strategy_concurrency": strategy_concurrency,
        "cell_concurrency": strategy_concurrency,
        "cell_timeout_seconds": cell_timeout_seconds or None,
        "cells": cells,
    }


def _cell_artifacts(cell: dict[str, Any], repo: Path, out: Path) -> dict[str, str]:
    if cell.get("memory_backend") in {"openviking", "none"}:
        run_dir = out / "memory_cells" / cell["run_label"]
        artifacts = {
            "summary": str(run_dir / f"{cell['run_label']}.summary.json"),
            "results": str(run_dir / f"{cell['run_label']}.json"),
        }
        if cell.get("memory_backend") == "none":
            return artifacts
        corpus_dir = Path(cell["corpus_dir"])
        artifacts["retrieval_trace"] = str(run_dir / f"{cell['run_label']}.retrieval_trace.jsonl")
        artifacts["corpus_manifest"] = str(corpus_dir / "corpus_manifest.json")
        return artifacts
    return {"results": str(repo / "data" / "simulations" / f"{cell['run_label']}.json")}


def _cell_metrics(cell: dict[str, Any], artifacts: dict[str, str]) -> dict[str, Any] | None:
    summary_path = Path(artifacts.get("summary", ""))
    if summary_path.is_file():
        summary = json.loads(summary_path.read_text(encoding="utf-8"))
        return summary.get("metrics")

    results_path = Path(artifacts["results"])
    if not results_path.is_file():
        return None
    return _metrics_from_tau2_results(results_path)


def _memory_corpus_key(cell: dict[str, Any]) -> str:
    return str(cell.get("corpus_key") or f"{cell['domain']}_{cell['corpus_id']}")


def _tau2_subprocess_env(repo: Path) -> dict[str, str]:
    env = os.environ.copy()
    src = repo / "src"
    pythonpath_entry = str(src if src.is_dir() else repo)
    existing = env.get("PYTHONPATH")
    env["PYTHONPATH"] = (
        pythonpath_entry if not existing else f"{pythonpath_entry}{os.pathsep}{existing}"
    )
    return env


def _prepare_memory_corpus(cell: dict[str, Any], repo: Path, out: Path) -> dict[str, Any]:
    key = _memory_corpus_key(cell)
    manifest_path = Path(cell["corpus_dir"]) / "corpus_manifest.json"
    if manifest_path.is_file():
        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
        cached_transcript_format = str(
            manifest.get("train_transcript_format") or TRAIN_TRANSCRIPT_OPENVIKING_TEXT
        )
        requested_transcript_format = str(
            cell.get("train_transcript_format") or TRAIN_TRANSCRIPT_OPENVIKING_TEXT
        )
        if cached_transcript_format != requested_transcript_format:
            raise RuntimeError(
                "cached corpus train_transcript_format mismatch for "
                f"{key}: {cached_transcript_format!r} != {requested_transcript_format!r}; "
                "use a distinct corpus_id or rebuild the corpus"
            )
        cached_include_system_prompt = bool(manifest.get("train_include_system_prompt") or False)
        requested_include_system_prompt = bool(cell.get("train_include_system_prompt") or False)
        if cached_include_system_prompt != requested_include_system_prompt:
            raise RuntimeError(
                "cached corpus train_include_system_prompt mismatch for "
                f"{key}: {cached_include_system_prompt!r} != {requested_include_system_prompt!r}; "
                "use a distinct corpus_id or rebuild the corpus"
            )
        cached_tool_output_max_chars = int(
            manifest.get("train_tool_output_max_chars") or DEFAULT_TRAIN_TOOL_OUTPUT_MAX_CHARS
        )
        requested_tool_output_max_chars = int(
            cell.get("train_tool_output_max_chars") or DEFAULT_TRAIN_TOOL_OUTPUT_MAX_CHARS
        )
        if cached_tool_output_max_chars != requested_tool_output_max_chars:
            raise RuntimeError(
                "cached corpus train_tool_output_max_chars mismatch for "
                f"{key}: {cached_tool_output_max_chars!r} != {requested_tool_output_max_chars!r}; "
                "use a distinct corpus_id or rebuild the corpus"
            )
        cached_skip_failed = bool(manifest.get("train_skip_failed_sessions") or False)
        requested_skip_failed = bool(cell.get("train_skip_failed_sessions") or False)
        if cached_skip_failed != requested_skip_failed:
            raise RuntimeError(
                "cached corpus train_skip_failed_sessions mismatch for "
                f"{key}: {cached_skip_failed!r} != {requested_skip_failed!r}; "
                "use a distinct corpus_id or rebuild the corpus"
            )
        row = {
            "domain": cell["domain"],
            "strategy_id": cell["strategy_id"],
            "corpus_id": str(cell.get("corpus_id") or cell["strategy_id"]),
            "corpus_key": key,
            "returncode": 0,
            "reused": True,
            "artifacts": {"corpus_manifest": str(manifest_path)},
        }
        write_json(out / "corpus_prepare_results" / f"{key}.json", row)
        print(f"[tau2] reusing corpus {key}", flush=True)
        return row
    command = list(cell["command"]) + ["--prepare-corpus-only"]
    print(f"[tau2] preparing corpus {key}", flush=True)
    completed = subprocess.run(
        command,
        cwd=repo,
        env=_tau2_subprocess_env(repo),
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        check=False,
    )
    row = {
        "domain": cell["domain"],
        "strategy_id": cell["strategy_id"],
        "corpus_id": str(cell.get("corpus_id") or cell["strategy_id"]),
        "corpus_key": key,
        "returncode": completed.returncode,
        "stdout_tail": completed.stdout[-4000:],
        "stderr_tail": completed.stderr[-4000:],
        "artifacts": {"corpus_manifest": str(Path(cell["corpus_dir"]) / "corpus_manifest.json")},
    }
    write_json(out / "corpus_prepare_results" / f"{key}.json", row)
    if completed.returncode != 0:
        raise RuntimeError(f"corpus prepare failed: {key} returncode={completed.returncode}")
    return row


def _prepare_memory_corpora(plan: dict[str, Any], repo: Path, out: Path) -> list[dict[str, Any]]:
    corpus_cells: dict[str, dict[str, Any]] = {}
    for cell in plan["cells"]:
        if cell.get("memory_backend") != "openviking" or not cell.get("train_required"):
            continue
        corpus_cells.setdefault(_memory_corpus_key(cell), cell)
    if not corpus_cells:
        return []

    worker_count = max(1, int(plan.get("corpus_prepare_concurrency") or 1))
    if worker_count == 1 or len(corpus_cells) == 1:
        return [_prepare_memory_corpus(cell, repo, out) for cell in corpus_cells.values()]

    rows: list[dict[str, Any]] = []
    with ThreadPoolExecutor(max_workers=worker_count) as executor:
        futures = {
            executor.submit(_prepare_memory_corpus, cell, repo, out): key
            for key, cell in corpus_cells.items()
        }
        for future in as_completed(futures):
            rows.append(future.result())
    return rows


def _summarize(rows: list[dict[str, Any]]) -> dict[str, Any]:
    def weighted(rows_for_group: list[dict[str, Any]]) -> dict[str, Any]:
        metric_rows = [row for row in rows_for_group if row.get("metrics")]
        sim_count = sum(int(row["metrics"].get("simulation_count") or 0) for row in metric_rows)
        reward_sum = sum(
            float(row["metrics"].get("avg_reward") or 0.0)
            * int(row["metrics"].get("simulation_count") or 0)
            for row in metric_rows
        )
        db_weighted_rows = [
            row
            for row in metric_rows
            if row["metrics"].get("db_match_rate") is not None
            and int(row["metrics"].get("simulation_count") or 0) > 0
        ]
        db_weight = sum(
            int(row["metrics"].get("simulation_count") or 0) for row in db_weighted_rows
        )
        db_sum = sum(
            float(row["metrics"]["db_match_rate"])
            * int(row["metrics"].get("simulation_count") or 0)
            for row in db_weighted_rows
        )
        return {
            "cell_count": len(rows_for_group),
            "completed_cell_count": len(metric_rows),
            "simulation_count": sim_count,
            "avg_reward": reward_sum / sim_count if sim_count else None,
            "db_match_rate": db_sum / db_weight if db_weight else None,
        }

    by_strategy: dict[str, dict[str, Any]] = {}
    for row in rows:
        strategy_id = row["strategy_id"]
        strategy_summary = by_strategy.setdefault(
            strategy_id,
            {
                "strategy_id": strategy_id,
                "domains": {},
                "task_weighted_total": {},
            },
        )
        strategy_summary["domains"].setdefault(row["domain"], []).append(row)

    for strategy_summary in by_strategy.values():
        all_rows = []
        for domain, domain_rows in list(strategy_summary["domains"].items()):
            strategy_summary["domains"][domain] = weighted(domain_rows)
            all_rows.extend(domain_rows)
        strategy_summary["task_weighted_total"] = weighted(all_rows)

    return {
        "schema_version": "openviking.tau2.scoreboard.v0",
        "strategies": by_strategy,
    }


def _execute_cell(
    cell: dict[str, Any], repo: Path, out: Path, cell_timeout: int | None
) -> dict[str, Any]:
    cell_result_path = out / "cell_results" / f"{cell['run_label']}.json"
    if cell_result_path.is_file():
        existing_row = json.loads(cell_result_path.read_text(encoding="utf-8"))
        if existing_row.get("returncode") == 0 and existing_row.get("metrics"):
            print(f"[tau2] skipping completed {cell['run_label']}", flush=True)
            return existing_row

    print(f"[tau2] running {cell['run_label']}", flush=True)
    try:
        completed = subprocess.run(
            cell["command"],
            cwd=repo,
            env=_tau2_subprocess_env(repo),
            text=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            check=False,
            timeout=cell_timeout,
        )
    except subprocess.TimeoutExpired as exc:
        stdout = exc.stdout or ""
        stderr = exc.stderr or ""
        if isinstance(stdout, bytes):
            stdout = stdout.decode(errors="replace")
        if isinstance(stderr, bytes):
            stderr = stderr.decode(errors="replace")
        row = {
            "run_label": cell["run_label"],
            "domain": cell["domain"],
            "strategy_id": cell["strategy_id"],
            "returncode": 124,
            "timed_out": True,
            "timeout_seconds": cell_timeout,
            "stdout_tail": stdout[-4000:],
            "stderr_tail": stderr[-4000:],
            "artifacts": _cell_artifacts(cell, repo, out),
            "metrics": None,
        }
        write_json(cell_result_path, row)
        return row

    row = {
        "run_label": cell["run_label"],
        "domain": cell["domain"],
        "strategy_id": cell["strategy_id"],
        "returncode": completed.returncode,
        "stdout_tail": completed.stdout[-4000:],
        "stderr_tail": completed.stderr[-4000:],
    }
    row["artifacts"] = _cell_artifacts(cell, repo, out)
    row["metrics"] = _cell_metrics(cell, row["artifacts"])
    write_json(cell_result_path, row)
    return row


def _execute_cells(plan: dict[str, Any], repo: Path, out: Path) -> list[dict[str, Any]]:
    policy_report = plan.get("simulator_policy") or {}
    if not policy_report.get("supported", False):
        raise RuntimeError(
            "configured user simulator policy is not supported by this TAU-2 checkout: "
            f"{policy_report}"
        )
    _prepare_memory_corpora(plan, repo, out)
    cells = []
    for cell in plan["cells"]:
        if not cell.get("executable"):
            raise RuntimeError(
                f"cell is not executable yet: {cell['run_label']} "
                f"(strategy_id={cell['strategy_id']}, adapter_status={cell.get('adapter_status')})"
            )
        cells.append(cell)

    cell_timeout = int(plan.get("cell_timeout_seconds") or 0) or None
    worker_count = max(
        1, int(plan.get("strategy_concurrency") or plan.get("cell_concurrency") or 1)
    )
    if worker_count == 1 or len(cells) == 1:
        return [_execute_cell(cell, repo, out, cell_timeout) for cell in cells]

    print(f"[tau2] running eval cells with concurrency={worker_count}", flush=True)
    rows: list[dict[str, Any]] = []
    with ThreadPoolExecutor(max_workers=worker_count) as executor:
        futures = {
            executor.submit(_execute_cell, cell, repo, out, cell_timeout): cell for cell in cells
        }
        for future in as_completed(futures):
            rows.append(future.result())
    return rows


def _execution_failures(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
    return [
        row
        for row in rows
        if row.get("returncode") != 0 or row.get("timed_out") or not row.get("metrics")
    ]


def _preflight(config: dict[str, Any], out: Path, *, strict: bool) -> int:
    errors: list[str] = []
    llm_env = normalize_litellm_env()
    tau2_info = tau2_context(config)
    policy_report = simulator_policy_report(config)
    if strict and not tau2_info["tau2_repo_exists"]:
        errors.append(f"missing TAU-2 repo: {tau2_info['tau2_repo']}")
    if strict and not tau2_info["tau2_cli_resolved"]:
        errors.append(f"missing TAU-2 CLI: {tau2_info['tau2_cli']}")
    if strict and not llm_env["has_api_key"]:
        errors.append("missing LLM API key: set OPENAI_API_KEY or ARK_API_KEY")
    if strict and not llm_env["has_base_url"]:
        errors.append(
            "missing OpenAI-compatible base URL: set OPENAI_API_BASE, OPENAI_BASE_URL, or ARK_BASE_URL"
        )
    if strict and not policy_report["supported"]:
        errors.append(
            "configured confirmation-aware user simulator policy requires a TAU-2 "
            f"checkout with the prompt fix: {policy_report['prompt_files']}"
        )
    split_rows = []
    fixture_rows = []
    for domain in domains(config):
        path = split_file(config, domain)
        exists = path.is_file()
        split_rows.append({"domain": domain, "path": str(path), "exists": exists})
        if strict and not exists:
            errors.append(f"missing split file for {domain}: {path}")
        fixed_first_user_file = _fixed_first_user_file(config, domain)
        fixture_exists = fixed_first_user_file.is_file() if fixed_first_user_file else False
        fixture_rows.append(
            {
                "domain": domain,
                "path": str(fixed_first_user_file) if fixed_first_user_file else None,
                "exists": fixture_exists,
            }
        )
        if strict and _require_fixed_first_user(config) and not fixture_exists:
            errors.append(
                "missing fixed-first-user fixture for "
                f"{domain}: set TAU2_{domain.upper()}_FIXED_FIRST_USER_FILE"
            )

    import_rows = []
    for module in ("openviking", "openviking_cli", "tau2"):
        ok = importlib.util.find_spec(module) is not None
        import_rows.append({"module": module, "ok": ok})
        if strict and not ok:
            errors.append(f"missing Python module: {module}")

    report = {
        "status": "failed" if errors else "ok",
        "strict": strict,
        "tau2": tau2_info,
        "eval_protocol": config.get("eval", {}).get("protocol"),
        "require_fixed_first_user": _require_fixed_first_user(config),
        "llm_env": llm_env,
        "simulator_policy": policy_report,
        "domains": domains(config),
        "strategies": strategy_ids(config),
        "imports": import_rows,
        "split_files": split_rows,
        "fixed_first_user_fixtures": fixture_rows,
        "errors": errors,
    }
    write_json(out / "preflight.json", report)
    if errors:
        for error in errors:
            print(f"[preflight][ERROR] {error}", file=sys.stderr)
        return 1
    print(f"[preflight][OK] wrote {out / 'preflight.json'}")
    return 0


def main() -> int:
    parser = argparse.ArgumentParser(description="Plan or run TAU-2 benchmark cells.")
    parser.add_argument(
        "--config", type=Path, default=Path(__file__).parents[1] / "config" / "no_memory.yaml"
    )
    parser.add_argument("--run-id", default=run_id())
    parser.add_argument(
        "--domain", action="append", help="Run only this configured domain; may be repeated."
    )
    parser.add_argument(
        "--repeat-count", type=int, help="Override benchmark.repeat_count for smoke runs."
    )
    parser.add_argument(
        "--cell-concurrency",
        type=int,
        help="Deprecated alias for --strategy-concurrency.",
    )
    parser.add_argument(
        "--strategy-concurrency",
        type=int,
        help="Override benchmark.strategy_concurrency for parallel matrix cells.",
    )
    parser.add_argument(
        "--strategy-id", action="append", help="Run only this strategy id; may be repeated."
    )
    parser.add_argument(
        "--task-id", action="append", help="Run only this TAU-2 task id; may be repeated."
    )
    parser.add_argument(
        "--num-tasks", type=int, help="Run the first N tasks from the selected split."
    )
    parser.add_argument(
        "--train-num-tasks", type=int, help="Train OpenViking memory on the first N train tasks."
    )
    parser.add_argument(
        "--preflight",
        action="store_true",
        help="Write a lightweight environment/config preflight report.",
    )
    parser.add_argument(
        "--strict-preflight",
        action="store_true",
        help="Fail if optional runtime imports or split files are missing.",
    )
    parser.add_argument("--plan-only", action="store_true", help="Only write run_plan.json.")
    parser.add_argument("--execute", action="store_true", help="Execute planned cells.")
    args = parser.parse_args()
    normalize_litellm_env()

    if args.plan_only and args.execute:
        raise SystemExit("--plan-only and --execute are mutually exclusive")
    if args.cell_concurrency is not None and args.cell_concurrency < 1:
        raise SystemExit("--cell-concurrency must be >= 1")
    if args.strategy_concurrency is not None and args.strategy_concurrency < 1:
        raise SystemExit("--strategy-concurrency must be >= 1")

    config = load_config(args.config)
    out = output_dir(config, args.run_id)
    out.mkdir(parents=True, exist_ok=True)
    if args.preflight or args.strict_preflight:
        preflight_status = _preflight(config, out, strict=args.strict_preflight)
        if args.strict_preflight and preflight_status != 0:
            return preflight_status

    plan = _build_plan(
        config,
        args.run_id,
        selected_domains=set(args.domain) if args.domain else None,
        selected_strategy_ids=set(args.strategy_id) if args.strategy_id else None,
        task_ids=args.task_id,
        num_tasks=args.num_tasks,
        train_num_tasks=args.train_num_tasks,
        repeat_count_override=args.repeat_count,
        cell_concurrency_override=args.cell_concurrency,
        strategy_concurrency_override=args.strategy_concurrency,
    )
    write_json(out / "run_plan.json", plan)
    write_json(out / "resolved_config.json", config)
    print(f"[tau2] wrote {out / 'run_plan.json'}")

    if args.execute:
        try:
            rows = _execute_cells(plan, tau2_repo(config), out)
            failures = _execution_failures(rows)
            plan["status"] = "failed" if failures else "succeeded"
            plan["executed_cell_count"] = len(rows)
            plan["failed_cell_count"] = len(failures)
            write_json(out / "run_plan.json", plan)
            write_json(out / "scoreboard.json", _summarize(rows))
            if failures:
                labels = ", ".join(str(row.get("run_label")) for row in failures[:5])
                raise RuntimeError(f"{len(failures)} cell(s) failed or incomplete: {labels}")
        except Exception as exc:
            plan["status"] = "failed"
            plan["error"] = str(exc)
            write_json(out / "run_plan.json", plan)
            print(f"[tau2][ERROR] {exc}", file=sys.stderr)
            return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())