Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
36.2 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""summarize_search_baseline.py — search baseline summary from Harbor trial dirs.

Walks one or more Harbor baseline output dirs (the dirs `scripts/run-baselines.sh`
writes against a search-task suite, e.g. the extracted `search-v2-accflag-*`
tarballs) and emits markdown tables per (agent, model):

  1. Reward — headline reward + 4 search subscores. Missing trials are
     counted as `reward=0` across all metrics (same infra-fail accounting
     as `summarize_baseline.py`).
  2. Score variants — per-rule comparison (strict F1 / lenient F1 / IoU /
     recall flavors) for the same trial corpus.
  3. Efficiency — mean turns / tools / tokens / wall-clock per task. Means
     are taken across non-missing trials only (zero-filling efficiency
     metrics for infra-fails would be misleading); the `n` column shows
     how many trials contributed.

Multi-trial input: when a single (agent, model) ran the same task more than
once (e.g. variance-study runs), all trials are kept and aggregated. Per-task
mean is taken first, then aggregated to the cohort. The `±σ` column on the
reward shows mean within-task stdev (= "if you reran the same model on the
same cohort, how much would the headline number bounce?"). Cells where every
task has only one trial (k=1) report `±0.000`.

Default cohort = `references/search-tasks.txt` (resolved relative to this
script's parent directory). Denominator is fixed by the cohort size; any task
not seen for a model counts as an infra fail.

Override with `--cohort-tasks-file PATH`, `--exclude TASKID,...`, or
`--no-cohort-check` to disable cohort filling and use trials-seen as the
denominator.

Multi-trial input: when more than one trial per (agent, model, task) is found
(e.g. a model rerun against the same cohort for variance analysis), all trials
are aggregated. Per-task mean is computed first, then aggregated to the
cohort. The reward column shows `mean ± σ` where σ is mean within-task
stdev — the rerun-jitter you'd expect from a fresh run.

Runs split across multiple job-dirs that don't overlap on tasks
(e.g. opus47-high + opus47-high-resume) merge cleanly with σ=0.

Model identity comes from `run_manifest.json` adjacent to each job-dir
(`agent.name` + `agent.model` + `reasoning.effort`). Falls back to a trial's
`result.json::config.agent.{name,model_name}` if no manifest is present.

Usage:
  uv run python scripts/summarize_search_baseline.py <root>... \\
      [--alias JOB_DIR=LABEL]... [--cohort-tasks-file PATH] \\
      [--exclude TASKID,...] [--no-cohort-check]

Examples:
  # Default — pin to canonical cohort, missing tasks counted as infra-fail
  uv run python scripts/summarize_search_baseline.py ~/Downloads/search-v2-accflag-*/

  # Merge a split run (opus47 ran in two batches)
  uv run python scripts/summarize_search_baseline.py \\
      ~/Downloads/search-v2-accflag-opus47-high \\
      ~/Downloads/search-v2-accflag-opus47-high-resume

  # Drop a flaky task ad-hoc
  uv run python scripts/summarize_search_baseline.py <root>... \\
      --exclude craft-tunix-c-066ae850

  # No cohort filling — denominator = trials seen
  uv run python scripts/summarize_search_baseline.py <root>... --no-cohort-check
"""

from __future__ import annotations

import argparse
import csv
import json
import sys
from collections import defaultdict
from datetime import datetime
from pathlib import Path

DEFAULT_COHORT_FILE = Path(__file__).resolve().parent.parent / "references" / "search-tasks.txt"

# Search reward.json subscores reported alongside the headline reward.
#
# As of the Apr 2026 verifier with strict-recall + lenient-precision +
# Jaccard IoU support:
#   navigation_score        = 0.5·file_f1 + 0.5·func_f1, where each F1 is
#                             strict-recall + lenient-precision.
#   navigation_score_lenient = same, but with alts crediting recall (capped).
#   navigation_recall       = pure 0.5·file_recall + 0.5·func_recall (strict).
#   navigation_recall_lenient = lenient variant.
#   navigation_iou          = 0.5·file_iou + 0.5·func_iou (Option B Jaccard).
#
# Older trial corpora carry recall-only `nav_score` in `navigation_score` and
# don't populate the precision/F1/IoU fields — those render as `—` in tables.
SUBSCORE_KEYS = (
    "navigation_score",
    "navigation_score_lenient",
    "navigation_recall",
    "navigation_recall_lenient",
    "navigation_iou",
    "assertion_coverage",
    "file_recall",
    "file_recall_lenient",
    "file_precision",
    "file_f1",
    "file_f1_lenient",
    "file_iou",
    "function_recall",
    "function_recall_lenient",
    "function_precision",
    "function_f1",
    "function_f1_lenient",
    "function_iou",
)

# Efficiency metrics pulled from reward.json::process_metrics + result.json.
EFFICIENCY_KEYS = (
    "agent_steps",  # turns
    "tool_call_count",
    "input_tokens",
    "cached_tokens",
    "output_tokens",
    "wall_clock_sec",
)


def _mean(xs: list[float]) -> float:
    """Arithmetic mean. Returns 0 on empty list (caller is responsible for that case)."""
    return sum(xs) / len(xs) if xs else 0.0


def _stdev(xs: list[float]) -> float:
    """Sample stdev (Bessel-corrected). Returns 0 for n<2."""
    if len(xs) < 2:
        return 0.0
    m = sum(xs) / len(xs)
    return (sum((x - m) ** 2 for x in xs) / (len(xs) - 1)) ** 0.5


def _parse_iso(ts: str | None) -> datetime | None:
    if not ts:
        return None
    try:
        # Harbor writes ISO-8601 with trailing 'Z' for UTC; datetime.fromisoformat
        # handles 'Z' from Python 3.11+.
        return datetime.fromisoformat(ts.replace("Z", "+00:00"))
    except ValueError:
        return None


def _wall_clock_sec(result: dict, process_metrics: dict) -> float | None:
    """Wall-clock for the agent's solve attempt.

    Prefer `result.json::agent_execution.{started_at,finished_at}` because it
    excludes environment_setup + verifier overhead and is present on every
    agent. Falls back to `process_metrics.execution_time_sec` (opencode/codex
    populate it; claude-code does not).
    """
    ax = result.get("agent_execution") or {}
    start = _parse_iso(ax.get("started_at"))
    end = _parse_iso(ax.get("finished_at"))
    if start and end:
        return (end - start).total_seconds()
    raw = process_metrics.get("execution_time_sec")
    return float(raw) if raw is not None else None


def find_trial_dirs(roots: list[Path]) -> list[Path]:
    """Yield every trial dir (containing `verifier/reward.json` + `result.json`)."""
    out: list[Path] = []
    for root in roots:
        if not root.exists():
            print(f"WARN: missing root {root}", file=sys.stderr)
            continue
        if (root / "verifier" / "reward.json").exists() and (root / "result.json").exists():
            out.append(root)
            continue
        for entry in root.rglob("verifier/reward.json"):
            trial = entry.parent.parent
            if (trial / "result.json").exists():
                out.append(trial)
    return out


def find_job_dirs(roots: list[Path]) -> list[Path]:
    """Yield every job-dir (containing `run_manifest.json`).

    Used to seed model rows even when a run produced zero readable trials —
    otherwise the cohort-mode infra-fail accounting silently drops the failed
    run instead of reporting it as 100% infra-fail.
    """
    out: list[Path] = []
    seen: set[Path] = set()
    for root in roots:
        if not root.exists():
            continue
        if (root / "run_manifest.json").is_file():
            if root not in seen:
                out.append(root)
                seen.add(root)
            continue
        for entry in root.rglob("run_manifest.json"):
            job_dir = entry.parent
            if job_dir not in seen:
                out.append(job_dir)
                seen.add(job_dir)
    return out


def _job_dir_identity(job_dir: Path) -> tuple[str, str] | None:
    """Return (agent, model_with_effort) from the job-dir's run_manifest.json.

    The model field is suffixed with `effort=X` when reasoning effort is set,
    so two runs of the same model at different efforts get distinct rows.
    Returns None if the manifest is missing or unparseable.
    """
    manifest_path = job_dir / "run_manifest.json"
    if not manifest_path.is_file():
        return None
    try:
        manifest = json.loads(manifest_path.read_text())
    except (OSError, json.JSONDecodeError):
        return None
    agent = manifest.get("agent") or {}
    name = agent.get("name") or ""
    model = agent.get("model") or ""
    effort = (manifest.get("reasoning") or {}).get("effort")
    if not name and not model:
        return None
    if effort:
        model = f"{model} / effort={effort}" if model else f"effort={effort}"
    return (name or "unknown", model or "unknown")


def _result_identity(result: dict) -> tuple[str, str]:
    """Fallback (agent, model) from `result.json::config.agent` when no manifest exists.

    Effort is appended to the model when present in `config.agent.kwargs.reasoning_effort`,
    matching the `_job_dir_identity` shape so manifest-fed and result-fed rows merge.
    """
    agent_cfg = ((result.get("config") or {}).get("agent")) or {}
    name = agent_cfg.get("name") or "unknown"
    model = agent_cfg.get("model_name") or "unknown"
    effort = (agent_cfg.get("kwargs") or {}).get("reasoning_effort")
    if effort and model != "unknown":
        model = f"{model} / effort={effort}"
    elif effort:
        model = f"effort={effort}"
    return (name, model)


def _apply_alias(value: str, base: tuple[str, str]) -> tuple[str, str]:
    """Apply an --alias value to a base (agent, model) identity.

    Alias forms:
      "agent / model"  -> overrides both
      "model"          -> overrides model only (keeps base agent)
    """
    if " / " in value:
        agent, model = value.split(" / ", 1)
        return (agent.strip(), model.strip())
    return (base[0], value.strip())


def load_trial(trial_dir: Path, aliases: dict[str, str]) -> dict | None:
    """Read one trial. Returns None if reward.json or result.json is unparseable.

    Supported trial-dir layouts (the alias key is the matching ancestor name):
      - `<root>/<job-dir>/<trial>/...`              run-baselines.sh + Harbor (k=1 baselines)
      - `<root>/<trial>/...`                        flat harbor-jobs dirs (k=N variance runs)
      - `<root>/<run-group>/<job-dir>/<trial>/...`  tools-side `summarize_baseline.py` shape

    Identity resolution order:
      1. nearest ancestor with `run_manifest.json` (canonical for run-baselines.sh)
         else `result.json::config.agent.{name,model_name,kwargs.reasoning_effort}`
      2. `--alias <ancestor-name>=LABEL` override — tries each ancestor name from
         `trial.parent` upward, first match wins. Lets aliases work across all
         layouts without requiring callers to know the depth.
    """
    try:
        reward = json.loads((trial_dir / "verifier" / "reward.json").read_text())
        result = json.loads((trial_dir / "result.json").read_text())
    except (OSError, json.JSONDecodeError):
        return None

    # Trial dir convention: `<task_name>-<7-char-shortuuid>`. Prefer
    # result.json's `task_name` field; fall back to stripping the suffix.
    task_name = result.get("task_name") or trial_dir.name.rsplit("-", 1)[0]

    # Walk up looking for the nearest run_manifest.json. Bound the walk at 4
    # levels to avoid hitting the filesystem root on weird inputs. The trial
    # itself is excluded — manifests live alongside or above the trial.
    manifest_dir: Path | None = None
    for ancestor in list(trial_dir.parents)[:4]:
        if (ancestor / "run_manifest.json").is_file():
            manifest_dir = ancestor
            break

    base = (_job_dir_identity(manifest_dir) if manifest_dir else None) or _result_identity(result)

    # Try aliases against each ancestor name from closest-to-trial outward.
    agent, model = base
    for ancestor in list(trial_dir.parents)[:4]:
        if ancestor.name in aliases:
            agent, model = _apply_alias(aliases[ancestor.name], base)
            break

    rec: dict = {
        "task": task_name,
        "agent": agent,
        "model": model,
        "reward": float(reward.get("reward") or 0.0),
    }
    # Use None for absent fields (older trial corpora pre-date the F1 verifier
    # change and lack file_precision/file_f1/function_precision/function_f1).
    # Aggregation skips None so old runs report `—` instead of a fake 0.0.
    for key in SUBSCORE_KEYS:
        v = reward.get(key)
        rec[key] = float(v) if v is not None else None

    process_metrics = reward.get("process_metrics") or {}
    rec["agent_steps"] = process_metrics.get("agent_steps")
    rec["tool_call_count"] = process_metrics.get("tool_call_count")
    rec["input_tokens"] = process_metrics.get("input_tokens")
    rec["cached_tokens"] = process_metrics.get("cached_tokens")
    rec["output_tokens"] = process_metrics.get("output_tokens")
    rec["wall_clock_sec"] = _wall_clock_sec(result, process_metrics)
    return rec


def _read_cohort_file(path: Path) -> set[str]:
    return {ln.strip() for ln in path.read_text().splitlines() if ln.strip() and not ln.startswith("#")}


def _load_provenance_map(tasks_dir: Path) -> dict[str, str]:
    """Walk `<tasks_dir>/<task_name>/provenance.json` and return task_name → parent_t2_task.

    Tasks without a provenance file or without a `parent_t2_task` field map to
    the empty string (so the CSV column is still populated, just blank). Used
    for the search-side CSV's parent_task_id column, enabling cross-summarizer
    correlation against the tools-side per-task CSV.
    """
    out: dict[str, str] = {}
    if not tasks_dir.is_dir():
        return out
    for task_dir in tasks_dir.iterdir():
        if not task_dir.is_dir():
            continue
        prov = task_dir / "provenance.json"
        if not prov.is_file():
            out[task_dir.name] = ""
            continue
        try:
            data = json.loads(prov.read_text())
        except (OSError, json.JSONDecodeError):
            out[task_dir.name] = ""
            continue
        out[task_dir.name] = data.get("parent_t2_task") or ""
    return out


# CSV column order. Stable so downstream scripts can rely on positions.
# Scoring columns mirror the SUBSCORE_KEYS the verifier writes; efficiency
# columns mirror the markdown table.
_CSV_HEADER = [
    "agent",
    "model",
    "task",
    "parent_task_id",
    "trial_index",
    "infra_fail",
    "reward",
    "navigation_score",
    "navigation_score_lenient",
    "navigation_recall",
    "navigation_recall_lenient",
    "navigation_iou",
    "assertion_coverage",
    "file_recall",
    "file_recall_lenient",
    "file_precision",
    "file_f1",
    "file_f1_lenient",
    "file_iou",
    "function_recall",
    "function_recall_lenient",
    "function_precision",
    "function_f1",
    "function_f1_lenient",
    "function_iou",
    "agent_steps",
    "tool_call_count",
    "input_tokens",
    "cached_tokens",
    "output_tokens",
    "wall_clock_sec",
]


def _csv_infra_row(agent: str, model: str, task: str, parent: str) -> list:
    """Build a CSV row for an infra-fail (no trial seen). Reward zero-filled
    per the cohort-mode convention; subscores and efficiency left blank."""
    row = [agent, model, task, parent, 0, 1, 0]
    # All metric columns blank for infra-fails — zero-filling subscores and
    # efficiency would bias downstream means and Pearson correlations.
    row.extend([""] * (len(_CSV_HEADER) - len(row)))
    return row


def _csv_trial_row(agent: str, model: str, task: str, parent: str, trial_index: int, rec: dict) -> list:
    """Build a CSV row for one observed trial. Missing optional fields blank."""

    def _v(key: str) -> str | float | int:
        v = rec.get(key)
        return "" if v is None else v

    return [
        agent,
        model,
        task,
        parent,
        trial_index,
        0,
        rec["reward"],
        _v("navigation_score"),
        _v("navigation_score_lenient"),
        _v("navigation_recall"),
        _v("navigation_recall_lenient"),
        _v("navigation_iou"),
        _v("assertion_coverage"),
        _v("file_recall"),
        _v("file_recall_lenient"),
        _v("file_precision"),
        _v("file_f1"),
        _v("file_f1_lenient"),
        _v("file_iou"),
        _v("function_recall"),
        _v("function_recall_lenient"),
        _v("function_precision"),
        _v("function_f1"),
        _v("function_f1_lenient"),
        _v("function_iou"),
        _v("agent_steps"),
        _v("tool_call_count"),
        _v("input_tokens"),
        _v("cached_tokens"),
        _v("output_tokens"),
        _v("wall_clock_sec"),
    ]


def _parse_excludes(values: list[str]) -> set[str]:
    out: set[str] = set()
    for v in values:
        for tok in v.split(","):
            tok = tok.strip()
            if tok:
                out.add(tok)
    return out


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
    ap.add_argument("roots", nargs="+", type=Path, help="Trial dirs OR parent dirs containing trials.")
    ap.add_argument(
        "--alias",
        action="append",
        default=[],
        metavar="JOB_DIR=LABEL",
        help="Override (agent, model) for trials whose job-dir basename matches "
        "JOB_DIR. LABEL forms: 'agent / model' (overrides both) or 'model' "
        "(overrides model only). Useful for vllm-served runs whose manifest "
        "carries model='model'. Repeatable.",
    )
    ap.add_argument(
        "--cohort-tasks-file",
        type=Path,
        default=None,
        help="Cohort definition file (one task name per line; '#' comments OK). "
        f"Default: references/{DEFAULT_COHORT_FILE.name}",
    )
    ap.add_argument(
        "--exclude",
        action="append",
        default=[],
        metavar="TASKID[,TASKID...]",
        help="Drop these task IDs from the cohort for this run. Repeatable.",
    )
    ap.add_argument(
        "--no-cohort-check",
        action="store_true",
        help="Disable cohort filling. Denominator becomes trials seen per model.",
    )
    ap.add_argument(
        "--csv",
        type=Path,
        default=None,
        metavar="PATH",
        help="Write per-trial CSV with one row per (agent, model, task, trial_index). "
        "Includes infra-fail rows (zero-filled, trial_index=0) so every cohort "
        "task is represented per (agent, model) cell. Use with --tasks-dir to "
        "populate the parent_task_id column for cross-summarizer correlation.",
    )
    ap.add_argument(
        "--tasks-dir",
        type=Path,
        default=None,
        metavar="PATH",
        help="Path to the harbor-tasks directory of the search dataset (e.g. "
        "~/projects/craft-bench/harbor-tasks/craft-search-from-v2/harbor-tasks). "
        "Used to look up each task's `provenance.json::parent_t2_task` for the "
        "CSV's parent_task_id column. Required when --csv is used; otherwise ignored.",
    )
    args = ap.parse_args()

    aliases: dict[str, str] = {}
    for a in args.alias:
        if "=" not in a:
            print(f"WARN: malformed --alias {a!r}; need JOB_DIR=LABEL", file=sys.stderr)
            continue
        k, v = a.split("=", 1)
        aliases[k.strip()] = v.strip()

    # CSV mode requires --tasks-dir for the parent_task_id lookup. Validating
    # upfront so we don't run the whole pipeline only to fail at the writer.
    parent_map: dict[str, str] = {}
    if args.csv:
        if args.tasks_dir is None:
            print(
                "ERROR: --csv requires --tasks-dir (path to the search dataset's harbor-tasks/)",
                file=sys.stderr,
            )
            return 2
        if not args.tasks_dir.is_dir():
            print(f"ERROR: --tasks-dir {args.tasks_dir} not found", file=sys.stderr)
            return 2
        parent_map = _load_provenance_map(args.tasks_dir)
        n_with_parent = sum(1 for v in parent_map.values() if v)
        print(
            f"Loaded provenance for {len(parent_map)} tasks "
            f"({n_with_parent} with parent_t2_task) from {args.tasks_dir}",
            file=sys.stderr,
        )

    excludes = _parse_excludes(args.exclude)

    cohort: set[str] | None = None
    cohort_label = ""
    if args.no_cohort_check:
        if args.cohort_tasks_file:
            print(
                "ERROR: --no-cohort-check and --cohort-tasks-file are mutually exclusive",
                file=sys.stderr,
            )
            return 2
        if excludes:
            print(
                "ERROR: --no-cohort-check and --exclude are mutually exclusive",
                file=sys.stderr,
            )
            return 2
    else:
        cohort_path = args.cohort_tasks_file or DEFAULT_COHORT_FILE
        if not cohort_path.is_file():
            print(f"ERROR: cohort file {cohort_path} not found", file=sys.stderr)
            print(
                "Hint: either commit a cohort file or pass --no-cohort-check.",
                file=sys.stderr,
            )
            return 2
        cohort = _read_cohort_file(cohort_path)
        if excludes:
            unknown = excludes - cohort
            if unknown:
                print(
                    f"WARN: --exclude tasks not present in cohort: {sorted(unknown)}",
                    file=sys.stderr,
                )
            applied = excludes & cohort
            cohort -= applied
            cohort_label = f"{cohort_path} ({len(cohort)} tasks, excludes: {len(applied)})"
        else:
            cohort_label = f"{cohort_path} ({len(cohort)} tasks)"

    # Seed `by_identity` from job-dir manifests first so a run that produced
    # zero readable trials still appears as a row of all infra-fails (cohort
    # mode) rather than vanishing from the report.
    #
    # Per (identity, task) we keep a LIST of trials, not a single record.
    # Repeat runs of the same model on the same task are kept for variance.
    by_identity: dict[tuple[str, str], dict[str, list[dict]]] = defaultdict(lambda: defaultdict(list))
    for job_dir in find_job_dirs(args.roots):
        base = _job_dir_identity(job_dir)
        if base is None:
            continue
        if job_dir.name in aliases:
            identity = _apply_alias(aliases[job_dir.name], base)
        else:
            identity = base
        # touch the entry so manifest-only runs (zero readable trials) still appear
        _ = by_identity[identity]

    trial_dirs = find_trial_dirs(args.roots)
    print(f"Found {len(trial_dirs)} trial dirs across {len(args.roots)} root(s)", file=sys.stderr)

    skipped = 0
    for td in trial_dirs:
        rec = load_trial(td, aliases)
        if rec is None:
            skipped += 1
            continue
        identity = (rec["agent"], rec["model"])
        by_identity[identity][rec["task"]].append(rec)
    if skipped:
        print(f"Skipped {skipped} unreadable trials", file=sys.stderr)

    if not by_identity:
        print("ERROR: no manifests or trials found", file=sys.stderr)
        return 2

    rows = []
    for identity in sorted(by_identity):
        per_task = by_identity[identity]
        seen = set(per_task)
        if cohort is not None:
            target = cohort
            missing = sorted(target - seen)
            extra = sorted(seen - target)
        else:
            target = seen
            missing = []
            extra = []

        n = len(target) if target else 0
        infra = 0
        # Per-cohort-task means + within-task stdevs, then aggregated. Reward
        # is cohort-divided (zeros for infra-fail tasks). Subscores skip Nones
        # so older trial corpora that lack F1/IoU fields don't fake-zero them.
        reward_task_means: list[float] = []
        reward_task_stdevs: list[float] = []
        sub_task_values: dict[str, list[tuple[float, float]]] = {k: [] for k in SUBSCORE_KEYS}
        # Efficiency stays simple-mean across all trials (no per-task fold) —
        # variance there is mostly noise of-no-interest; we just want a
        # representative cost figure.
        eff_sums: dict[str, float] = {k: 0.0 for k in EFFICIENCY_KEYS}
        eff_counts: dict[str, int] = {k: 0 for k in EFFICIENCY_KEYS}
        # k_per_task tracks trial-count-per-cohort-task so the table can show
        # whether the dataset is k=1 (single-shot) or k=multi (variance run).
        k_per_task: list[int] = []

        for t in target:
            recs = per_task.get(t) or []
            if not recs:
                infra += 1
                continue
            k_per_task.append(len(recs))
            # Reward: per-task mean + stdev across trials.
            r_vals = [r["reward"] for r in recs]
            reward_task_means.append(_mean(r_vals))
            reward_task_stdevs.append(_stdev(r_vals))
            # Subscores: per-task (mean, stdev) over non-None values.
            for k in SUBSCORE_KEYS:
                vals = [r.get(k) for r in recs]
                vals = [float(v) for v in vals if v is not None]
                if vals:
                    sub_task_values[k].append((_mean(vals), _stdev(vals)))
            for k in EFFICIENCY_KEYS:
                for r in recs:
                    v = r.get(k)
                    if v is None:
                        continue
                    eff_sums[k] += float(v)
                    eff_counts[k] += 1

        # Cohort-mode reward stays cohort-divided (zeros for infra-fail tasks).
        # mean reward = average of per-task means, with infra-fails zero-filled
        # so the headline matches the existing convention.
        reward_mean = (sum(reward_task_means)) / n if n else 0.0
        # Within-task stdev, averaged across observed (non-infra) tasks. This is
        # the per-rerun-jitter on the headline. Multiplying by 0 for infra-fails
        # would distort the signal, so we average across observed tasks only.
        reward_stdev = _mean(reward_task_stdevs) if reward_task_stdevs else 0.0

        sub_means: dict[str, float | None] = {}
        sub_stdevs: dict[str, float | None] = {}
        for k, pairs in sub_task_values.items():
            if not pairs:
                sub_means[k] = None
                sub_stdevs[k] = None
                continue
            sub_means[k] = _mean([m for m, _ in pairs])
            sub_stdevs[k] = _mean([s for _, s in pairs])

        eff_means = {k: (eff_sums[k] / eff_counts[k] if eff_counts[k] else None) for k in EFFICIENCY_KEYS}
        eff_n = max(eff_counts.values()) if eff_counts else 0

        k_min = min(k_per_task) if k_per_task else 0
        k_max = max(k_per_task) if k_per_task else 0
        rows.append(
            {
                "agent": identity[0],
                "model": identity[1],
                "n": n,
                "infra": infra,
                "trials_seen": len(seen),
                "missing": len(missing),
                "extra": len(extra),
                "eff_n": eff_n,
                "eff": eff_means,
                "reward": reward_mean,
                "reward_stdev": reward_stdev,
                "sub": sub_means,
                "sub_stdev": sub_stdevs,
                "k_min": k_min,
                "k_max": k_max,
                "trial_count": sum(k_per_task),
            }
        )

    print()
    if cohort_label:
        print(f"# Search baseline — cohort: **{cohort_label}**")
    else:
        print("# Search baseline — `--no-cohort-check` (denominator = trials seen per model)")
    print()

    def _f(v: float | None, decimals: int = 3) -> str:
        return "—" if v is None else f"{v:.{decimals}f}"

    def _ms(mean: float | None, stdev: float | None, decimals: int = 3) -> str:
        """Render `mean ± stdev`. `—` if mean is None; drops `± 0.000` for k=1 cells."""
        if mean is None:
            return "—"
        if not stdev:
            return f"{mean:.{decimals}f}"
        return f"{mean:.{decimals}f} ± {stdev:.{decimals}f}"

    def _trio(sub: dict, key_f1: str, key_p: str, key_r: str, bold_f1: bool = False) -> str:
        f1 = sub.get(key_f1)
        p = sub.get(key_p)
        r = sub.get(key_r)
        f1_str = _f(f1)
        if bold_f1 and f1 is not None:
            f1_str = f"**{f1_str}**"
        return f"{f1_str} ({_f(p, 2)} / {_f(r, 2)})"

    def _kstr(r: dict) -> str:
        """Render trial-count column: `k=N` if uniform, `k=min..max` otherwise."""
        if r["k_min"] == r["k_max"]:
            return f"k={r['k_min']}"
        return f"k={r['k_min']}..{r['k_max']}"

    def _winners(values: list[float | None], higher_is_better: bool) -> set[int]:
        """Return row indices that hold the best value for this column.

        Skips None and NaN. Ties bold all tied rows. Returns empty set if no
        comparable values exist (column entirely `—`) — caller renders normally.
        """
        comparable = [(i, v) for i, v in enumerate(values) if v is not None]
        if not comparable:
            return set()
        best = (
            max(comparable, key=lambda iv: iv[1])[1]
            if higher_is_better
            else min(comparable, key=lambda iv: iv[1])[1]
        )
        # Float equality is fine here — winners are derived from the same
        # cohort-mean computation, no rounding-vs-comparison drift.
        return {i for i, v in comparable if v == best}

    def _bold(s: str, mark: bool) -> str:
        # Bold the value but leave any trailing `± stdev` alone; adding the
        # markers around the whole string would bold the σ too which reads as
        # noisy in the table.
        if not mark or s == "—":
            return s
        if " ± " in s:
            mean_part, stdev_part = s.split(" ± ", 1)
            return f"**{mean_part}** ± {stdev_part}"
        return f"**{s}**"

    # Per-column winner sets. Higher-is-better for reward + nav + assert + F1s.
    win_reward = _winners([r["reward"] for r in rows], higher_is_better=True)
    win_nav = _winners([r["sub"].get("navigation_score") for r in rows], higher_is_better=True)
    win_assert = _winners([r["sub"].get("assertion_coverage") for r in rows], higher_is_better=True)
    win_file_f1 = _winners([r["sub"].get("file_f1") for r in rows], higher_is_better=True)
    win_func_f1 = _winners([r["sub"].get("function_f1") for r in rows], higher_is_better=True)

    print("| Agent | Model | Reward | Nav | Assert | File F1 (P / R) | Func F1 (P / R) | n | k |")
    print("|---|---|---:|---:|---:|---:|---:|---:|---:|")
    for i, r in enumerate(rows):
        scored = r["n"] - r["infra"]
        sub = r["sub"]
        sd = r["sub_stdev"]
        reward = _bold(_ms(r["reward"], r["reward_stdev"]), i in win_reward)
        nav = _bold(_ms(sub.get("navigation_score"), sd.get("navigation_score")), i in win_nav)
        assertc = _bold(_ms(sub.get("assertion_coverage"), sd.get("assertion_coverage")), i in win_assert)
        file_cell = _trio(sub, "file_f1", "file_precision", "file_recall", bold_f1=i in win_file_f1)
        func_cell = _trio(
            sub, "function_f1", "function_precision", "function_recall", bold_f1=i in win_func_f1
        )
        print(
            f"| `{r['agent']}` | `{r['model']}` | {reward} | {nav} | {assertc} "
            f"| {file_cell} | {func_cell} | {scored} | {_kstr(r)} |"
        )
    print()

    print("## Score variants (mean ± within-task stdev)")
    print()
    print("Strict = strict-recall + lenient-precision F1 mix (current `navigation_score`).  ")
    print("Lenient = alts credit recall too (capped at 1.0).  ")
    print("Recall = pure recall mean (legacy pre-F1 metric).  ")
    print("IoU = Jaccard with alts in numerator only, primary in denominator (Option B).  ")
    print("σ = within-task stdev across repeat trials, averaged over the cohort. `0.000` for k=1 runs.")
    print()
    win_strict = win_nav  # already computed
    win_lenient = _winners([r["sub"].get("navigation_score_lenient") for r in rows], higher_is_better=True)
    win_recall = _winners([r["sub"].get("navigation_recall") for r in rows], higher_is_better=True)
    win_recall_l = _winners([r["sub"].get("navigation_recall_lenient") for r in rows], higher_is_better=True)
    win_iou = _winners([r["sub"].get("navigation_iou") for r in rows], higher_is_better=True)

    print("| Agent | Model | Strict F1 | Lenient F1 | Recall (strict) | Recall (lenient) | IoU | n | k |")
    print("|---|---|---:|---:|---:|---:|---:|---:|---:|")
    for i, r in enumerate(rows):
        scored = r["n"] - r["infra"]
        sub = r["sub"]
        sd = r["sub_stdev"]
        strict = _bold(_ms(sub.get("navigation_score"), sd.get("navigation_score")), i in win_strict)
        lenient = _bold(
            _ms(sub.get("navigation_score_lenient"), sd.get("navigation_score_lenient")),
            i in win_lenient,
        )
        recall = _bold(_ms(sub.get("navigation_recall"), sd.get("navigation_recall")), i in win_recall)
        recall_l = _bold(
            _ms(sub.get("navigation_recall_lenient"), sd.get("navigation_recall_lenient")),
            i in win_recall_l,
        )
        iou = _bold(_ms(sub.get("navigation_iou"), sd.get("navigation_iou")), i in win_iou)
        print(
            f"| `{r['agent']}` | `{r['model']}` | {strict} | {lenient} | {recall} "
            f"| {recall_l} | {iou} | {scored} | {_kstr(r)} |"
        )
    print()

    print("## Efficiency (mean per task, non-missing trials only)")
    print()
    print("| Agent | Model | Turns | Tools | Input tok | Cached tok | Output tok | Wall (s) | n |")
    print("|---|---|---:|---:|---:|---:|---:|---:|---:|")
    # Efficiency is lower-is-better — bold the min in each column.
    eff_winners = {k: _winners([r["eff"][k] for r in rows], higher_is_better=False) for k in EFFICIENCY_KEYS}

    def _fmt_eff(v: float | None, decimals: int) -> str:
        if v is None:
            return "—"
        if decimals == 0:
            return f"{int(round(v)):,}"
        return f"{v:,.{decimals}f}"

    eff_specs = [
        ("agent_steps", 1),
        ("tool_call_count", 1),
        ("input_tokens", 0),
        ("cached_tokens", 0),
        ("output_tokens", 0),
        ("wall_clock_sec", 1),
    ]
    for i, r in enumerate(rows):
        eff = r["eff"]
        cells = [_bold(_fmt_eff(eff[key], dec), i in eff_winners[key]) for key, dec in eff_specs]
        print(
            f"| `{r['agent']}` "
            f"| `{r['model']}` "
            f"| {cells[0]} "
            f"| {cells[1]} "
            f"| {cells[2]} "
            f"| {cells[3]} "
            f"| {cells[4]} "
            f"| {cells[5]} "
            f"| {r['eff_n']} |"
        )
    print()

    for r in rows:
        identity = f"{r['agent']} / {r['model']}"
        if cohort and r["missing"]:
            print(
                f"⚠ `{identity}` is missing {r['missing']} of {len(cohort)} cohort tasks "
                f"(counted as infra-fail with reward=0).",
                file=sys.stderr,
            )
        if cohort and r["extra"]:
            print(
                f"ℹ `{identity}` has {r['extra']} trials outside the cohort "
                f"(ignored in headline; pass --no-cohort-check to include).",
                file=sys.stderr,
            )

    # Per-trial CSV. One row per (agent, model, task, trial_index) over the
    # cohort (or trials-seen if --no-cohort-check). Infra-fail rows are
    # emitted with trial_index=0 and zero-filled scoring fields so every
    # cohort task is represented per (agent, model) cell — matches the
    # tools-side convention. Efficiency cells stay blank for infra-fails
    # (zero-filling them would bias downstream means).
    if args.csv:
        target_tasks = cohort if cohort is not None else None
        with args.csv.open("w", newline="") as fh:
            w = csv.writer(fh)
            w.writerow(_CSV_HEADER)
            for identity in sorted(by_identity):
                agent, model = identity
                per_task = by_identity[identity]
                tasks_to_emit = target_tasks if target_tasks is not None else set(per_task)
                for task in sorted(tasks_to_emit):
                    parent = parent_map.get(task, "")
                    recs = per_task.get(task) or []
                    if not recs:
                        # Infra-fail: one zero-filled row.
                        w.writerow(_csv_infra_row(agent, model, task, parent))
                        continue
                    for trial_index, rec in enumerate(recs):
                        w.writerow(_csv_trial_row(agent, model, task, parent, trial_index, rec))
        print(f"Wrote per-trial CSV → {args.csv}", file=sys.stderr)

    return 0


if __name__ == "__main__":
    sys.exit(main())