Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
32.9 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""deep_dive_e2e_trajectories.py — exhaustive per-trial analyses of e2e
trajectories on the v2b cohort.

Twelve analyses (A1-A12) producing a single per-trial enriched CSV and a
summary markdown. The 12 angles are documented in
`docs/analyses/may01-search-vs-e2e-paper-framing.md` and the response thread that
prompted them. Briefly:

  A1 — 2x2 read-enough × resolved confusion matrix (Jiantao's framing)
  A2 — total examined-files distribution by outcome
  A3 — "read beyond gold" — adjacent files / test files / docs files
  A4 — "effective gold" derived from successful-agent behavior, rescore
  A5 — probe-then-edit ratio (Read tool calls vs Edit/apply_patch)
  A6 — edit-locality scaling: examined files vs gold-patch size
  A7 — failure-mode attribution to task properties
  A8 — edit/test iteration count
  A9 — subtask decomposition (TodoWrite/task) usage
  A10 — thrashing signatures: same file written multiple times, edits reverted
  A11 — intermediate-state replay (skipped: reconstruction infeasible from
        trajectory alone without running tests)
  A12 — self-introspection: did the agent's text mention the right files
        before editing?

Inputs: the four per-model implicit-search CSVs + the e2e trial dirs +
patch-gold + v2b harbor-tasks dir. Outputs:

  docs/data/v2b-deep-dive-per-trial.csv      — every trial × every metric
  docs/data/v2b-deep-dive-summary.md         — aggregate findings, per-model
                                               splits, top examples per
                                               analysis
"""

from __future__ import annotations

import argparse
import csv
import json
import re
import statistics as st
import sys
from collections import Counter, defaultdict
from dataclasses import asdict, dataclass
from pathlib import Path

# ---------------------------------------------------------------------------
# Tool-name conventions (carried from score_e2e_implicit_search.py)
# ---------------------------------------------------------------------------

CODEX_BASH = "exec_command"
CODEX_PATCH = "apply_patch"
CODEX_STDIN = "write_stdin"

CC_BASH = "Bash"
CC_READ = "Read"
CC_EDIT = "Edit"
CC_WRITE = "Write"
CC_MULTI_EDIT = "MultiEdit"
CC_NOTEBOOK = "NotebookEdit"
CC_GREP = "Grep"
CC_GLOB = "Glob"
CC_TODO = "TodoWrite"

OC_BASH = "bash"
OC_READ = "read"
OC_EDIT = "edit"
OC_WRITE = "write"
OC_GREP = "grep"
OC_GLOB = "glob"
OC_TASK = "task"
OC_TODO = "todowrite"

EDIT_TOOLS = {CODEX_PATCH, CC_EDIT, CC_WRITE, CC_MULTI_EDIT, CC_NOTEBOOK, OC_EDIT, OC_WRITE}
READ_TOOLS = {CC_READ, OC_READ}  # codex's exec_command is multi-purpose; counted via bash
BASH_TOOLS = {CODEX_BASH, CC_BASH, OC_BASH}
GREP_GLOB_TOOLS = {CC_GREP, CC_GLOB, OC_GREP, OC_GLOB}
TODO_TOOLS = {CC_TODO, OC_TODO, OC_TASK}

# ---------------------------------------------------------------------------
# Trajectory loaders (copy of score_e2e_implicit_search.py adapter)
# ---------------------------------------------------------------------------


def _load_trajectory(trial_dir: Path) -> dict | None:
    p = trial_dir / "agent" / "trajectory.json"
    if p.is_file():
        try:
            return json.loads(p.read_text())
        except (OSError, json.JSONDecodeError):
            return None
    cc = trial_dir / "agent" / "claude-code.txt"
    if cc.is_file():
        try:
            return _adapt_claude_code(cc)
        except (OSError, json.JSONDecodeError):
            return None
    return None


def _adapt_claude_code(p: Path) -> dict:
    steps: list[dict] = []
    pending: dict[str, dict] = {}
    sid = 0
    with p.open() as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                ev = json.loads(line)
            except json.JSONDecodeError:
                continue
            if ev.get("type") == "assistant":
                msg = ev.get("message") or {}
                content = msg.get("content") or []
                text_buf: list[str] = []
                for blk in content:
                    btype = blk.get("type")
                    if btype == "text":
                        text_buf.append(blk.get("text") or "")
                    elif btype == "tool_use":
                        sid += 1
                        step = {
                            "step_id": sid,
                            "source": "agent",
                            "message": "\n".join(text_buf),
                            "tool_calls": [
                                {
                                    "tool_call_id": blk.get("id"),
                                    "function_name": blk.get("name", ""),
                                    "arguments": blk.get("input") or {},
                                }
                            ],
                        }
                        text_buf = []
                        pending[blk.get("id") or ""] = step
                        steps.append(step)
            elif ev.get("type") == "user":
                msg = ev.get("message") or {}
                content = msg.get("content") or []
                if isinstance(content, list):
                    for blk in content:
                        if blk.get("type") == "tool_result":
                            tcid = blk.get("tool_use_id")
                            payload = blk.get("content")
                            if isinstance(payload, list):
                                txt = "\n".join(p.get("text", "") for p in payload if isinstance(p, dict))
                            else:
                                txt = str(payload or "")
                            if tcid in pending:
                                pending[tcid].setdefault("observation", {}).setdefault("results", []).append(
                                    {"source_call_id": tcid, "content": txt}
                                )
    return {"schema_version": "claude-code-adapted", "agent": {"name": "claude-code"}, "steps": steps}


# ---------------------------------------------------------------------------
# Utility: file path normalization
# ---------------------------------------------------------------------------

_STRIP_PREFIXES = ("/repo/", "repo/", "/code/", "code/", "./")


def _normalize_file(path: str) -> str:
    p = path.strip().lower()
    for prefix in _STRIP_PREFIXES:
        if p.startswith(prefix):
            p = p[len(prefix) :]
            break
    return p.rstrip("/")


_PATH_LIKE_RE = re.compile(r"(?:[\w./_-]+/)+[\w_-]+\.\w+")


def _paths_in(text: str) -> set[str]:
    out: set[str] = set()
    for m in _PATH_LIKE_RE.finditer(text or ""):
        c = m.group(0)
        if c.startswith(("http://", "https://")):
            continue
        out.add(_normalize_file(c))
    return out


# ---------------------------------------------------------------------------
# Per-trial extractor: collect all the signals from one trial
# ---------------------------------------------------------------------------


@dataclass
class TrialSignals:
    """Everything we need from one trial. Most are computed once here; A1-A12
    pivot off these fields plus the implicit-search CSV row."""

    # Identity
    model: str = ""
    task: str = ""
    trial_dir: str = ""
    e2e_resolved: int = 0  # 0 or 1

    # Outcome details
    f2p_passed: int = 0
    f2p_total: int = 0
    p2p_passed: int = 0
    p2p_total: int = 0

    # Carried from implicit-search CSV
    exam_file_recall: float = 0.0
    exam_function_recall: float = 0.0
    comm_file_recall: float = 0.0
    n_examined_files: int = 0
    n_committed_files: int = 0
    gold_n_files: int = 0
    gold_n_functions: int = 0

    # NEW signals from trajectory walk
    n_steps: int = 0
    n_read_calls: int = 0  # explicit Read/read tool
    n_bash_calls: int = 0
    n_edit_calls: int = 0  # Edit/Write/apply_patch/edit/write
    n_grep_glob_calls: int = 0
    n_todo_calls: int = 0  # TodoWrite/todowrite/task
    # Multi-edit on same file (thrashing signal)
    files_edited_once: int = 0
    files_edited_2plus: int = 0  # potential thrashing
    # Test invocations: Bash commands matching pytest/pytest-xdist/python -m unittest
    n_test_invocations: int = 0
    # First-edit step / first-test-invocation step (probe-then-edit ratio)
    first_edit_step: int = 0  # 0 if no edits ever
    first_test_step: int = 0
    # Total LOC of agent edits (approx.) — same as classify_localized_failures
    agent_diff_lines: int = 0
    # Files examined that are NOT in gold (over-broad)
    n_examined_nongold: int = 0
    # Files examined matching test patterns
    n_examined_test_files: int = 0
    # Files examined matching docs / README
    n_examined_doc_files: int = 0
    # Self-introspection: did the agent's text mention any gold function name
    # BEFORE the first edit?
    pre_edit_mention_count: int = 0  # gold functions mentioned in agent text before first edit
    pre_edit_diagnosis_text: str = ""  # excerpt of agent text from before first edit (max 600 chars)

    # Gold-patch size (lines added+removed)
    gold_diff_lines: int = 0


def _gold_patch_size(task_dir: Path) -> int:
    p = task_dir / "solution" / "changes.patch"
    if not p.is_file():
        return 0
    n = 0
    for line in p.read_text(errors="replace").splitlines():
        if line.startswith(("+", "-")) and not line.startswith(("+++ ", "--- ")):
            n += 1
    return n


def _committed_file_in_step(fn: str, args: dict) -> str | None:
    """Return the file path the agent committed to, if any."""
    if fn == CODEX_PATCH:
        # Codex DSL — first '*** Update File:' or '*** Add File:'
        text = (args.get("input") or "") + "\n" + (args.get("patch") or "")
        m = re.search(r"^\*\*\* (?:Update|Add|Delete) File:\s*(\S+)\s*$", text, re.MULTILINE)
        if m:
            return _normalize_file(m.group(1))
    elif fn in (CC_EDIT, CC_WRITE, CC_MULTI_EDIT, CC_NOTEBOOK):
        fp = args.get("file_path") or args.get("notebook_path")
        if fp:
            return _normalize_file(fp)
    elif fn in (OC_EDIT, OC_WRITE):
        fp = args.get("filePath")
        if fp:
            return _normalize_file(fp)
    return None


_TEST_INVOKE_RE = re.compile(r"\b(?:pytest|python\s+-m\s+(?:pytest|unittest)|tox|nose2)\b")
_TEST_FILE_RE = re.compile(r"(?:^|/)(?:test_[\w_]+|conftest|tests?)\.py$|(?:^|/)tests?/")
_DOC_FILE_RE = re.compile(r"\.(?:md|rst|txt)$|(?:^|/)docs?/", re.IGNORECASE)


def _agent_diff_size(traj: dict) -> int:
    n = 0
    for step in traj.get("steps") or []:
        for tc in step.get("tool_calls") or []:
            fn = tc.get("function_name", "")
            args = tc.get("arguments") or {}
            if fn == CODEX_PATCH:
                for line in (args.get("input") or "").splitlines():
                    if line.startswith(("+", "-")) and not line.startswith(("+++ ", "--- ")):
                        n += 1
                for line in (args.get("patch") or "").splitlines():
                    if line.startswith(("+", "-")) and not line.startswith(("+++ ", "--- ")):
                        n += 1
            elif fn in (CC_EDIT, CC_MULTI_EDIT):
                for k in ("old_string", "new_string"):
                    s = args.get(k) or ""
                    n += s.count("\n") + (1 if s else 0)
            elif fn == CC_WRITE:
                s = args.get("content") or ""
                n += s.count("\n") + (1 if s else 0)
            elif fn in (OC_EDIT, OC_WRITE):
                for k in ("oldString", "newString", "content"):
                    s = args.get(k) or ""
                    n += s.count("\n") + (1 if s else 0)
    return n


def _build_trial_signals(
    csv_row: dict, trial_dir: Path, task_dir: Path | None, gold_files: set[str], gold_funcs: list[str]
) -> TrialSignals:
    """Walk one trial's trajectory and gather all the per-trial signals."""
    rj_path = trial_dir / "verifier" / "reward.json"
    rj: dict = {}
    if rj_path.is_file():
        try:
            rj = json.loads(rj_path.read_text())
        except (OSError, json.JSONDecodeError):
            rj = {}

    sig = TrialSignals(
        model=csv_row["agent"] + "/" + csv_row["model"][:80],
        task=csv_row["task"],
        trial_dir=str(trial_dir),
        e2e_resolved=1 if csv_row["e2e_resolved"] == "1" else 0,
        f2p_passed=int(rj.get("f2p_passed") or 0),
        f2p_total=int(rj.get("f2p_total") or 0),
        p2p_passed=int(rj.get("p2p_passed") or 0),
        p2p_total=int(rj.get("p2p_total") or 0),
        exam_file_recall=float(csv_row["exam_file_recall"] or 0),
        exam_function_recall=float(csv_row["exam_function_recall"] or 0),
        comm_file_recall=float(csv_row["comm_file_recall"] or 0),
        n_examined_files=int(csv_row["n_examined_files"] or 0),
        n_committed_files=int(csv_row["n_committed_files"] or 0),
        gold_n_files=int(csv_row["gold_n_files"] or 0),
        gold_n_functions=int(csv_row["gold_n_functions"] or 0),
    )
    if task_dir is not None and task_dir.is_dir():
        sig.gold_diff_lines = _gold_patch_size(task_dir)

    traj = _load_trajectory(trial_dir)
    if traj is None:
        return sig

    sig.agent_diff_lines = _agent_diff_size(traj)
    steps = traj.get("steps") or []
    sig.n_steps = len(steps)

    # Collect: all examined files (path-extracted), all committed files (per
    # tool call), tool counts, first-edit step, first-test step, and pre-edit
    # text.
    examined_files: set[str] = set()
    edits_per_file: Counter = Counter()
    pre_edit_text_buf: list[str] = []
    first_edit_step = 0
    first_test_step = 0
    gold_func_bare = {gf.strip().rsplit(".", 1)[-1] for gf in gold_funcs if gf.strip()}

    for i, step in enumerate(steps, 1):
        for tc in step.get("tool_calls") or []:
            fn = tc.get("function_name", "")
            args = tc.get("arguments") or {}

            if fn in EDIT_TOOLS:
                sig.n_edit_calls += 1
                committed = _committed_file_in_step(fn, args)
                if committed:
                    edits_per_file[committed] += 1
                if first_edit_step == 0:
                    first_edit_step = i
            elif fn in READ_TOOLS:
                sig.n_read_calls += 1
                fp = args.get("file_path") or args.get("filePath")
                if fp:
                    examined_files.add(_normalize_file(fp))
            elif fn in BASH_TOOLS:
                sig.n_bash_calls += 1
                cmd = args.get("command") or args.get("cmd") or ""
                examined_files.update(_paths_in(cmd))
                if _TEST_INVOKE_RE.search(cmd):
                    sig.n_test_invocations += 1
                    if first_test_step == 0:
                        first_test_step = i
            elif fn in GREP_GLOB_TOOLS:
                sig.n_grep_glob_calls += 1
            elif fn in TODO_TOOLS:
                sig.n_todo_calls += 1

        # Observation paths (rg/grep output)
        obs = step.get("observation") or {}
        for r in obs.get("results") or []:
            content = r.get("content") or ""
            examined_files.update(_paths_in(content))

        # Agent text up to (but not including) the first edit
        msg = step.get("message") or ""
        if first_edit_step == 0 and msg:
            pre_edit_text_buf.append(msg)

    # Files-edited-multiple-times (thrashing)
    sig.files_edited_once = sum(1 for c in edits_per_file.values() if c == 1)
    sig.files_edited_2plus = sum(1 for c in edits_per_file.values() if c >= 2)

    sig.first_edit_step = first_edit_step
    sig.first_test_step = first_test_step

    # Examined-file analytics
    sig.n_examined_nongold = len(examined_files - gold_files)
    sig.n_examined_test_files = sum(1 for f in examined_files if _TEST_FILE_RE.search(f))
    sig.n_examined_doc_files = sum(1 for f in examined_files if _DOC_FILE_RE.search(f))

    # Self-introspection: how many gold function names appear in pre-edit text
    pre_edit_full = "\n".join(pre_edit_text_buf)
    if pre_edit_full and gold_func_bare:
        seen = set()
        for bare in gold_func_bare:
            if bare and re.search(rf"\b{re.escape(bare)}\b", pre_edit_full):
                seen.add(bare)
        sig.pre_edit_mention_count = len(seen)
        sig.pre_edit_diagnosis_text = pre_edit_full[-600:]

    return sig


# ---------------------------------------------------------------------------
# Cross-trial analyses (operate on the population of TrialSignals)
# ---------------------------------------------------------------------------


def _safe_mean(xs: list[float]) -> float:
    return st.mean(xs) if xs else 0.0


def analysis_a1_2x2(sigs: list[TrialSignals], threshold: float = 0.8) -> dict:
    """A1: read-enough × resolved confusion matrix.

    'Read-enough' = exam_file_recall >= threshold."""
    out: dict = {"by_model": {}, "overall": {}}
    by_model: dict[str, dict] = defaultdict(lambda: dict(rt=0, rf=0, nt=0, nf=0))
    for s in sigs:
        cell_read = s.exam_file_recall >= threshold
        cell_succ = s.e2e_resolved == 1
        key = (
            "rt"
            if (cell_read and cell_succ)
            else "rf"
            if (cell_read and not cell_succ)
            else "nt"
            if (not cell_read and cell_succ)
            else "nf"
        )
        by_model[s.model][key] += 1
    for m in sorted(by_model):
        c = by_model[m]
        c["read_enough_rate"] = (c["rt"] + c["rf"]) / max(1, sum(c[k] for k in ("rt", "rf", "nt", "nf")))
        c["resolved_given_read_enough"] = c["rt"] / max(1, c["rt"] + c["rf"])
        c["resolved_given_read_poor"] = c["nt"] / max(1, c["nt"] + c["nf"])
        out["by_model"][m] = c
    # Overall
    overall = {"rt": 0, "rf": 0, "nt": 0, "nf": 0}
    for c in by_model.values():
        for k in ("rt", "rf", "nt", "nf"):
            overall[k] += c[k]
    overall["resolved_given_read_enough"] = overall["rt"] / max(1, overall["rt"] + overall["rf"])
    overall["resolved_given_read_poor"] = overall["nt"] / max(1, overall["nt"] + overall["nf"])
    out["overall"] = overall
    out["threshold"] = threshold
    return out


def analysis_a2_total_files(sigs: list[TrialSignals]) -> dict:
    """A2: total examined-files distribution by outcome, per model."""
    out: dict = {"by_model": {}}
    by_model: dict[str, dict] = defaultdict(lambda: {"resolved": [], "failed": []})
    for s in sigs:
        bucket = "resolved" if s.e2e_resolved == 1 else "failed"
        by_model[s.model][bucket].append(s.n_examined_files)
    for m in sorted(by_model):
        d = by_model[m]
        out["by_model"][m] = {
            "resolved_mean": _safe_mean(d["resolved"]),
            "resolved_median": st.median(d["resolved"]) if d["resolved"] else 0,
            "resolved_n": len(d["resolved"]),
            "failed_mean": _safe_mean(d["failed"]),
            "failed_median": st.median(d["failed"]) if d["failed"] else 0,
            "failed_n": len(d["failed"]),
        }
    return out


def analysis_a3_read_beyond_gold(sigs: list[TrialSignals]) -> dict:
    """A3: did the agent read adjacent files (test files, docs, files outside gold)?"""
    out: dict = {"by_model_outcome": {}}
    by: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
    for s in sigs:
        out_label = "resolved" if s.e2e_resolved == 1 else "failed"
        by[(s.model, out_label)]["nongold"].append(s.n_examined_nongold)
        by[(s.model, out_label)]["test"].append(s.n_examined_test_files)
        by[(s.model, out_label)]["doc"].append(s.n_examined_doc_files)
    for (m, o), data in by.items():
        out["by_model_outcome"][f"{m}/{o}"] = {k: _safe_mean(v) for k, v in data.items()}
    return out


def analysis_a4_effective_gold(
    sigs: list[TrialSignals], all_examined: dict[str, dict[str, set[str]]]
) -> dict:
    """A4: "effective gold" = union of files read by every successful trial of
    each task. Per-task: how many extra files are in effective gold beyond
    patch-derived gold? Then rescore failed trials against effective gold."""
    out = {"per_task_extras": [], "summary": {}}
    extras_sizes = []
    for task, by_outcome in all_examined.items():
        succ_examined = by_outcome.get("resolved_examined_union") or set()
        gold_files = by_outcome.get("gold_files") or set()
        if not succ_examined:
            continue
        # Files all successful agents read but aren't in gold
        adjacent = succ_examined - gold_files
        extras_sizes.append(len(adjacent))
        out["per_task_extras"].append(
            {
                "task": task,
                "gold_n": len(gold_files),
                "succ_examined_n": len(succ_examined),
                "adjacent_n": len(adjacent),
            }
        )
    if extras_sizes:
        out["summary"] = {
            "tasks_with_resolved": len(extras_sizes),
            "mean_adjacent_files": _safe_mean(extras_sizes),
            "median_adjacent_files": st.median(extras_sizes),
        }
    return out


def analysis_a5_probe_then_edit(sigs: list[TrialSignals]) -> dict:
    """A5: probe-then-edit ratio. How many steps before first edit?"""
    out: dict = {"by_model_outcome": {}}
    by: dict[tuple[str, str], list[float]] = defaultdict(list)
    by_ratio: dict[tuple[str, str], list[float]] = defaultdict(list)
    for s in sigs:
        if s.first_edit_step == 0:
            continue
        out_label = "resolved" if s.e2e_resolved == 1 else "failed"
        # Probe-fraction: steps before first edit / total steps
        ratio = s.first_edit_step / max(1, s.n_steps)
        by[(s.model, out_label)].append(s.first_edit_step)
        by_ratio[(s.model, out_label)].append(ratio)
    for (m, o), vals in by.items():
        out["by_model_outcome"][f"{m}/{o}"] = {
            "mean_first_edit_step": _safe_mean(vals),
            "mean_probe_fraction": _safe_mean(by_ratio[(m, o)]),
            "n": len(vals),
        }
    return out


def analysis_a6_edit_locality(sigs: list[TrialSignals]) -> dict:
    """A6: examined-files vs gold-patch size. Do agents adapt search to scope?"""
    # Pearson correlation per model, plus scatter data
    import math

    out: dict = {"by_model": {}}
    by: dict[str, list[tuple[int, int]]] = defaultdict(list)
    for s in sigs:
        if s.gold_diff_lines and s.n_examined_files:
            by[s.model].append((s.gold_diff_lines, s.n_examined_files))
    for m, pairs in by.items():
        if len(pairs) < 3:
            continue
        xs = [p[0] for p in pairs]
        ys = [p[1] for p in pairs]
        mx = sum(xs) / len(xs)
        my = sum(ys) / len(ys)
        num = sum((x - mx) * (y - my) for x, y in pairs)
        denx = math.sqrt(sum((x - mx) ** 2 for x in xs))
        deny = math.sqrt(sum((y - my) ** 2 for y in ys))
        r = num / (denx * deny) if (denx and deny) else 0.0
        out["by_model"][m] = {"pearson_r": round(r, 3), "n": len(pairs), "mean_examined": round(my, 1)}
    return out


def analysis_a7_failure_modes_by_task(sigs: list[TrialSignals]) -> dict:
    """A7: what task properties correlate with F2P_NONE vs F2P_PARTIAL?
    Within the failed-AND-fully-localized subset only."""
    failures = [s for s in sigs if s.e2e_resolved == 0 and s.exam_file_recall >= 1.0]
    out: dict = {"n": len(failures), "buckets": {}}
    # Partition into F2P_NONE vs F2P_PARTIAL
    f2p_none = [s for s in failures if s.f2p_total > 0 and s.f2p_passed == 0]
    f2p_partial = [s for s in failures if s.f2p_total > 0 and 0 < s.f2p_passed < s.f2p_total]
    # Mean task properties per bucket
    for label, bucket in [("F2P_NONE", f2p_none), ("F2P_PARTIAL", f2p_partial)]:
        if not bucket:
            continue
        out["buckets"][label] = {
            "n": len(bucket),
            "mean_gold_n_files": _safe_mean([s.gold_n_files for s in bucket]),
            "mean_gold_diff_lines": _safe_mean([s.gold_diff_lines for s in bucket]),
            "mean_gold_n_functions": _safe_mean([s.gold_n_functions for s in bucket]),
            "mean_agent_diff_lines": _safe_mean([s.agent_diff_lines for s in bucket]),
        }
    return out


def analysis_a8_iteration(sigs: list[TrialSignals]) -> dict:
    """A8: edit-test iteration count. n_test_invocations + n_edit_calls."""
    out: dict = {"by_model_outcome": {}}
    by: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
    for s in sigs:
        out_label = "resolved" if s.e2e_resolved == 1 else "failed"
        by[(s.model, out_label)]["edits"].append(s.n_edit_calls)
        by[(s.model, out_label)]["tests"].append(s.n_test_invocations)
        by[(s.model, out_label)]["steps"].append(s.n_steps)
    for (m, o), data in by.items():
        out["by_model_outcome"][f"{m}/{o}"] = {
            "mean_edits": _safe_mean(data["edits"]),
            "mean_test_runs": _safe_mean(data["tests"]),
            "mean_steps": _safe_mean(data["steps"]),
            "n": len(data["edits"]),
        }
    return out


def analysis_a9_decomposition(sigs: list[TrialSignals]) -> dict:
    """A9: TodoWrite/task usage by outcome."""
    out: dict = {"by_model_outcome": {}}
    by: dict[tuple[str, str], list[int]] = defaultdict(list)
    for s in sigs:
        out_label = "resolved" if s.e2e_resolved == 1 else "failed"
        by[(s.model, out_label)].append(s.n_todo_calls)
    for (m, o), vals in by.items():
        out["by_model_outcome"][f"{m}/{o}"] = {
            "mean_todo_calls": _safe_mean(vals),
            "frac_with_decomposition": sum(1 for v in vals if v > 0) / max(1, len(vals)),
            "n": len(vals),
        }
    return out


def analysis_a10_thrashing(sigs: list[TrialSignals]) -> dict:
    """A10: files edited multiple times — possible thrashing."""
    out: dict = {"by_model_outcome": {}}
    by: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
    for s in sigs:
        out_label = "resolved" if s.e2e_resolved == 1 else "failed"
        total_edited = s.files_edited_once + s.files_edited_2plus
        thrash_rate = s.files_edited_2plus / total_edited if total_edited else 0
        by[(s.model, out_label)]["thrash_rate"].append(thrash_rate)
        by[(s.model, out_label)]["files_2plus"].append(s.files_edited_2plus)
    for (m, o), data in by.items():
        out["by_model_outcome"][f"{m}/{o}"] = {
            "mean_thrash_rate": _safe_mean(data["thrash_rate"]),
            "mean_files_edited_2plus": _safe_mean(data["files_2plus"]),
            "n": len(data["thrash_rate"]),
        }
    return out


def analysis_a12_self_introspection(sigs: list[TrialSignals]) -> dict:
    """A12: did the agent's pre-edit text mention any gold function name?"""
    out: dict = {"by_model_outcome": {}}
    by: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
    for s in sigs:
        out_label = "resolved" if s.e2e_resolved == 1 else "failed"
        by[(s.model, out_label)]["mention_count"].append(s.pre_edit_mention_count)
        by[(s.model, out_label)]["any_mention"].append(1 if s.pre_edit_mention_count else 0)
    for (m, o), data in by.items():
        out["by_model_outcome"][f"{m}/{o}"] = {
            "mean_mention_count": _safe_mean(data["mention_count"]),
            "frac_any_mention": _safe_mean(data["any_mention"]),
            "n": len(data["mention_count"]),
        }
    return out


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
    ap.add_argument(
        "--implicit-csvs", nargs="+", type=Path, required=True, help="Per-model implicit-search CSVs."
    )
    ap.add_argument(
        "--e2e-roots", nargs="+", type=Path, required=True, help="One or more dirs holding e2e job dirs."
    )
    ap.add_argument("--patch-gold", type=Path, required=True, help="references/v2b-patch-gold.json")
    ap.add_argument(
        "--tasks-dir", type=Path, required=True, help="v2b harbor-tasks dir (for solution/changes.patch)."
    )
    ap.add_argument("--output-csv", type=Path, required=True, help="Per-trial enriched CSV.")
    ap.add_argument("--output-md", type=Path, required=True, help="Aggregate findings markdown.")
    args = ap.parse_args()

    patch_gold = json.loads(args.patch_gold.read_text())
    print(f"Loaded patch-gold for {len(patch_gold)} v2b tasks", file=sys.stderr)

    def _find_trial(trial_path: str) -> Path | None:
        for root in args.e2e_roots:
            cand = root / "jobs" / trial_path
            if cand.is_dir():
                return cand
            cand2 = root / trial_path
            if cand2.is_dir():
                return cand2
        return None

    sigs: list[TrialSignals] = []
    skipped = 0
    for csv_path in args.implicit_csvs:
        with csv_path.open() as f:
            for row in csv.DictReader(f):
                trial_dir = _find_trial(row["trial_dir"])
                if trial_dir is None:
                    skipped += 1
                    continue
                gold = patch_gold.get(row["task"])
                if gold is None:
                    skipped += 1
                    continue
                gold_files = {_normalize_file(f) for f in (gold.get("files") or [])}
                gold_funcs = list(gold.get("functions") or [])
                task_dir = args.tasks_dir / row["task"]
                sig = _build_trial_signals(row, trial_dir, task_dir, gold_files, gold_funcs)
                sigs.append(sig)
    print(f"Built {len(sigs)} trial signal records ({skipped} skipped)", file=sys.stderr)

    # Per-trial CSV
    args.output_csv.parent.mkdir(parents=True, exist_ok=True)
    with args.output_csv.open("w", newline="") as fh:
        if sigs:
            w = csv.DictWriter(fh, fieldnames=list(asdict(sigs[0])))
            w.writeheader()
            for s in sigs:
                w.writerow(asdict(s))
    print(f"Wrote per-trial CSV → {args.output_csv}", file=sys.stderr)

    # Effective-gold builder for A4 (needs full examined-file sets per
    # successful trial). Approximation: use n_examined_files etc.; the union
    # itself requires re-walking trajectories. For v1 we use what we already
    # have aggregated.
    a4 = {"summary": {}, "per_task_extras": []}
    by_task: dict[str, dict[str, list[TrialSignals]]] = defaultdict(lambda: {"r": [], "f": []})
    for s in sigs:
        by_task[s.task]["r" if s.e2e_resolved else "f"].append(s)
    extras_sizes = []
    for task, d in by_task.items():
        if not d["r"]:
            continue
        # Approximation: mean examined-files among successful trials,
        # minus gold-n-files, gives a (rough) "extra adjacent" count
        succ_mean_exam = _safe_mean([s.n_examined_files for s in d["r"]])
        gold_n = d["r"][0].gold_n_files
        adjacent = max(0, succ_mean_exam - gold_n)
        extras_sizes.append(adjacent)
        a4["per_task_extras"].append(
            {
                "task": task,
                "gold_n": gold_n,
                "succ_mean_examined": round(succ_mean_exam, 1),
                "approx_adjacent": round(adjacent, 1),
            }
        )
    if extras_sizes:
        a4["summary"] = {
            "tasks_with_resolved": len(extras_sizes),
            "mean_approx_adjacent": round(_safe_mean(extras_sizes), 1),
            "median_approx_adjacent": round(st.median(extras_sizes), 1),
        }

    findings = {
        "n_trials": len(sigs),
        "n_models": len({s.model for s in sigs}),
        "a1_2x2_thr0.8": analysis_a1_2x2(sigs, threshold=0.8),
        "a1_2x2_thr1.0": analysis_a1_2x2(sigs, threshold=1.0),
        "a2_total_files": analysis_a2_total_files(sigs),
        "a3_read_beyond_gold": analysis_a3_read_beyond_gold(sigs),
        "a4_effective_gold": a4,
        "a5_probe_then_edit": analysis_a5_probe_then_edit(sigs),
        "a6_edit_locality": analysis_a6_edit_locality(sigs),
        "a7_failure_mode_by_task": analysis_a7_failure_modes_by_task(sigs),
        "a8_iteration": analysis_a8_iteration(sigs),
        "a9_decomposition": analysis_a9_decomposition(sigs),
        "a10_thrashing": analysis_a10_thrashing(sigs),
        "a12_self_introspection": analysis_a12_self_introspection(sigs),
    }

    # Dump structured findings beside the markdown for downstream tooling
    json_out = args.output_md.with_suffix(".json")
    json_out.write_text(json.dumps(findings, indent=2, default=str))
    print(f"Wrote structured findings → {json_out}", file=sys.stderr)

    # Brief stdout summary
    print()
    print("=== A1 — read-enough × resolved (threshold=0.8) ===")
    o = findings["a1_2x2_thr0.8"]["overall"]
    print(f"  read-enough∩resolved={o['rt']}, read-enough∩failed={o['rf']},")
    print(f"  read-poor∩resolved={o['nt']}, read-poor∩failed={o['nf']}")
    p_re = o["resolved_given_read_enough"]
    p_rp = o["resolved_given_read_poor"]
    print(f"  P(resolved | read-enough) = {p_re:.3f}")
    print(f"  P(resolved | read-poor)   = {p_rp:.3f}")

    print("\n=== A6 — edit-locality scaling (Pearson r per model) ===")
    for m, d in findings["a6_edit_locality"]["by_model"].items():
        print(f"  {m[:60]:<60}  r={d['pearson_r']:+.3f}  n={d['n']}")

    return 0


if __name__ == "__main__":
    sys.exit(main())