Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
20.8 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""classify_localized_failures.py — automatic failure-mode typology for
fully-localized e2e failures.

Population: every (model, task) trial pair where the e2e agent achieved
`exam_file_recall == 1.0` (saw every gold-edited file) AND `e2e_resolved == 0`
(failed verification). 64 such pairs across codex55 / opus47 / haiku45 /
qwen36 on the v2b cohort.

For each pair, this script extracts a structured per-trial record by reading
the trial's `verifier/reward.json`, `verifier/test-stdout.txt`, the agent's
trajectory (codex `apply_patch` payloads, claude-code `Edit`/`Write` tool
calls, opencode `edit`/`write`), and the gold patch. From those, it computes
rule-based failure-mode flags and a primary failure class.

Failure-mode flags (multi-label):

  PATCH_APPLY_FAILED      — verifier output indicates patch couldn't be applied
                            (rare in practice — agents call apply tools directly,
                            not raw `git apply`)
  P2P_REGRESSION          — at least one previously-passing test now fails
  F2P_PARTIAL             — flipped some F2P tests but not all
  F2P_NONE                — no F2P tests now pass (no forward progress)
  TYPE_OR_IMPORT_ERROR    — pytest collected zero tests OR exited with
                            ImportError (signals broken module-level code)
  COMMITTED_NARROWER      — committed_file_recall < 1.0 despite
                            examined_file_recall = 1.0
                            (saw all the right files, edited fewer than gold)
  COMMITTED_BROADER       — agent edited files NOT in the gold set
                            (over-eager fix touching unrelated code)
  AGENT_DIFF_TINY         — agent's diff is <30% the size of gold patch
                            (insufficient edit volume — likely wrong approach)
  AGENT_DIFF_HUGE         — agent's diff is >300% gold patch size
                            (over-engineering / unrelated changes)
  EXIT_NONZERO            — agent process exited non-zero before completing

A primary class is assigned by precedence:
  PATCH_APPLY_FAILED > TYPE_OR_IMPORT_ERROR > P2P_REGRESSION > F2P_NONE >
  F2P_PARTIAL > (none — fallback)

Output: per-trial JSON records in --output-jsonl, one record per line. Plus a
tabular summary (CSV) of class distribution per model.

Usage:
  uv run python scripts/classify_localized_failures.py \\
      --implicit-csvs /tmp/e2e-implicit-search.csv /tmp/e2e-opus.csv \\
                      /tmp/e2e-haiku.csv /tmp/e2e-qwen.csv \\
      --e2e-roots /tmp/e2e-codex-full /tmp/e2e-opus /tmp/e2e-haiku /tmp/e2e-qwen \\
      --tasks-dir ~/projects/craft-bench/harbor-tasks/craft-taskgen-v2b \\
      --output-jsonl docs/data/v2b-localized-failure-typology.jsonl \\
      --output-summary docs/data/v2b-localized-failure-summary.csv
"""

from __future__ import annotations

import argparse
import csv
import json
import re
import sys
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from pathlib import Path

# Failure-mode codes
PATCH_APPLY_FAILED = "PATCH_APPLY_FAILED"
P2P_REGRESSION = "P2P_REGRESSION"
F2P_PARTIAL = "F2P_PARTIAL"
F2P_NONE = "F2P_NONE"
TYPE_OR_IMPORT_ERROR = "TYPE_OR_IMPORT_ERROR"
COMMITTED_NARROWER = "COMMITTED_NARROWER"
COMMITTED_BROADER = "COMMITTED_BROADER"
AGENT_DIFF_TINY = "AGENT_DIFF_TINY"
AGENT_DIFF_HUGE = "AGENT_DIFF_HUGE"
EXIT_NONZERO = "EXIT_NONZERO"

PRIMARY_CLASS_PRECEDENCE = [
    PATCH_APPLY_FAILED,
    TYPE_OR_IMPORT_ERROR,
    P2P_REGRESSION,
    F2P_NONE,
    F2P_PARTIAL,
]

ALL_FLAGS = [
    PATCH_APPLY_FAILED,
    P2P_REGRESSION,
    F2P_PARTIAL,
    F2P_NONE,
    TYPE_OR_IMPORT_ERROR,
    COMMITTED_NARROWER,
    COMMITTED_BROADER,
    AGENT_DIFF_TINY,
    AGENT_DIFF_HUGE,
    EXIT_NONZERO,
]


# ---------------------------------------------------------------------------
# Diff size estimation
# ---------------------------------------------------------------------------


def _gold_patch_size(task_dir: Path) -> int:
    """Lines added + removed in the gold patch."""
    p = task_dir / "solution" / "changes.patch"
    if not p.is_file():
        return 0
    n = 0
    for line in p.read_text(errors="replace").splitlines():
        if line.startswith(("+", "-")) and not line.startswith(("+++ ", "--- ")):
            n += 1
    return n


def _agent_diff_size_from_trajectory(traj: dict) -> int:
    """Estimate the agent's total edit volume in lines added+removed.

    Three sources, by tool:
      codex apply_patch: `input` is the codex DSL — count `+`/`-` lines
      claude-code Edit/Write/MultiEdit — count newlines in old/new strings
      opencode edit/write — count newlines in oldString/newString/content
    """
    n = 0
    for step in traj.get("steps") or []:
        for tc in step.get("tool_calls") or []:
            fn = tc.get("function_name", "")
            args = tc.get("arguments") or {}
            if fn == "apply_patch":
                # codex DSL
                for line in (args.get("input") or "").splitlines():
                    if line.startswith(("+", "-")) and not line.startswith(("+++ ", "--- ")):
                        n += 1
                # unified-diff fallback
                for line in (args.get("patch") or "").splitlines():
                    if line.startswith(("+", "-")) and not line.startswith(("+++ ", "--- ")):
                        n += 1
            elif fn in ("Edit", "MultiEdit"):
                for k in ("old_string", "new_string"):
                    s = args.get(k) or ""
                    n += s.count("\n") + (1 if s else 0)
            elif fn == "Write":
                s = args.get("content") or ""
                n += s.count("\n") + (1 if s else 0)
            elif fn in ("edit", "write"):
                for k in ("oldString", "newString", "content"):
                    s = args.get(k) or ""
                    n += s.count("\n") + (1 if s else 0)
    return n


def _load_trajectory(trial_dir: Path) -> dict | None:
    """Load ATIF trajectory or adapt claude-code.txt. Same as in score_e2e_implicit_search.py
    but inlined here for self-containment."""
    p = trial_dir / "agent" / "trajectory.json"
    if p.is_file():
        try:
            return json.loads(p.read_text())
        except (OSError, json.JSONDecodeError):
            return None
    cc = trial_dir / "agent" / "claude-code.txt"
    if cc.is_file():
        try:
            return _adapt_claude_code(cc)
        except (OSError, json.JSONDecodeError):
            return None
    return None


def _adapt_claude_code(p: Path) -> dict:
    steps: list[dict] = []
    pending: dict[str, dict] = {}
    sid = 0
    with p.open() as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                ev = json.loads(line)
            except json.JSONDecodeError:
                continue
            if ev.get("type") == "assistant":
                msg = ev.get("message") or {}
                content = msg.get("content") or []
                text_buf: list[str] = []
                for blk in content:
                    btype = blk.get("type")
                    if btype == "text":
                        text_buf.append(blk.get("text") or "")
                    elif btype == "tool_use":
                        sid += 1
                        step = {
                            "step_id": sid,
                            "source": "agent",
                            "message": "\n".join(text_buf),
                            "tool_calls": [
                                {
                                    "tool_call_id": blk.get("id"),
                                    "function_name": blk.get("name", ""),
                                    "arguments": blk.get("input") or {},
                                }
                            ],
                        }
                        text_buf = []
                        pending[blk.get("id") or ""] = step
                        steps.append(step)
            elif ev.get("type") == "user":
                msg = ev.get("message") or {}
                content = msg.get("content") or []
                if isinstance(content, list):
                    for blk in content:
                        if blk.get("type") == "tool_result":
                            tcid = blk.get("tool_use_id")
                            payload = blk.get("content")
                            if isinstance(payload, list):
                                txt = "\n".join(p.get("text", "") for p in payload if isinstance(p, dict))
                            else:
                                txt = str(payload or "")
                            if tcid in pending:
                                pending[tcid].setdefault("observation", {}).setdefault("results", []).append(
                                    {"source_call_id": tcid, "content": txt}
                                )
    return {
        "schema_version": "claude-code-adapted",
        "agent": {"name": "claude-code"},
        "steps": steps,
    }


# ---------------------------------------------------------------------------
# Verifier output parsing
# ---------------------------------------------------------------------------

_PATCH_APPLY_FAIL_RE = re.compile(
    r"(error: patch failed|patch does not apply|fatal: corrupt patch|cannot apply patch)",
    re.IGNORECASE,
)
_IMPORT_ERROR_RE = re.compile(r"^(ImportError|ModuleNotFoundError|SyntaxError):", re.MULTILINE)
_COLLECTION_ERROR_RE = re.compile(r"errors? during collection|0 items collected", re.IGNORECASE)


def _check_verifier_output(test_stdout: str) -> tuple[bool, bool]:
    """Return (patch_apply_failed, type_or_import_error) flags."""
    patch_apply = bool(_PATCH_APPLY_FAIL_RE.search(test_stdout))
    type_or_import = bool(_IMPORT_ERROR_RE.search(test_stdout)) or bool(
        _COLLECTION_ERROR_RE.search(test_stdout)
    )
    return (patch_apply, type_or_import)


# ---------------------------------------------------------------------------
# Per-trial classifier
# ---------------------------------------------------------------------------


@dataclass
class TrialRecord:
    model: str
    task: str
    trial_path: str

    # Verifier outcome
    f2p_passed: int = 0
    f2p_total: int = 0
    p2p_passed: int = 0
    p2p_total: int = 0
    f2p_failed: list[str] = field(default_factory=list)
    p2p_failed: list[str] = field(default_factory=list)

    # Implicit-search numbers (carried from input CSV)
    exam_file_recall: float = 0.0
    exam_function_recall: float = 0.0
    comm_file_recall: float = 0.0
    comm_function_recall: float = 0.0
    n_examined_files: int = 0
    n_committed_files: int = 0
    gold_n_files: int = 0
    gold_n_functions: int = 0

    # Diff sizes
    gold_diff_lines: int = 0
    agent_diff_lines: int = 0
    agent_diff_ratio: float = 0.0  # agent / gold

    # Flags
    flags: list[str] = field(default_factory=list)
    primary_class: str = ""

    def to_dict(self) -> dict:
        d = self.__dict__.copy()
        return d


def _classify(rec: TrialRecord) -> None:
    """Apply classification rules. Mutates `rec.flags` and `rec.primary_class`."""
    flags: set[str] = set()

    # F2P-based outcome flags
    if rec.f2p_total == 0:
        flags.add(TYPE_OR_IMPORT_ERROR)  # collected nothing — likely import-time crash
    else:
        if rec.f2p_passed == 0:
            flags.add(F2P_NONE)
        elif rec.f2p_passed < rec.f2p_total:
            flags.add(F2P_PARTIAL)
    # P2P regression: known failures > 0
    if rec.p2p_failed:
        flags.add(P2P_REGRESSION)

    # Localization width
    if rec.exam_file_recall >= 1.0 and rec.comm_file_recall < 1.0:
        flags.add(COMMITTED_NARROWER)
    if rec.n_committed_files > rec.gold_n_files:
        flags.add(COMMITTED_BROADER)

    # Diff-size flags
    if rec.gold_diff_lines > 0:
        ratio = rec.agent_diff_lines / rec.gold_diff_lines
        rec.agent_diff_ratio = round(ratio, 3)
        if ratio < 0.30:
            flags.add(AGENT_DIFF_TINY)
        elif ratio > 3.0:
            flags.add(AGENT_DIFF_HUGE)

    rec.flags = sorted(flags)

    # Primary class by precedence (first match wins)
    for c in PRIMARY_CLASS_PRECEDENCE:
        if c in flags:
            rec.primary_class = c
            return
    rec.primary_class = "UNCLASSIFIED"


# ---------------------------------------------------------------------------
# Population builder
# ---------------------------------------------------------------------------


def _build_trial_record(csv_row: dict, trial_dir: Path, task_dir: Path | None) -> TrialRecord | None:
    """Read all the artifacts and build a TrialRecord."""
    rj_path = trial_dir / "verifier" / "reward.json"
    if not rj_path.is_file():
        return None
    try:
        rj = json.loads(rj_path.read_text())
    except (OSError, json.JSONDecodeError):
        return None
    # f2p_failed/p2p_failed live in reward-details.json under the new
    # SCORE_PY_TEMPLATE (split out for harbor>=0.13.1 pydantic compat).
    details_path = trial_dir / "verifier" / "reward-details.json"
    if details_path.is_file():
        try:
            details = json.loads(details_path.read_text())
            for key in ("f2p_failed", "p2p_failed"):
                if key in details and key not in rj:
                    rj[key] = details[key]
        except (OSError, json.JSONDecodeError):
            pass

    # Implicit-search numbers from the CSV row
    rec = TrialRecord(
        model=csv_row["agent"] + "/" + csv_row["model"][:80],
        task=csv_row["task"],
        trial_path=str(trial_dir),
        f2p_passed=int(rj.get("f2p_passed") or 0),
        f2p_total=int(rj.get("f2p_total") or 0),
        p2p_passed=int(rj.get("p2p_passed") or 0),
        p2p_total=int(rj.get("p2p_total") or 0),
        f2p_failed=list(rj.get("f2p_failed") or []),
        p2p_failed=list(rj.get("p2p_failed") or []),
        exam_file_recall=float(csv_row["exam_file_recall"] or 0),
        exam_function_recall=float(csv_row["exam_function_recall"] or 0),
        comm_file_recall=float(csv_row["comm_file_recall"] or 0),
        comm_function_recall=float(csv_row["comm_function_recall"] or 0),
        n_examined_files=int(csv_row["n_examined_files"] or 0),
        n_committed_files=int(csv_row["n_committed_files"] or 0),
        gold_n_files=int(csv_row["gold_n_files"] or 0),
        gold_n_functions=int(csv_row["gold_n_functions"] or 0),
    )

    # Gold diff size from task_dir
    if task_dir is not None and task_dir.is_dir():
        rec.gold_diff_lines = _gold_patch_size(task_dir)

    # Agent diff size from trajectory
    traj = _load_trajectory(trial_dir)
    if traj is not None:
        rec.agent_diff_lines = _agent_diff_size_from_trajectory(traj)

    # Verifier-output text flags
    test_stdout = trial_dir / "verifier" / "test-stdout.txt"
    if test_stdout.is_file():
        try:
            text = test_stdout.read_text(errors="replace")
            patch_fail, type_err = _check_verifier_output(text)
            if patch_fail:
                rec.flags.append(PATCH_APPLY_FAILED)
            if type_err:
                rec.flags.append(TYPE_OR_IMPORT_ERROR)
        except OSError:
            pass

    _classify(rec)
    return rec


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
    ap.add_argument(
        "--implicit-csvs",
        nargs="+",
        type=Path,
        required=True,
        help="One or more CSVs from score_e2e_implicit_search.py (per-model).",
    )
    ap.add_argument(
        "--e2e-roots",
        nargs="+",
        type=Path,
        required=True,
        help="One or more roots holding the e2e job dirs (matched against trial_dir column).",
    )
    ap.add_argument(
        "--tasks-dir",
        type=Path,
        required=True,
        help="v2b harbor-tasks dir (for solution/changes.patch lookup).",
    )
    ap.add_argument(
        "--output-jsonl",
        type=Path,
        required=True,
        help="Per-trial structured records, one JSON per line.",
    )
    ap.add_argument(
        "--output-summary",
        type=Path,
        required=True,
        help="Aggregate distribution CSV.",
    )
    ap.add_argument(
        "--include-all-failures",
        action="store_true",
        help="If set, include all e2e_resolved=0 trials (not only fully-localized ones). "
        "Default: only include trials with exam_file_recall=1.0.",
    )
    args = ap.parse_args()

    # Pre-resolve trial_dir → absolute path. The trial_dir column is a relative
    # path under one of the e2e-roots; we walk all roots to find the right one.
    def _find_trial(trial_path: str) -> Path | None:
        for root in args.e2e_roots:
            cand = root / "jobs" / trial_path
            if cand.is_dir():
                return cand
            cand2 = root / trial_path
            if cand2.is_dir():
                return cand2
        return None

    records: list[TrialRecord] = []
    skipped = 0
    for csv_path in args.implicit_csvs:
        with csv_path.open() as f:
            for row in csv.DictReader(f):
                if row["e2e_resolved"] != "0":
                    continue
                if not args.include_all_failures and float(row["exam_file_recall"] or 0) < 1.0:
                    continue
                trial_dir = _find_trial(row["trial_dir"])
                if trial_dir is None:
                    skipped += 1
                    continue
                task_dir = args.tasks_dir / row["task"]
                rec = _build_trial_record(row, trial_dir, task_dir)
                if rec is None:
                    skipped += 1
                    continue
                records.append(rec)

    print(f"Built {len(records)} trial records ({skipped} skipped)", file=sys.stderr)

    # Write per-trial JSONL
    args.output_jsonl.parent.mkdir(parents=True, exist_ok=True)
    with args.output_jsonl.open("w") as fh:
        for rec in records:
            fh.write(json.dumps(rec.to_dict(), default=str) + "\n")
    print(f"Wrote per-trial records → {args.output_jsonl}", file=sys.stderr)

    # Aggregate distribution
    by_model_class: dict[str, Counter[str]] = defaultdict(Counter)
    by_model_flag: dict[str, Counter[str]] = defaultdict(Counter)
    overall_class: Counter[str] = Counter()
    overall_flag: Counter[str] = Counter()
    by_model_n: Counter[str] = Counter()
    for rec in records:
        m = rec.model
        by_model_n[m] += 1
        by_model_class[m][rec.primary_class] += 1
        overall_class[rec.primary_class] += 1
        for f in rec.flags:
            by_model_flag[m][f] += 1
            overall_flag[f] += 1

    args.output_summary.parent.mkdir(parents=True, exist_ok=True)
    with args.output_summary.open("w", newline="") as fh:
        w = csv.writer(fh)
        # Header: model, n_trials, primary class counts, flag counts
        models = sorted(by_model_n)
        header = ["row_kind", "label"] + models + ["overall"]
        w.writerow(header)

        # Trial counts per model
        w.writerow(["trial_count", "n"] + [by_model_n[m] for m in models] + [sum(by_model_n.values())])

        # Primary class distribution
        for cls in PRIMARY_CLASS_PRECEDENCE + ["UNCLASSIFIED"]:
            row = ["primary_class", cls]
            for m in models:
                row.append(by_model_class[m].get(cls, 0))
            row.append(overall_class.get(cls, 0))
            w.writerow(row)

        # Flag distribution (multi-label, so doesn't sum to 100%)
        for flag in ALL_FLAGS:
            row = ["flag", flag]
            for m in models:
                row.append(by_model_flag[m].get(flag, 0))
            row.append(overall_flag.get(flag, 0))
            w.writerow(row)

    print(f"Wrote summary → {args.output_summary}", file=sys.stderr)

    # Brief stdout summary
    print(f"\n=== Primary failure-class distribution (n={sum(by_model_n.values())}) ===")
    for cls in PRIMARY_CLASS_PRECEDENCE + ["UNCLASSIFIED"]:
        n = overall_class.get(cls, 0)
        if n == 0:
            continue
        pct = 100.0 * n / max(1, sum(by_model_n.values()))
        print(f"  {cls:<28} {n:>3}  ({pct:>5.1f}%)")
    print("\n=== Flag distribution (multi-label) ===")
    for flag in ALL_FLAGS:
        n = overall_flag.get(flag, 0)
        if n == 0:
            continue
        pct = 100.0 * n / max(1, sum(by_model_n.values()))
        print(f"  {flag:<28} {n:>3}  ({pct:>5.1f}%)")
    print("\n=== Per-model primary-class breakdown ===")
    print(f"  {'model':<60} " + " ".join(f"{c[:10]:>10}" for c in PRIMARY_CLASS_PRECEDENCE))
    for m in models:
        cells = " ".join(f"{by_model_class[m].get(c, 0):>10}" for c in PRIMARY_CLASS_PRECEDENCE)
        print(f"  {m:<60} {cells}")

    return 0


if __name__ == "__main__":
    sys.exit(main())