Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
19.9 KB
Raw Blame
#!/usr/bin/env python3
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Analyze whether meaningful candidate interfaces confound pass rates."""

from __future__ import annotations

# ruff: noqa: E501,I001

import argparse
import ast
import csv
import json
import re
from collections import defaultdict
from pathlib import Path
from statistics import mean, median
from typing import Iterable

DEFAULT_STATE = Path("harbor-tasks/craft-tools-v3a/runs/new_pipeline_0427/state.json")
DEFAULT_ENRICHED = Path("docs/analyses/data/swebench-pro/findings/task_outcomes_enriched.csv")
DEFAULT_SWEBENCH_PRO = Path("swebench_pro.jsonl")
DEFAULT_OUTPUT_CSV = Path("docs/analyses/data/swebench-pro/findings/interface_confounders_enriched.csv")
DEFAULT_REPORT = Path("docs/analyses/data/swebench-pro/findings/interface_confounders_report.md")

NO_INTERFACE_RE = re.compile(
    r"\b(no|not)\b.*\bnew\b.*\b(public\s+)?interfaces?\b|"
    r"\bthere are no new interfaces?\b",
    re.IGNORECASE,
)

NO_REQUIREMENTS_RE = re.compile(
    r"^(no\s+(new\s+)?requirements?\.?|"
    r"there are no\s+(new\s+)?requirements?\.?|"
    r"no explicit requirements?\.?|not applicable\.?|n/?a|none|null)$",
    re.IGNORECASE,
)


def text_value(value: object) -> str:
    if not isinstance(value, str):
        return ""
    return value.strip().strip("\"'").strip()


def raw_non_empty(value: object) -> bool:
    if value is None:
        return False
    if isinstance(value, str):
        return bool(text_value(value))
    if isinstance(value, (list, dict, tuple, set)):
        return bool(value)
    return True


def is_interface_placeholder(value: object) -> bool:
    return bool(NO_INTERFACE_RE.search(text_value(value)))


def is_requirement_placeholder(value: object) -> bool:
    return bool(NO_REQUIREMENTS_RE.match(text_value(value)))


def bool_value(value: str) -> bool:
    return value.strip().lower() == "true"


def int_value(value: str) -> int:
    if value.strip() == "":
        return 0
    return int(float(value))


def pct(num: float, den: float) -> str:
    if den == 0:
        return ""
    return f"{num / den * 100:.1f}%"


def number(value: float) -> str:
    return f"{value:.1f}"


def summary(values: list[int]) -> dict[str, str]:
    if not values:
        return {"mean": "", "median": "", "p75": "", "max": ""}
    ordered = sorted(values)
    p75_idx = min(len(ordered) - 1, int(round((len(ordered) - 1) * 0.75)))
    return {
        "mean": number(mean(values)),
        "median": number(median(values)),
        "p75": number(ordered[p75_idx]),
        "max": str(max(values)),
    }


def markdown_table(rows: list[dict[str, object]], columns: list[str]) -> list[str]:
    if not rows:
        return ["(no rows)"]
    lines = [
        "| " + " | ".join(columns) + " |",
        "| " + " | ".join("---" for _ in columns) + " |",
    ]
    for row in rows:
        lines.append("| " + " | ".join(str(row.get(column, "")) for column in columns) + " |")
    return lines


def group_rows(rows: Iterable[dict[str, object]], keys: tuple[str, ...]) -> list[dict[str, object]]:
    groups: dict[tuple[object, ...], list[dict[str, object]]] = defaultdict(list)
    for row in rows:
        groups[tuple(row[key] for key in keys)].append(row)

    result: list[dict[str, object]] = []
    for key, group in sorted(groups.items()):
        total = len(group)
        passed = sum(1 for row in group if row["agent_success_bool"])
        required_total = summary([int(row["agent_required_tests_total"]) for row in group])
        f2p_total = summary([int(row["swebench_fail_to_pass_total"]) for row in group])
        result.append(
            {
                **dict(zip(keys, key)),
                "n": total,
                "passed": passed,
                "pass_rate": pct(passed, total),
                "avg_required_total": required_total["mean"],
                "avg_swebench_f2p_total": f2p_total["mean"],
            }
        )
    return result


def matched_interface_adjustment(
    rows: list[dict[str, object]],
) -> tuple[list[dict[str, object]], dict[str, object]]:
    """Compare interface groups inside matched repo/eval/alignment/F2P cells."""

    cell_keys = ("repo", "new_eval_verdict", "alignment_verdict", "required_total_bucket")
    by_cell: dict[tuple[object, ...], dict[str, list[dict[str, object]]]] = defaultdict(
        lambda: defaultdict(list)
    )
    for row in rows:
        by_cell[tuple(row[key] for key in cell_keys)][row["interface_kind"]].append(row)

    matched_rows: list[dict[str, object]] = []
    meaningful_actual_passes = 0
    meaningful_total = 0
    meaningful_expected_at_placeholder_rates = 0.0

    for cell, by_kind in sorted(by_cell.items()):
        if "meaningful" not in by_kind or "placeholder_no_new" not in by_kind:
            continue
        placeholder = by_kind["placeholder_no_new"]
        meaningful = by_kind["meaningful"]
        placeholder_n = len(placeholder)
        placeholder_pass = sum(1 for row in placeholder if row["agent_success_bool"])
        meaningful_n = len(meaningful)
        meaningful_pass = sum(1 for row in meaningful if row["agent_success_bool"])
        placeholder_rate = placeholder_pass / placeholder_n

        meaningful_total += meaningful_n
        meaningful_actual_passes += meaningful_pass
        meaningful_expected_at_placeholder_rates += placeholder_rate * meaningful_n

        matched_rows.append(
            {
                "repo": cell[0],
                "eval": cell[1],
                "alignment": cell[2],
                "f2p_bucket": cell[3],
                "placeholder": f"{placeholder_pass}/{placeholder_n} ({pct(placeholder_pass, placeholder_n)})",
                "meaningful": f"{meaningful_pass}/{meaningful_n} ({pct(meaningful_pass, meaningful_n)})",
            }
        )

    summary_row = {
        "matched_meaningful_total": meaningful_total,
        "matched_meaningful_actual": meaningful_actual_passes,
        "matched_meaningful_actual_rate": pct(meaningful_actual_passes, meaningful_total),
        "expected_passes_at_placeholder_cell_rates": number(meaningful_expected_at_placeholder_rates),
        "expected_rate_at_placeholder_cell_rates": pct(
            meaningful_expected_at_placeholder_rates, meaningful_total
        ),
    }
    return matched_rows, summary_row


def load_interface_metadata(state_path: Path) -> dict[str, dict[str, object]]:
    with state_path.open() as f:
        state = json.load(f)

    metadata: dict[str, dict[str, object]] = {}
    for task_id, task in state["tasks"].items():
        source_metadata = task.get("candidate_data", {}).get("source_metadata", {}) or {}
        interface = source_metadata.get("interface")
        requirements = source_metadata.get("requirements")

        interface_raw = raw_non_empty(interface)
        interface_placeholder = interface_raw and is_interface_placeholder(interface)
        requirement_raw = raw_non_empty(requirements)
        requirement_placeholder = requirement_raw and is_requirement_placeholder(requirements)

        metadata[task_id] = {
            "interface_raw_non_empty": interface_raw,
            "interface_placeholder_no_new": interface_placeholder,
            "interface_meaningful": interface_raw and not interface_placeholder,
            "interface_kind": "meaningful"
            if interface_raw and not interface_placeholder
            else "placeholder_no_new",
            "requirements_raw_non_empty": requirement_raw,
            "requirements_placeholder": requirement_placeholder,
            "requirements_meaningful": requirement_raw and not requirement_placeholder,
            "interface_char_count": len(text_value(interface)),
            "requirements_char_count": len(text_value(requirements)),
        }
    return metadata


def parse_serialized_list(raw: object) -> list[str]:
    if not raw:
        return []
    if not isinstance(raw, str):
        return list(raw or [])
    try:
        return json.loads(raw)
    except Exception:
        return ast.literal_eval(raw)


def load_swebench_counts(swebench_pro_path: Path) -> dict[str, dict[str, int]]:
    counts: dict[str, dict[str, int]] = {}
    with swebench_pro_path.open() as f:
        for line in f:
            if not line.strip():
                continue
            obj = json.loads(line)
            counts[obj["instance_id"]] = {
                "fail_to_pass_total": len(parse_serialized_list(obj.get("fail_to_pass"))),
                "pass_to_pass_total": len(parse_serialized_list(obj.get("pass_to_pass"))),
            }
    return counts


def load_rows(
    enriched_path: Path,
    interface_metadata: dict[str, dict[str, object]],
    swebench_counts: dict[str, dict[str, int]],
) -> list[dict[str, object]]:
    with enriched_path.open() as f:
        reader = csv.DictReader(f)
        rows = []
        for row in reader:
            task_id = row["task_id"]
            if task_id not in interface_metadata:
                raise ValueError(f"{task_id} is missing from state metadata")
            instance_id = row["swebench_instance_id"]
            if instance_id not in swebench_counts:
                raise ValueError(f"{instance_id} is missing from swebench_pro counts")
            merged: dict[str, object] = {**row, **interface_metadata[task_id]}
            merged["agent_success_bool"] = bool_value(row["agent_success"])
            merged["agent_required_tests_total"] = int_value(row["agent_f2p_tests_total"])
            merged["agent_required_tests_passed"] = int_value(row["agent_f2p_tests_passed"])
            merged["required_total_bucket"] = row["f2p_total_bucket"]
            merged["swebench_fail_to_pass_total"] = swebench_counts[instance_id]["fail_to_pass_total"]
            merged["swebench_pass_to_pass_total"] = swebench_counts[instance_id]["pass_to_pass_total"]
            merged["required_total_matches_swebench"] = (
                merged["agent_required_tests_total"]
                == merged["swebench_fail_to_pass_total"] + merged["swebench_pass_to_pass_total"]
            )
            for column in (
                "source_lines_changed",
                "test_lines_changed",
                "source_file_count",
                "test_file_count",
                "edit_count",
                "edited_file_count",
            ):
                merged[column] = int_value(row[column])
            rows.append(merged)
    return rows


def write_csv(rows: list[dict[str, object]], output_path: Path) -> None:
    output_path.parent.mkdir(parents=True, exist_ok=True)
    fieldnames = [
        "task_id",
        "repo",
        "new_eval_verdict",
        "alignment_verdict",
        "agent_success",
        "agent_required_tests_passed",
        "agent_required_tests_total",
        "swebench_fail_to_pass_total",
        "swebench_pass_to_pass_total",
        "required_total_matches_swebench",
        "required_total_bucket",
        "interface_kind",
        "interface_meaningful",
        "interface_placeholder_no_new",
        "requirements_meaningful",
        "requirements_placeholder",
        "interface_char_count",
        "requirements_char_count",
        "source_lines_changed",
        "test_lines_changed",
        "source_file_count",
        "test_file_count",
        "is_multi_file",
        "is_multi_package",
        "is_refactoring",
    ]
    with output_path.open("w", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        writer.writeheader()
        for row in rows:
            writer.writerow({field: row.get(field, "") for field in fieldnames})


def make_report(
    rows: list[dict[str, object]],
    matched_rows: list[dict[str, object]],
    matched_summary: dict[str, object],
    report_path: Path,
    output_csv: Path,
) -> None:
    report_path.parent.mkdir(parents=True, exist_ok=True)
    meaningful = [row for row in rows if row["interface_kind"] == "meaningful"]
    placeholder = [row for row in rows if row["interface_kind"] == "placeholder_no_new"]

    def pass_summary(group: list[dict[str, object]]) -> tuple[int, int, str]:
        passed = sum(1 for row in group if row["agent_success_bool"])
        return passed, len(group), pct(passed, len(group))

    summary_rows = []
    for label, group in (("meaningful", meaningful), ("placeholder_no_new", placeholder)):
        passed, total, rate = pass_summary(group)
        required_total = summary([int(row["agent_required_tests_total"]) for row in group])
        f2p_total = summary([int(row["swebench_fail_to_pass_total"]) for row in group])
        source_lines = summary([int(row["source_lines_changed"]) for row in group])
        test_lines = summary([int(row["test_lines_changed"]) for row in group])
        leaked_count = sum(1 for row in group if row["alignment_verdict"] == "leaked")
        ok_count = sum(1 for row in group if row["alignment_verdict"] == "ok")
        summary_rows.append(
            {
                "interface_kind": label,
                "n": total,
                "passed": passed,
                "pass_rate": rate,
                "leaked_share": pct(leaked_count, total),
                "ok_share": pct(ok_count, total),
                "avg_required_total": required_total["mean"],
                "avg_swebench_f2p_total": f2p_total["mean"],
                "avg_source_lines": source_lines["mean"],
                "avg_test_lines": test_lines["mean"],
            }
        )

    requirement_placeholder_count = sum(1 for row in rows if row["requirements_placeholder"])
    requirement_meaningful_count = sum(1 for row in rows if row["requirements_meaningful"])

    top_repo_rows = group_rows(rows, ("repo", "interface_kind"))
    by_alignment_rows = group_rows(rows, ("alignment_verdict", "interface_kind"))
    by_eval_rows = group_rows(rows, ("new_eval_verdict", "interface_kind"))
    by_f2p_bucket_rows = group_rows(rows, ("f2p_total_bucket", "interface_kind"))
    by_alignment_f2p_rows = group_rows(rows, ("alignment_verdict", "interface_kind", "f2p_total_bucket"))

    lines = [
        "# Interface Field Confounder Analysis",
        "",
        f"Enriched CSV: `{output_csv}`",
        "",
        "Raw `interface` is non-empty for every task, so this report uses the semantic split:",
        "`meaningful` vs `placeholder_no_new`.",
        "",
        "## Headline",
        "",
    ]
    lines.extend(
        markdown_table(
            summary_rows,
            [
                "interface_kind",
                "n",
                "passed",
                "pass_rate",
                "leaked_share",
                "ok_share",
                "avg_required_total",
                "avg_swebench_f2p_total",
                "avg_source_lines",
                "avg_test_lines",
            ],
        )
    )
    lines.extend(
        [
            "",
            "Requirements are not useful as a splitter in this state file: "
            f"{requirement_meaningful_count}/{len(rows)} are semantically meaningful and "
            f"{requirement_placeholder_count}/{len(rows)} are placeholder-like.",
            "",
            "## By Alignment Verdict",
            "",
        ]
    )
    lines.extend(
        markdown_table(
            by_alignment_rows,
            [
                "alignment_verdict",
                "interface_kind",
                "n",
                "passed",
                "pass_rate",
                "avg_required_total",
                "avg_swebench_f2p_total",
            ],
        )
    )
    lines.extend(["", "## By Required Test Count Bucket", ""])
    lines.extend(
        markdown_table(
            by_f2p_bucket_rows,
            [
                "f2p_total_bucket",
                "interface_kind",
                "n",
                "passed",
                "pass_rate",
                "avg_required_total",
                "avg_swebench_f2p_total",
            ],
        )
    )
    lines.extend(["", "## By Alignment Verdict And Required Test Bucket", ""])
    lines.extend(
        markdown_table(
            by_alignment_f2p_rows,
            [
                "alignment_verdict",
                "interface_kind",
                "f2p_total_bucket",
                "n",
                "passed",
                "pass_rate",
                "avg_required_total",
                "avg_swebench_f2p_total",
            ],
        )
    )
    lines.extend(["", "## By Repo", ""])
    lines.extend(
        markdown_table(
            top_repo_rows,
            [
                "repo",
                "interface_kind",
                "n",
                "passed",
                "pass_rate",
                "avg_required_total",
                "avg_swebench_f2p_total",
            ],
        )
    )
    lines.extend(["", "## By Eval Verdict", ""])
    lines.extend(
        markdown_table(
            by_eval_rows,
            [
                "new_eval_verdict",
                "interface_kind",
                "n",
                "passed",
                "pass_rate",
                "avg_required_total",
                "avg_swebench_f2p_total",
            ],
        )
    )
    lines.extend(["", "## Matched Cell Check", ""])
    lines.extend(
        [
            "Matched cells use `(repo, eval verdict, alignment verdict, required test-count bucket)`. "
            "This asks whether meaningful-interface tasks still underperform after comparing "
            "only against placeholder-interface tasks in the same broad strata.",
            "",
            "| metric | value |",
            "| --- | --- |",
        ]
    )
    for key, value in matched_summary.items():
        lines.append(f"| {key} | {value} |")
    lines.extend(["", "Matched cells:"])
    lines.extend(
        markdown_table(
            matched_rows,
            ["repo", "eval", "alignment", "f2p_bucket", "placeholder", "meaningful"],
        )
    )
    lines.extend(
        [
            "",
            "## Interpretation",
            "",
            "- Raw pass rate barely changes by interface kind: meaningful is 57.9% and placeholder-no-new is 56.8%. So meaningful interface metadata is not an overall negative predictor by itself.",
            "- The interesting interaction is with alignment verdict: leaked + meaningful interface is 43.2%, while leaked + placeholder-no-new is 63.6%; OK + meaningful is 65.2%. The leaked underperformance is concentrated in the meaningful-interface slice.",
            "- Required test count is a plausible confounder inside that slice: leaked + meaningful averages 112 required tests, while OK + meaningful averages 53. But required count alone is not sufficient, since leaked + placeholder-no-new averages 123 required tests and still passes at 63.6%.",
            "- The matched cell check is noisy because many strata are small, but it does not support a stable overall penalty for meaningful interfaces. Matched meaningful-interface tasks pass at 60.5%, versus 51.8% expected if they followed placeholder rates in the same cells.",
            "- Requirements do not provide a usable confounder split here because every candidate has meaningful requirements text.",
        ]
    )

    report_path.write_text("\n".join(lines) + "\n")


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--state", type=Path, default=DEFAULT_STATE)
    parser.add_argument("--enriched", type=Path, default=DEFAULT_ENRICHED)
    parser.add_argument("--swebench-pro", type=Path, default=DEFAULT_SWEBENCH_PRO)
    parser.add_argument("--output-csv", type=Path, default=DEFAULT_OUTPUT_CSV)
    parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
    args = parser.parse_args()

    interface_metadata = load_interface_metadata(args.state)
    swebench_counts = load_swebench_counts(args.swebench_pro)
    rows = load_rows(args.enriched, interface_metadata, swebench_counts)
    matched_rows, matched_summary = matched_interface_adjustment(rows)
    write_csv(rows, args.output_csv)
    make_report(rows, matched_rows, matched_summary, args.report, args.output_csv)

    print(f"rows: {len(rows)}")
    print(f"wrote: {args.output_csv}")
    print(f"wrote: {args.report}")


if __name__ == "__main__":
    main()