Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
18.6 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""regress_e2e_resolution.py — multivariate logistic regression on e2e success.

Models:
  e2e_resolved ~ behavioral predictors + trial-length controls + task-complexity
                 controls + per-model fixed effects

Predictors are normalized to be comparable across models (rates, fractions,
recalls). Model fixed effects absorb the codex-vs-others tool-call-volume
difference (codex's exec_command runs rg/grep, inflating examined-file
counts; claude-code's Read is one file each).

Reports for each predictor:
  - β coefficient (log-odds change per unit; SD-units for continuous)
  - SE (Hessian-based)
  - 95% CI
  - z-statistic
  - approximate two-sided p-value (normal approximation; n=364 makes this
    fine)

Three model variants:
  1. POOLED — single regression with model fixed effects
  2. PER-MODEL — one regression per model (no FE), to diagnose heterogeneity
  3. FIRST-HALF — same as pooled but using only first-half-of-trial signals,
     to control for "success ends the trial early" reverse-causality

Logistic regression is fit by IRLS (Newton-Raphson on the log-likelihood)
with ridge regularization (λ=0.01) for numerical stability. No statsmodels
dep — implementation is ~30 lines and is correct for our scale (n=364,
p≤15).

Usage:
  uv run python scripts/regress_e2e_resolution.py \\
      --per-trial-csv docs/data/v2b-deep-dive-per-trial.csv \\
      --first-half-csv docs/data/v2b-first-half-per-trial.csv \\
      --output-md docs/data/v2b-regression-results.md
"""

from __future__ import annotations

import argparse
import csv
import json
import math
import sys
from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path

import numpy as np

# ---------------------------------------------------------------------------
# Logistic regression via IRLS with L2 regularization
# ---------------------------------------------------------------------------


def _sigmoid(z: np.ndarray) -> np.ndarray:
    # numerically stable
    return np.where(z >= 0, 1 / (1 + np.exp(-z)), np.exp(z) / (1 + np.exp(z)))


@dataclass
class LogitResult:
    feature_names: list[str]
    coef: np.ndarray  # shape (p+1,) — intercept first
    se: np.ndarray  # standard errors
    z: np.ndarray
    p: np.ndarray
    ci_lo: np.ndarray  # 95% CI lower for log-odds
    ci_hi: np.ndarray
    n: int
    n_resolved: int
    log_likelihood: float
    null_log_likelihood: float  # intercept-only
    pseudo_r2: float  # McFadden's
    accuracy: float  # in-sample classification accuracy at 0.5 threshold


def fit_logistic(
    X: np.ndarray,  # shape (n, p) — without intercept
    y: np.ndarray,  # shape (n,) — 0/1
    feature_names: list[str],
    ridge: float = 0.01,
    max_iter: int = 100,
    tol: float = 1e-8,
) -> LogitResult:
    """L2-regularized logistic regression by IRLS.

    Loss: sum_i [-y_i log p_i - (1-y_i) log(1-p_i)] + 0.5 * ridge * ||β[1:]||²
    (intercept is not regularized).
    """
    n, p = X.shape
    # Add intercept column
    X1 = np.column_stack([np.ones(n), X])
    beta = np.zeros(p + 1)

    # Build regularization mask (no penalty on intercept)
    reg = np.full(p + 1, ridge)
    reg[0] = 0.0

    prev_ll = -np.inf
    for it in range(max_iter):
        z = X1 @ beta
        pi = _sigmoid(z)
        # Gradient: X^T (y - pi) - reg * beta
        grad = X1.T @ (y - pi) - reg * beta
        # Hessian: -X^T diag(pi*(1-pi)) X - diag(reg)
        W = pi * (1 - pi)
        # Avoid singularity
        W = np.clip(W, 1e-9, None)
        H = -(X1.T * W) @ X1 - np.diag(reg)
        # Newton step: beta -= H^{-1} grad
        try:
            step = np.linalg.solve(H, grad)
        except np.linalg.LinAlgError:
            step = np.linalg.lstsq(H, grad, rcond=None)[0]
        beta_new = beta - step
        # Log-likelihood for convergence check
        z_new = X1 @ beta_new
        log_p = -np.logaddexp(0.0, -z_new)
        log_1_p = -np.logaddexp(0.0, z_new)
        ll = float(np.sum(y * log_p + (1 - y) * log_1_p)) - 0.5 * float(reg @ (beta_new**2))
        if abs(ll - prev_ll) < tol:
            beta = beta_new
            break
        prev_ll = ll
        beta = beta_new

    # Standard errors from the (negative) Hessian inverse
    z = X1 @ beta
    pi = _sigmoid(z)
    W = pi * (1 - pi)
    W = np.clip(W, 1e-9, None)
    info = (X1.T * W) @ X1 + np.diag(reg)
    try:
        cov = np.linalg.inv(info)
    except np.linalg.LinAlgError:
        cov = np.linalg.pinv(info)
    se = np.sqrt(np.maximum(np.diag(cov), 0.0))

    z_stat = np.where(se > 0, beta / se, 0.0)
    # two-sided p via normal CDF
    from math import erf as _erf
    from math import sqrt as _sqrt

    p_vals = np.array([2 * (1 - 0.5 * (1 + _erf(abs(zz) / _sqrt(2)))) for zz in z_stat])
    ci_lo = beta - 1.96 * se
    ci_hi = beta + 1.96 * se

    # Final log-likelihood (without ridge term — for pseudo R²)
    log_p = -np.logaddexp(0.0, -z)
    log_1_p = -np.logaddexp(0.0, z)
    ll_full = float(np.sum(y * log_p + (1 - y) * log_1_p))
    # Null model: intercept only
    p_null = float(np.mean(y))
    ll_null = (
        float(np.sum(y) * math.log(p_null) + np.sum(1 - y) * math.log(1 - p_null)) if 0 < p_null < 1 else 0.0
    )
    pseudo_r2 = 1 - ll_full / ll_null if ll_null < 0 else 0.0
    pred = (pi >= 0.5).astype(int)
    accuracy = float(np.mean(pred == y))

    return LogitResult(
        feature_names=["(intercept)"] + feature_names,
        coef=beta,
        se=se,
        z=z_stat,
        p=p_vals,
        ci_lo=ci_lo,
        ci_hi=ci_hi,
        n=n,
        n_resolved=int(np.sum(y)),
        log_likelihood=ll_full,
        null_log_likelihood=ll_null,
        pseudo_r2=pseudo_r2,
        accuracy=accuracy,
    )


# ---------------------------------------------------------------------------
# Feature engineering
# ---------------------------------------------------------------------------

MODEL_LABELS = {
    "codex": "codex55",
    "claude-code": "opus47",
    "haiku": "haiku45",
    "qwen": "qwen36",
}


def _short_model(model_str: str) -> str:
    s = model_str.lower()
    if "gpt-5.5" in s:
        return "codex55"
    if "opus-4-7" in s:
        return "opus47"
    if "haiku-4-5" in s:
        return "haiku45"
    if "qwen" in s or "vllm/model" in s:
        return "qwen36"
    return "unknown"


def _build_features(rows: list[dict]) -> tuple[np.ndarray, np.ndarray, list[str], list[str]]:
    """Build the design matrix.

    Predictors (continuous, standardized to mean=0 sd=1 across the pooled
    population so coefficients are interpretable in SD-units):
        tests_per_edit
        tests_per_step
        examined_file_recall
        comm_file_recall
        n_examined_files (log-transformed)
        frac_test_files
        agent_diff_size_ratio (clipped at 10x)
        probe_fraction
        thrash_rate
        pre_edit_mention_rate
        n_steps (log-transformed)
        gold_diff_lines (log-transformed)

    Plus 3 model dummy variables (codex55 is reference; opus/haiku/qwen
    each get a dummy).
    """
    feats: list[list[float]] = []
    y_list: list[int] = []
    skipped = 0
    for r in rows:
        model = _short_model(r["model"])
        if model == "unknown":
            skipped += 1
            continue
        try:
            n_steps = max(1, int(float(r["n_steps"])))
            n_edits = max(0, int(float(r["n_edit_calls"])))
            n_tests = max(0, int(float(r["n_test_invocations"])))
            n_exam = max(0, int(float(r["n_examined_files"])))
            n_test_files = max(0, int(float(r["n_examined_test_files"])))
            files_e1 = max(0, int(float(r["files_edited_once"])))
            files_e2 = max(0, int(float(r["files_edited_2plus"])))
            agent_diff = max(0, int(float(r["agent_diff_lines"])))
            gold_diff = max(0, int(float(r["gold_diff_lines"])))
            first_edit = max(0, int(float(r["first_edit_step"])))
            pre_mention = max(0, int(float(r["pre_edit_mention_count"])))
            gold_n_funcs = max(1, int(float(r["gold_n_functions"])))
            exam_recall = float(r["exam_file_recall"])
            comm_recall = float(r["comm_file_recall"])
        except (KeyError, ValueError):
            skipped += 1
            continue

        # Skip degenerate trials (no edits AND no test invocations)
        if n_edits == 0 and n_tests == 0:
            skipped += 1
            continue

        tests_per_edit = n_tests / max(1, n_edits)
        tests_per_step = n_tests / n_steps
        frac_test = n_test_files / max(1, n_exam)
        diff_ratio = min(agent_diff / max(1, gold_diff), 10.0)
        probe_frac = first_edit / n_steps if first_edit else 1.0  # if no edit ever, "all probe"
        thrash = files_e2 / max(1, files_e1 + files_e2)
        mention_rate = min(pre_mention / gold_n_funcs, 1.0)
        log_exam = math.log1p(n_exam)
        log_steps = math.log1p(n_steps)
        log_gold_diff = math.log1p(gold_diff)

        feats.append(
            [
                tests_per_edit,
                tests_per_step,
                exam_recall,
                comm_recall,
                log_exam,
                frac_test,
                diff_ratio,
                probe_frac,
                thrash,
                mention_rate,
                log_steps,
                log_gold_diff,
                # Model dummies (codex55 reference)
                1.0 if model == "opus47" else 0.0,
                1.0 if model == "haiku45" else 0.0,
                1.0 if model == "qwen36" else 0.0,
            ]
        )
        y_list.append(1 if r["e2e_resolved"] == "1" else 0)

    feature_names = [
        "tests_per_edit",
        "tests_per_step",
        "exam_file_recall",
        "comm_file_recall",
        "log_n_examined_files",
        "frac_test_files",
        "agent_diff_ratio",
        "probe_fraction",
        "thrash_rate",
        "pre_edit_mention_rate",
        "log_n_steps",
        "log_gold_diff_lines",
        "is_opus47",
        "is_haiku45",
        "is_qwen36",
    ]

    X = np.array(feats, dtype=float)
    y = np.array(y_list, dtype=float)
    if skipped:
        print(f"[features] skipped {skipped} rows (degenerate or unknown model)", file=sys.stderr)

    # Standardize the continuous features (first 12) for interpretability
    cont_idx = list(range(12))
    standardized_names = list(feature_names)
    means = np.zeros(X.shape[1])
    sds = np.ones(X.shape[1])
    for i in cont_idx:
        mu = X[:, i].mean()
        sd = X[:, i].std()
        means[i] = mu
        sds[i] = sd if sd > 0 else 1.0
        X[:, i] = (X[:, i] - mu) / sds[i]
        standardized_names[i] = feature_names[i] + " (sd)"

    return X, y, feature_names, standardized_names


# ---------------------------------------------------------------------------
# Reporting
# ---------------------------------------------------------------------------


def _format_result(result: LogitResult, title: str) -> list[str]:
    out: list[str] = []
    out.append(f"### {title}")
    out.append("")
    out.append(
        f"_n = {result.n}, n_resolved = {result.n_resolved} "
        f"({result.n_resolved / result.n * 100:.1f}%); "
        f"in-sample accuracy at 0.5 = {result.accuracy:.3f}; "
        f"McFadden pseudo-R² = {result.pseudo_r2:.3f}_"
    )
    out.append("")
    out.append("| Feature | β | SE | z | p | 95% CI |")
    out.append("|---|---:|---:|---:|---:|---:|")
    for i, name in enumerate(result.feature_names):
        b = result.coef[i]
        se = result.se[i]
        z = result.z[i]
        p = result.p[i]
        lo = result.ci_lo[i]
        hi = result.ci_hi[i]
        stars = "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else ""
        out.append(f"| `{name}` | {b:+.3f}{stars} | {se:.3f} | {z:+.2f} | {p:.4f} | [{lo:+.3f}, {hi:+.3f}] |")
    return out


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
    ap.add_argument(
        "--per-trial-csv",
        type=Path,
        required=True,
        help="docs/data/v2b-deep-dive-per-trial.csv",
    )
    ap.add_argument(
        "--first-half-csv",
        type=Path,
        default=None,
        help="Optional first-half-trials CSV (for reverse-causality check). If absent, skipped.",
    )
    ap.add_argument("--output-md", type=Path, required=True)
    args = ap.parse_args()

    rows = list(csv.DictReader(args.per_trial_csv.open()))
    print(f"Loaded {len(rows)} trial rows", file=sys.stderr)

    X, y, feature_names, std_names = _build_features(rows)
    print(f"Design matrix: n={X.shape[0]}, p={X.shape[1]}, n_resolved={int(y.sum())}", file=sys.stderr)

    # MODEL 1: pooled
    pooled = fit_logistic(X, y, std_names)

    # MODEL 2: per-model (drop the model dummies; refit on each subset)
    # We need un-standardized model assignment to subset
    by_model: dict[str, list[int]] = defaultdict(list)
    for i, r in enumerate(rows):
        m = _short_model(r["model"])
        if m != "unknown":
            by_model[m].append(i)
    # But X has been built including dummies and standardized using the pooled
    # population. Refit per-model on that same X (without dummies) using only
    # the subset of rows.
    per_model_results: dict[str, LogitResult] = {}
    n_dummies = 3  # last 3 columns
    # Re-derive subset mapping on the post-skip X. Since _build_features
    # skips some rows, the indices don't match. Easier: rebuild per-model.
    for m in ("codex55", "opus47", "haiku45", "qwen36"):
        sub_rows = [r for r in rows if _short_model(r["model"]) == m]
        if len(sub_rows) < 30:
            continue
        Xm, ym, _, std_m = _build_features(sub_rows)
        # Drop the 3 dummy columns (all 0 for this single-model fit)
        Xm_nd = Xm[:, :-n_dummies]
        cont_m = std_m[:-n_dummies]
        if int(ym.sum()) < 5 or int((1 - ym).sum()) < 5:
            print(f"[per-model {m}] insufficient class balance, skipping", file=sys.stderr)
            continue
        per_model_results[m] = fit_logistic(Xm_nd, ym, cont_m)

    # MODEL 3: first-half
    first_half_result: LogitResult | None = None
    if args.first_half_csv and args.first_half_csv.is_file():
        rows_h = list(csv.DictReader(args.first_half_csv.open()))
        print(f"Loaded {len(rows_h)} first-half trial rows", file=sys.stderr)
        Xh, yh, _, std_h = _build_features(rows_h)
        first_half_result = fit_logistic(Xh, yh, std_h)

    # Write markdown
    out: list[str] = []
    out.append("# E2E resolution: multivariate logistic regression")
    out.append("")
    out.append(
        "Logistic regression of `e2e_resolved ∈ {0,1}` on behavioral predictors. "
        "Continuous features standardized to SD=1 so coefficients are comparable. "
        "Model dummies in the pooled regression (`codex55` is the reference category). "
        "L2 regularization with λ=0.01."
    )
    out.append("")
    out.append("## How to read this")
    out.append("")
    out.append(
        "- `β` is the change in log-odds of resolution per 1 SD increase (for continuous) "
        "or per category change (for dummies). e.g. β=+0.3 means +1 SD raises log-odds by 0.3, "
        "or odds-ratio = exp(0.3) ≈ 1.35.\n"
        "- The pooled model is the headline. Per-model regressions diagnose heterogeneity.\n"
        "- The first-half regression controls for 'success ends the trial early': it uses only "
        "  signals from the first half of each trial.\n"
        "- Stars: *** p<0.001, ** p<0.01, * p<0.05.\n"
        "- McFadden pseudo-R² is interpretable as 'fraction of null deviance explained'. Values "
        "  above ~0.2 indicate strong fit for binary outcomes."
    )
    out.append("")

    out.append("## Pooled regression with model fixed effects")
    out.append("")
    out.extend(_format_result(pooled, "Pooled (all 4 models, n=" + str(pooled.n) + ")"))
    out.append("")

    out.append("## Per-model regressions")
    out.append("")
    for m in ("codex55", "opus47", "haiku45", "qwen36"):
        if m in per_model_results:
            out.extend(_format_result(per_model_results[m], m))
            out.append("")

    if first_half_result is not None:
        out.append("## First-half-only regression (reverse-causality control)")
        out.append("")
        out.append(
            "Same predictors, but using only signals from steps before the median step of each trial. "
            "Controls for 'successful trials end early' confounding."
        )
        out.append("")
        out.extend(_format_result(first_half_result, "First-half only (n=" + str(first_half_result.n) + ")"))
        out.append("")

    args.output_md.parent.mkdir(parents=True, exist_ok=True)
    args.output_md.write_text("\n".join(out) + "\n")
    print(f"Wrote regression results → {args.output_md}", file=sys.stderr)

    # Also dump as JSON for downstream tooling
    json_out = args.output_md.with_suffix(".json")
    json_data = {
        "pooled": {
            "feature_names": pooled.feature_names,
            "coef": pooled.coef.tolist(),
            "se": pooled.se.tolist(),
            "z": pooled.z.tolist(),
            "p": pooled.p.tolist(),
            "n": pooled.n,
            "n_resolved": pooled.n_resolved,
            "pseudo_r2": pooled.pseudo_r2,
            "accuracy": pooled.accuracy,
        },
        "per_model": {
            m: {
                "feature_names": r.feature_names,
                "coef": r.coef.tolist(),
                "p": r.p.tolist(),
                "n": r.n,
                "n_resolved": r.n_resolved,
                "pseudo_r2": r.pseudo_r2,
            }
            for m, r in per_model_results.items()
        },
    }
    if first_half_result is not None:
        json_data["first_half"] = {
            "feature_names": first_half_result.feature_names,
            "coef": first_half_result.coef.tolist(),
            "p": first_half_result.p.tolist(),
            "n": first_half_result.n,
            "pseudo_r2": first_half_result.pseudo_r2,
        }
    json_out.write_text(json.dumps(json_data, indent=2))
    print(f"Wrote regression JSON → {json_out}", file=sys.stderr)

    # Stdout summary
    print()
    print("=== Top-5 features by |β| in pooled regression ===")
    pairs = sorted(zip(pooled.feature_names, pooled.coef, pooled.p), key=lambda x: abs(x[1]), reverse=True)
    for name, b, p in pairs[:8]:
        if name == "(intercept)":
            continue
        stars = "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else ""
        print(f"  {name:<28} β = {b:+.3f}{stars}  (p = {p:.4f})")

    return 0


if __name__ == "__main__":
    sys.exit(main())