Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
17.8 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Prototype: score harbor-lab trajectories against the proposed easiness heuristics.

Two modes:

  --mode fixtures (default)
    Reads a deep-dive fixture bundle (tmp2/dd-fixtures-20260422/) with
    pre-captured harbor_lab/hlab_*.md outputs. Small cohort, quick turnaround.

  --mode run_dir
    Reads a live pipeline run directory (e.g. harbor-tasks/craft-tools-v4/runs/<ts>/)
    and its `state.json`, shells out to harbor-lab for each task that made it to
    Opus triage, and scores the trajectory. Designed to run on the VM with live
    artifacts. Needs `harbor-lab` on PATH (or $HARBOR_LAB pointing to the binary)
    and the jobs/ tree still present.

Either mode produces objective flags derived from already-captured harbor-lab
outputs (tool-sequence, edits, metrics). No LLM calls, no pipeline integration
— this is exploratory analysis to calibrate thresholds before wiring into the
pipeline as `TaskState.easiness_flag_efficiency` (task i).

Heuristics (objective, reproducible). **All are gated on reward=1.0** — a
failing trial can't be "too easy" regardless of trajectory shape.

Calibrated against the 2026-04-17 run (43 tasks, 29 eligible):

STRONG (rare recipe-writing — calibrated to be never-firing on legitimate work):
    no_exploration    : total Grep+Read < 10.  (Cohort min was 9; <10 picks
                        up tasks that essentially didn't explore at all.)
    zero_iteration    : num_pytest_runs == 0.  (The "run-once-to-confirm"
                        pattern at the end is normal; >= 1 means the agent
                        at least verified its work.)

MEDIUM (statistical outliers, p10 thresholds):
    fast_wall_time    : wall <= global p10 of the cohort.
    low_turns         : total tool calls <= global p10 of the cohort.

DROPPED (didn't pan out on real data):
    few_edits         : num_edits <= 2 — fires on 100% false positives in
                        calibration (minimal patches to complex tasks).

Auto-reject rule:
    auto_reject = reward == 1.0 AND (
                    no_exploration OR zero_iteration
                    OR (fast_wall_time AND low_turns)
                  )

Soft flag rule (individual p10 without corroboration — human review queue):
    soft_flag   = reward == 1.0 AND NOT auto_reject AND (fast_wall_time OR low_turns)

Usage:
    uv run python scripts/analyze-easiness.py \\
        --fixtures tmp2/dd-fixtures-20260422 \\
        --output easiness_analysis.csv
"""

from __future__ import annotations

import argparse
import csv
import json
import os
import re
import shutil
import subprocess
import sys
from dataclasses import asdict, dataclass
from pathlib import Path


@dataclass
class TrajectoryStats:
    task_id: str
    bucket: str
    opus_score: str
    num_edits: int
    num_tool_calls: int
    num_greps: int
    num_reads: int
    num_bash: int
    num_pytest_runs: int
    exploration_before_first_edit: int
    wall_time_s: float | None
    reward: float | None
    f2p_passed: int | None
    f2p_total: int | None


def _read_or_empty(p: Path) -> str:
    try:
        return p.read_text(errors="replace")
    except OSError:
        return ""


_EDITS_COUNT_RE = re.compile(r"\(claude-code,\s*(?P<n>\d+)\s*edits?\)")
_TOOL_ROW_RE = re.compile(r"^\|\s*\d+\s*\|\s*(?P<tool>\[?\w+\]?)\s*\|\s*(?P<args>.*?)\s*\|$")
_WALL_TIME_RE = re.compile(r"Mean agent execution\s*\|\s*([\d.]+)s")


def _parse_tool_sequence(md: str) -> tuple[list[tuple[str, str]], int]:
    """Return [(tool, args), ...] and the total-calls count from the header."""
    calls: list[tuple[str, str]] = []
    total_calls = 0
    total_m = re.search(r"\(claude-code,\s*(?P<n>\d+)\s*calls?\)", md)
    if total_m:
        total_calls = int(total_m.group("n"))
    for line in md.splitlines():
        m = _TOOL_ROW_RE.match(line)
        if m:
            calls.append((m.group("tool"), m.group("args")))
    return calls, total_calls


def _is_pytest_bash(args: str) -> bool:
    a = args.lower()
    return "pytest" in a


def _stats_from_outputs(
    *,
    task_id: str,
    bucket: str,
    opus_score: str,
    edits_md: str,
    tool_seq_md: str,
    metrics_md: str,
    reward_json_text: str,
) -> TrajectoryStats:
    """Parse harbor-lab output + reward.json text into a TrajectoryStats. The
    source of each string (fixture file vs live subprocess) is the caller's
    concern."""
    m = _EDITS_COUNT_RE.search(edits_md)
    num_edits = int(m.group("n")) if m else 0

    calls, total_calls = _parse_tool_sequence(tool_seq_md)

    num_greps = sum(1 for t, _ in calls if t == "Grep")
    num_reads = sum(1 for t, _ in calls if t == "Read")
    num_bash = sum(1 for t, _ in calls if t == "Bash")
    num_pytest = sum(1 for t, a in calls if t == "Bash" and _is_pytest_bash(a))

    exploration_before_first_edit = 0
    for t, _ in calls:
        if t == "Edit":
            break
        if t in ("Grep", "Read"):
            exploration_before_first_edit += 1

    wall_m = _WALL_TIME_RE.search(metrics_md)
    wall_time_s = float(wall_m.group(1)) if wall_m else None

    reward: float | None = None
    f2p_passed: int | None = None
    f2p_total: int | None = None
    if reward_json_text.strip():
        try:
            d = json.loads(reward_json_text)
            reward = d.get("reward")
            f2p_passed = d.get("f2p_passed")
            f2p_total = d.get("f2p_total")
        except json.JSONDecodeError:
            pass

    return TrajectoryStats(
        task_id=task_id,
        bucket=bucket,
        opus_score=opus_score,
        num_edits=num_edits,
        num_tool_calls=total_calls,
        num_greps=num_greps,
        num_reads=num_reads,
        num_bash=num_bash,
        num_pytest_runs=num_pytest,
        exploration_before_first_edit=exploration_before_first_edit,
        wall_time_s=wall_time_s,
        reward=reward,
        f2p_passed=f2p_passed,
        f2p_total=f2p_total,
    )


def stats_for_sample(sample_dir: Path) -> TrajectoryStats:
    """Fixture-mode: read pre-captured harbor_lab/hlab_*.md + trial/verifier/reward.json."""
    info = json.loads(_read_or_empty(sample_dir / "fixture_info.json") or "{}")
    return _stats_from_outputs(
        task_id=info.get("task_id", sample_dir.name),
        bucket=info.get("bucket", "?"),
        opus_score=info.get("opus_score", ""),
        edits_md=_read_or_empty(sample_dir / "harbor_lab" / "hlab_edits.md"),
        tool_seq_md=_read_or_empty(sample_dir / "harbor_lab" / "hlab_tool_sequence.md"),
        metrics_md=_read_or_empty(sample_dir / "harbor_lab" / "hlab_metrics.md"),
        reward_json_text=_read_or_empty(sample_dir / "trial" / "verifier" / "reward.json"),
    )


def _resolve_harbor_lab() -> str:
    env = os.environ.get("HARBOR_LAB")
    if env and os.access(env, os.X_OK):
        return env
    on_path = shutil.which("harbor-lab")
    if on_path:
        return on_path
    # Known checkout locations: $HOME/Documents/vscode/harbor-lab (macOS dev
    # machines) and /data/projects/harbor-lab (craftbench VMs). Both use
    # uv-managed .venv/bin/harbor-lab.
    candidates = [
        os.path.expanduser("~/Documents/vscode/harbor-lab/.venv/bin/harbor-lab"),
        "/data/projects/harbor-lab/.venv/bin/harbor-lab",
    ]
    for p in candidates:
        if os.path.isfile(p) and os.access(p, os.X_OK):
            return p
    raise RuntimeError(
        "harbor-lab binary not found. Install harbor-lab, set $HARBOR_LAB, "
        "or put it on PATH. Tried: $HARBOR_LAB, PATH, and "
        f"{candidates}"
    )


def _run_hlab(bin_path: str, *args: str, timeout: int = 60) -> str:
    try:
        r = subprocess.run([bin_path, *args], capture_output=True, text=True, timeout=timeout)
        return r.stdout
    except (subprocess.TimeoutExpired, OSError):
        return ""


def stats_for_live_task(task_id: str, task_state: dict, hlab_bin: str) -> TrajectoryStats | None:
    """Run-dir mode: shell out to harbor-lab to capture stats from a live trial dir.

    Returns None if the task has no Opus trial or if harbor-lab can't find it.
    """
    opus_trial_dir = task_state.get("opus_trial_dir", "")
    if not opus_trial_dir:
        return None
    # harbor-lab takes the JOB dir (parent of the trial dir)
    job_dir = str(Path(opus_trial_dir).parent)
    if not Path(job_dir).is_dir():
        return None

    edits_md = _run_hlab(hlab_bin, "edits", f"{job_dir}/", "--format", "markdown")
    # IMPORTANT: no --tail — we need the full trajectory to count
    # Grep+Read calls before the first Edit, which appears near the start.
    tool_seq_md = _run_hlab(hlab_bin, "tool-sequence", f"{job_dir}/", "--text", "--format", "markdown")
    metrics_md = _run_hlab(hlab_bin, "metrics", f"{job_dir}/", "--format", "markdown")

    reward_json_text = ""
    reward_path = Path(opus_trial_dir) / "verifier" / "reward.json"
    if reward_path.is_file():
        try:
            reward_json_text = reward_path.read_text(errors="replace")
        except OSError:
            pass

    stage = task_state.get("stage", "")
    return _stats_from_outputs(
        task_id=task_id,
        bucket=stage,
        opus_score=task_state.get("opus_score", ""),
        edits_md=edits_md,
        tool_seq_md=tool_seq_md,
        metrics_md=metrics_md,
        reward_json_text=reward_json_text,
    )


def _percentile(values: list[float], p: float) -> float | None:
    """Simple no-interp percentile. Returns None on empty input."""
    vs = sorted(v for v in values if v is not None)
    if not vs:
        return None
    idx = int(max(0, min(len(vs) - 1, round(p / 100.0 * (len(vs) - 1)))))
    return vs[idx]


def classify(stats: TrajectoryStats, wall_p10: float | None, turns_p10: float | None) -> dict:
    """Apply the proposed easiness heuristics. All gated on reward=1.0 —
    a failing trial cannot be "too easy" regardless of trajectory shape.
    Returns flag booleans + human-readable reasons.

    `wall_p10` and `turns_p10` are the 10th-percentile thresholds from the
    cohort (only the bottom decile trips the medium flags — calibrated
    against the 2026-04-17 run where p20 over-flagged at 41%).
    """
    flags: dict = {
        "no_exploration": False,
        "zero_iteration": False,
        "fast_wall_time": False,
        "low_turns": False,
        "strong": False,
        "medium_combined": False,
        "auto_reject_candidate": False,
        "soft_flag_only": False,
    }
    reasons: list[str] = []

    if stats.reward != 1.0:
        return {"flags": flags, "reasons": ["reward != 1.0, skipped"]}

    total_exploration = stats.num_greps + stats.num_reads
    no_exploration = total_exploration < 10
    flags["no_exploration"] = no_exploration
    if no_exploration:
        reasons.append(f"only {total_exploration} Grep+Read (<10)")

    zero_iteration = stats.num_pytest_runs == 0
    flags["zero_iteration"] = zero_iteration
    if zero_iteration:
        reasons.append("no pytest runs in trajectory")

    fast_wall = False
    if stats.wall_time_s is not None and wall_p10 is not None:
        fast_wall = stats.wall_time_s <= wall_p10
        if fast_wall:
            reasons.append(f"wall {stats.wall_time_s:.0f}s <= p10 {wall_p10:.0f}s")
    flags["fast_wall_time"] = fast_wall

    low_turns = False
    if stats.num_tool_calls and turns_p10 is not None:
        low_turns = stats.num_tool_calls <= turns_p10
        if low_turns:
            reasons.append(f"{stats.num_tool_calls} calls <= p10 {turns_p10:.0f}")
    flags["low_turns"] = low_turns

    flags["strong"] = no_exploration or zero_iteration
    flags["medium_combined"] = fast_wall and low_turns
    flags["auto_reject_candidate"] = flags["strong"] or flags["medium_combined"]
    flags["soft_flag_only"] = not flags["auto_reject_candidate"] and (fast_wall or low_turns)

    return {"flags": flags, "reasons": reasons}


def _load_fixture_mode_stats(fixtures_dir: Path) -> list[TrajectoryStats]:
    samples_dir = fixtures_dir / "samples"
    if not samples_dir.is_dir():
        print(f"ERROR: {samples_dir} not found", file=sys.stderr)
        sys.exit(1)
    return [stats_for_sample(d) for d in sorted(samples_dir.iterdir()) if d.is_dir()]


def _load_run_dir_stats(run_dir: Path) -> list[TrajectoryStats]:
    state_path = run_dir / "state.json"
    if not state_path.is_file():
        print(f"ERROR: {state_path} not found", file=sys.stderr)
        sys.exit(1)
    try:
        hlab_bin = _resolve_harbor_lab()
    except RuntimeError as e:
        print(f"ERROR: {e}", file=sys.stderr)
        sys.exit(1)

    print(f"Loading {state_path}...")
    state = json.loads(state_path.read_text())
    tasks = state.get("tasks", {})

    # Eligible tasks: made it to Opus triage or past. Ignore evaluated/rejected earlier.
    eligible_stages = {
        "opus_smoke_tested",
        "opus_triaged",
        "haiku_smoke_tested",
        "accepted",
        "needs_fix",
    }
    eligible = [
        (tid, t) for tid, t in tasks.items() if t.get("stage") in eligible_stages and t.get("opus_trial_dir")
    ]
    print(f"Scoring {len(eligible)} tasks via harbor-lab at {hlab_bin}")
    print(f"  eligible stages: {sorted(eligible_stages)}")

    out: list[TrajectoryStats] = []
    for i, (tid, tstate) in enumerate(eligible, 1):
        if i % 10 == 0:
            print(f"  ... {i}/{len(eligible)}")
        s = stats_for_live_task(tid, tstate, hlab_bin)
        if s is not None:
            out.append(s)
    print(f"Scored {len(out)} tasks")
    return out


def main() -> int:
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument(
        "--mode",
        choices=["fixtures", "run_dir"],
        default="fixtures",
        help="fixtures: read pre-captured bundle. run_dir: shell out to harbor-lab on a live run.",
    )
    parser.add_argument(
        "--fixtures",
        default="tmp2/dd-fixtures-20260422",
        help="(mode=fixtures) Path to the deep-dive fixture bundle",
    )
    parser.add_argument(
        "--run-dir",
        default="",
        help="(mode=run_dir) Path to a pipeline run dir, e.g. harbor-tasks/craft-tools-v4/runs/<ts>/",
    )
    parser.add_argument(
        "--output",
        default="easiness_analysis.csv",
        help="CSV path (empty to skip)",
    )
    args = parser.parse_args()

    if args.mode == "fixtures":
        stats_list = _load_fixture_mode_stats(Path(args.fixtures))
    else:
        if not args.run_dir:
            print("ERROR: --mode run_dir requires --run-dir <path>", file=sys.stderr)
            return 1
        stats_list = _load_run_dir_stats(Path(args.run_dir))

    # Global percentile thresholds. Medium-flag threshold is p10 (bottom
    # decile) — p20 over-flagged in calibration against the 2026-04-17 run.
    # Only the `reward=1.0` cohort contributes to the distribution, since
    # those are the tasks the flags actually apply to.
    eligible_stats = [s for s in stats_list if s.reward == 1.0]
    wall_values = [s.wall_time_s for s in eligible_stats if s.wall_time_s is not None]
    turn_values = [float(s.num_tool_calls) for s in eligible_stats if s.num_tool_calls]
    wall_p10 = _percentile(wall_values, 10)
    turns_p10 = _percentile(turn_values, 10)

    wall_med = _percentile(wall_values, 50)
    turns_med = _percentile(turn_values, 50)
    print(f"Cohort: {len(stats_list)} samples ({len(eligible_stats)} eligible reward=1.0)")
    if wall_values and wall_p10 is not None and wall_med is not None:
        print(
            f"  wall_time_s: min={min(wall_values):.0f}, p10={wall_p10:.0f}, "
            f"median={wall_med:.0f}, max={max(wall_values):.0f}"
        )
    if turn_values and turns_p10 is not None and turns_med is not None:
        print(
            f"  tool_calls : min={int(min(turn_values))}, p10={int(turns_p10)}, "
            f"median={int(turns_med)}, max={int(max(turn_values))}"
        )
    print()

    rows: list[dict] = []
    for s in stats_list:
        c = classify(s, wall_p10, turns_p10)
        flagged = c["flags"].get("auto_reject_candidate", False)
        soft = c["flags"].get("soft_flag_only", False)
        eligible = s.reward == 1.0
        if not eligible:
            marker = "skip-r!=1"
        elif flagged:
            marker = "AUTO-REJECT"
        elif soft:
            marker = "soft-flag"
        else:
            marker = "clean"

        wall_str = f"{s.wall_time_s:.0f}s" if s.wall_time_s is not None else "?"
        print(f"  [{marker:>11}] {s.bucket:<22} {s.task_id}")
        print(
            f"    score={s.opus_score}  reward={s.reward}  edits={s.num_edits}  "
            f"calls={s.num_tool_calls}  greps={s.num_greps}  reads={s.num_reads}  "
            f"pytest={s.num_pytest_runs}  explore_pre_edit={s.exploration_before_first_edit}  "
            f"wall={wall_str}"
        )
        if c["reasons"]:
            print(f"    flags: {'; '.join(c['reasons'])}")

        rows.append(
            {
                **asdict(s),
                **{f"flag_{k}": v for k, v in c["flags"].items()},
                "reasons": "; ".join(c["reasons"]),
                "marker": marker,
            }
        )

    # Summary (eligible = reward=1.0 only — easiness heuristics don't apply to failing trials)
    n_total = len(rows)
    n_eligible = sum(1 for r in rows if r["marker"] != "skip-r!=1")
    n_auto = sum(1 for r in rows if r["marker"] == "AUTO-REJECT")
    n_soft = sum(1 for r in rows if r["marker"] == "soft-flag")
    n_clean = sum(1 for r in rows if r["marker"] == "clean")
    print()
    print(
        f"Summary (of {n_total} total): {n_eligible} eligible (reward=1.0) → "
        f"{n_auto} auto-reject, {n_soft} soft-flag, {n_clean} clean"
    )

    if args.output:
        with open(args.output, "w", newline="") as f:
            writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
            writer.writeheader()
            writer.writerows(rows)
        print(f"\nWrote CSV: {args.output}")

    return 0


if __name__ == "__main__":
    sys.exit(main())