Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
12.1 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Replay the triage judges over existing harbor trajectories — no Docker, no smoke.

The full pipeline runs smoke (Harbor + Docker) before triage, which is slow and
needs the task's container. When we already have harbor trial trajectories from a
prior evaluation (e.g. the codex + GPT-5.5 run over the v2b tasks), we can feed
those trials straight into the two triage *judges* — the Opus per-test deep dive
(skip/keep on each failing reference test) and the cross-family GPT fairness
review (task-level severity) — and collect their verdicts in bulk.

This mirrors the judge calls in ``steps.py::_run_triage_one`` (the reward<1 path)
exactly, but runs them **side-effect-free**: it does NOT write ``f2p_skip.txt``,
rescore the trial, regenerate the instruction, or re-smoke. It only reads the
existing trial dir + the task dir and records what the judges say.

Primary use: re-running the *bumped* judges (Opus 4.8 deep dive + GPT-5.5 fairness
review) over tasks that previously passed fairness, to measure how the model bump
shifts verdicts (see the GPT-5.5-stricter calibration note).

Usage:
  uv run python scripts/triage-replay.py \
      --job-dir   <harbor job dir with per-task trial subdirs> \
      --task-root <dir containing the built task dirs, e.g. craft-bench/harbor-tasks/craft-taskgen-v2> \
      --tasks-file references/v2b-tasks.txt \
      --out triage-replay.csv

Each trial subdir is matched to its task dir via ``task_name`` in result.json.
"""

from __future__ import annotations

import argparse
import asyncio
import csv
import json
import sys
from pathlib import Path

import craft_taskgen.config as _cfg
from craft_taskgen import llm_judge
from craft_taskgen.config import PipelineProfile, Stage, TaskState
from craft_taskgen.prompts import DEEP_DIVE_SCHEMA, deep_dive_prompt
from craft_taskgen.steps import (
    _fetch_deep_dive_context,
    _load_actually_failed_tests,
    _load_skipped_tests,
    _run_fairness_review_one,
)


def _read_task_name(trial_dir: Path) -> str | None:
    """Read the harbor task_name from a trial dir's result.json."""
    result = trial_dir / "result.json"
    if not result.is_file():
        return None
    try:
        return json.loads(result.read_text()).get("task_name")
    except (json.JSONDecodeError, OSError):
        return None


def _reward_of(trial_dir: Path) -> float | None:
    """Best-effort reward read for filtering/labelling (handles both reward.json eras)."""
    for fname in ("reward-details.json", "reward.json"):
        p = trial_dir / "verifier" / fname
        if p.is_file():
            try:
                return float(json.loads(p.read_text()).get("reward", 0.0))
            except (json.JSONDecodeError, OSError, TypeError, ValueError):
                continue
    return None


def _discover(job_dir: Path, task_root: Path, allow: set[str] | None) -> list[dict]:
    """Map each trial subdir to its built task dir. Returns work items."""
    items: list[dict] = []
    unmatched: list[str] = []
    for trial_dir in sorted(p for p in job_dir.iterdir() if p.is_dir()):
        name = _read_task_name(trial_dir)
        if not name:
            continue
        if allow is not None and name not in allow:
            continue
        # Built task dir is named exactly task_name somewhere under task_root.
        matches = [d for d in task_root.rglob(name) if (d / "task.toml").is_file()]
        if not matches:
            unmatched.append(name)
            continue
        items.append({"task_name": name, "trial_dir": str(trial_dir), "task_dir": str(matches[0])})
    if unmatched:
        print(f"  [warn] {len(unmatched)} trial(s) had no matching task dir under {task_root}:")
        for n in unmatched[:10]:
            print(f"           {n}")
    return items


async def _replay_one(item: dict, sem: asyncio.Semaphore) -> dict:
    """Run the two triage judges on one trial; record verdicts only."""
    task_dir, trial_dir, name = item["task_dir"], item["trial_dir"], item["task_name"]
    async with sem:
        row: dict = {"task_name": name, "reward": _reward_of(Path(trial_dir))}
        try:
            ctx = await _fetch_deep_dive_context(task_dir, trial_dir)
            dd_prompt = deep_dive_prompt(
                instruction_md=ctx["instruction_md"],
                reward_json=ctx["reward_json"],
                verify_output_tail=ctx["verify_output_tail"],
                postmerge_test_bodies=ctx["postmerge_test_bodies"],
                harbor_lab_errors=ctx["harbor_lab_errors"],
                harbor_lab_edits=ctx["harbor_lab_edits"],
                harbor_lab_tool_sequence=ctx["harbor_lab_tool_sequence"],
                harbor_lab_metrics=ctx["harbor_lab_metrics"],
                f2p_tests=ctx["f2p_tests"],
                p2p_tests=ctx["p2p_tests"],
                f2p_skip=ctx["f2p_skip"],
                p2p_skip=ctx["p2p_skip"],
                triage_history="",
            )
            # Minimal task object — _run_fairness_review_one only uses it for usage logging.
            stub = TaskState(
                task_id=name,
                repo="replay",
                commit_sha="replay",
                description="triage replay",
                base_sha="replay",
                merge_base_sha="replay",
                stage=Stage.OPUS_SMOKE_TESTED,
                task_dir=task_dir,
            )
            dd_judge, reviewer = await asyncio.gather(
                llm_judge.judge(prompt=dd_prompt, schema=DEEP_DIVE_SCHEMA, model=_cfg.LLM_STEP_MODEL),
                _run_fairness_review_one(stub, ctx, "Opus"),
            )
        except Exception as e:  # noqa: BLE001 — record per-task failure, keep the batch going
            row["error"] = f"{type(e).__name__}: {e}"
            print(f"  [err] {name}: {row['error']}")
            return row

        # Deep dive — mirror steps.py::_run_triage_one filtering so counts are
        # production-faithful:
        #   (1) drop verdicts on tests ALREADY in f2p_skip.txt/p2p_skip.txt — they
        #       still run and still report FAILED, so the judge re-nominates them
        #       every pass; production drops them. Without this, dd_skip is inflated
        #       by re-nominations of already-excluded tests.
        #   (2) drop verdicts on tests that didn't actually fail in this trial.
        raw_failures = list(dd_judge.result.get("failures", []))
        row["dd_raw_skip"] = sum(1 for f in raw_failures if f.get("classification") == "skip")
        already_skipped = _load_skipped_tests(task_dir)
        failures = [f for f in raw_failures if f.get("test_name", "") not in already_skipped]
        row["dd_already_skipped"] = len(already_skipped)
        actually_failed = _load_actually_failed_tests(trial_dir)
        if actually_failed is not None:
            failures = [f for f in failures if f.get("test_name", "") in actually_failed]
        classes = [f.get("classification", "") for f in failures]
        row["dd_failures"] = len(failures)
        row["dd_skip"] = sum(1 for c in classes if c == "skip")  # NEW skips (not already excluded)
        row["dd_keep"] = sum(1 for c in classes if c == "keep")
        row["dd_assessment"] = (dd_judge.result.get("overall_assessment", "") or "")[:200]

        # Fairness review
        sev = (reviewer or {}).get("severity", "") or ""
        quote = ((reviewer or {}).get("evidence_quote", "") or "").strip()
        test = ((reviewer or {}).get("evidence_test", "") or "").strip()
        row["reviewer_severity"] = sev
        row["reviewer_flag"] = sev not in ("", "none")
        row["reviewer_triggers_regen"] = sev == "major" and bool(quote) and bool(test)
        row["reviewer_evidence_test"] = test
        row["reviewer_evidence_quote"] = quote[:300]
        row["reviewer_reason"] = ((reviewer or {}).get("reason", "") or "")[:300]
        print(
            f"  [ok] {name}: reward={row['reward']} | DD {row['dd_keep']}keep/{row['dd_skip']}skip "
            f"of {row['dd_failures']} | fairness={sev or 'none'}"
            + (" [MAJOR->regen]" if row["reviewer_triggers_regen"] else "")
        )
        return row


async def _run(items: list[dict], concurrency: int) -> list[dict]:
    sem = asyncio.Semaphore(concurrency)
    return await asyncio.gather(*[_replay_one(it, sem) for it in items])


def _summarize(rows: list[dict]) -> None:
    ok = [r for r in rows if "error" not in r]
    errs = [r for r in rows if "error" in r]
    print("\n" + "=" * 60)
    print("TRIAGE REPLAY SUMMARY")
    print("=" * 60)
    print(f"  tasks: {len(rows)}  ({len(ok)} judged, {len(errs)} errored)")
    sev_dist: dict[str, int] = {}
    for r in ok:
        sev_dist[r["reviewer_severity"] or "none"] = sev_dist.get(r["reviewer_severity"] or "none", 0) + 1
    print(f"  fairness severity: {dict(sorted(sev_dist.items()))}")
    regen = [r for r in ok if r.get("reviewer_triggers_regen")]
    print(f"  would trigger Build regen (major + quote + test): {len(regen)}")
    for r in regen:
        print(f"      {r['task_name']}: {r['reviewer_evidence_test']}")
    any_skip = [r for r in ok if r.get("dd_skip", 0) > 0]
    print(f"  tasks with >=1 deep-dive 'skip' verdict (unfair test): {len(any_skip)}")
    for r in any_skip:
        print(f"      {r['task_name']}: {r['dd_skip']} skip / {r['dd_keep']} keep")


def main() -> None:
    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("--job-dir", required=True, help="Harbor job dir with per-task trial subdirs")
    p.add_argument("--task-root", required=True, help="Dir containing built task dirs (searched recursively)")
    p.add_argument("--tasks-file", help="Optional newline-delimited task-name allowlist (e.g. v2b-tasks.txt)")
    p.add_argument("--profile", help="TOML profile (else config.py defaults — Opus 4.8 + GPT-5.5)")
    p.add_argument("--alignment-model", help="Override LLM_ALIGNMENT_MODEL (fairness judge)")
    p.add_argument("--deep-dive-model", help="Override LLM_STEP_MODEL (deep-dive judge)")
    p.add_argument("--concurrency", type=int, default=6, help="Parallel judge pairs (default 6)")
    p.add_argument("--limit", type=int, help="Process only the first N matched trials")
    p.add_argument("--out", default="triage-replay.csv", help="CSV output path")
    args = p.parse_args()

    _cfg._load_env()
    if args.profile:
        PipelineProfile.from_toml(args.profile).apply()
    if args.alignment_model:
        _cfg.LLM_ALIGNMENT_MODEL = args.alignment_model
    if args.deep_dive_model:
        _cfg.LLM_STEP_MODEL = args.deep_dive_model

    job_dir = Path(args.job_dir)
    task_root = Path(args.task_root)
    if not job_dir.is_dir():
        sys.exit(f"ERROR: --job-dir not a directory: {job_dir}")
    if not task_root.is_dir():
        sys.exit(f"ERROR: --task-root not a directory: {task_root}")

    allow: set[str] | None = None
    if args.tasks_file:
        allow = {ln.strip() for ln in Path(args.tasks_file).read_text().splitlines() if ln.strip()}

    print(f"deep-dive model: {_cfg.LLM_STEP_MODEL}")
    print(f"fairness model:  {_cfg.LLM_ALIGNMENT_MODEL}")
    items = _discover(job_dir, task_root, allow)
    if args.limit:
        items = items[: args.limit]
    print(f"matched {len(items)} trials to task dirs; running judges (concurrency={args.concurrency})...\n")
    if not items:
        sys.exit("ERROR: no trials matched — check --job-dir / --task-root / --tasks-file")

    rows = asyncio.run(_run(items, args.concurrency))

    fields = [
        "task_name",
        "reward",
        "dd_failures",
        "dd_keep",
        "dd_skip",
        "dd_raw_skip",
        "dd_already_skipped",
        "reviewer_severity",
        "reviewer_flag",
        "reviewer_triggers_regen",
        "reviewer_evidence_test",
        "reviewer_evidence_quote",
        "reviewer_reason",
        "dd_assessment",
        "error",
    ]
    with open(args.out, "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore")
        w.writeheader()
        for r in rows:
            w.writerow(r)
    _summarize(rows)
    print(f"\nCSV written to {args.out}")


if __name__ == "__main__":
    main()