Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
10.9 KB
Raw Blame
#!/usr/bin/env python3
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Join a planner job dir and an implementer job dir into a single report.

Input: two harbor job dirs from the iterative planning pipeline.
  - Planner job dir: trials contain plan.md + planner reward (file recall).
  - Implementer job dir: trials contain F2P reward.

Output (written to --output):
  - reward.txt: single float, mean F2P across all implementer trials.
  - results.json: full per-task breakdown.
  - report.md: human-readable table.

Trials are joined by task_name (from result.json), not trial dir name,
because harbor appends a random suffix per trial.

Usage:
    python planning/05_join_scores.py \\
        --planner-job jobs/e2e-foo/planner/<harbor_subdir> \\
        --impl-job    jobs/e2e-foo/implementer/<harbor_subdir> \\
        --output      jobs/e2e-foo/results
"""

import argparse
import json
import logging
import statistics
import sys
from pathlib import Path

logger = logging.getLogger(__name__)


def _resolve_job_dir(path: Path) -> Path:
    """Accept either a harbor parent dir (single timestamped child) or the inner dir."""
    if not path.exists():
        raise FileNotFoundError(f"job dir does not exist: {path}")
    if (path / "job.log").exists():
        return path
    children = [c for c in path.iterdir() if c.is_dir() and (c / "job.log").exists()]
    if len(children) == 1:
        return children[0]
    if len(children) == 0:
        raise FileNotFoundError(f"no harbor job dir found under {path}")
    raise ValueError(
        f"{path} has multiple harbor job dirs; pass the specific one: {[c.name for c in children]}"
    )


def _index_trials(job_dir: Path) -> dict[str, Path]:
    """Map task_name -> trial dir, using result.json as the source of truth."""
    index: dict[str, Path] = {}
    for child in sorted(job_dir.iterdir()):
        if not child.is_dir():
            continue
        result_path = child / "result.json"
        if not result_path.exists():
            continue
        try:
            result = json.loads(result_path.read_text())
        except json.JSONDecodeError:
            logger.warning("skipping %s: invalid result.json", child.name)
            continue
        task_name = result.get("task_name")
        if not task_name:
            continue
        if task_name in index:
            logger.warning("duplicate task %s in %s, keeping first", task_name, job_dir)
            continue
        index[task_name] = child
    return index


def _read_reward(trial_dir: Path) -> float | None:
    result_path = trial_dir / "result.json"
    if not result_path.exists():
        return None
    try:
        result = json.loads(result_path.read_text())
    except json.JSONDecodeError:
        return None
    rewards = (result.get("verifier_result") or {}).get("rewards") or {}
    reward = rewards.get("reward")
    return float(reward) if reward is not None else None


def _read_verifier_details(trial_dir: Path) -> dict:
    details_path = trial_dir / "verifier" / "results.json"
    if not details_path.exists():
        return {}
    try:
        return json.loads(details_path.read_text())
    except json.JSONDecodeError:
        return {}


def _extract_f2p_p2p(details: dict) -> tuple[dict, dict]:
    """Normalize the F2P/P2P section across the old flat and new nested formats."""
    f2p_block = details.get("f2p")
    if isinstance(f2p_block, dict):
        f2p_detail = {
            "passed": f2p_block.get("passed"),
            "total": f2p_block.get("total"),
            "score": f2p_block.get("score"),
            "failed_tests": f2p_block.get("failed_tests") or [],
        }
    else:
        passed = details.get("fail_to_pass")
        total = details.get("total_functional")
        score = passed / total if passed is not None and total not in (None, 0) else None
        f2p_detail = {
            "passed": passed,
            "total": total,
            "score": round(score, 3) if score is not None else None,
            "failed_tests": details.get("still_failing_tests") or [],
        }

    p2p_block = details.get("p2p")
    if isinstance(p2p_block, dict):
        p2p_detail = {
            "passed": p2p_block.get("passed"),
            "total": p2p_block.get("total"),
            "score": p2p_block.get("score"),
            "failed_tests": p2p_block.get("failed_tests") or [],
        }
    else:
        p2p_detail = {
            "passed": None,
            "total": None,
            "score": None,
            "failed_tests": details.get("regression_tests") or [],
        }
    return f2p_detail, p2p_detail


def _read_agent_config(trial_dir: Path) -> dict:
    result_path = trial_dir / "result.json"
    if not result_path.exists():
        return {}
    try:
        result = json.loads(result_path.read_text())
    except json.JSONDecodeError:
        return {}
    config = result.get("config") or {}
    agent = config.get("agent") or {}
    return {
        "name": agent.get("name"),
        "model": agent.get("model_name"),
        "kwargs": agent.get("kwargs") or {},
    }


def join_scores(planner_job: Path, impl_job: Path, output_dir: Path) -> dict:
    planner_job = _resolve_job_dir(planner_job)
    impl_job = _resolve_job_dir(impl_job)

    planner_trials = _index_trials(planner_job)
    impl_trials = _index_trials(impl_job)

    all_tasks = sorted(set(planner_trials) | set(impl_trials))
    if not all_tasks:
        raise RuntimeError(f"no task trials found under {planner_job} or {impl_job}")

    planner_config = next((_read_agent_config(t) for t in planner_trials.values()), {})
    impl_config = next((_read_agent_config(t) for t in impl_trials.values()), {})

    per_task: dict[str, dict] = {}
    for task in all_tasks:
        planner_trial = planner_trials.get(task)
        impl_trial = impl_trials.get(task)

        plan_recall = _read_reward(planner_trial) if planner_trial else None
        reward = _read_reward(impl_trial) if impl_trial else None
        impl_details = _read_verifier_details(impl_trial) if impl_trial else {}
        f2p_detail, p2p_detail = _extract_f2p_p2p(impl_details)

        per_task[task] = {
            "plan_recall": plan_recall,
            "reward": reward if reward is not None else 0.0,
            "f2p": f2p_detail,
            "p2p": p2p_detail,
            "planner_ok": planner_trial is not None and plan_recall is not None,
            "impl_ok": impl_trial is not None and reward is not None,
        }

    reward_values = [v["reward"] for v in per_task.values()]
    f2p_fractions = [v["f2p"]["score"] for v in per_task.values() if v["f2p"]["score"] is not None]
    p2p_fractions = [v["p2p"]["score"] for v in per_task.values() if v["p2p"]["score"] is not None]
    recall_values = [v["plan_recall"] for v in per_task.values() if v["plan_recall"] is not None]

    summary = {
        "pass_rate": round(statistics.mean(reward_values), 4) if reward_values else 0.0,
        "n_passed": sum(1 for v in reward_values if v >= 1.0),
        "n_tasks": len(all_tasks),
        "mean_f2p_fraction": (round(statistics.mean(f2p_fractions), 4) if f2p_fractions else None),
        "mean_p2p_fraction": (round(statistics.mean(p2p_fractions), 4) if p2p_fractions else None),
        "mean_plan_recall": (round(statistics.mean(recall_values), 4) if recall_values else None),
        "n_planner_ok": sum(1 for v in per_task.values() if v["planner_ok"]),
        "n_impl_ok": sum(1 for v in per_task.values() if v["impl_ok"]),
    }

    report = {
        "summary": summary,
        "planner": planner_config,
        "implementer": impl_config,
        "job_dirs": {
            "planner": str(planner_job),
            "implementer": str(impl_job),
        },
        "per_task": per_task,
    }

    output_dir.mkdir(parents=True, exist_ok=True)
    (output_dir / "reward.txt").write_text(f"{summary['pass_rate']:.4f}\n")
    (output_dir / "results.json").write_text(json.dumps(report, indent=2) + "\n")
    (output_dir / "report.md").write_text(_render_markdown(report))

    return report


def _render_markdown(report: dict) -> str:
    summary = report["summary"]
    planner = report["planner"]
    impl = report["implementer"]

    def _pct(value: float | None) -> str:
        return f"{value:.1%}" if value is not None else "n/a"

    lines = [
        "# Iterative Planning E2E Report",
        "",
        f"- Planner: `{planner.get('name')}` / `{planner.get('model')}`",
        f"- Implementer: `{impl.get('name')}` / `{impl.get('model')}`",
        f"- Tasks: {summary['n_tasks']} "
        f"(planner ok: {summary['n_planner_ok']}, impl ok: {summary['n_impl_ok']})",
        f"- **Pass rate (binary reward): {summary['pass_rate']:.4f} "
        f"({summary['n_passed']}/{summary['n_tasks']})**",
        f"- Mean F2P fraction: {_pct(summary['mean_f2p_fraction'])}",
        f"- Mean P2P fraction: {_pct(summary['mean_p2p_fraction'])}",
        "- Mean plan recall: "
        + (f"{summary['mean_plan_recall']:.4f}" if summary["mean_plan_recall"] is not None else "n/a"),
        "",
        "| Task | Reward | F2P | P2P | Plan recall | Regressions | Planner | Impl |",
        "|------|--------|-----|-----|-------------|-------------|---------|------|",
    ]
    for task, v in sorted(report["per_task"].items()):
        reward = f"{v['reward']:.1f}"
        f2p = v["f2p"]
        f2p_str = (
            f"{f2p['passed']}/{f2p['total']} ({f2p['score']:.1%})"
            if f2p["passed"] is not None and f2p["total"] is not None and f2p["score"] is not None
            else "-"
        )
        p2p = v["p2p"]
        p2p_str = (
            f"{p2p['passed']}/{p2p['total']} ({p2p['score']:.1%})"
            if p2p["passed"] is not None and p2p["total"] is not None and p2p["score"] is not None
            else "-"
        )
        recall = f"{v['plan_recall']:.3f}" if v["plan_recall"] is not None else "-"
        regressions = len(p2p.get("failed_tests") or [])
        planner_ok = "ok" if v["planner_ok"] else "FAIL"
        impl_ok = "ok" if v["impl_ok"] else "FAIL"
        lines.append(
            f"| {task} | {reward} | {f2p_str} | {p2p_str} | {recall} | "
            f"{regressions} | {planner_ok} | {impl_ok} |"
        )
    lines.append("")
    return "\n".join(lines)


def main() -> int:
    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--planner-job", type=Path, required=True)
    parser.add_argument("--impl-job", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    args = parser.parse_args()

    report = join_scores(args.planner_job, args.impl_job, args.output)
    summary = report["summary"]
    print(
        f"pass rate: {summary['pass_rate']:.4f} "
        f"({summary['n_passed']}/{summary['n_tasks']}) "
        f"| impl ok: {summary['n_impl_ok']}/{summary['n_tasks']} "
        f"| wrote: {args.output}"
    )
    return 0


if __name__ == "__main__":
    sys.exit(main())