Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
9.9 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Harbor smoke test runner and trial diagnostics."""

from __future__ import annotations

import asyncio
import json
import os
from pathlib import Path

from craft_taskgen.config import TaskState

# Repo root (…/craft-taskgen) — sibling resolution for the codex model catalog.
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
# Filtered single-model codex catalogs are cached here (cwd-relative, same as
# the jobs/ dir harbor writes to). Reused across runs; cheap to regenerate.
_CODEX_CATALOG_DIR = Path("jobs") / ".codex-catalogs"


def _agent_version(agent: str) -> str | None:
    """Pinned CLI version for a Harbor agent, or None if not version-pinned."""
    from craft_taskgen.adapters._docker import CLAUDE_CODE_VERSION, CODEX_VERSION, OPENCODE_VERSION

    return {
        "claude-code": CLAUDE_CODE_VERSION,
        "codex": CODEX_VERSION,
        "opencode": OPENCODE_VERSION,
    }.get(agent)


def _filtered_codex_catalog(model: str) -> str | None:
    """Write a single-model codex catalog for `model`; return its path (or None).

    Codex only emits reasoning tokens for slugs it recognizes as
    reasoning-capable; gateway slugs aren't in its built-in catalog, so harbor's
    codex patch reads CODEX_MODEL_CATALOG_JSON and injects the catalog. Harbor
    inlines the file content into a docker-exec argv, so we filter to one model
    to stay under Linux's per-arg MAX_ARG_STRLEN (the full catalog is ~150 KB).
    Returns None when the source catalog has no row for `model` — codex then
    runs without reasoning (matches scripts/run-baselines.sh behavior).
    """
    src = _REPO_ROOT / "patches" / "codex-model-catalog.json"
    if not src.is_file():
        return None
    data = json.loads(src.read_text())
    matches = [m for m in data.get("models", []) if m.get("slug") == model]
    if not matches:
        return None
    _CODEX_CATALOG_DIR.mkdir(parents=True, exist_ok=True)
    out = _CODEX_CATALOG_DIR / f"{model.replace('/', '-')}.json"
    out.write_text(json.dumps({"models": matches}, indent=2))
    return str(out)


def _build_smoke_cmd(
    task_dir: str,
    agent: str,
    model: str,
    reasoning_effort: str,
    job_name: str,
) -> tuple[list[str], dict[str, str]]:
    """Build the ``harbor run`` argv + env overrides for an agent smoke trial.

    Per-agent wiring mirrors scripts/run-baselines.sh (the canonical launcher):
    version pin, codex reasoning catalog, reasoning_effort kwarg, and
    claude-code's plan-mode tool restriction. API keys/base URLs are read by
    harbor's agents from the host environment (loaded from .env), so they are
    not passed here. Returns (cmd, env_overrides); env_overrides is layered on
    top of os.environ by the caller.
    """
    cmd = [
        ".venv/bin/harbor",
        "run",
        # Auto-confirm Harbor's host-env-access prompt (added in the 331dcba3
        # pin). The smoke step runs harbor as a non-interactive subprocess, so
        # without this the prompt receives no input and harbor aborts before
        # creating the job dir (surfacing as a context-free NO_TRIAL). Mirrors
        # the --yes that scripts/run-baselines.sh already passes.
        "--yes",
        "-p",
        task_dir,
        "-a",
        agent,
        "-m",
        model,
        "--job-name",
        job_name,
    ]
    env_overrides: dict[str, str] = {}

    version = _agent_version(agent)
    if version:
        cmd += ["--agent-kwarg", f"version={version}"]

    if agent == "claude-code":
        cmd += ["--agent-kwarg", "disallowed_tools=EnterPlanMode,ExitPlanMode"]

    if agent == "codex":
        catalog = _filtered_codex_catalog(model)
        if catalog:
            env_overrides["CODEX_MODEL_CATALOG_JSON"] = catalog
        else:
            print(f"    -> WARNING: no codex catalog row for {model}; reasoning will be absent")

    effort = reasoning_effort
    if not effort:
        from craft_taskgen.baselines.reasoning_defaults import effort_for

        effort = effort_for(agent, model) or ""
    if effort:
        cmd += ["--agent-kwarg", f"reasoning_effort={effort}"]

    return cmd, env_overrides


def _find_trial_in_job(job_dir: str) -> str | None:
    """Find the trial subdirectory within a job directory."""
    if not os.path.isdir(job_dir):
        return None
    for entry in os.listdir(job_dir):
        trial_path = os.path.join(job_dir, entry)
        if os.path.isdir(trial_path):
            return trial_path
    return None


def _read_trial_diagnostics(trial_dir: str) -> dict:
    """Read diagnostic info from a trial directory. Returns dict with key metrics."""
    diag: dict = {}

    # reward.json — score and test counts
    reward_path = os.path.join(trial_dir, "verifier", "reward.json")
    if os.path.isfile(reward_path):
        with open(reward_path) as f:
            reward_data = json.load(f)
        diag["reward"] = reward_data.get("reward", 0.0)
        f2p_passed = reward_data.get("f2p_passed", 0)
        p2p_passed = reward_data.get("p2p_passed", 0)
        f2p_total = reward_data.get("f2p_total", 0)
        p2p_total = reward_data.get("p2p_total", 0)
        diag["ref_passed"] = f2p_passed + p2p_passed
        diag["ref_total"] = f2p_total + p2p_total
        diag["ref_failed"] = diag["ref_total"] - diag["ref_passed"]
        diag["f2p_passed"] = f2p_passed
        diag["f2p_total"] = f2p_total
        diag["p2p_passed"] = p2p_passed
        diag["p2p_total"] = p2p_total

    # result.json — agent execution metadata (Harbor ATIF format)
    result_path = os.path.join(trial_dir, "result.json")
    if os.path.isfile(result_path):
        with open(result_path) as f:
            result_data = json.load(f)
        agent_result = result_data.get("agent_result") or {}
        diag["input_tokens"] = agent_result.get("n_input_tokens") or 0
        diag["output_tokens"] = agent_result.get("n_output_tokens") or 0
        diag["cache_tokens"] = agent_result.get("n_cache_tokens") or 0
        # Duration from timestamps
        agent_exec = result_data.get("agent_execution") or {}
        started = agent_exec.get("started_at", "")
        finished = agent_exec.get("finished_at", "")
        if started and finished:
            from datetime import datetime as _dt

            try:
                t0 = _dt.fromisoformat(started.replace("Z", "+00:00"))
                t1 = _dt.fromisoformat(finished.replace("Z", "+00:00"))
                diag["duration_s"] = (t1 - t0).total_seconds()
            except ValueError:
                diag["duration_s"] = 0
        config = result_data.get("config") or {}
        agent_cfg = config.get("agent") or {}
        diag["model"] = agent_cfg.get("model_name", "")
        diag["exception_info"] = result_data.get("exception_info")

    # Detect infra failure: agent produced no output tokens
    diag["infra_failure"] = diag.get("output_tokens", 0) == 0 and diag.get("exception_info") is not None

    # verify_full_output.txt — first 500 chars for diagnostic
    full_output_path = os.path.join(trial_dir, "verifier", "verify_full_output.txt")
    if os.path.isfile(full_output_path):
        with open(full_output_path) as f:
            diag["verify_output_head"] = f.read()[:500]

    # exception.txt — agent crash info
    exception_path = os.path.join(trial_dir, "exception.txt")
    if os.path.isfile(exception_path):
        with open(exception_path) as f:
            diag["exception"] = f.read()[:500]
        diag["infra_failure"] = True

    return diag


async def _run_smoke_async(
    task: TaskState,
    model: str,
    label: str,
    *,
    agent: str = "claude-code",
    reasoning_effort: str = "",
) -> dict:
    """Async smoke test with deterministic --job-name to avoid race conditions."""
    task_name = Path(task.task_dir).name if task.task_dir else task.task_id
    # Always unique job name — prevents Harbor from reusing stale job dirs
    # and ensures harbor-lab only sees one trial per analysis
    from datetime import datetime as _dt

    ts = _dt.now().strftime("%H%M%S")
    job_name = f"smoke-{label.lower()}-{task_name}-{ts}"

    cmd, env_overrides = _build_smoke_cmd(task.task_dir, agent, model, reasoning_effort, job_name)
    print(f"    -> {label} smoke: agent={agent} model={model}")
    env = {**os.environ, **env_overrides} if env_overrides else None

    try:
        proc = await asyncio.create_subprocess_exec(
            *cmd,
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE,
            env=env,
        )
        await asyncio.wait_for(proc.communicate(), timeout=1800)
    except asyncio.TimeoutError:
        proc.kill()
        await proc.wait()
        print(f"    -> {label} TIMEOUT (>30 min)")
        return {"timeout": True}

    # Look up by job name instead of "latest"
    job_dir = os.path.join("jobs", job_name)
    trial_dir = _find_trial_in_job(job_dir) if os.path.isdir(job_dir) else None

    if not trial_dir:
        print(f"    -> {label}: Could not find trial directory in {job_dir}")
        return {"no_trial": True}

    diag = _read_trial_diagnostics(trial_dir)

    score = f"{diag.get('ref_passed', '?')}/{diag.get('ref_total', '?')}"
    f2p_str = f"F2P {diag.get('f2p_passed', '?')}/{diag.get('f2p_total', '?')}"
    p2p_str = f"P2P {diag.get('p2p_passed', '?')}/{diag.get('p2p_total', '?')}"
    score_detail = f"{f2p_str}, {p2p_str}"
    tokens = f"{diag.get('input_tokens') or 0:,}in/{diag.get('output_tokens') or 0:,}out"
    duration = f"{diag.get('duration_s') or 0:.0f}s"
    print(f"    -> {label}: {score_detail} (reward={diag.get('reward', '?')}) | {tokens} | {duration}")

    if diag.get("infra_failure"):
        print("    -> WARNING: Infra failure detected")
        if diag.get("exception"):
            print(f"       Exception: {diag['exception'][:150]}")

    diag["trial_dir"] = trial_dir
    diag["score"] = score
    diag["score_detail"] = score_detail
    return diag