Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
36.1 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Pre-flight checks for long-running pipeline invocations.

Run `craft-taskgen-preflight --candidates 'candidates/*.json'` before a long,
unattended run to catch missing repos, broken auth, or low disk space before
wasting wall time.

Exits 0 if all checks pass; 1 otherwise. Each check prints PASS/FAIL with a short hint.
"""

from __future__ import annotations

import argparse
import glob
import json
import os
import shutil
import subprocess
import sys
from pathlib import Path


def _print(label: str, ok: bool, detail: str = "") -> None:
    mark = "PASS" if ok else "FAIL"
    line = f"  [{mark}] {label}"
    if detail:
        line += f" — {detail}"
    print(line)


def _print_warn(label: str, detail: str) -> None:
    """Non-fatal warning — prints but doesn't affect pass/fail counts."""
    print(f"  [WARN] {label} — {detail}")


def _print_info(label: str, detail: str) -> None:
    """Informational line — prints but doesn't affect pass/fail counts."""
    print(f"  [INFO] {label} — {detail}")


def check_claude_cli() -> bool:
    """Claude CLI is installed and returns a valid response to a trivial prompt.

    Uses `_cfg.CLAUDE_CMD` (the pinned binary if installed, else system claude)
    so preflight reports the version the pipeline will actually call.
    """
    import craft_taskgen.config as _cfg

    cmd = _cfg.CLAUDE_CMD
    if cmd == "claude" and not shutil.which("claude"):
        _print("claude CLI on PATH", False, "install Claude Code CLI")
        return False
    try:
        result = subprocess.run(
            [cmd, "--version"],
            capture_output=True,
            text=True,
            timeout=15,
        )
    except (subprocess.TimeoutExpired, OSError) as e:
        _print("claude --version", False, f"{e}")
        return False
    if result.returncode != 0:
        _print("claude --version", False, result.stderr.strip()[:120])
        return False
    version_line = result.stdout.strip().splitlines()[0] if result.stdout else ""
    suffix = f" (pinned at {cmd})" if cmd != "claude" else " (system)"
    _print("claude CLI", True, version_line + suffix)
    return True


def check_gh_cli() -> bool:
    """gh CLI present + authenticated (needed only for mining, not pipeline runs)."""
    if not shutil.which("gh"):
        _print("gh CLI on PATH", False, "needed for craft-taskgen-mine; skip if candidates already mined")
        return False
    try:
        result = subprocess.run(
            ["gh", "auth", "status"],
            capture_output=True,
            text=True,
            timeout=10,
        )
    except (subprocess.TimeoutExpired, OSError) as e:
        _print("gh auth", False, f"{e}")
        return False
    ok = result.returncode == 0
    _print("gh auth", ok, "Logged in" if ok else "run `gh auth login`")
    return ok


def check_docker() -> bool:
    """Docker daemon reachable."""
    if not shutil.which("docker"):
        _print("docker on PATH", False, "install Docker")
        return False
    try:
        result = subprocess.run(
            ["docker", "info", "--format", "{{.ServerVersion}}"],
            capture_output=True,
            text=True,
            timeout=10,
        )
    except (subprocess.TimeoutExpired, OSError) as e:
        _print("docker info", False, f"{e}")
        return False
    ok = result.returncode == 0
    _print(
        "docker daemon",
        ok,
        f"server {result.stdout.strip()}" if ok else result.stderr.strip()[:120],
    )
    return ok


def check_harbor() -> bool:
    """Harbor venv exists with runnable executable (smoke-test step needs this)."""
    harbor_bin = Path(".venv/bin/harbor")
    if not harbor_bin.is_file():
        _print("Harbor executable", False, "expected .venv/bin/harbor — run `uv sync`")
        return False
    try:
        result = subprocess.run(
            [str(harbor_bin), "--help"],
            capture_output=True,
            text=True,
            timeout=15,
        )
    except (subprocess.TimeoutExpired, OSError) as e:
        _print("Harbor --help", False, f"{e}")
        return False
    ok = result.returncode == 0
    _print("Harbor runnable", ok, "ready" if ok else result.stderr.strip()[:120])
    return ok


def check_harbor_lab() -> bool:
    """harbor-lab binary resolvable and runnable. Deep-dive step shells out to it
    to pull `errors`, `edits`, `tool-sequence`, and `metrics` from a trial. The
    resolver honors $HARBOR_LAB, then PATH, then the pinned pydev venv."""
    try:
        from craft_taskgen.steps import _resolve_harbor_lab_bin
    except ImportError as e:
        _print("harbor-lab resolver", False, f"import error: {e}")
        return False

    try:
        bin_path = _resolve_harbor_lab_bin()
    except RuntimeError as e:
        _print("harbor-lab binary", False, str(e)[:160])
        return False

    try:
        result = subprocess.run(
            [bin_path, "--help"],
            capture_output=True,
            text=True,
            timeout=15,
        )
    except (subprocess.TimeoutExpired, OSError) as e:
        _print("harbor-lab --help", False, f"{e}")
        return False
    ok = result.returncode == 0
    _print("harbor-lab runnable", ok, f"at {bin_path}" if ok else result.stderr.strip()[:120])
    return ok


def check_gh_rate_limit(needed: int) -> bool:
    """Ensure GitHub core API has enough remaining calls for a mining run.

    The miner issues roughly 3 calls per repo (list merged PRs paged + per-PR diff stats).
    Pass `needed` as an estimate (e.g. 3 * number of repos to mine) so we fail early
    if the user is about to stall the miner on a rate-limit wall.
    """
    if not shutil.which("gh"):
        _print("gh API rate limit", False, "gh CLI not on PATH")
        return False
    try:
        result = subprocess.run(
            ["gh", "api", "rate_limit", "--jq", ".rate.remaining"],
            capture_output=True,
            text=True,
            timeout=10,
        )
    except (subprocess.TimeoutExpired, OSError) as e:
        _print("gh API rate limit", False, f"{e}")
        return False
    if result.returncode != 0:
        _print("gh API rate limit", False, result.stderr.strip()[:120])
        return False
    try:
        remaining = int(result.stdout.strip())
    except ValueError:
        _print("gh API rate limit", False, f"unparseable: {result.stdout!r}")
        return False
    ok = remaining >= needed
    _print(
        f"gh API rate limit (need ~{needed})",
        ok,
        f"{remaining} calls remaining this hour"
        + ("" if ok else "; wait or authenticate a higher-limit token"),
    )
    return ok


def check_docker_memory(min_gb: int = 8) -> None:
    """Warn (not fail) if Docker daemon has <min_gb of memory allocated.

    At concurrency 6 the pipeline can have 6 Docker containers running at once
    during F2P/P2P classify; low-memory Docker Desktop configs will OOM.
    """
    try:
        result = subprocess.run(
            ["docker", "info", "--format", "{{.MemTotal}}"],
            capture_output=True,
            text=True,
            timeout=10,
        )
    except (subprocess.TimeoutExpired, OSError):
        return  # docker already failed its own check; stay quiet
    if result.returncode != 0:
        return
    try:
        mem_bytes = int(result.stdout.strip())
    except ValueError:
        return
    mem_gb = mem_bytes / (1024**3)
    if mem_gb < min_gb:
        _print_warn(
            "Docker memory",
            f"{mem_gb:.1f}GB allocated; concurrency 6 may OOM. "
            f"Raise to {min_gb}GB+ in Docker Desktop → Settings → Resources.",
        )
    else:
        _print("Docker memory", True, f"{mem_gb:.1f}GB allocated")


def check_repos_csv(csv_path: str) -> bool:
    """Validate a repos CSV used by `craft-taskgen-mine --repos-csv`."""
    path = Path(csv_path)
    if not path.is_file():
        _print(f"repos CSV {csv_path}", False, "file not found")
        return False
    try:
        with open(path) as f:
            header = f.readline().strip().split(",")
            n_rows = sum(1 for _ in f)
    except OSError as e:
        _print(f"repos CSV {csv_path}", False, f"{e}")
        return False
    required = {"short_name", "github_repo"}
    missing = required - set(header)
    if missing:
        _print(f"repos CSV {csv_path}", False, f"missing columns: {', '.join(sorted(missing))}")
        return False
    _print(f"repos CSV {csv_path}", True, f"{n_rows} repos")
    return True


def check_claude_auth(model: str | None = None) -> bool:
    """End-to-end validation of the `claude -p` path through the gateway.

    Builds the same env `gateway.build_gateway_env` builds, shells out to
    `claude -p`, then asserts the response's `modelUsage` key looks gateway-shaped
    (e.g. `aws/anthropic/...`) — NOT a bare Anthropic-direct name like `opus`
    or `claude-opus-4-6`. A mismatch means the request leaked to OAuth/direct.
    """
    import craft_taskgen.config as _cfg
    from craft_taskgen.gateway import build_gateway_env

    if not model:
        _print("claude -p auth", False, "no model provided (pass --profile with llm_step_model set)")
        return False
    try:
        env = build_gateway_env(model)
    except RuntimeError as e:
        _print("claude -p auth", False, f"gateway env: {e}")
        return False

    cmd = [_cfg.CLAUDE_CMD, "-p", "hi", "--output-format", "json", "--max-turns", "1"]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30, env=env)
    except (subprocess.TimeoutExpired, OSError) as e:
        _print("claude -p auth", False, f"{e}")
        return False
    if result.returncode != 0:
        hint = result.stderr.strip().splitlines()[0] if result.stderr.strip() else "no stderr"
        _print("claude -p auth", False, hint[:120])
        return False
    try:
        out = json.loads(result.stdout)
    except json.JSONDecodeError:
        _print("claude -p auth", False, "response was not JSON")
        return False
    if out.get("is_error"):
        _print("claude -p auth", False, f"is_error=True subtype={out.get('subtype', '?')}")
        return False
    model_usage = out.get("modelUsage") or {}
    model_used = next(iter(model_usage.keys()), "") if model_usage else ""
    if not model_used:
        _print("claude -p auth", False, "response had no modelUsage — cannot verify gateway routing")
        return False
    # Gateway-shaped model IDs start with a provider prefix like `aws/`, `gcp/`, `azure/`.
    # Anthropic-direct names look like `claude-opus-4-6` or short aliases like `opus`.
    if "/" not in model_used:
        _print(
            "claude -p auth",
            False,
            f"model {model_used!r} is NOT gateway-shaped — request routed via OAuth/direct. "
            "Check ANTHROPIC_API_KEY / ANTHROPIC_BASE_URL in .env.",
        )
        return False
    _print("claude -p auth", True, f"gateway ok ({model_used})")
    return True


def _post_openai_compat(
    label: str,
    base_url: str,
    api_key: str,
    path: str,
    payload: dict,
) -> bool:
    """POST to an OpenAI-compatible endpoint and report PASS/FAIL via _print.

    Shared body for check_litellm_endpoint (/chat/completions) and
    check_responses_endpoint (/responses). The two differ only in path +
    payload shape.
    """
    import urllib.error
    import urllib.request

    try:
        req = urllib.request.Request(
            f"{base_url.rstrip('/')}{path}",
            data=json.dumps(payload).encode(),
            headers={
                "Authorization": f"Bearer {api_key}",
                "content-type": "application/json",
            },
            method="POST",
        )
        with urllib.request.urlopen(req, timeout=20) as resp:
            resp.read()
    except urllib.error.HTTPError as e:
        body = ""
        try:
            body = e.read().decode()[:200]
        except Exception:
            pass
        _print(label, False, f"HTTP {e.code} — {body or e.reason}")
        return False
    except (urllib.error.URLError, OSError) as e:
        _print(label, False, f"{e}")
        return False
    _print(label, True, f"{payload['model']} reachable")
    return True


def check_litellm_endpoint(
    test_model: str | None = None,
    base_url: str | None = None,
    api_key: str | None = None,
) -> bool:
    """POST to /chat/completions to catch "key exists but can't access this model" —
    a per-key policy failure LiteLLM gateways return only when you actually call.

    Reads OPENAI_BASE_URL (preferred, matches gateway.py / llm_judge) or
    OPENAI_API_BASE (legacy) when base_url / api_key are None.
    """
    api_key = (api_key or os.environ.get("OPENAI_API_KEY", "")).strip()
    base_url = (
        base_url or os.environ.get("OPENAI_BASE_URL", "") or os.environ.get("OPENAI_API_BASE", "")
    ).strip()
    if not api_key or not base_url:
        _print("LiteLLM endpoint", False, "OPENAI_API_KEY or OPENAI_BASE_URL missing in .env")
        return False
    if not test_model:
        _print("LiteLLM endpoint", False, "no test model provided (pass --profile)")
        return False
    return _post_openai_compat(
        "LiteLLM endpoint",
        base_url,
        api_key,
        "/chat/completions",
        {"model": test_model, "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]},
    )


def check_anthropic_endpoint(test_model: str | None = None) -> bool:
    """Validate ANTHROPIC_API_KEY against ANTHROPIC_BASE_URL with a minimal request.

    This is the path Harbor uses during smoke tests (claude_code.py reads these vars).
    `test_model` should be the profile's haiku_model so we exercise a model the
    key actually has policy access to.
    """
    api_key = os.environ.get("ANTHROPIC_API_KEY", "").strip()
    base_url = os.environ.get("ANTHROPIC_BASE_URL", "").strip().rstrip("/")
    if not api_key:
        _print("Anthropic endpoint", False, "ANTHROPIC_API_KEY missing in .env")
        return False
    if not base_url:
        _print("Anthropic endpoint", False, "ANTHROPIC_BASE_URL missing in .env")
        return False
    if not test_model:
        _print("Anthropic endpoint", False, "no test model provided (pass --profile)")
        return False
    try:
        import urllib.error
        import urllib.request

        payload = json.dumps(
            {
                "model": test_model,
                "max_tokens": 1,
                "messages": [{"role": "user", "content": "hi"}],
            }
        ).encode()
        # Send both auth headers. LiteLLM-style gateways (NVIDIA inference-api)
        # expect Authorization: Bearer; real Anthropic expects x-api-key.
        # Sending both covers both cases without a fallback round-trip.
        req = urllib.request.Request(
            f"{base_url}/v1/messages",
            data=payload,
            headers={
                "x-api-key": api_key,
                "Authorization": f"Bearer {api_key}",
                "anthropic-version": "2023-06-01",
                "content-type": "application/json",
            },
            method="POST",
        )
        with urllib.request.urlopen(req, timeout=20) as resp:
            resp.read()
    except urllib.error.HTTPError as e:
        # 401/403 = auth failure. 404 = bad base_url or path. 400 = model/body
        # quirk but auth+routing work (accept as PASS with note).
        body = ""
        try:
            body = e.read().decode()[:200]
        except Exception:
            pass
        if e.code in (401, 403):
            _print("Anthropic endpoint", False, f"HTTP {e.code} — {body or 'key rejected'}")
            return False
        if e.code == 404:
            _print("Anthropic endpoint", False, f"HTTP 404 — check ANTHROPIC_BASE_URL: {base_url}")
            return False
        _print("Anthropic endpoint", True, f"HTTP {e.code} (auth + routing ok)")
        return True
    except (urllib.error.URLError, OSError) as e:
        _print("Anthropic endpoint", False, f"{e}")
        return False
    _print("Anthropic endpoint", True, "ok")
    return True


def check_env_file() -> bool:
    """`.env` exists and the gateway-only keys are set to non-placeholder values.

    Hard-fails if ANTHROPIC_API_KEY or ANTHROPIC_BASE_URL are missing — OAuth
    fallback on this machine would charge the user's personal budget.
    """
    if not Path(".env").is_file():
        _print(".env file", False, "copy .env.example to .env and fill in API keys")
        return False
    values: dict[str, str] = {}
    with open(".env") as f:
        for line in f:
            line = line.strip()
            if not line or line.startswith("#") or "=" not in line:
                continue
            key, _, value = line.partition("=")
            values[key.strip()] = value.strip().strip("\"'")

    placeholders = {"", "your-api-key-here", "sk-your-key-here", "changeme"}

    # OPENAI_* are required by `llm_judge` for every direct-API dispatch (all
    # models normalize to an `openai/` litellm route). Missing either key
    # passes preflight today but crashes on the first evaluate/build call.
    openai_key = values.get("OPENAI_API_KEY", "")
    openai_base = values.get("OPENAI_BASE_URL", "")
    if openai_key in placeholders:
        _print(".env OPENAI_API_KEY", False, "missing or placeholder; llm_judge will fail")
        return False
    if openai_base in placeholders:
        _print(".env OPENAI_BASE_URL", False, "missing or placeholder; llm_judge will fail")
        return False

    # Gateway-only policy: both of these must be set. Missing either means the
    # claude CLI will silently fall back to the OAuth token in ~/.claude and
    # charge the user's personal budget.
    anthropic_key = values.get("ANTHROPIC_API_KEY", "") or values.get("ANTHROPIC_AUTH_TOKEN", "")
    anthropic_base = values.get("ANTHROPIC_BASE_URL", "")
    if anthropic_key in placeholders:
        _print(
            ".env ANTHROPIC_API_KEY",
            False,
            "missing or placeholder. OAuth fallback is forbidden on this machine.",
        )
        return False
    if anthropic_base in placeholders:
        _print(
            ".env ANTHROPIC_BASE_URL",
            False,
            "missing or placeholder. OAuth fallback is forbidden on this machine.",
        )
        return False
    _print(".env file", True, "gateway keys present")
    return True


def check_disk_space(min_gb: int = 30) -> bool:
    """Free disk space on the current filesystem."""
    try:
        usage = shutil.disk_usage(".")
    except OSError as e:
        _print("disk usage", False, f"{e}")
        return False
    free_gb = usage.free / (1024**3)
    ok = free_gb >= min_gb
    _print(f"disk space >= {min_gb}GB", ok, f"{free_gb:.1f}GB free")
    return ok


def check_harbor_patches_applied() -> bool:
    """Verify patches/harbor-agent-patches.diff is applied to the installed harbor.

    `uv sync --reinstall-package harbor` silently resets the source tree,
    dropping our codex/opencode/claude-code patches. Unpatched codex routes
    to api.openai.com — only caught after a failed baseline.
    """
    import sysconfig

    patch_file = Path("patches/harbor-agent-patches.diff")
    site_packages = Path(sysconfig.get_paths()["purelib"])
    if not patch_file.is_file():
        _print("Harbor patches", False, f"{patch_file} not found")
        return False
    if not site_packages.is_dir():
        _print("Harbor patches", False, f"{site_packages} not found — run `uv sync`")
        return False

    added_by_file: dict[str, list[str]] = {}
    current_target: str | None = None
    with open(patch_file) as f:
        for raw in f:
            line = raw.rstrip("\n")
            if line.startswith("+++ b/"):
                current_target = line[len("+++ b/") :]
                added_by_file.setdefault(current_target, [])
            elif line.startswith("+") and not line.startswith("+++") and current_target:
                added = line[1:]
                if added.strip():
                    added_by_file[current_target].append(added)

    if not added_by_file:
        _print("Harbor patches", False, "could not parse any hunks from diff")
        return False

    missing: list[str] = []
    for rel_path, added_lines in added_by_file.items():
        target = site_packages / rel_path
        if not target.is_file():
            missing.append(f"{rel_path} (file not found)")
            continue
        # Exact-line match so an added line "foo" can't match "foo.bar()" via substring.
        target_lines = set(target.read_text().splitlines())
        for added in added_lines:
            if added not in target_lines:
                missing.append(f"{rel_path}: missing {added.strip()[:60]!r}")
                break

    if missing:
        _print(
            "Harbor patches",
            False,
            f"not applied ({len(missing)}/{len(added_by_file)} files) — "
            f"first: {missing[0]}. Run: patch -d {site_packages} "
            "-p1 < patches/harbor-agent-patches.diff",
        )
        return False
    _print("Harbor patches", True, f"applied across {len(added_by_file)} files")
    return True


def check_responses_endpoint(test_model: str | None = None) -> bool:
    """POST to /responses — the endpoint codex-family models (gpt-5.x-codex) require.

    Those models don't accept /chat/completions, so a LiteLLM check alone
    misses codex-specific auth/model-access failures.
    """
    api_key = os.environ.get("OPENAI_API_KEY", "").strip()
    base_url = (os.environ.get("OPENAI_BASE_URL", "") or os.environ.get("OPENAI_API_BASE", "")).strip()
    if not api_key or not base_url:
        _print("Responses endpoint", False, "OPENAI_API_KEY or OPENAI_BASE_URL missing in .env")
        return False
    if not test_model:
        _print("Responses endpoint", False, "no test model provided")
        return False
    return _post_openai_compat(
        "Responses endpoint",
        base_url,
        api_key,
        "/responses",
        {"model": test_model, "input": "hi", "max_output_tokens": 16},
    )


def _is_sanity_only(agents: list[str]) -> bool:
    """True if every agent is a sanity-only (non-LLM) agent."""
    from craft_taskgen.baselines import SANITY_AGENT_NAMES

    return bool(agents) and all(a in SANITY_AGENT_NAMES for a in agents)


def check_baseline_readiness(backend: str, model: str | None, agents: list[str]) -> list[bool]:
    """Compose infrastructure + per-agent-endpoint checks for the baseline launcher."""
    # Report the resolved reasoning_effort for each agent we'll launch. Not a
    # check (can't fail), just visibility so the run's settings are in the log.
    from craft_taskgen.baselines import OUTPUT_TOKEN_CAP, SANITY_AGENT_NAMES, effort_for

    # Sanity agents (oracle, nop) don't talk to an LLM, so the reasoning
    # / cap / endpoint checks below don't apply.
    llm_agents = [a for a in agents if a not in SANITY_AGENT_NAMES]
    sanity_only = _is_sanity_only(agents)

    for agent in llm_agents:
        override = os.environ.get("REASONING_EFFORT_OVERRIDE")
        if override is not None:
            label = "set" if override else "disabled"
            _print_info(f"{agent} reasoning_effort override", f"{override!r} ({label})")
        else:
            effort = effort_for(agent, model)
            if effort:
                _print_info(f"{agent} reasoning_effort", f"{effort} (from reasoning_defaults)")
            else:
                _print_info(
                    f"{agent} reasoning_effort",
                    f"no row for ({agent}, {model}); agent-native default will apply",
                )

    # Report the output-token cap each agent will receive. Codex has no
    # working knob (openai/codex#4138) and runs uncapped; qwen-coder has
    # a settings.json knob but harbor doesn't plumb it.
    for agent in llm_agents:
        if agent == "codex":
            _print_info(f"{agent} output_cap", "UNCAPPED (openai/codex#4138)")
        elif agent == "qwen-coder":
            _print_info(f"{agent} output_cap", "UNCAPPED (no harbor plumbing for ~/.qwen/settings.json)")
        elif agent == "pi":
            _print_info(f"{agent} output_cap", "UNCAPPED (no env var for max_tokens)")
        elif agent in ("claude-code", "opencode"):
            _print_info(f"{agent} output_cap", f"{OUTPUT_TOKEN_CAP} tokens")
        else:
            _print_info(f"{agent} output_cap", f"{OUTPUT_TOKEN_CAP} (unknown agent; may not be honored)")

    for agent in agents:
        if agent in SANITY_AGENT_NAMES:
            _print_info(f"{agent} sanity agent", "no LLM endpoint check (oracle/nop)")

    results: list[bool] = [
        check_harbor(),
        check_harbor_patches_applied(),
        check_docker(),
        check_disk_space(min_gb=30),
    ]

    # Sanity-only runs need infra checks (harbor/docker/disk/patches) but
    # none of the per-agent LLM endpoint reachability — they don't talk
    # to an LLM. Return early; .env is irrelevant when no key is needed.
    if sanity_only:
        return results

    if backend == "gateway":
        results.append(check_env_file())
        # Short-circuit on the known aws/+claude-code breakage — otherwise we'd
        # still hit the gateway and record a second, redundant failure.
        if "claude-code" in agents and model.startswith("aws/"):
            _print(
                "claude-code + aws/ gateway model",
                False,
                "claude-code is temporarily broken against aws/ gateway endpoints. "
                "Use an azure/ model (e.g. azure/anthropic/claude-opus-4-6).",
            )
            results.append(False)
        elif "claude-code" in agents:
            results.append(check_anthropic_endpoint(test_model=model))
        if "codex" in agents:
            results.append(check_responses_endpoint(test_model=model))
        if "opencode" in agents:
            results.append(check_litellm_endpoint(test_model=model))
        if "qwen-coder" in agents:
            # Qwen Code CLI hits /v1/chat/completions via the OpenAI Node SDK
            # against OPENAI_BASE_URL. Same reachability check as opencode.
            results.append(check_litellm_endpoint(test_model=model))
        if "pi" in agents:
            # Pi (nvidia provider) hits /v1/chat/completions via the openai-
            # completions adapter against the baseUrl planted in models.json
            # (which the launcher seeds from OPENAI_BASE_URL).
            results.append(check_litellm_endpoint(test_model=model))
        return results

    # backend == "vllm" (argparse enforces the choice set)
    base_url = os.environ.get("VLLM_BASE_URL", "").strip()
    if not base_url:
        _print("vLLM backend", False, "VLLM_BASE_URL not set in env")
        results.append(False)
        return results
    _print("vLLM backend", True, base_url)
    # Note: `localhost`/`127.0.0.1` are rewritten to `host.docker.internal`
    # before forwarding to the container (see scripts/run-baselines.sh), which
    # also passes --extra-docker-compose patches/compose-overrides/host-gateway.yaml
    # to map host.docker.internal → host-gateway so the container can reach it.
    if "claude-code" in agents:
        _print_warn(
            "claude-code + vLLM",
            "vLLM does not speak the Anthropic protocol; claude-code will not work "
            "against this backend unless you've put an Anthropic-compatible shim in front.",
        )
    # opencode requires `provider/model_id` format (split at first "/"); the
    # `provider/` prefix is consumed by opencode's config machinery and only
    # model_id is forwarded to the upstream server. Strip it before preflight
    # so we hit vLLM with the name vLLM knows.
    test_model = model.split("/", 1)[1] if "opencode" in agents and "/" in model else model
    results.append(
        check_litellm_endpoint(
            test_model=test_model,
            base_url=base_url,
            api_key=os.environ.get("VLLM_API_KEY", "EMPTY"),
        )
    )
    return results


def check_repos_exist(candidate_files: list[str]) -> bool:
    """Every candidate file's repo directory exists under repos/."""
    if not candidate_files:
        _print("candidate repos", False, "no candidate files provided (use --candidates)")
        return False
    missing: list[str] = []
    shas_to_check: list[tuple[str, str]] = []  # (repo, base_sha)
    for fpath in candidate_files:
        repo = Path(fpath).stem
        repo_path = Path("repos") / repo
        if not repo_path.is_dir():
            missing.append(repo)
            continue
        # Sample one base_sha from the candidates to verify it's reachable
        try:
            with open(fpath) as f:
                data = json.load(f)
        except (OSError, json.JSONDecodeError):
            continue
        cands = data.get("candidates", [])
        if cands:
            base_sha = cands[0].get("merge_base_sha") or cands[0].get("base_sha")
            if base_sha:
                shas_to_check.append((str(repo_path), base_sha))

    if missing:
        _print("repos/ directories", False, f"missing: {', '.join(missing[:5])}")
        return False
    _print("repos/ directories", True, f"{len(candidate_files)} repos present")

    # Verify base SHAs resolve in their clones
    unreachable: list[str] = []
    for repo_path, sha in shas_to_check:
        try:
            result = subprocess.run(
                ["git", "-C", repo_path, "cat-file", "-t", sha],
                capture_output=True,
                text=True,
                timeout=10,
            )
        except (subprocess.TimeoutExpired, OSError):
            unreachable.append(f"{repo_path}@{sha[:8]}")
            continue
        if result.returncode != 0:
            unreachable.append(f"{repo_path}@{sha[:8]}")

    if unreachable:
        _print(
            "base_sha reachable in clones",
            False,
            f"missing: {', '.join(unreachable[:3])} (run `git fetch --all` in those repos)",
        )
        return False
    if shas_to_check:
        _print("base_sha reachable", True, f"verified {len(shas_to_check)} sample SHAs")
    return True


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Pre-flight checks for long-running pipeline invocations",
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument(
        "--candidates",
        nargs="*",
        default=[],
        help="Candidate JSON glob(s) to verify repos/ for (e.g. 'candidates/*.json')",
    )
    parser.add_argument(
        "--repos-csv",
        type=str,
        default=None,
        help="Validate a repos CSV and check GH rate limit for mining it (e.g. references/repo_list.csv)",
    )
    parser.add_argument("--min-disk-gb", type=int, default=30, help="Minimum free disk in GB")
    parser.add_argument(
        "--min-docker-gb",
        type=int,
        default=8,
        help="Docker memory warning threshold in GB (non-fatal)",
    )
    parser.add_argument("--skip-gh", action="store_true", help="Skip gh CLI check")
    parser.add_argument("--skip-harbor", action="store_true", help="Skip Harbor check")
    parser.add_argument("--skip-harbor-lab", action="store_true", help="Skip harbor-lab check")
    parser.add_argument(
        "--check-endpoints",
        action="store_true",
        help="Make live API calls to validate auth for Claude CLI, LiteLLM, and Anthropic endpoints "
        "(recommended before long unattended runs; costs fractions of a cent)",
    )
    parser.add_argument(
        "--profile",
        type=str,
        default=None,
        help="TOML profile to load (used with --check-endpoints to verify the profile's "
        "opus/haiku/llm_step_model names resolve at the LiteLLM endpoint)",
    )
    parser.add_argument(
        "--check-baselines",
        action="store_true",
        help="Preflight for scripts/run-baselines.sh (Harbor, Docker, disk, and endpoint reachability "
        "for the selected backend/model/agents). Incompatible with --candidates/--repos-csv.",
    )
    parser.add_argument(
        "--backend",
        choices=["gateway", "vllm"],
        default="gateway",
        help="For --check-baselines: which inference backend to verify.",
    )
    parser.add_argument(
        "--model",
        type=str,
        default=None,
        help="For --check-baselines: the model ID the baselines will run against.",
    )
    parser.add_argument(
        "--agents",
        type=str,
        default="claude-code",
        help=(
            "For --check-baselines: comma-separated agent names "
            "(claude-code, codex, opencode, qwen-coder, pi)."
        ),
    )
    args = parser.parse_args()

    if args.check_baselines and (args.candidates or args.repos_csv):
        print("--check-baselines is incompatible with --candidates/--repos-csv")
        return 1
    # Sanity agents (oracle, nop) don't talk to an LLM; --model is
    # only required when at least one LLM-driven agent is in the list.
    if args.check_baselines and not args.model:
        parsed_agents = [a.strip() for a in args.agents.split(",") if a.strip()]
        if not _is_sanity_only(parsed_agents):
            print("--check-baselines requires --model (unless --agents is sanity-only: oracle/nop)")
            return 1

    candidate_files: list[str] = []
    for pattern in args.candidates:
        candidate_files.extend(glob.glob(pattern))

    # Load .env so endpoint checks can read OPENAI_API_KEY etc.
    from craft_taskgen.config import _load_env

    _load_env()

    if args.check_baselines:
        agents = [a.strip() for a in args.agents.split(",") if a.strip()]
        print("Baseline preflight")
        print("=" * 60)
        checks = check_baseline_readiness(args.backend, args.model, agents)
        print("=" * 60)
        passed = sum(1 for c in checks if c)
        total = len(checks)
        if passed == total:
            print(f"All {total} checks passed. Ready to launch baselines.")
            return 0
        print(f"{passed}/{total} checks passed. Fix failures before launching.")
        return 1

    # Load profile if given — tells endpoint check which models to actually call
    profile_haiku_model: str | None = None
    profile_llm_step_model: str | None = None
    if args.profile:
        from craft_taskgen.config import PipelineProfile

        profile = PipelineProfile.from_toml(args.profile)
        profile_haiku_model = profile.haiku_model
        profile_llm_step_model = profile.llm_step_model or None
    elif args.check_endpoints:
        print(
            "  [warn] --check-endpoints without --profile: endpoint checks need "
            "a profile to know which models to test"
        )
        return 1

    print("Pre-flight checks")
    print("=" * 60)

    checks = [
        check_claude_cli(),
        check_docker(),
        check_env_file(),
        check_disk_space(args.min_disk_gb),
    ]
    # Docker memory is advisory (warn-only), not in pass/fail totals
    check_docker_memory(args.min_docker_gb)

    if candidate_files:
        checks.append(check_repos_exist(candidate_files))
    else:
        print("  [skip] repos/ check — no --candidates provided")

    if args.repos_csv:
        csv_ok = check_repos_csv(args.repos_csv)
        checks.append(csv_ok)
        if csv_ok and not args.skip_gh:
            # Estimate API calls: miner issues ~3 per repo (listing + per-PR stats).
            with open(args.repos_csv) as f:
                n_repos = sum(1 for _ in f) - 1  # minus header
            checks.append(check_gh_rate_limit(needed=3 * max(n_repos, 1)))

    if not args.skip_gh:
        checks.append(check_gh_cli())
    if not args.skip_harbor:
        checks.append(check_harbor())
    if not args.skip_harbor_lab:
        checks.append(check_harbor_lab())

    if args.check_endpoints:
        checks.append(check_claude_auth(model=profile_llm_step_model))
        # Test LiteLLM with summary model (Sonnet for fix-loop summaries)
        checks.append(check_litellm_endpoint(test_model="aws/anthropic/bedrock-claude-sonnet-4-6"))
        # Test Anthropic endpoint with the profile's haiku model (cheapest/fastest)
        checks.append(check_anthropic_endpoint(test_model=profile_haiku_model))

    print("=" * 60)
    passed = sum(1 for c in checks if c)
    total = len(checks)
    if passed == total:
        print(f"All {total} checks passed. Ready to launch.")
        return 0
    print(f"{passed}/{total} checks passed. Fix failures before launching a long run.")
    return 1


if __name__ == "__main__":
    sys.exit(main())