Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
18.9 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Per-run reproducibility manifest for Switchyard Harbor baselines.

The manifest is written before Harbor starts and finalized after Harbor exits.
It intentionally stays stdlib-only so ``benchmark/run-baseline.sh`` can call it
with ``python3`` without forcing an editable Switchyard build.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import shutil
import socket
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

SCHEMA_VERSION = 1
UTC = timezone.utc


def _iso_timestamp() -> str:
    return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")


def _run(cmd: list[str], cwd: Path | None = None) -> tuple[int, str] | None:
    try:
        result = subprocess.run(
            cmd,
            cwd=cwd,
            capture_output=True,
            text=True,
            timeout=10,
            check=False,
        )
    except (OSError, subprocess.SubprocessError):
        return None
    return result.returncode, result.stdout.strip()


def _repo_root(path: Path) -> Path:
    cwd = path if path.is_dir() else path.parent
    result = _run(["git", "rev-parse", "--show-toplevel"], cwd=cwd)
    if result is None or result[0] != 0 or not result[1]:
        return cwd
    return Path(result[1])


def _git_value(args: list[str], cwd: Path) -> str | None:
    result = _run(["git", *args], cwd=cwd)
    if result is None or result[0] != 0:
        return None
    return result[1] or None


def _git_dirty(cwd: Path) -> bool | None:
    result = _run(["git", "status", "--porcelain"], cwd=cwd)
    if result is None or result[0] != 0:
        return None
    return bool(result[1])


def _tree_kind(dirty: bool | None) -> str:
    if dirty is None:
        return "not-git"
    return "git-dirty" if dirty else "git-clean"


def _harbor_version(harbor_command: list[str] | None = None) -> str | None:
    command = harbor_command or ["harbor"]
    result = _run([*command, "--version"])
    if result is None or result[0] != 0:
        return None
    return result[1].splitlines()[0] if result[1] else None


def _file_digest(path: Path) -> bytes:
    hasher = hashlib.sha256()
    with path.open("rb") as file:
        for chunk in iter(lambda: file.read(1024 * 1024), b""):
            hasher.update(chunk)
    return hasher.digest()


def path_digest(path: Path) -> str:
    """Return a deterministic sha256 digest for a file or directory."""
    try:
        resolved = path.resolve()
        hasher = hashlib.sha256()
        if resolved.is_file():
            hasher.update(resolved.name.encode())
            hasher.update(_file_digest(resolved))
            return f"sha256:{hasher.hexdigest()}"
        if resolved.is_dir():
            for item in sorted(p for p in resolved.rglob("*") if p.is_file()):
                rel = item.relative_to(resolved).as_posix()
                file_hash = _file_digest(item).hex()
                hasher.update(f"{rel}\n{file_hash}\n".encode())
            return f"sha256:{hasher.hexdigest()}"
    except OSError:
        return "sha256:unknown"
    return "sha256:missing"


def dataset_fingerprint(
    dataset: str | None,
    task_list_file: Path | None = None,
    harbor_path: Path | None = None,
) -> str:
    """Fingerprint the Harbor dataset selector plus optional task-list file/path."""
    payload: dict[str, Any] = {"dataset": dataset}
    if harbor_path is not None:
        payload["harbor_path"] = str(harbor_path.resolve())
        payload["harbor_path_digest"] = path_digest(harbor_path)
    if task_list_file is not None:
        payload["task_list_file"] = str(task_list_file.resolve())
        payload["task_list_digest"] = path_digest(task_list_file)
    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
    return f"sha256:{hashlib.sha256(encoded).hexdigest()}"


def snapshot_server_config(source: Path | None, run_dir: Path) -> Path | None:
    """Copy the Rust server configuration used for the run."""
    if source is None:
        return None
    if not source.is_file():
        raise FileNotFoundError(source)

    dest = run_dir.resolve() / "server_config" / source.name
    dest.parent.mkdir(parents=True, exist_ok=True)
    shutil.copyfile(source, dest)
    return dest


def snapshot_dataset_manifest(source: Path | None, run_dir: Path) -> Path | None:
    """Copy a generated local dataset manifest into the run directory."""
    if source is None:
        return None
    manifest = source / "switchyard_dataset_manifest.json"
    if not manifest.is_file():
        return None

    dest = run_dir.resolve() / "dataset" / manifest.name
    dest.parent.mkdir(parents=True, exist_ok=True)
    shutil.copyfile(manifest, dest)
    return dest


def _json_arg(value: str, default: Any) -> Any:
    if not value:
        return default
    try:
        return json.loads(value)
    except json.JSONDecodeError:
        return default


def _opt(value: str | None) -> str | None:
    return value if value else None


def _opt_int(value: int | None) -> int | None:
    return value if value is not None and value >= 0 else None


def _bool_arg(value: str | bool | None) -> bool:
    if isinstance(value, bool):
        return value
    return (value or "").strip().lower() in {"1", "true", "yes", "on"}


def write_manifest(
    output_path: Path,
    harbor_command: list[str] | None = None,
    **fields: Any,
) -> None:
    repo = _repo_root(Path(__file__).resolve())
    dirty = _git_dirty(repo)
    run_meta: dict[str, Any] = {
        "schema_version": SCHEMA_VERSION,
        "timestamp": _iso_timestamp(),
        "hostname": socket.gethostname(),
        "git_branch": _git_value(["branch", "--show-current"], cwd=repo),
        "git_sha": _git_value(["rev-parse", "HEAD"], cwd=repo),
        "git_dirty": dirty,
        "git_tree_kind": _tree_kind(dirty),
        "harbor_command": harbor_command or ["harbor"],
        "harbor_version": _harbor_version(harbor_command),
        "launcher_argv": fields.pop("launcher_argv", []),
    }
    manifest = {"run": run_meta, **fields}
    output_path.parent.mkdir(parents=True, exist_ok=True)
    output_path.write_text(json.dumps(manifest, indent=2, sort_keys=False) + "\n")


def _find_harbor_result(job_dir: Path | None) -> Path | None:
    if job_dir is None:
        return None
    direct = job_dir / "result.json"
    if direct.is_file():
        return direct
    candidates = sorted(job_dir.glob("*/result.json"), key=lambda p: p.stat().st_mtime)
    return candidates[-1] if candidates else None


def _find_artifact(job_dir: Path | None, artifact_name: str) -> Path | None:
    if job_dir is None or not artifact_name:
        return None
    direct = job_dir / artifact_name
    if direct.is_file():
        return direct
    candidates = sorted(job_dir.rglob(artifact_name), key=lambda p: p.stat().st_mtime)
    return candidates[-1] if candidates else None


def _copy_if_present(source: Path | None, dest: Path | None) -> str:
    if source is None or dest is None or not source.is_file():
        return "missing"
    try:
        dest.parent.mkdir(parents=True, exist_ok=True)
        if source.resolve() != dest.resolve():
            shutil.copyfile(source, dest)
    except OSError:
        return "missing"
    return "present"


def finalize_manifest(
    path: Path,
    *,
    harbor_rc: int | None,
    harbor_job_dir: Path | None = None,
    server_metrics: Path | None = None,
    routing_stats: Path | None = None,
) -> int:
    """Copy run artifacts into place and record outcomes.

    Returns 0 on success, 1 when the manifest is missing.
    """
    if not path.is_file():
        print(f"ERROR: manifest not found: {path}")
        return 1

    manifest = json.loads(path.read_text())
    outcomes = manifest.setdefault("outcomes", {})

    harbor_dest = Path(outcomes["harbor_result_json"]) if outcomes.get("harbor_result_json") else None
    harbor_source = _find_harbor_result(harbor_job_dir)
    outcomes["harbor_result_json_status"] = _copy_if_present(harbor_source, harbor_dest)
    if harbor_job_dir is not None:
        outcomes["harbor_job_dir"] = str(harbor_job_dir.resolve())

    metrics_dest = (
        Path(outcomes["server_metrics_prom"]) if outcomes.get("server_metrics_prom") else None
    )
    if outcomes.get("server_metrics_prom_status") != "not-requested":
        if server_metrics is None and metrics_dest is not None:
            server_metrics = metrics_dest
        outcomes["server_metrics_prom_status"] = _copy_if_present(server_metrics, metrics_dest)

    stats_dest = (
        Path(outcomes["routing_stats_json"]) if outcomes.get("routing_stats_json") else None
    )
    if outcomes.get("routing_stats_json_status") != "not-requested":
        if routing_stats is None and stats_dest is not None:
            routing_stats = stats_dest
        outcomes["routing_stats_json_status"] = _copy_if_present(routing_stats, stats_dest)

    closed_book = manifest.setdefault("closed_book", {})
    if isinstance(closed_book, dict) and closed_book.get("proxy_strip_log"):
        strip_dest = Path(closed_book["proxy_strip_log"])
        artifact_path = closed_book.get("proxy_strip_artifact") or strip_dest.name
        strip_source = None
        for artifact_name in dict.fromkeys(
            [Path(str(artifact_path)).name, strip_dest.name, "strip.jsonl"]
        ):
            strip_source = _find_artifact(harbor_job_dir, artifact_name)
            if strip_source is not None:
                break
        closed_book["proxy_strip_log_status"] = _copy_if_present(strip_source, strip_dest)

    outcomes["harbor_rc"] = harbor_rc
    outcomes["completed_at"] = _iso_timestamp()
    path.write_text(json.dumps(manifest, indent=2, sort_keys=False) + "\n")
    return 0


def _apply_extra(body: dict[str, Any], entries: list[str]) -> int:
    for entry in entries:
        if "=" not in entry or "." not in entry.split("=", 1)[0]:
            print(f"ERROR: --extra must be section.key=value, got {entry!r}")
            return 2
        lhs, raw = entry.split("=", 1)
        section, key = lhs.split(".", 1)
        try:
            parsed: Any = json.loads(raw)
        except json.JSONDecodeError:
            parsed = raw
        target = body.setdefault(section, {})
        if not isinstance(target, dict):
            print(f"ERROR: --extra target {section!r} is not an object")
            return 2
        target[key] = parsed
    return 0


def _cli_main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(prog="run_manifest")
    sub = parser.add_subparsers(dest="command")

    write = sub.add_parser("write")
    write.add_argument("--output", type=Path, required=True)
    write.add_argument("--launcher-argv-json", default="[]")
    write.add_argument("--server-preset", required=True)
    write.add_argument("--server-mode", default="")
    write.add_argument("--server-url", default="")
    write.add_argument("--server-port", type=int, default=0)
    write.add_argument("--server-argv-json", default="[]")
    write.add_argument("--server-config-json", default="{}")
    write.add_argument("--harbor-server-url", default="")
    write.add_argument("--harbor-base-url", default="")
    write.add_argument("--upstream-base-url", default="")
    write.add_argument("--upstream-api-key-env", default="")
    write.add_argument("--server-config", type=Path, default=None)
    write.add_argument("--route-model", default="")
    write.add_argument("--harbor-command-json", default="[]")
    write.add_argument("--dataset-label", default="")
    write.add_argument("--harbor-path", type=Path, default=None)
    write.add_argument("--codex-model-catalog", type=Path, default=None)
    write.add_argument("--agent", required=True)
    write.add_argument("--harbor-model", required=True)
    write.add_argument("--reasoning-effort", default="")
    write.add_argument("--n-concurrent", type=int, required=True)
    write.add_argument("--max-retries", type=int, required=True)
    write.add_argument("--agent-timeout-multiplier", required=True)
    write.add_argument("--n-tasks", type=int, default=-1)
    write.add_argument("--task-id", default="")
    write.add_argument("--task-list-file", type=Path, default=None)
    write.add_argument("--harbor-extra-json", default="[]")
    write.add_argument("--closed-book-mode", default="closed")
    write.add_argument("--closed-book-gateway-enforced", default="0")
    write.add_argument("--closed-book-hosted-tools-disabled", default="0")
    write.add_argument("--closed-book-proxy-strip-artifact", default="")
    write.add_argument("--harbor-patch-json", default="{}")
    write.add_argument("--agent-versions-json", default="{}")
    write.add_argument("--run-dir", type=Path, required=True)
    write.add_argument("--log-path", type=Path, required=True)
    write.add_argument("--harbor-result-json", type=Path, required=True)
    write.add_argument("--server-metrics-prom", type=Path, default=None)
    write.add_argument("--server-metrics-status", default="not-requested")
    write.add_argument("--routing-stats-json", type=Path, required=True)
    write.add_argument("--routing-stats-status", default="predicted")
    write.add_argument("--extra", action="append", default=[])

    finalize = sub.add_parser("finalize")
    finalize.add_argument("--manifest", type=Path, required=True)
    finalize.add_argument("--harbor-rc", type=int, default=None)
    finalize.add_argument("--harbor-job-dir", type=Path, default=None)
    finalize.add_argument("--server-metrics", type=Path, default=None)
    finalize.add_argument("--routing-stats", type=Path, default=None)

    ns = parser.parse_args(argv)
    if ns.command == "finalize":
        return finalize_manifest(
            ns.manifest,
            harbor_rc=ns.harbor_rc,
            harbor_job_dir=ns.harbor_job_dir,
            server_metrics=ns.server_metrics,
            routing_stats=ns.routing_stats,
        )
    if ns.command != "write":
        parser.print_help()
        return 2

    task_list = ns.task_list_file.resolve() if ns.task_list_file else None
    server_config = ns.server_config.resolve() if ns.server_config else None
    harbor_path = ns.harbor_path.resolve() if ns.harbor_path else None
    codex_model_catalog = (
        ns.codex_model_catalog.resolve() if ns.codex_model_catalog else None
    )
    run_dir = ns.run_dir.resolve()
    try:
        server_config_snapshot = snapshot_server_config(server_config, run_dir)
        dataset_manifest_snapshot = snapshot_dataset_manifest(harbor_path, run_dir)
    except OSError as exc:
        print(f"ERROR: failed to snapshot run inputs: {exc}")
        return 2
    harbor: dict[str, Any] = {
        "dataset": _opt(ns.dataset_label),
        "path": str(harbor_path) if harbor_path else None,
        "path_digest": path_digest(harbor_path) if harbor_path else None,
        "dataset_fingerprint": dataset_fingerprint(ns.dataset_label, task_list, harbor_path),
        "agent": ns.agent,
        "model": ns.harbor_model,
        "reasoning_effort": _opt(ns.reasoning_effort),
        "n_concurrent": ns.n_concurrent,
        "max_retries": ns.max_retries,
        "agent_timeout_multiplier": ns.agent_timeout_multiplier,
        "n_tasks": _opt_int(ns.n_tasks),
        "task_id": _opt(ns.task_id),
        "task_list_file": str(task_list) if task_list else None,
        "task_list_digest": path_digest(task_list) if task_list else None,
        "codex_model_catalog": (
            str(codex_model_catalog) if codex_model_catalog else None
        ),
        "codex_model_catalog_digest": (
            path_digest(codex_model_catalog) if codex_model_catalog else None
        ),
        "extra_args": _json_arg(ns.harbor_extra_json, []),
    }
    closed_book_mode = ns.closed_book_mode
    proxy_strip_log = run_dir / "proxy_strip_log.jsonl"
    closed_book: dict[str, Any] = {
        "mode": closed_book_mode,
        "gateway_enforced": _bool_arg(ns.closed_book_gateway_enforced),
        "hosted_tools_disabled": _bool_arg(ns.closed_book_hosted_tools_disabled),
        "proxy_strip_artifact": _opt(ns.closed_book_proxy_strip_artifact),
        "proxy_strip_log": str(proxy_strip_log),
        "proxy_strip_log_status": (
            "predicted" if ns.closed_book_proxy_strip_artifact else "not-requested"
        ),
        "verifier_egress": (
            "open-via-authenticated-proxy"
            if closed_book_mode == "closed"
            else "open-via-proxy"
        ),
        "agent_versions": _json_arg(ns.agent_versions_json, {}),
        "dataset_manifest_snapshot": (
            str(dataset_manifest_snapshot) if dataset_manifest_snapshot else None
        ),
        "dataset_manifest_snapshot_digest": (
            path_digest(dataset_manifest_snapshot) if dataset_manifest_snapshot else None
        ),
    }
    body: dict[str, Any] = {
        "server": {
            "preset": ns.server_preset,
            "mode": _opt(ns.server_mode),
            "url": _opt(ns.server_url),
            "port": ns.server_port or None,
            "argv": _json_arg(ns.server_argv_json, []),
            "config": _json_arg(ns.server_config_json, {}),
            "harbor_server_url": _opt(ns.harbor_server_url),
            "harbor_base_url": _opt(ns.harbor_base_url),
            "upstream_base_url": _opt(ns.upstream_base_url),
            "upstream_api_key_env": _opt(ns.upstream_api_key_env),
            "server_config": str(server_config) if server_config else None,
            "server_config_digest": path_digest(server_config) if server_config else None,
            "server_config_snapshot": (
                str(server_config_snapshot) if server_config_snapshot else None
            ),
            "server_config_snapshot_digest": (
                path_digest(server_config_snapshot) if server_config_snapshot else None
            ),
            "route_model": _opt(ns.route_model),
        },
        "harbor": harbor,
        "harbor_patch": _json_arg(ns.harbor_patch_json, {}),
        "closed_book": closed_book,
        "determinism": {
            "PYTHONHASHSEED": "0",
            "LC_ALL": "C.UTF-8",
        },
        "outcomes": {
            "run_dir": str(run_dir),
            "log_path": str(ns.log_path.resolve()),
            "harbor_result_json": str(ns.harbor_result_json.resolve()),
            "harbor_result_json_status": "predicted",
            "server_metrics_prom": (
                str(ns.server_metrics_prom.resolve()) if ns.server_metrics_prom else None
            ),
            "server_metrics_prom_status": ns.server_metrics_status,
            "routing_stats_json": str(ns.routing_stats_json.resolve()),
            "routing_stats_json_status": ns.routing_stats_status,
            "harbor_rc": None,
        },
    }
    extra_rc = _apply_extra(body, ns.extra)
    if extra_rc:
        return extra_rc
    write_manifest(
        ns.output,
        launcher_argv=_json_arg(ns.launcher_argv_json, []),
        harbor_command=_json_arg(ns.harbor_command_json, ["harbor"]),
        **body,
    )
    return 0


if __name__ == "__main__":  # pragma: no cover
    raise SystemExit(_cli_main())