Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
4.9 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from __future__ import annotations

import csv
import json

import pandas as pd

from scripts.agent_pass_rate_matrix import build_joined_df, build_matrix, pretty_summary


def _write_state(path) -> None:
    path.write_text(
        json.dumps(
            {
                "tasks": {
                    "task-1": {
                        "task_id": "task-1",
                        "stage": "promising",
                        "alignment_verdict": "ok",
                        "candidate_data": {
                            "source_task_id": "instance_1",
                            "source_metadata": {"instance_id": "instance_1"},
                        },
                    },
                    "task-2": {
                        "task_id": "task-2",
                        "stage": "rejected",
                        "alignment_verdict": "leaked",
                        "candidate_data": {
                            "source_task_id": "instance_2",
                            "source_metadata": {"instance_id": "instance_2"},
                        },
                    },
                }
            }
        )
    )


def _write_agent_runs(path) -> None:
    with path.open("w", newline="") as f:
        writer = csv.DictWriter(
            f,
            fieldnames=[
                "metadata.instance_id",
                "metadata.model_name",
                "metadata.resolved",
            ],
        )
        writer.writeheader()
        writer.writerow(
            {
                "metadata.instance_id": "instance_1",
                "metadata.model_name": "Model A -- 10222025",
                "metadata.resolved": "true",
            }
        )
        writer.writerow(
            {
                "metadata.instance_id": "instance_1",
                "metadata.model_name": "Model B",
                "metadata.resolved": "false",
            }
        )
        writer.writerow(
            {
                "metadata.instance_id": "instance_2",
                "metadata.model_name": "Model A -- 10222025",
                "metadata.resolved": "false",
            }
        )
        writer.writerow(
            {
                "metadata.instance_id": "instance_2",
                "metadata.model_name": "Model B",
                "metadata.resolved": "true",
            }
        )


def test_build_matrix_groups_by_stage_alignment_and_model(tmp_path) -> None:
    state_path = tmp_path / "state.json"
    runs_path = tmp_path / "runs.csv"
    _write_state(state_path)
    _write_agent_runs(runs_path)

    joined = build_joined_df(runs_path, state_path)
    matrix = build_matrix(joined)

    row = matrix[(matrix["stage"] == "promising") & (matrix["alignment_verdict"] == "ok")].iloc[0]
    assert row["matched_agent_runs"] == 2
    assert row["matched_instances"] == 1
    assert row["Model A pass_rate"] == 1.0
    assert row["Model A runs"] == 1
    assert row["Model B pass_rate"] == 0.0
    assert row["Model B runs"] == 1

    row = matrix[(matrix["stage"] == "rejected") & (matrix["alignment_verdict"] == "leaked")].iloc[0]
    assert row["Model A pass_rate"] == 0.0
    assert row["Model B pass_rate"] == 1.0


def test_build_matrix_can_emit_percent_without_counts() -> None:
    joined = pd.DataFrame(
        [
            {
                "instance_id": "instance_1",
                "model_label": "Model A",
                "resolved": True,
                "stage": "promising",
                "alignment_verdict": "ok",
            },
            {
                "instance_id": "instance_2",
                "model_label": "Model A",
                "resolved": False,
                "stage": "promising",
                "alignment_verdict": "ok",
            },
        ]
    )

    matrix = build_matrix(joined, include_counts=False, as_percent=True)

    row = matrix[(matrix["stage"] == "promising") & (matrix["alignment_verdict"] == "ok")].iloc[0]
    assert row["Model A pass_rate"] == 50.0
    assert "Model A runs" not in matrix.columns


def test_pretty_summary_prints_model_blocks() -> None:
    joined = pd.DataFrame(
        [
            {
                "instance_id": "instance_1",
                "model_label": "Claude Opus 4.5",
                "resolved": True,
                "stage": "accepted",
                "alignment_verdict": "ok",
            },
            {
                "instance_id": "instance_2",
                "model_label": "Claude Opus 4.5",
                "resolved": False,
                "stage": "rejected",
                "alignment_verdict": "narrow_tests",
            },
        ]
    )

    text = pretty_summary(joined)

    assert "-----Claude Opus 4.5-----" in text
    assert "overall: 50.0% (1/2)" in text
    assert "Rejected: 0.0% (0/1)" in text
    assert "Accepted: 100.0% (1/1)" in text
    assert "Narrow test: 0.0% (0/1)" in text
    assert "OK: 100.0% (1/1)" in text