Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
16.6 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from __future__ import annotations

import json
from pathlib import Path
from types import SimpleNamespace

from craft_taskgen import swebench_alignment


def _write_candidate_file(tmp_path):
    path = tmp_path / "teleport.json"
    path.write_text(
        json.dumps(
            {
                "repo": "teleport",
                "candidates": [
                    {
                        "sha": "abc123",
                        "merge_base_sha": "base123",
                        "source_task_id": "instance_1",
                        "source_metadata": {
                            "problem_statement": "Fix the login flow.",
                            "requirements": "- Preserve existing login behavior.",
                            "interface": "Keep the public login API unchanged.",
                        },
                    },
                    {
                        "sha": "def456",
                        "merge_base_sha": "base456",
                        "source_task_id": "instance_2",
                        "source_metadata": {
                            "problem_statement": "Fix the logout flow.",
                            "requirements": "- Preserve existing logout behavior.",
                            "interface": "Keep the public logout API unchanged.",
                        },
                    },
                ],
            }
        )
    )
    return path


def _write_state_file(tmp_path):
    path = tmp_path / "state.json"
    path.write_text(
        json.dumps(
            {
                "created": "2026-01-01T00:00:00",
                "last_updated": "2026-01-01T00:00:00",
                "run_dir": str(tmp_path),
                "profile_data": {},
                "run_info": {},
                "tasks": {
                    "teleport-abc123": {
                        "task_id": "teleport-abc123",
                        "repo": "teleport",
                        "commit_sha": "abc123",
                        "description": "Fix login",
                        "base_sha": "base123",
                        "merge_base_sha": "base123",
                        "stage": "candidate",
                        "candidate_data": {
                            "source_task_id": "instance_1",
                            "source_metadata": {
                                "problem_statement": "Fix the login flow.",
                                "requirements": "- Preserve existing login behavior.",
                                "interface": "Keep the public login API unchanged.",
                            },
                        },
                    }
                },
            }
        )
    )
    return path


def test_swebench_alignment_happy_path(tmp_path, monkeypatch) -> None:
    state_path = _write_state_file(tmp_path)
    output_path = tmp_path / "out.jsonl"

    def fake_build_context(candidate, repos_dir):
        assert repos_dir == "repos"
        assert candidate.source_task_id == "instance_1"
        return {
            "instruction_md": candidate.problem_statement,
            "reference_test_bodies": [("tests/test_login.py", "def test_login(): pass\n")],
            "diff": "diff --git a/x b/x",
            "diff_truncated": False,
        }

    async def fake_judge(*, prompt, schema, model):
        assert "Fix the login flow." in prompt
        assert schema == swebench_alignment.ALIGNMENT_SCHEMA
        assert model == "test-model"
        return SimpleNamespace(
            result={"verdict": "ok", "reason": "looks aligned", "leakage_evidence": [], "v4_audit": {}},
            usage={"input_tokens": 10, "output_tokens": 5},
            model="openai/test-model",
            latency_s=1.25,
        )

    monkeypatch.setattr(swebench_alignment, "_build_context", fake_build_context)
    monkeypatch.setattr(swebench_alignment.llm_judge, "judge", fake_judge)
    monkeypatch.setattr(
        "sys.argv",
        [
            "craft-taskgen-swebench-align",
            "--state-json",
            str(state_path),
            "--output",
            str(output_path),
            "--model",
            "test-model",
            "--instance-id",
            "instance_1",
        ],
    )

    rc = swebench_alignment.main()

    assert rc == 0
    rows = [json.loads(line) for line in output_path.read_text().splitlines()]
    assert len(rows) == 1
    assert rows[0]["status"] == "ok"
    assert rows[0]["verdict"] == "ok"
    assert rows[0]["reference_test_paths"] == ["tests/test_login.py"]
    state = json.loads(state_path.read_text())
    task = state["tasks"]["teleport-abc123"]
    assert task["alignment_verdict"] == "ok"
    assert task["alignment_reason"] == "looks aligned"


def test_swebench_alignment_promotes_ok_promising_tasks(tmp_path, monkeypatch) -> None:
    state_path = _write_state_file(tmp_path)
    output_path = tmp_path / "out.jsonl"

    def fake_build_context(candidate, repos_dir):
        return {
            "instruction_md": candidate.problem_statement,
            "reference_test_bodies": [("tests/test_login.py", "def test_login(): pass\n")],
            "diff": "diff --git a/x b/x",
            "diff_truncated": False,
        }

    async def fake_judge(*, prompt, schema, model):
        return SimpleNamespace(
            result={"verdict": "ok", "reason": "looks aligned", "leakage_evidence": [], "v4_audit": {}},
            usage={"input_tokens": 10, "output_tokens": 5},
            model="openai/test-model",
            latency_s=1.25,
        )

    state = json.loads(state_path.read_text())
    state["tasks"]["teleport-abc123"]["stage"] = "promising"
    state_path.write_text(json.dumps(state))

    monkeypatch.setattr(swebench_alignment, "_build_context", fake_build_context)
    monkeypatch.setattr(swebench_alignment.llm_judge, "judge", fake_judge)
    monkeypatch.setattr(
        "sys.argv",
        [
            "craft-taskgen-swebench-align",
            "--state-json",
            str(state_path),
            "--output",
            str(output_path),
            "--model",
            "test-model",
            "--promote-ok",
        ],
    )

    rc = swebench_alignment.main()

    assert rc == 0
    state = json.loads(state_path.read_text())
    task = state["tasks"]["teleport-abc123"]
    assert task["stage"] == "alignment_checked"
    assert task["task_dir"]
    assert (tmp_path / Path(task["task_dir"]).name / "instruction.md").read_text() == "Fix the login flow."
    assert task["instruction_words"] == 4


def test_swebench_alignment_include_interface(tmp_path, monkeypatch) -> None:
    state_path = _write_state_file(tmp_path)
    output_path = tmp_path / "out.jsonl"

    def fake_build_context(candidate, repos_dir):
        assert candidate.requirements == "- Preserve existing login behavior."
        assert candidate.interface == "Keep the public login API unchanged."
        return {
            "instruction_md": candidate.problem_statement,
            "reference_test_bodies": [("tests/test_login.py", "def test_login(): pass\n")],
            "diff": "diff --git a/x b/x",
            "diff_truncated": False,
        }

    async def fake_judge(*, prompt, schema, model):
        assert "Fix the login flow." in prompt
        assert "## Requirements" not in prompt
        assert "## Interface\nKeep the public login API unchanged." in prompt
        return SimpleNamespace(
            result={"verdict": "ok", "reason": "looks aligned", "leakage_evidence": [], "v4_audit": {}},
            usage={"input_tokens": 10, "output_tokens": 5},
            model="openai/test-model",
            latency_s=1.25,
        )

    state = json.loads(state_path.read_text())
    state["tasks"]["teleport-abc123"]["stage"] = "promising"
    state_path.write_text(json.dumps(state))

    monkeypatch.setattr(swebench_alignment, "_build_context", fake_build_context)
    monkeypatch.setattr(swebench_alignment.llm_judge, "judge", fake_judge)
    monkeypatch.setattr(
        "sys.argv",
        [
            "craft-taskgen-swebench-align",
            "--state-json",
            str(state_path),
            "--output",
            str(output_path),
            "--model",
            "test-model",
            "--promote-ok",
            "--include-interface",
        ],
    )

    rc = swebench_alignment.main()

    assert rc == 0
    rows = [json.loads(line) for line in output_path.read_text().splitlines()]
    assert rows[0]["instruction_source"] == "problem_statement+interface"
    assert rows[0]["interface_included"] is True
    state = json.loads(state_path.read_text())
    task = state["tasks"]["teleport-abc123"]
    assert Path(task["task_dir"], "instruction.md").read_text() == (
        "Fix the login flow.\n\n## Interface\nKeep the public login API unchanged."
    )


def test_swebench_alignment_include_requirements_and_interface(tmp_path, monkeypatch) -> None:
    state_path = _write_state_file(tmp_path)
    output_path = tmp_path / "out.jsonl"

    def fake_build_context(candidate, repos_dir):
        assert candidate.requirements == "- Preserve existing login behavior."
        assert candidate.interface == "Keep the public login API unchanged."
        return {
            "instruction_md": candidate.problem_statement,
            "reference_test_bodies": [("tests/test_login.py", "def test_login(): pass\n")],
            "diff": "diff --git a/x b/x",
            "diff_truncated": False,
        }

    async def fake_judge(*, prompt, schema, model):
        assert "Fix the login flow." in prompt
        assert "## Requirements\n- Preserve existing login behavior." in prompt
        assert "## Interface\nKeep the public login API unchanged." in prompt
        assert prompt.index("## Requirements") < prompt.index("## Interface")
        return SimpleNamespace(
            result={"verdict": "ok", "reason": "looks aligned", "leakage_evidence": [], "v4_audit": {}},
            usage={"input_tokens": 10, "output_tokens": 5},
            model="openai/test-model",
            latency_s=1.25,
        )

    state = json.loads(state_path.read_text())
    state["tasks"]["teleport-abc123"]["stage"] = "promising"
    state_path.write_text(json.dumps(state))

    monkeypatch.setattr(swebench_alignment, "_build_context", fake_build_context)
    monkeypatch.setattr(swebench_alignment.llm_judge, "judge", fake_judge)
    monkeypatch.setattr(
        "sys.argv",
        [
            "craft-taskgen-swebench-align",
            "--state-json",
            str(state_path),
            "--output",
            str(output_path),
            "--model",
            "test-model",
            "--promote-ok",
            "--include-requirements",
            "--include-interface",
        ],
    )

    rc = swebench_alignment.main()

    assert rc == 0
    rows = [json.loads(line) for line in output_path.read_text().splitlines()]
    assert rows[0]["instruction_source"] == "problem_statement+requirements+interface"
    assert rows[0]["requirements_included"] is True
    assert rows[0]["interface_included"] is True
    state = json.loads(state_path.read_text())
    task = state["tasks"]["teleport-abc123"]
    assert Path(task["task_dir"], "instruction.md").read_text() == (
        "Fix the login flow.\n\n"
        "## Requirements\n- Preserve existing login behavior.\n\n"
        "## Interface\nKeep the public login API unchanged."
    )


def test_swebench_alignment_does_not_revive_rejected_tasks(tmp_path, monkeypatch) -> None:
    state_path = _write_state_file(tmp_path)
    output_path = tmp_path / "out.jsonl"

    def fake_build_context(candidate, repos_dir):
        return {
            "instruction_md": candidate.problem_statement,
            "reference_test_bodies": [("tests/test_login.py", "def test_login(): pass\n")],
            "diff": "diff --git a/x b/x",
            "diff_truncated": False,
        }

    async def fake_judge(*, prompt, schema, model):
        return SimpleNamespace(
            result={"verdict": "ok", "reason": "looks aligned", "leakage_evidence": [], "v4_audit": {}},
            usage={"input_tokens": 10, "output_tokens": 5},
            model="openai/test-model",
            latency_s=1.25,
        )

    state = json.loads(state_path.read_text())
    state["tasks"]["teleport-abc123"]["stage"] = "rejected"
    state_path.write_text(json.dumps(state))

    monkeypatch.setattr(swebench_alignment, "_build_context", fake_build_context)
    monkeypatch.setattr(swebench_alignment.llm_judge, "judge", fake_judge)
    monkeypatch.setattr(
        "sys.argv",
        [
            "craft-taskgen-swebench-align",
            "--state-json",
            str(state_path),
            "--output",
            str(output_path),
            "--model",
            "test-model",
            "--promote-ok",
        ],
    )

    rc = swebench_alignment.main()

    assert rc == 0
    state = json.loads(state_path.read_text())
    task = state["tasks"]["teleport-abc123"]
    assert task["stage"] == "rejected"
    assert task.get("task_dir", "") == ""


def test_swebench_alignment_promote_existing_ok(tmp_path, monkeypatch) -> None:
    state_path = _write_state_file(tmp_path)
    state = json.loads(state_path.read_text())
    state["tasks"]["teleport-abc123"]["stage"] = "promising"
    state["tasks"]["teleport-abc123"]["alignment_verdict"] = "ok"
    state["tasks"]["teleport-abc123"]["alignment_reason"] = "looks aligned"
    state_path.write_text(json.dumps(state))

    monkeypatch.setattr(
        "sys.argv",
        [
            "craft-taskgen-swebench-align",
            "--state-json",
            str(state_path),
            "--promote-existing-ok",
        ],
    )

    rc = swebench_alignment.main()

    assert rc == 0
    state = json.loads(state_path.read_text())
    task = state["tasks"]["teleport-abc123"]
    assert task["stage"] == "alignment_checked"
    assert Path(task["task_dir"], "instruction.md").read_text() == "Fix the login flow."


def test_swebench_alignment_continues_on_error(tmp_path, monkeypatch) -> None:
    candidate_path = _write_candidate_file(tmp_path)
    output_path = tmp_path / "out.jsonl"

    def fake_build_context(candidate, repos_dir):
        if candidate.source_task_id == "instance_1":
            raise ValueError("missing git clone")
        return {
            "instruction_md": candidate.problem_statement,
            "reference_test_bodies": [("tests/test_logout.py", "def test_logout(): pass\n")],
            "diff": "diff --git a/y b/y",
            "diff_truncated": False,
        }

    async def fake_judge(*, prompt, schema, model):
        return SimpleNamespace(
            result={
                "verdict": "leaked",
                "reason": "too specific",
                "leakage_evidence": ["Fix logout"],
                "v4_audit": {},
            },
            usage={"input_tokens": 7, "output_tokens": 3},
            model="openai/test-model",
            latency_s=0.5,
        )

    monkeypatch.setattr(swebench_alignment, "_build_context", fake_build_context)
    monkeypatch.setattr(swebench_alignment.llm_judge, "judge", fake_judge)
    monkeypatch.setattr(
        "sys.argv",
        [
            "craft-taskgen-swebench-align",
            "--candidates",
            str(candidate_path),
            "--output",
            str(output_path),
            "--model",
            "test-model",
        ],
    )

    rc = swebench_alignment.main()

    assert rc == 0
    rows = [json.loads(line) for line in output_path.read_text().splitlines()]
    assert len(rows) == 2
    assert rows[0]["status"] == "context_error"
    assert rows[0]["error"] == "missing git clone"
    assert rows[1]["status"] == "ok"
    assert rows[1]["verdict"] == "leaked"


def test_swebench_alignment_dry_run_writes_prompt(tmp_path, monkeypatch) -> None:
    state_path = _write_state_file(tmp_path)
    output_path = tmp_path / "out.jsonl"

    def fake_build_context(candidate, repos_dir):
        return {
            "instruction_md": candidate.problem_statement,
            "reference_test_bodies": [("tests/test_login.py", "def test_login(): pass\n")],
            "diff": "diff --git a/x b/x",
            "diff_truncated": False,
        }

    monkeypatch.setattr(swebench_alignment, "_build_context", fake_build_context)
    monkeypatch.setattr(
        "sys.argv",
        [
            "craft-taskgen-swebench-align",
            "--state-json",
            str(state_path),
            "--output",
            str(output_path),
            "--dry-run",
        ],
    )

    rc = swebench_alignment.main()

    assert rc == 0
    rows = [json.loads(line) for line in output_path.read_text().splitlines()]
    assert len(rows) == 1
    assert rows[0]["status"] == "dry_run"
    assert "Fix the login flow." in rows[0]["prompt"]