Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
8.3 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Tests for the native search Harbor adapter."""

from __future__ import annotations

import json
import os
import tempfile
from pathlib import Path

import pytest

from craft_taskgen.adapters.search_native import converter

MANIFEST = {
    "aiohttp": {
        "url": "https://github.com/aio-libs/aiohttp.git",
        "commit": "abc123def456",
    },
}


def _make_candidate(
    uuid: str = "d1e5bec3abcdef01",
    repo: str = "aiohttp",
    tier: str = "medium",
    instruction: str = "Where does aiohttp decompress response bodies?",
    files: list[str] | None = None,
    functions: list[str] | None = None,
    explanation: str = "The decompression happens in compression_utils.py.",
) -> dict:
    return {
        "task": {
            "id": uuid,
            "repo": repo,
            "tier": tier,
            "instruction": instruction,
            "gold_answer": {
                "files": files if files is not None else ["aiohttp/compression_utils.py"],
                "functions": functions if functions is not None else ["aiohttp.compression_utils.decompress"],
                "explanation": explanation,
            },
        }
    }


def _make_manifest_file(tmp: str) -> str:
    path = os.path.join(tmp, "manifest.json")
    with open(path, "w") as f:
        json.dump(MANIFEST, f)
    return path


def _make_candidates_tree(tmp: str, candidates: list[tuple[str, dict]]) -> str:
    """Write candidates to `{candidates_dir}/{repo}/{uuid}.json`."""
    root = os.path.join(tmp, "candidates")
    for uuid, cand in candidates:
        repo = cand["task"]["repo"]
        repo_dir = os.path.join(root, repo)
        os.makedirs(repo_dir, exist_ok=True)
        with open(os.path.join(repo_dir, f"{uuid}.json"), "w") as f:
            json.dump(cand, f)
    return root


# ---------------------------------------------------------------------------
# convert_task
# ---------------------------------------------------------------------------


def test_convert_task_produces_expected_layout():
    with tempfile.TemporaryDirectory() as tmp:
        candidate = _make_candidate()
        out = os.path.join(tmp, "harbor")
        os.makedirs(out, exist_ok=True)

        task_dir = converter.convert_task(candidate, MANIFEST, out)
        tid = os.path.basename(task_dir)
        assert tid == "craft-aiohttp-d1e5bec3"

        # Directory structure
        assert os.path.isfile(os.path.join(task_dir, "task.toml"))
        assert os.path.isfile(os.path.join(task_dir, "instruction.md"))
        assert os.path.isfile(os.path.join(task_dir, "environment", "Dockerfile"))
        assert os.path.isfile(os.path.join(task_dir, "tests", "test.sh"))
        assert os.path.isfile(os.path.join(task_dir, "tests", "test_runner.py"))
        assert os.path.isfile(os.path.join(task_dir, "tests", "gold_answer.json"))
        assert os.path.isfile(os.path.join(task_dir, "solution", "solve.sh"))


def test_dockerfile_substitutes_repo_url_and_commit():
    with tempfile.TemporaryDirectory() as tmp:
        candidate = _make_candidate()
        out = os.path.join(tmp, "harbor")
        os.makedirs(out, exist_ok=True)

        task_dir = converter.convert_task(candidate, MANIFEST, out)
        with open(os.path.join(task_dir, "environment", "Dockerfile")) as f:
            dockerfile = f.read()

        assert "https://github.com/aio-libs/aiohttp.git" in dockerfile
        assert "abc123def456" in dockerfile
        assert "{repo_url}" not in dockerfile
        assert "{repo_commit}" not in dockerfile


def test_gold_answer_and_solve_sh_contain_gold():
    with tempfile.TemporaryDirectory() as tmp:
        candidate = _make_candidate(
            files=["a.py", "b.py"],
            functions=["mod.f1", "mod.f2"],
            explanation="hello",
        )
        out = os.path.join(tmp, "harbor")
        os.makedirs(out, exist_ok=True)

        task_dir = converter.convert_task(candidate, MANIFEST, out)

        with open(os.path.join(task_dir, "tests", "gold_answer.json")) as f:
            gold = json.load(f)
        assert gold["files"] == ["a.py", "b.py"]
        assert gold["functions"] == ["mod.f1", "mod.f2"]

        with open(os.path.join(task_dir, "solution", "solve.sh")) as f:
            solve = f.read()
        assert '"files"' in solve
        assert "a.py" in solve
        assert "mod.f1" in solve


def test_task_toml_reflects_tier_timeout():
    with tempfile.TemporaryDirectory() as tmp:
        out = os.path.join(tmp, "harbor")
        os.makedirs(out, exist_ok=True)

        task_dir = converter.convert_task(_make_candidate(tier="easy"), MANIFEST, out)
        with open(os.path.join(task_dir, "task.toml")) as f:
            toml = f.read()
        assert 'difficulty = "easy"' in toml
        assert "timeout_sec = 600" in toml


def test_missing_repo_in_manifest_raises():
    with tempfile.TemporaryDirectory() as tmp:
        candidate = _make_candidate(repo="not-in-manifest")
        out = os.path.join(tmp, "harbor")
        os.makedirs(out, exist_ok=True)

        with pytest.raises(ValueError, match="not found in manifest"):
            converter.convert_task(candidate, MANIFEST, out)


def test_missing_required_fields_raise():
    with tempfile.TemporaryDirectory() as tmp:
        out = os.path.join(tmp, "harbor")
        os.makedirs(out, exist_ok=True)

        # Missing top-level 'task'
        with pytest.raises(ValueError, match="missing 'task' key"):
            converter.convert_task({"other": {}}, MANIFEST, out)

        # Missing gold_answer
        bad = _make_candidate()
        del bad["task"]["gold_answer"]
        with pytest.raises(ValueError, match="gold_answer"):
            converter.convert_task(bad, MANIFEST, out)


# ---------------------------------------------------------------------------
# run_convert
# ---------------------------------------------------------------------------


def test_run_convert_writes_registry_and_tasks():
    with tempfile.TemporaryDirectory() as tmp:
        candidates_dir = _make_candidates_tree(
            tmp,
            [
                ("aaaaaaaabbbbb", _make_candidate(uuid="aaaaaaaabbbbb")),
                ("ccccccccddddd", _make_candidate(uuid="ccccccccddddd")),
            ],
        )
        manifest_path = _make_manifest_file(tmp)
        out = os.path.join(tmp, "harbor")

        result = converter.run_convert(candidates_dir, manifest_path, out)
        assert result["converted"] == 2
        assert set(result["task_ids"]) == {"craft-aiohttp-aaaaaaaa", "craft-aiohttp-cccccccc"}

        with open(os.path.join(out, "registry.json")) as f:
            registry = json.load(f)
        assert registry[0]["name"] == "craft-search"
        assert set(registry[0]["tasks"]) == set(result["task_ids"])


def test_run_convert_respects_limit():
    with tempfile.TemporaryDirectory() as tmp:
        candidates_dir = _make_candidates_tree(
            tmp,
            [
                ("aaaaaaaa11111", _make_candidate(uuid="aaaaaaaa11111")),
                ("aaaaaaaa22222", _make_candidate(uuid="aaaaaaaa22222")),
                ("aaaaaaaa33333", _make_candidate(uuid="aaaaaaaa33333")),
            ],
        )
        manifest_path = _make_manifest_file(tmp)
        out = os.path.join(tmp, "harbor")

        result = converter.run_convert(candidates_dir, manifest_path, out, limit=2)
        assert result["converted"] == 2


def test_run_convert_archives_stale_tasks():
    with tempfile.TemporaryDirectory() as tmp:
        # Seed output dir with a stale task from a previous run
        out = os.path.join(tmp, "harbor")
        stale_tid = "craft-aiohttp-oldstale"
        os.makedirs(os.path.join(out, stale_tid), exist_ok=True)
        Path(out, stale_tid, "marker.txt").touch()

        candidates_dir = _make_candidates_tree(tmp, [("newnewnewnew", _make_candidate(uuid="newnewnewnew"))])
        manifest_path = _make_manifest_file(tmp)

        result = converter.run_convert(candidates_dir, manifest_path, out)
        assert result["archived"] == [stale_tid]

        # Stale task moved to a `_stale_<ts>/` subdir
        assert not os.path.isdir(os.path.join(out, stale_tid))
        stale_subdirs = [d for d in os.listdir(out) if d.startswith("_stale_")]
        assert len(stale_subdirs) == 1
        assert os.path.isdir(os.path.join(out, stale_subdirs[0], stale_tid))