Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

22.8 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Exercise the inherited Harbor bootstrap, including package selection."""

import asyncio
import re
import subprocess
from pathlib import Path

import pytest

pytest.importorskip("harbor")

from harbor.agents.factory import AgentFactory
from harbor.agents.installed.base import BaseInstalledAgent
from harbor.agents.installed.claude_code import ClaudeCode
from harbor.agents.installed.codex import Codex
from harbor.agents.installed.opencode import OpenCode
from harbor.environments.base import ExecResult
from harbor.models.trial.config import AgentConfig
from harbor.utils.import_path import import_symbol

from skillevaluator.tier3 import commands
from skillevaluator.tier3.harbor import DOCKER_AGENT_IMPORT_PATHS, adapter
from skillevaluator.tier3.harbor.installed_agents import (
    SkillEvaluatorClaudeCode,
    SkillEvaluatorCodex,
    SkillEvaluatorOpenCode,
)
from skillevaluator.tier3.harbor.local_agents import SkillEvaluatorClaudeCode as ContainerClaudeCode
from skillevaluator.tier3.harbor.local_agents import (
    SkillEvaluatorGatewayCodex,
    SkillEvaluatorGatewayOpenCode,
    SkillEvaluatorLocalClaudeCode,
    SkillEvaluatorLocalCodex,
    SkillEvaluatorNvidiaBuildClaudeCode,
    SkillEvaluatorNvidiaBuildCodex,
)


class _BootstrapEnvironment:
    default_user = None

    def __init__(
        self,
        *,
        manager: str = "apt-get",
        system_node: bool = False,
        ca_bundle: bool = True,
        missing_commands: tuple[str, ...] = ("node", "npm"),
        installed: bool = False,
        installed_version: str = "1.2.3",
        probe_result: ExecResult | None = None,
        probe_error: Exception | None = None,
    ) -> None:
        self.commands: list[str] = []
        self.exec_kwargs: list[dict[str, object]] = []
        self.manager = manager
        self.system_node = system_node
        self.ca_bundle = ca_bundle
        self.missing_commands = missing_commands
        self.installed = installed
        self.installed_version = installed_version
        self.probe_result = probe_result
        self.probe_error = probe_error

    @property
    def probes(self) -> list[str]:
        return [command for command in self.commands if command.startswith("if ") and "printf" in command]

    @property
    def package_installs(self) -> list[str]:
        return [
            command
            for command in self.commands
            if any(text in command for text in ("apt-get install", "dnf install", "yum install", "apk add"))
        ]

    async def exec(self, command: str, **_kwargs: object) -> ExecResult:
        self.commands.append(command)
        self.exec_kwargs.append(_kwargs)
        if command in (Codex._INSTALL_CHECK_COMMAND, ClaudeCode._INSTALL_CHECK_COMMAND):
            return ExecResult(return_code=0 if self.installed else 1, stdout="", stderr="")
        if command in (Codex._INSTALL_VERSION_COMMAND, ClaudeCode._INSTALL_VERSION_COMMAND):
            version = (
                f"codex-cli {self.installed_version}"
                if command == Codex._INSTALL_VERSION_COMMAND
                else f"{self.installed_version} (Claude Code)"
            )
            return ExecResult(return_code=0 if self.installed else 1, stdout=version, stderr="")
        if command.startswith("command -v ") and command.endswith(" >/dev/null 2>&1"):
            executable = command.split()[2]
            return ExecResult(return_code=1 if executable in self.missing_commands else 0, stdout="", stderr="")
        if command.startswith("for manager in "):
            return ExecResult(return_code=0, stdout=self.manager, stderr="")
        if command.startswith("if ") and "printf" in command:
            if self.probe_error is not None:
                raise self.probe_error
            if self.probe_result is not None:
                return self.probe_result
            if self.system_node:
                return ExecResult(return_code=0, stdout="system", stderr="")
            return ExecResult(return_code=0, stdout="bundled" if self.ca_bundle else "bundled-without-ca", stderr="")
        return ExecResult(return_code=0, stdout="", stderr="")


def _vendor_install(agent_class: type) -> str:
    """The text of the agent CLI install in Harbor's installer command."""
    return "npm i -g opencode-ai" if issubclass(agent_class, OpenCode) else "npm install -g"


def test_gateway_codex_debian_bootstrap_does_not_install_system_node(tmp_path: Path) -> None:
    environment = _BootstrapEnvironment()
    agent = SkillEvaluatorGatewayCodex(logs_dir=tmp_path, model_name="openai/test-model")

    asyncio.run(agent.install(environment))

    assert environment.package_installs == [], f"Unexpected system package install: {environment.package_installs}"
    assert any("nvm install 22" in command for command in environment.commands)


@pytest.mark.parametrize("agent_class", [SkillEvaluatorCodex, SkillEvaluatorClaudeCode, SkillEvaluatorOpenCode])
@pytest.mark.parametrize("manager", ["apt-get", "dnf"])
def test_native_bootstrap_avoids_system_node_and_keeps_vendor_installer(
    tmp_path: Path, agent_class: type, manager: str
) -> None:
    environment = _BootstrapEnvironment(manager=manager)
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")

    asyncio.run(agent.install(environment))

    assert environment.package_installs == []
    assert len(environment.probes) == 1
    probe_index = environment.commands.index(environment.probes[0])
    assert environment.exec_kwargs[probe_index]["user"] == "root"
    if agent_class is SkillEvaluatorCodex:
        assert "ldd --version 2>&1 | grep -qi musl || [ -f /etc/alpine-release ]" in environment.probes[0]
        assert any("nvm install 22" in command for command in environment.commands)
        assert any("ln -sf" in command for command in environment.commands)
    elif agent_class is SkillEvaluatorOpenCode:
        assert "ldd --version 2>&1 | grep -qi musl || [ -f /etc/alpine-release ]" in environment.probes[0]
        assert any("nvm install 22" in command for command in environment.commands)
        assert any("npm i -g opencode-ai" in command for command in environment.commands)
    else:
        assert "command -v apk" in environment.probes[0]
        assert any("https://downloads.claude.ai/claude-code-releases/bootstrap.sh" in c for c in environment.commands)


@pytest.mark.parametrize("agent_class", [SkillEvaluatorCodex, SkillEvaluatorClaudeCode, SkillEvaluatorOpenCode])
@pytest.mark.parametrize(
    ("manager", "expected_install"),
    [
        ("apt-get", "set -o pipefail; apt-get update && apt-get install -y ca-certificates"),
        ("dnf", "set -o pipefail; dnf install -y ca-certificates"),
    ],
)
def test_bundled_installer_without_ca_bundle_installs_ca_certificates(
    tmp_path: Path, agent_class: type, manager: str, expected_install: str
) -> None:
    # Without npm, nothing else pulls in the CA bundle the vendor downloads need.
    environment = _BootstrapEnvironment(manager=manager, ca_bundle=False)
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")

    asyncio.run(agent.install(environment))

    assert len(environment.probes) == 1
    assert "[ -s /etc/ssl/certs/ca-certificates.crt ]" in environment.probes[0]
    assert "[ -s /etc/pki/tls/certs/ca-bundle.crt ]" in environment.probes[0]
    assert environment.package_installs == [expected_install]


@pytest.mark.parametrize("agent_class", [SkillEvaluatorCodex, SkillEvaluatorClaudeCode, SkillEvaluatorOpenCode])
@pytest.mark.parametrize("manager", ["apt-get", "apk"])
def test_system_npm_installer_still_provisions_node_and_npm(tmp_path: Path, agent_class: type, manager: str) -> None:
    environment = _BootstrapEnvironment(manager=manager, system_node=True)
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")

    asyncio.run(agent.install(environment))

    assert len(environment.package_installs) == 1
    assert "nodejs npm" in environment.package_installs[0]
    assert environment.package_installs[0].startswith(
        "set -o pipefail; apt-get update" if manager == "apt-get" else "set -o pipefail; apk add"
    )
    assert any(_vendor_install(agent_class) in command for command in environment.commands)


@pytest.mark.parametrize(
    ("agent_class", "missing_command", "expected_package"),
    [
        (SkillEvaluatorCodex, "rg", "ripgrep"),
        (SkillEvaluatorClaudeCode, "pgrep", "procps"),
        (SkillEvaluatorOpenCode, "stdbuf", "coreutils"),
    ],
)
def test_native_bootstrap_preserves_other_system_dependencies(
    tmp_path: Path, agent_class: type, missing_command: str, expected_package: str
) -> None:
    environment = _BootstrapEnvironment(missing_commands=("node", "npm", missing_command))
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")

    asyncio.run(agent.install(environment))

    assert environment.package_installs == [f"set -o pipefail; apt-get update && apt-get install -y {expected_package}"]


@pytest.mark.parametrize("agent_class", [SkillEvaluatorCodex, SkillEvaluatorClaudeCode])
@pytest.mark.parametrize("version", [None, "1.2.3"])
def test_existing_matching_cli_skips_probe_and_bootstrap(
    tmp_path: Path, agent_class: type, version: str | None
) -> None:
    environment = _BootstrapEnvironment(installed=True)
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model", version=version)

    asyncio.run(agent.install(environment))

    assert len(environment.commands) == 1
    assert environment.probes == []
    assert environment.package_installs == []


@pytest.mark.parametrize("agent_class", [SkillEvaluatorCodex, SkillEvaluatorClaudeCode])
def test_mismatching_cli_version_runs_bootstrap_with_requested_version(tmp_path: Path, agent_class: type) -> None:
    environment = _BootstrapEnvironment(installed=True, installed_version="1.2.2")
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model", version="1.2.3")

    asyncio.run(agent.install(environment))

    assert environment.package_installs == []
    assert len(environment.probes) == 1
    expected = "@openai/[email protected]" if agent_class is SkillEvaluatorCodex else "bash -s -- 1.2.3"
    assert any(expected in command for command in environment.commands)


@pytest.mark.parametrize("agent_class", [SkillEvaluatorCodex, SkillEvaluatorClaudeCode, SkillEvaluatorOpenCode])
@pytest.mark.parametrize(
    ("return_code", "stdout"),
    [(1, "bundled"), (137, ""), (0, ""), (0, "unrecognized")],
)
def test_unknown_installer_platform_stops_without_installing(
    tmp_path: Path, agent_class: type, return_code: int, stdout: str
) -> None:
    environment = _BootstrapEnvironment(probe_result=ExecResult(return_code=return_code, stdout=stdout, stderr=""))
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")

    with pytest.raises(RuntimeError, match="Could not determine"):
        asyncio.run(agent.install(environment))

    assert environment.package_installs == []
    assert not any(_vendor_install(agent_class) in command for command in environment.commands)


@pytest.mark.parametrize("agent_class", [SkillEvaluatorCodex, SkillEvaluatorClaudeCode, SkillEvaluatorOpenCode])
def test_platform_probe_transport_errors_propagate(tmp_path: Path, agent_class: type) -> None:
    environment = _BootstrapEnvironment(probe_error=TimeoutError("probe transport timeout"))
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")

    with pytest.raises(TimeoutError, match="probe transport timeout"):
        asyncio.run(agent.install(environment))

    assert environment.package_installs == []


@pytest.mark.parametrize("agent_class", [SkillEvaluatorCodex, SkillEvaluatorClaudeCode, SkillEvaluatorOpenCode])
def test_non_node_dependency_requests_skip_platform_detection(tmp_path: Path, agent_class: type) -> None:
    environment = _BootstrapEnvironment(missing_commands=("git",), probe_error=AssertionError("unneeded probe"))
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")

    asyncio.run(agent.ensure_system_dependencies(environment, ("git",)))

    assert environment.package_installs == ["set -o pipefail; apt-get update && apt-get install -y git"]
    assert environment.probes == []


@pytest.mark.parametrize("agent_class", [SkillEvaluatorCodex, SkillEvaluatorClaudeCode, SkillEvaluatorOpenCode])
def test_native_bootstrap_preserves_unknown_dependency_validation(tmp_path: Path, agent_class: type) -> None:
    environment = _BootstrapEnvironment()
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")

    with pytest.raises(ValueError, match="Unknown system dependencies: unknown-package"):
        asyncio.run(agent.ensure_system_dependencies(environment, ("nodejs", "npm", "unknown-package")))

    assert len(environment.probes) == 1
    assert environment.package_installs == []


@pytest.mark.parametrize(
    ("agent_name", "agent_class", "model_name", "reported_model_name", "reported_provider"),
    [
        ("codex", SkillEvaluatorCodex, "openai/test-model", "test-model", "openai"),
        ("claude-code", ContainerClaudeCode, "anthropic/test-model", "test-model", "anthropic"),
        # Harbor 0.25 keeps OpenCode's native provider/model selector intact.
        ("opencode", SkillEvaluatorOpenCode, "nvidia/test-model", "nvidia/test-model", None),
    ],
)
@pytest.mark.parametrize("config_field", ["name", "import_path"])
def test_docker_import_routes_resolve_and_preserve_report_identity(
    tmp_path: Path,
    agent_name: str,
    agent_class: type,
    model_name: str,
    reported_model_name: str,
    reported_provider: str | None,
    config_field: str,
) -> None:
    config = AgentConfig(
        **{config_field: DOCKER_AGENT_IMPORT_PATHS[agent_name]},
        model_name=model_name,
        kwargs={"version": "1.2.3"},
    )

    assert AgentFactory.get_agent_class_from_config(config) is agent_class
    agent = AgentFactory.create_agent_from_config(config, logs_dir=tmp_path)

    assert type(agent) is agent_class
    assert agent.name() == agent_name
    info = agent.to_agent_info()
    assert info.name == agent_name
    assert info.version == "1.2.3"
    assert info.model_info is not None
    assert info.model_info.name == reported_model_name
    assert info.model_info.provider == reported_provider


@pytest.mark.parametrize(
    "agent_class",
    [
        SkillEvaluatorGatewayCodex,
        SkillEvaluatorNvidiaBuildCodex,
        SkillEvaluatorNvidiaBuildClaudeCode,
        ContainerClaudeCode,
        SkillEvaluatorGatewayOpenCode,
    ],
)
def test_gateway_and_nvidia_wrappers_use_repaired_bootstrap(tmp_path: Path, agent_class: type) -> None:
    environment = _BootstrapEnvironment()
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")

    asyncio.run(agent.install(environment))

    assert environment.package_installs == []
    assert len(environment.probes) == 1
    assert any(_vendor_install(agent_class) in command for command in environment.commands)


@pytest.mark.parametrize("agent_class", [SkillEvaluatorLocalCodex, SkillEvaluatorLocalClaudeCode])
def test_local_agents_keep_existing_version_check_without_install(tmp_path: Path, agent_class: type) -> None:
    environment = _BootstrapEnvironment(installed=True)
    agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")

    asyncio.run(agent.install(environment))

    assert len(environment.commands) == 1
    assert environment.commands[0].endswith(
        "codex --version" if agent_class is SkillEvaluatorLocalCodex else "claude --version"
    )
    assert environment.package_installs == []
    assert environment.probes == []


# Debian marks coreutils Essential and the python:3.12-slim base installs
# ca-certificates, so every generated image has both without listing them.
_SLIM_BASE_APT_PACKAGES = frozenset({"coreutils", "ca-certificates"})


def _skill(root: Path) -> Path:
    root.mkdir()
    (root / "SKILL.md").write_text("---\nname: skill\ndescription: Test fixture.\n---\n", encoding="utf-8")
    return root


def _apt_packages(dockerfile: str) -> set[str]:
    """The packages named by a generated Dockerfile's ``apt-get install`` command."""
    flags = "install -y --no-install-recommends"
    command = next(line for line in dockerfile.replace("\\\n", " ").splitlines() if flags in line)
    return set(command.split(flags, 1)[1].split("&&", 1)[0].replace("\\", " ").split())


def _generated_dockerfiles(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, str]:
    """Every Dockerfile SkillEvaluator writes on the python:3.12-slim base, by its writer."""
    built: list[str] = []

    def fake_run(cmd: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]:
        if cmd[:3] == ["docker", "image", "inspect"]:
            return subprocess.CompletedProcess(cmd, 1, "", "")
        built.append((Path(cmd[-1]) / "Dockerfile").read_text(encoding="utf-8"))
        return subprocess.CompletedProcess(cmd, 0, "", "")

    monkeypatch.setattr(adapter.subprocess, "run", fake_run)
    assert adapter.build_eval_base_image(_skill(tmp_path / "base-skill")).startswith("skillevaluator-base:")

    task = tmp_path / "task"
    adapter._write_dockerfile(
        task_dir=task,
        skill_path=_skill(tmp_path / "task-skill"),
        reference_skills_dir=None,
        workspace_skill_paths=None,
        has_skill=True,
    )

    environment = tmp_path / "environment"
    environment.mkdir()
    adapter._write_default_environment_dockerfile(
        environment, base_image="", agent_config_lines=[], include_input=False
    )

    starter_skill = _skill(tmp_path / "starter-skill")
    assert (
        commands.init_harbor_task(
            starter_skill, force=False, case_id="case-001", mode="default", language="python", with_config=False
        )
        == 0
    )

    return {
        "build_eval_base_image": built[0],
        "_write_dockerfile": (task / "environment" / "Dockerfile").read_text(encoding="utf-8"),
        "_write_default_environment_dockerfile": (environment / "Dockerfile").read_text(encoding="utf-8"),
        "init-harbor-task": (starter_skill / "evals/harbor/case-001/environment/Dockerfile").read_text(
            encoding="utf-8"
        ),
    }


@pytest.mark.parametrize("agent_name", ["claude-code", "codex", "opencode"])
def test_generated_images_satisfy_docker_agent_setup_dependencies(
    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, agent_name: str
) -> None:
    """Harbor's agent setup apt-installs, in every trial, each prerequisite the image lacks."""
    requested: list[tuple[str, ...]] = []

    async def record(self: BaseInstalledAgent, environment: object, dependencies: tuple[str, ...]) -> None:
        requested.append(tuple(dependencies))

    # Record what reaches Harbor's package step after the wrapper's installer-branch filter.
    monkeypatch.setattr(BaseInstalledAgent, "ensure_system_dependencies", record)
    agent_class = import_symbol(DOCKER_AGENT_IMPORT_PATHS[agent_name])
    agent = agent_class(logs_dir=tmp_path / "logs", model_name="openai/test-model")
    asyncio.run(agent.install(_BootstrapEnvironment()))
    assert requested
    needed = {
        package
        for dependencies in requested
        for dependency in dependencies
        for package in BaseInstalledAgent.SYSTEM_PACKAGES[dependency].packages["apt-get"]
    }
    assert not {"nodejs", "npm"} & needed

    for writer, dockerfile in _generated_dockerfiles(tmp_path, monkeypatch).items():
        assert dockerfile.startswith("FROM python:3.12-slim\n"), writer
        image = _apt_packages(dockerfile) | _SLIM_BASE_APT_PACKAGES
        assert needed - image == set(), writer


def _docker_setup_note_paragraphs() -> list[str]:
    """The paragraphs of the eval-datasets note on Docker agent setup packages, whitespace collapsed."""
    docs = (Path(__file__).resolve().parents[2] / "docs" / "eval-datasets.mdx").read_text(encoding="utf-8")
    notes = [block.split("</Note>", 1)[0] for block in docs.split("<Note>")[1:]]
    (note,) = [note for note in notes if "`procps`" in note]
    paragraphs = re.split(r"\n[ \t]*\n", note.strip())
    return [" ".join(paragraph.split()) for paragraph in paragraphs]


def _docker_setup_note() -> str:
    """The eval-datasets note on Docker agent setup packages, with whitespace collapsed."""
    return " ".join(_docker_setup_note_paragraphs())


def test_custom_dockerfiles_reach_procps_only_through_the_shared_base_image_rebase(tmp_path: Path) -> None:
    """A custom Dockerfile keeps its own base, and so needs procps, unless it is rebased."""
    custom_from = "FROM debian:bookworm-slim"
    first_lines: dict[str, str] = {}
    for name, kwargs in {
        # The runner passes an empty base_image unless harbor.base_image_mode is reuse or rebuild.
        "default": {},
        "preserve": {"base_image": "skillevaluator-base:test", "custom_dockerfile_mode": "preserve"},
        "rebase": {"base_image": "skillevaluator-base:test", "custom_dockerfile_mode": "rebase"},
    }.items():
        skill = _skill(tmp_path / f"{name}-skill")
        (skill / "evals" / "environment").mkdir(parents=True)
        (skill / "evals" / "environment" / "Dockerfile").write_text(f"{custom_from}\nRUN true\n", encoding="utf-8")
        task = tmp_path / f"{name}-task"
        adapter._write_dockerfile(
            task_dir=task,
            skill_path=skill,
            reference_skills_dir=None,
            workspace_skill_paths=None,
            has_skill=True,
            **kwargs,
        )
        first_lines[name] = (task / "environment" / "Dockerfile").read_text(encoding="utf-8").splitlines()[0]

    assert first_lines == {
        "default": custom_from,
        "preserve": custom_from,
        "rebase": "FROM skillevaluator-base:test",
    }
    note = _docker_setup_note()
    assert "A custom `evals/environment/Dockerfile` keeps its own base image, so it should install `procps` too" in note
    assert (
        "unless `harbor.base_image_mode` is `reuse` or `rebuild` and `harbor.custom_dockerfile_mode` is `rebase`"
        in note
    )


def test_docker_setup_note_gives_procps_guidance_its_own_paragraph() -> None:
    """Custom Dockerfile authors find the procps guidance without reading the Node.js scope first."""
    node_scope, procps = _docker_setup_note_paragraphs()

    assert "`nodejs` and `npm`" in node_scope
    assert "`scripts/apt-packages.txt`" in node_scope
    assert "procps" not in node_scope
    assert procps.startswith(
        "The auto-generated image and the `init-harbor-task` starter Dockerfile also include `procps`"
    )
    assert "nvm" not in procps
    assert "`harbor.custom_dockerfile_mode` is `rebase`" in procps


def test_docker_setup_note_scopes_the_npx_mcp_example_to_claude_code() -> None:
    """Codex and OpenCode stdio MCP servers inherit nvm's npx; only Claude Code has no Node.js."""
    note = _docker_setup_note()

    assert "including stdio MCP servers" in note
    assert 'Node.js with Claude Code, for example `command = "npx ..."`' in note
    assert "or `gcc`, `g++`, or `make` with any agent" in note