Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

5.8 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Behavior tests for bundled Tier 3 reference scripts."""

from __future__ import annotations

import importlib.util
import os
import subprocess
import sys
from pathlib import Path
from types import ModuleType
from typing import ClassVar

import pytest

REFERENCE_SKILLS = Path(__file__).parents[1] / "src/skillevaluator/tier3/reference_skills"
SCRIPTS = {
    "call_api": REFERENCE_SKILLS / "api-caller/scripts/call_api.py",
    "parse_openapi": REFERENCE_SKILLS / "api-caller/scripts/parse_openapi.py",
    "tasks": REFERENCE_SKILLS / "task-list/scripts/tasks.py",
}


class _SuccessfulResponse:
    """Minimal context-managed response used by request-body tests."""

    status = 200
    headers: ClassVar[dict[str, str]] = {}

    def __enter__(self):
        return self

    def __exit__(self, *_args: object) -> None:
        return None

    @staticmethod
    def read() -> bytes:
        return b"{}"


def _run_script(name: str, *args: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]:
    process_env = os.environ.copy()
    if env:
        process_env.update(env)
    return subprocess.run(
        [sys.executable, str(SCRIPTS[name]), *args],
        check=False,
        capture_output=True,
        text=True,
        env=process_env,
    )


def _load_script(name: str) -> ModuleType:
    spec = importlib.util.spec_from_file_location(f"reference_script_{name}", SCRIPTS[name])
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    scripts_dir = str(SCRIPTS[name].parent)
    sys.path.insert(0, scripts_dir)
    try:
        spec.loader.exec_module(module)
    finally:
        sys.path.remove(scripts_dir)
    return module


def test_task_content_containing_error_label_remains_successful(tmp_path: Path) -> None:
    result = _run_script(
        "tasks",
        "add",
        "--content",
        "Investigate Error: logs",
        env={"TASK_LIST_STATE_PATH": str(tmp_path / "todo.md")},
    )

    assert result.returncode == 0
    assert result.stdout == "Added [task_1]: Investigate Error: logs\n"
    assert result.stderr == ""


@pytest.mark.parametrize(
    ("content", "operation", "status", "checkbox"),
    [
        ("Investigate (pending) labels after merge", None, "pending", " "),
        ("Investigate (in_progress) labels after merge", None, "pending", " "),
        ("Investigate (completed) labels after merge", None, "pending", " "),
        ("Investigate (cancelled) labels after merge", None, "pending", " "),
        ("Check (completed)", "complete", "completed", "x"),
        ("Check (completed)", "cancel", "cancelled", "~"),
    ],
)
def test_task_status_words_survive_persistence(
    content: str, operation: str | None, status: str, checkbox: str, tmp_path: Path
) -> None:
    state_path = tmp_path / "todo.md"
    env = {"TASK_LIST_STATE_PATH": str(state_path)}

    added = _run_script("tasks", "add", "--content", content, env=env)
    if operation:
        updated = _run_script("tasks", operation, "--task_id", "task_1", env=env)
        assert updated.returncode == 0
        assert updated.stderr == ""
    listed = _run_script("tasks", "list", env=env)
    state, error = _load_script("tasks")._load_state(state_path)

    assert added.returncode == listed.returncode == 0
    assert added.stderr == listed.stderr == ""
    assert error is None
    assert state["tasks"]["task_1"] == {
        "id": "task_1",
        "content": content,
        "status": status,
    }
    if status != "cancelled":
        assert content in listed.stdout
    assert f"- [{checkbox}] task_1: {content} ({status})" in state_path.read_text(encoding="utf-8")


@pytest.mark.parametrize(
    ("name", "args", "expected_error"),
    [
        ("call_api", ("--url", "https://example.invalid", "--headers", "not-json"), "Invalid input"),
        ("call_api", (), "No URL provided"),
        ("parse_openapi", (), "Usage:"),
        ("tasks", ("add",), "Error: Content is required"),
    ],
)
def test_handled_input_failures_write_only_to_stderr(
    name: str, args: tuple[str, ...], expected_error: str, tmp_path: Path
) -> None:
    env = {"TASK_LIST_STATE_PATH": str(tmp_path / "todo.md")} if name == "tasks" else None

    result = _run_script(name, *args, env=env)

    assert result.returncode == 1
    assert result.stdout == ""
    assert expected_error in result.stderr
    assert "Traceback" not in result.stderr


@pytest.mark.parametrize(
    ("body", "encoded"),
    [
        ({}, b"{}"),
        ([], b"[]"),
        (0, b"0"),
        (False, b"false"),
        ("", b'""'),
        (None, b"null"),
    ],
)
def test_call_api_preserves_explicit_falsey_json_bodies(
    body: object,
    encoded: bytes,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    module = _load_script("call_api")
    requests: list[object] = []

    def fake_urlopen(request: object, *, timeout: int):
        assert timeout == 30
        requests.append(request)
        return _SuccessfulResponse()

    monkeypatch.setattr(module, "urlopen", fake_urlopen)

    result = module.make_request("https://example.invalid", method="POST", data=body)

    assert result["success"] is True
    assert len(requests) == 1
    assert requests[0].data == encoded


def test_call_api_omits_body_only_when_data_is_absent(monkeypatch: pytest.MonkeyPatch) -> None:
    module = _load_script("call_api")
    requests: list[object] = []

    def fake_urlopen(request: object, *, timeout: int):
        requests.append(request)
        return _SuccessfulResponse()

    monkeypatch.setattr(module, "urlopen", fake_urlopen)

    result = module.make_request("https://example.invalid")

    assert result["success"] is True
    assert len(requests) == 1
    assert requests[0].data is None