Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Provider diagnostics expose actionable protocol fields, never raw data."""
import json
import httpx
import pytest
from openai import APIConnectionError, APIStatusError, APITimeoutError
from skillevaluator.inference import LLMClientError
from skillevaluator.inference.diagnostics import llm_failure_diagnostic, safe_llm_labels, safe_llm_model
@pytest.mark.parametrize("status", [400, 401, 403, 404, 410, 422, 429, 500])
def test_http_status_without_echoing_response_or_request(status):
request = httpx.Request("POST", "https://[email protected]/v1/chat/completions")
response = httpx.Response(status, request=request)
exception = APIStatusError(
"message-private-token",
response=response,
body={"error": {"message": "body-private-token", "code": "model_not_found", "param": "model"}},
)
diagnostic = llm_failure_diagnostic(exception)
assert f"HTTP {status}" in diagnostic
assert "code=model_not_found" in diagnostic
assert "param=model" in diagnostic
assert "private-token" not in diagnostic
assert "example.invalid" not in diagnostic
@pytest.mark.parametrize("status", [404, 410])
def test_unavailable_model_points_to_the_model_setting(status):
response = httpx.Response(status, request=httpx.Request("POST", "https://example.invalid"))
diagnostic = llm_failure_diagnostic(APIStatusError("x", response=response, body=None))
assert "default may not be callable" in diagnostic
assert "SKILL_EVAL_LLM_MODEL" in diagnostic
assert "llm-model" in diagnostic
@pytest.mark.parametrize(
"body",
[
None,
[],
"private-token",
{"error": "private-token"},
{"error": {"code": "private-token", "param": "private-token", "type": "private-token"}},
{"error": {"code": [], "param": {}, "type": ["invalid_request_error"]}},
],
)
def test_unknown_or_malformed_provider_fields_are_not_echoed(body):
request = httpx.Request("POST", "https://example.invalid")
exception = APIStatusError("private-token", response=httpx.Response(400, request=request), body=body)
diagnostic = llm_failure_diagnostic(exception)
assert "HTTP 400" in diagnostic
assert "private-token" not in diagnostic
assert "code=" not in diagnostic
assert "param=" not in diagnostic
assert "type=" not in diagnostic
@pytest.mark.parametrize(
"exception, expected",
[
(APIConnectionError(request=httpx.Request("POST", "https://[email protected]")), "connection"),
(APITimeoutError(request=httpx.Request("POST", "https://[email protected]")), "timed out"),
(LLMClientError("LLM returned invalid JSON: Raw: private-token"), "structured-output"),
(json.JSONDecodeError("private-token", "private-token", 0), "structured-output"),
(ValueError("private-token"), "structured-output"),
(RuntimeError("private-token"), "Unexpected"),
],
)
def test_transport_and_response_errors_do_not_echo_raw_content(exception, expected):
diagnostic = llm_failure_diagnostic(exception)
assert expected in diagnostic
assert "private-token" not in diagnostic
@pytest.mark.parametrize(
"model",
[
"https://[email protected]",
"secret\nnew line",
"sk-secret",
"nvapi-secret",
"m" * 161,
None,
],
)
def test_model_labels_reject_urls_tokens_and_control_text(model):
assert safe_llm_labels("nv_build", model) == ("nv_build", "unresolved")
assert safe_llm_model(model) == "unresolved"
def test_model_labels_preserve_real_model_ids_and_bound_unknown_provider():
assert safe_llm_labels("nv_build", "nvidia/nemotron-3.5-lightning-30b-a3b") == (
"nv_build",
"nvidia/nemotron-3.5-lightning-30b-a3b",
)
assert safe_llm_labels("private-token", "anthropic.claude-model-v1:0") == (
"unresolved",
"anthropic.claude-model-v1:0",
)