Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Tests for EmbeddingClient -- public OpenAI-compatible API wrapper."""
from __future__ import annotations
import math
from dataclasses import dataclass
from unittest.mock import MagicMock, patch
import pytest
from skillevaluator.embedding.client import (
EmbeddingClient,
SimilarityConfigError,
_average_pool,
_fixed_size_chunks,
_split_by_headings,
_split_into_chunks,
normalize_embedding_vector,
unit_vector_similarity,
)
class TestCosineSimilarity:
def test_identical_vectors(self) -> None:
vec = [1.0, 2.0, 3.0]
assert EmbeddingClient.cosine_similarity(vec, vec) == pytest.approx(1.0)
def test_orthogonal_vectors(self) -> None:
a = [1.0, 0.0]
b = [0.0, 1.0]
assert EmbeddingClient.cosine_similarity(a, b) == pytest.approx(0.0)
def test_opposite_vectors(self) -> None:
a = [1.0, 0.0]
b = [-1.0, 0.0]
assert EmbeddingClient.cosine_similarity(a, b) == pytest.approx(-1.0)
def test_zero_vector_returns_zero(self) -> None:
a = [0.0, 0.0]
b = [1.0, 2.0]
assert EmbeddingClient.cosine_similarity(a, b) == 0.0
def test_similar_vectors(self) -> None:
a = [1.0, 1.0]
b = [1.0, 0.9]
score = EmbeddingClient.cosine_similarity(a, b)
assert 0.99 < score < 1.0
def test_matches_exactly_summed_reference(self) -> None:
a = [math.sin(index * 0.37) * (index % 7 + 1) for index in range(2_048)]
b = [math.cos(index * 0.11) - 0.25 for index in range(2_048)]
norm_a = math.sqrt(math.fsum(value * value for value in a))
norm_b = math.sqrt(math.fsum(value * value for value in b))
expected = math.fsum((x / norm_a) * (y / norm_b) for x, y in zip(a, b, strict=True))
assert EmbeddingClient.cosine_similarity(a, b) == pytest.approx(expected, abs=1e-15)
def test_equal_nonzero_vectors_score_exactly_one(self) -> None:
vector = [-5.479683767155414, -6.166558151604646]
unit = normalize_embedding_vector(vector)
assert math.sumprod(unit, unit) < 1.0 # Rounding alone would hide the duplicate at --threshold 1.
assert unit_vector_similarity(unit, list(unit)) == 1.0
assert EmbeddingClient.cosine_similarity(vector, list(vector)) == 1.0
assert unit_vector_similarity([0.0, 0.0], [0.0, 0.0]) == 0.0
def test_normalized_vectors_reuse_the_same_score(self) -> None:
a, b = [3.0, 4.0, 0.0], [4.0, 3.0, 0.0]
unit_a, unit_b = normalize_embedding_vector(a), normalize_embedding_vector(b)
assert unit_a == [0.6, 0.8, 0.0]
assert unit_vector_similarity(unit_a, unit_b) == EmbeddingClient.cosine_similarity(a, b)
assert normalize_embedding_vector([0.0, 0.0], allow_zero=True) == [0.0, 0.0]
with pytest.raises(SimilarityConfigError, match="zero vector"):
normalize_embedding_vector([0.0, 0.0])
@dataclass
class _FakeEmbeddingItem:
embedding: list[float]
index: int
@dataclass
class _FakeEmbeddingResponse:
data: list[_FakeEmbeddingItem]
def _make_fake_response(vectors: list[list[float]]) -> _FakeEmbeddingResponse:
return _FakeEmbeddingResponse(
data=[_FakeEmbeddingItem(embedding=vector, index=index) for index, vector in enumerate(vectors)]
)
class TestEmbed:
def test_openai_provider_uses_public_openai_endpoint(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "openai")
monkeypatch.setenv("SKILL_EVAL_EMBEDDING_PROVIDER", "openai")
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
mock_openai = MagicMock()
with patch("openai.OpenAI", return_value=mock_openai) as mock_cls:
EmbeddingClient()._get_client()
mock_cls.assert_called_once_with(api_key="test-key", base_url="https://api.openai.com/v1")
def test_explicit_retry_http_client_and_timeout_reach_the_sdk(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "openai")
monkeypatch.setenv("SKILL_EVAL_EMBEDDING_PROVIDER", "openai")
monkeypatch.setenv("OPENAI_API_KEY", "test-key")
http_client = object()
with patch("openai.OpenAI", return_value=MagicMock()) as mock_cls:
EmbeddingClient(max_retries=0, http_client=http_client, timeout=5.0)._get_client()
mock_cls.assert_called_once_with(
api_key="test-key",
base_url="https://api.openai.com/v1",
max_retries=0,
http_client=http_client,
timeout=5.0,
)
def test_embed_returns_vectors(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("NVIDIA_API_KEY", "test-key")
client = EmbeddingClient()
expected = [[0.1, 0.2], [0.3, 0.4]]
mock_openai = MagicMock()
mock_openai.embeddings.create.return_value = _make_fake_response(expected)
with patch("openai.OpenAI", return_value=mock_openai):
result = client.embed(["hello", "world"])
assert result == expected
mock_openai.embeddings.create.assert_called_once_with(
model="nvidia/nemotron-3-embed-1b",
input=["hello", "world"],
encoding_format="float",
extra_body={"input_type": "passage"},
)
@pytest.mark.parametrize("model", ["text-embedding-3-small", "custom-embedding-model"])
def test_other_models_keep_standard_openai_request(self, model: str) -> None:
client = EmbeddingClient(model=model, api_key="test-key", base_url="http://127.0.0.1:12345/v1")
sdk = MagicMock()
sdk.embeddings.create.return_value = _make_fake_response([[1.0, 0.0]])
client._client = sdk
assert client.embed(["text"]) == [[1.0, 0.0]]
sdk.embeddings.create.assert_called_once_with(model=model, input=["text"], encoding_format="float")
@pytest.mark.parametrize("model", [None, "publisher/nvidia/nemotron-3-embed-1b"])
def test_nvidia_default_at_explicit_endpoint_includes_passage_type(self, model) -> None:
client = EmbeddingClient(model=model, api_key="test-key", base_url="http://127.0.0.1:12345/v1")
sdk = MagicMock()
sdk.embeddings.create.return_value = _make_fake_response([[1.0, 0.0]])
client._client = sdk
client.embed(["text"])
assert sdk.embeddings.create.call_args.kwargs["model"] == (model or "nvidia/nemotron-3-embed-1b")
assert sdk.embeddings.create.call_args.kwargs["extra_body"] == {"input_type": "passage"}
@pytest.mark.parametrize("status", [400, 401, 403, 404, 410, 429, 500])
def test_service_error_is_actionable_without_echoing_response(self, status: int) -> None:
import httpx
from openai import APIStatusError
client = EmbeddingClient(model="test-model", api_key="test-key", base_url="http://127.0.0.1:12345/v1")
sdk = MagicMock()
request = httpx.Request("POST", "http://127.0.0.1:12345/v1/embeddings?secret-marker")
response = httpx.Response(status, request=request, json={"detail": "secret-marker"})
sdk.embeddings.create.side_effect = APIStatusError(
"secret-marker", response=response, body={"secret-marker": True}
)
client._client = sdk
with pytest.raises(SimilarityConfigError) as error:
client.embed(["private input text"])
assert f"HTTP {status}" in str(error.value)
assert "secret-marker" not in str(error.value)
assert "private input text" not in str(error.value)
if status in {404, 410}:
assert "SKILL_EVAL_EMBEDDING_MODEL" in str(error.value)
def test_connection_error_does_not_echo_endpoint_credentials(self) -> None:
import httpx
from openai import APIConnectionError
client = EmbeddingClient(model="test-model", api_key="test-key", base_url="http://127.0.0.1:12345/v1")
sdk = MagicMock()
request = httpx.Request("POST", "http://127.0.0.1:12345/v1/embeddings?secret-marker")
sdk.embeddings.create.side_effect = APIConnectionError(message="secret-marker", request=request)
client._client = sdk
with pytest.raises(SimilarityConfigError, match="connection") as error:
client.embed(["private input text"])
assert "secret-marker" not in str(error.value)
def test_invalid_response_index_does_not_echo_provider_content(self) -> None:
client = EmbeddingClient(model="test-model", api_key="test-key", base_url="http://127.0.0.1:12345/v1")
sdk = MagicMock()
sdk.embeddings.create.return_value = _FakeEmbeddingResponse(
data=[_FakeEmbeddingItem(embedding=[1.0, 0.0], index="private-input-marker")]
)
client._client = sdk
with pytest.raises(SimilarityConfigError, match="index") as error:
client.embed(["text"])
assert "private-input-marker" not in str(error.value)
def test_embed_reorders_out_of_order_provider_response_by_index(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("NVIDIA_API_KEY", "test-key")
client = EmbeddingClient()
first = [1.0, 0.0]
second = [0.0, 1.0]
mock_openai = MagicMock()
mock_openai.embeddings.create.return_value = _FakeEmbeddingResponse(
data=[
_FakeEmbeddingItem(embedding=second, index=1),
_FakeEmbeddingItem(embedding=first, index=0),
]
)
with patch("openai.OpenAI", return_value=mock_openai):
result = client.embed(["first", "second"])
assert result == [first, second]
@pytest.mark.parametrize(
"items",
[
[
_FakeEmbeddingItem(embedding=[1.0, 0.0], index=0),
_FakeEmbeddingItem(embedding=[0.0, 1.0], index=0),
],
[_FakeEmbeddingItem(embedding=[1.0, 0.0], index=2)],
],
ids=["duplicate-index", "out-of-range-index"],
)
def test_embed_rejects_invalid_provider_response_indices(
self,
monkeypatch: pytest.MonkeyPatch,
items: list[_FakeEmbeddingItem],
) -> None:
monkeypatch.setenv("NVIDIA_API_KEY", "test-key")
mock_openai = MagicMock()
mock_openai.embeddings.create.return_value = _FakeEmbeddingResponse(data=items)
with (
patch("openai.OpenAI", return_value=mock_openai),
pytest.raises(SimilarityConfigError, match="index"),
):
EmbeddingClient().embed(["first", "second"])
@pytest.mark.parametrize("vector", [[math.nan, 0.0], [1e308, 1e308]])
def test_embed_rejects_unsafe_provider_vectors(
self,
monkeypatch: pytest.MonkeyPatch,
vector: list[float],
) -> None:
monkeypatch.setenv("NVIDIA_API_KEY", "test-key")
mock_openai = MagicMock()
mock_openai.embeddings.create.return_value = _make_fake_response([vector])
with (
patch("openai.OpenAI", return_value=mock_openai),
pytest.raises(SimilarityConfigError, match=r"finite|magnitude|stable"),
):
EmbeddingClient().embed(["unsafe"])
def test_embed_single_delegates(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("NVIDIA_API_KEY", "test-key")
client = EmbeddingClient()
expected = [0.5, 0.6, 0.7]
mock_openai = MagicMock()
mock_openai.embeddings.create.return_value = _make_fake_response([expected])
with patch("openai.OpenAI", return_value=mock_openai):
result = client.embed_single("test text")
assert result == expected
def test_embed_empty_list_returns_empty(self) -> None:
client = EmbeddingClient(api_key="unused")
assert client.embed([]) == []
def test_missing_api_key_raises(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("NVIDIA_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
monkeypatch.delenv("SKILL_EVAL_LLM_PROVIDER", raising=False)
monkeypatch.delenv("SKILL_EVAL_EMBEDDING_PROVIDER", raising=False)
with pytest.raises(SimilarityConfigError, match="SKILL_EVAL_EMBEDDING_PROVIDER"):
EmbeddingClient().embed(["test"])
def test_env_embedding_model_is_honored_without_explicit_model(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""SKILL_EVAL_EMBEDDING_MODEL wins when no explicit model is given.
This is the documented contract in docs/configuration.mdx and what
makes custom OpenAI-compatible endpoints (including local servers)
usable for Tier 2.
"""
monkeypatch.setenv("SKILL_EVAL_EMBEDDING_PROVIDER", "openai-compatible")
monkeypatch.setenv("SKILL_EVAL_EMBEDDING_BASE_URL", "http://localhost:11434/v1")
monkeypatch.setenv("SKILL_EVAL_EMBEDDING_MODEL", "nomic-embed-text")
monkeypatch.setenv("SKILL_EVAL_EMBEDDING_API_KEY", "local-no-key")
assert EmbeddingClient().model == "nomic-embed-text"
class TestEmbedChunked:
def test_embed_chunked_averages_vectors(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("NVIDIA_API_KEY", "test-key")
client = EmbeddingClient()
mock_openai = MagicMock()
mock_openai.embeddings.create.return_value = _make_fake_response([[1.0, 0.0], [0.0, 1.0]])
with patch("openai.OpenAI", return_value=mock_openai):
result = client.embed_chunked("## Section 1\nfoo\n## Section 2\nbar")
assert result == pytest.approx([0.5, 0.5])
@pytest.mark.parametrize("overlap", [10, 11])
def test_embed_chunked_rejects_non_progressing_windows_before_provider_call(
self,
monkeypatch: pytest.MonkeyPatch,
overlap: int,
) -> None:
monkeypatch.setenv("NVIDIA_API_KEY", "test-key")
client = EmbeddingClient()
with (
patch.object(client, "embed") as mock_embed,
pytest.raises(SimilarityConfigError, match="overlap must be smaller"),
):
client.embed_chunked("x" * 100, chunk_size=10, overlap=overlap)
mock_embed.assert_not_called()
@pytest.mark.parametrize(
("chunk_size", "overlap"),
[
(True, 0),
(10.0, 0),
(10, True),
(10, 1.5),
(10, math.nan),
],
ids=["bool-size", "float-size", "bool-overlap", "float-overlap", "nan-overlap"],
)
def test_embed_chunked_rejects_non_integer_windows_before_provider_call(
self,
monkeypatch: pytest.MonkeyPatch,
chunk_size: object,
overlap: object,
) -> None:
monkeypatch.setenv("NVIDIA_API_KEY", "test-key")
client = EmbeddingClient()
with (
patch.object(client, "embed") as mock_embed,
pytest.raises(SimilarityConfigError, match="must be an integer"),
):
client.embed_chunked("x" * 100, chunk_size=chunk_size, overlap=overlap) # type: ignore[arg-type]
mock_embed.assert_not_called()
class TestSplitIntoChunks:
def test_short_text_single_chunk(self) -> None:
text = "A short sentence."
chunks = _split_into_chunks(text, chunk_size=100, overlap=10)
assert chunks == [text]
def test_split_by_headings(self) -> None:
text = "Preamble text\n## Section A\nContent A\n## Section B\nContent B"
sections = _split_by_headings(text)
assert len(sections) == 3
assert "Preamble" in sections[0]
assert "Section A" in sections[1]
assert "Section B" in sections[2]
def test_fallback_fixed_size(self) -> None:
text = "x" * 100
chunks = _fixed_size_chunks(text, max_chars=30, overlap_chars=5)
assert len(chunks) >= 3
for chunk in chunks:
assert len(chunk) <= 30
def test_empty_text_returns_original(self) -> None:
chunks = _split_into_chunks("", chunk_size=100, overlap=10)
assert chunks == [""]
@pytest.mark.parametrize(
("chunk_size", "overlap", "message"),
[
(0, 0, "chunk size must be greater than zero"),
(10, -1, "chunk overlap must not be negative"),
(10, 10, "overlap must be smaller than the chunk size"),
(10, 11, "overlap must be smaller than the chunk size"),
],
)
def test_rejects_invalid_chunk_windows(self, chunk_size: int, overlap: int, message: str) -> None:
with pytest.raises(SimilarityConfigError, match=message):
_split_into_chunks("x" * 100, chunk_size=chunk_size, overlap=overlap)
def test_heading_split_preserves_heading_with_content(self) -> None:
text = "# Title\nSome intro\n## Part 1\nDetails"
sections = _split_by_headings(text)
assert any("# Title" in s for s in sections)
assert any("## Part 1" in s for s in sections)
class TestAveragePool:
def test_single_vector(self) -> None:
assert _average_pool([[1.0, 2.0]]) == [1.0, 2.0]
def test_two_vectors(self) -> None:
result = _average_pool([[1.0, 0.0], [0.0, 1.0]])
assert result == pytest.approx([0.5, 0.5])
def test_empty_returns_empty(self) -> None:
assert _average_pool([]) == []