Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Embedding client for public OpenAI-compatible providers.
Wraps the OpenAI-compatible API to generate vector embeddings for
similarity comparison. Supports both single-string and chunked
full-body embedding modes.
"""
from __future__ import annotations
import math
import re
import sys
from dataclasses import replace
from typing import Any
from skillevaluator.constants import (
SIMILARITY_CHUNK_OVERLAP,
SIMILARITY_CHUNK_SIZE,
SIMILARITY_DEFAULT_MODEL,
)
from skillevaluator.logging_config import get_logger
from skillevaluator.provider_config import (
EMBEDDING_DEFAULT_NVIDIA,
ProviderConfig,
ProviderConfigurationError,
resolve_embedding_provider,
)
logger = get_logger(__name__)
class SimilarityConfigError(Exception):
"""Raised when configuration or provider failures prevent embedding comparison."""
MAX_EMBEDDING_VECTOR_DIMENSION = 65_536
def validate_embedding_vector(
vector: object,
expected_dimension: int | None = None,
*,
context: str = "Embedding provider",
) -> int:
"""Validate a provider vector before it reaches similarity arithmetic."""
values, _norm = _validated_vector_values(
vector,
expected_dimension,
context=context,
allow_zero=False,
)
return len(values)
def normalize_embedding_vector(
vector: object,
expected_dimension: int | None = None,
*,
context: str = "Embedding provider",
allow_zero: bool = False,
) -> list[float]:
"""Validate a vector and return a unit-vector copy for repeated comparisons.
Zero vectors remain zero when ``allow_zero`` is enabled, matching
:meth:`EmbeddingClient.cosine_similarity`.
"""
values, norm = _validated_vector_values(
vector,
expected_dimension,
context=context,
allow_zero=allow_zero,
)
if norm == 0.0:
return values
return [value / norm for value in values]
def unit_vector_similarity(left: list[float], right: list[float]) -> float:
"""Return the cosine similarity of two same-width normalized vectors.
Pairwise scans normalize each vector once with :func:`normalize_embedding_vector`
instead of revalidating and renormalizing both vectors for every pair.
Equal nonzero vectors score exactly 1.0 so rounding cannot hide exact
duplicates at ``--threshold 1``; zero vectors still score 0.0.
"""
if left == right:
return 1.0 if any(left) else 0.0
score = math.sumprod(left, right)
if not math.isfinite(score):
raise SimilarityConfigError("Cosine similarity produced a non-finite result.")
return max(-1.0, min(1.0, score))
class EmbeddingClient:
"""Public OpenAI-compatible client for generating text embeddings.
Uses the OpenAI SDK because NVIDIA Build exposes an OpenAI-compatible API.
Client construction is deferred until the first embed() call so that
importing this module never requires an API key.
"""
def __init__(
self,
model: str | None = None,
base_url: str | None = None,
api_key: str | None = None,
*,
max_retries: int | None = None,
http_client: Any = None,
timeout: float | None = None,
) -> None:
self._model = model
self._base_url = base_url
self._api_key = api_key
# None keeps the SDK defaults. The SDK ignores an http_client's own
# timeout when it equals httpx's 5-second default, so readiness probes
# pass their bound explicitly.
self._max_retries = max_retries
self._http_client = http_client
self._timeout = timeout
self._client: Any = None
self._provider_config: ProviderConfig | None = None
@property
def model(self) -> str:
"""The embedding model identifier."""
return self._model or self._resolved_config().model
def _resolved_config(self) -> ProviderConfig:
if self._provider_config is not None:
return self._provider_config
if self._api_key or self._base_url:
model = self._model or SIMILARITY_DEFAULT_MODEL
self._provider_config = ProviderConfig(
provider="openai-compatible",
model=model,
api_key=self._api_key,
base_url=self._base_url,
litellm_model=f"openai/{model}",
)
return self._provider_config
try:
config = resolve_embedding_provider()
except ProviderConfigurationError as exc:
raise SimilarityConfigError(str(exc)) from exc
if self._model:
config = replace(config, model=self._model, litellm_model=f"openai/{self._model}")
self._provider_config = config
return config
def _get_client(self) -> Any:
"""Lazily construct the OpenAI client on first use."""
if self._client is not None:
return self._client
config = self._resolved_config()
if not config.api_key or not config.base_url:
raise SimilarityConfigError(f"No embedding API key or base URL resolved for {config.provider}.")
try:
from openai import OpenAI
except ImportError as exc:
raise SimilarityConfigError(
"The 'openai' package is required for similarity checks. Install it with: pip install openai"
) from exc
client_kwargs: dict[str, Any] = {"api_key": config.api_key, "base_url": config.base_url}
if self._max_retries is not None:
client_kwargs["max_retries"] = self._max_retries
if self._http_client is not None:
client_kwargs["http_client"] = self._http_client
if self._timeout is not None:
client_kwargs["timeout"] = self._timeout
self._client = OpenAI(**client_kwargs)
return self._client
def embed(self, texts: list[str]) -> list[list[float]]:
"""Generate embeddings for a batch of texts.
Args:
texts: Strings to embed. Empty list returns empty list.
Returns:
List of embedding vectors, one per input text.
"""
if not texts:
return []
client = self._get_client()
from openai import APIConnectionError, APIStatusError
config = self._resolved_config()
kwargs: dict[str, Any] = {}
if (
config.provider == "nv_build"
or self.model == EMBEDDING_DEFAULT_NVIDIA
or self.model.endswith("/" + EMBEDDING_DEFAULT_NVIDIA)
):
# Deduplication compares documents symmetrically, not queries to documents.
kwargs["extra_body"] = {"input_type": "passage"}
try:
response = client.embeddings.create(
model=self.model,
input=texts,
encoding_format="float",
**kwargs,
)
except APIStatusError as exc:
# Provider responses can echo input text, credentials, or private URLs.
# Preserve the status and recovery action without forwarding the body.
status = exc.status_code
if status in (404, 410):
action = (
"The embedding model or endpoint is unavailable. "
"Check SKILL_EVAL_EMBEDDING_MODEL against your provider's active models "
"and verify the configured endpoint."
)
elif status in (401, 403):
action = "Check the embedding provider's API key and access to the configured model."
elif status == 429:
action = "The embedding provider is rate-limiting requests. Check your quota and retry later."
else:
action = "Check the embedding provider's availability and model configuration before retrying."
raise SimilarityConfigError(f"Embedding request failed (HTTP {status}). {action}") from exc
except APIConnectionError as exc:
raise SimilarityConfigError(
"Embedding connection failed. Check the configured endpoint and network, then retry."
) from exc
data = list(response.data)
if len(data) != len(texts):
raise SimilarityConfigError(
f"Embedding response index set is incomplete: received {len(data)} vectors for {len(texts)} inputs."
)
ordered: list[list[float] | None] = [None] * len(texts)
expected_dimension: int | None = None
for item in data:
index = getattr(item, "index", None)
if type(index) is not int or not 0 <= index < len(texts):
raise SimilarityConfigError("Embedding response index must be an integer within the input range.")
if ordered[index] is not None:
raise SimilarityConfigError(f"Embedding response contains duplicate index {index}.")
vector = getattr(item, "embedding", None)
dimension = validate_embedding_vector(
vector,
expected_dimension,
context=f"Embedding response at index {index}",
)
if expected_dimension is None:
expected_dimension = dimension
ordered[index] = vector
if any(vector is None for vector in ordered):
raise SimilarityConfigError("Embedding response indices are incomplete.")
return [vector for vector in ordered if vector is not None]
def embed_single(self, text: str) -> list[float]:
"""Generate an embedding for a single text string."""
return self.embed([text])[0]
def embed_chunked(
self,
text: str,
chunk_size: int = SIMILARITY_CHUNK_SIZE,
overlap: int = SIMILARITY_CHUNK_OVERLAP,
) -> list[float]:
"""Split text into chunks, embed each, and average-pool.
Used by --full-body mode for documents that may exceed the
model's context window.
Args:
text: Full document text.
chunk_size: Approximate token count per chunk.
overlap: Token overlap between adjacent chunks.
Returns:
Single averaged embedding vector.
"""
chunks = _split_into_chunks(text, chunk_size, overlap)
vectors = self.embed(chunks)
return _average_pool(vectors)
@staticmethod
def cosine_similarity(vec_a: list[float], vec_b: list[float]) -> float:
"""Compute cosine similarity between two vectors.
Uses stdlib math only -- no numpy dependency required.
Returns:
Similarity score in [-1.0, 1.0]. Returns 0.0 for zero vectors.
Raises:
SimilarityConfigError: If vectors have different dimensionality.
"""
if len(vec_a) != len(vec_b):
raise SimilarityConfigError(
f"Vector dimension mismatch: {len(vec_a)} vs {len(vec_b)}. "
"This usually means the embeddings were produced by different models "
"or a stale cache is being used."
)
unit_a = normalize_embedding_vector(
vec_a,
len(vec_a),
context="First similarity",
allow_zero=True,
)
unit_b = normalize_embedding_vector(
vec_b,
len(vec_a),
context="Second similarity",
allow_zero=True,
)
return unit_vector_similarity(unit_a, unit_b)
# ---------------------------------------------------------------------------
# Private helpers
# ---------------------------------------------------------------------------
_HEADING_PATTERN = re.compile(r"^#{1,6}\s", re.MULTILINE)
# Rough chars-per-token estimate for English markdown
_CHARS_PER_TOKEN = 4
def _split_into_chunks(
text: str,
chunk_size: int = SIMILARITY_CHUNK_SIZE,
overlap: int = SIMILARITY_CHUNK_OVERLAP,
) -> list[str]:
"""Split markdown text into chunks for embedding.
Strategy:
1. Split on markdown headings (## ) as natural boundaries.
2. If any section still exceeds chunk_size tokens, fall back to
fixed-size overlapping windows.
3. Guarantee at least one chunk is returned.
"""
if type(chunk_size) is not int:
raise SimilarityConfigError("Embedding chunk size must be an integer.")
if type(overlap) is not int:
raise SimilarityConfigError("Embedding chunk overlap must be an integer.")
if chunk_size <= 0:
raise SimilarityConfigError("Embedding chunk size must be greater than zero.")
if overlap < 0:
raise SimilarityConfigError("Embedding chunk overlap must not be negative.")
if overlap >= chunk_size:
raise SimilarityConfigError("Embedding chunk overlap must be smaller than the chunk size.")
max_chars = chunk_size * _CHARS_PER_TOKEN
overlap_chars = overlap * _CHARS_PER_TOKEN
sections = _split_by_headings(text)
chunks: list[str] = []
for section in sections:
if len(section) <= max_chars:
chunks.append(section)
else:
chunks.extend(_fixed_size_chunks(section, max_chars, overlap_chars))
return chunks or [text]
def _split_by_headings(text: str) -> list[str]:
"""Split text at markdown heading boundaries, keeping headings with their content."""
positions = [m.start() for m in _HEADING_PATTERN.finditer(text)]
if not positions:
return [text] if text.strip() else []
sections: list[str] = []
if positions[0] > 0:
preamble = text[: positions[0]].strip()
if preamble:
sections.append(preamble)
for i, start in enumerate(positions):
end = positions[i + 1] if i + 1 < len(positions) else len(text)
section = text[start:end].strip()
if section:
sections.append(section)
return sections
def _fixed_size_chunks(text: str, max_chars: int, overlap_chars: int) -> list[str]:
"""Fall back to fixed-size overlapping windows."""
chunks: list[str] = []
start = 0
while start < len(text):
end = start + max_chars
chunk = text[start:end].strip()
if chunk:
chunks.append(chunk)
start += max_chars - overlap_chars
return chunks
def _average_pool(vectors: list[list[float]]) -> list[float]:
"""Average-pool a list of vectors into a single vector."""
if not vectors:
return []
dimension: int | None = None
for index, vector in enumerate(vectors):
dimension = validate_embedding_vector(
vector,
dimension,
context=f"Embedding vector {index}",
)
assert dimension is not None
count = len(vectors)
pooled = [math.fsum(vector[index] for vector in vectors) / count for index in range(dimension)]
validate_embedding_vector(pooled, dimension, context="Pooled embedding")
return pooled
def _validated_vector_values(
vector: object,
expected_dimension: int | None,
*,
context: str,
allow_zero: bool,
) -> tuple[list[float], float]:
if not isinstance(vector, list) or not vector:
raise SimilarityConfigError(f"{context} vector must be a non-empty list.")
if len(vector) > MAX_EMBEDDING_VECTOR_DIMENSION:
raise SimilarityConfigError(f"{context} vector dimension exceeds {MAX_EMBEDDING_VECTOR_DIMENSION}.")
if expected_dimension is not None and len(vector) != expected_dimension:
raise SimilarityConfigError(
f"{context} vector dimension mismatch: {len(vector)}; expected {expected_dimension}."
)
maximum_component = math.sqrt(sys.float_info.max / len(vector))
values: list[float] = []
norm = 0.0
for value in vector:
if not isinstance(value, (int, float)) or isinstance(value, bool):
raise SimilarityConfigError(f"{context} vector values must be finite numbers.")
try:
numeric = float(value)
except (OverflowError, ValueError) as exc:
raise SimilarityConfigError(f"{context} vector values must be finite numbers.") from exc
if not math.isfinite(numeric):
raise SimilarityConfigError(f"{context} vector values must be finite numbers.")
if abs(numeric) > maximum_component:
raise SimilarityConfigError(f"{context} vector magnitude is too large for numerically stable similarity.")
values.append(numeric)
norm = math.hypot(norm, numeric)
if not math.isfinite(norm):
raise SimilarityConfigError(f"{context} vector magnitude is not numerically stable.")
if norm == 0.0 and not allow_zero:
raise SimilarityConfigError(f"{context} vector must not be a zero vector.")
return values, norm