Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(hermes): validate the selected Build credential during preflight (#583)
Christopher Kevine7eb419b6e4b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3396a621..fb035442 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -332,6 +332,10 @@ All notable changes to SkillEvaluator are documented in this file.side of the `@`, which still leaves a social handle such as`Jane Doe (@jdoe)`, a bare `@jdoe` or `Platform Team @ Example` alone, andnow also catches a non-ASCII local part.+- Hermes Docker runs with NVIDIA Build now validate `NVIDIA_API_KEY` during+ Harbor preflight, matching the credential used by the runtime instead of+ inheriting an OpenAI key requirement. Unrelated provider keys do not satisfy+ the check.- An advisory carried from a passing skill into a collection result now namesthe skill it came from, in its location and in `metadata.collection_skill`.The findings were preserved but attributed to nothing, so a collection rundiff --git a/src/skillevaluator/tier3/harbor/hermes_agent.py b/src/skillevaluator/tier3/harbor/hermes_agent.pyindex 4a0d9aca..dfc99374 100644--- a/src/skillevaluator/tier3/harbor/hermes_agent.py+++ b/src/skillevaluator/tier3/harbor/hermes_agent.py@@ -8,14 +8,17 @@ from __future__ import annotationsimport jsonimport osimport shlex-from typing import TYPE_CHECKING+from collections.abc import Mapping+from typing import TYPE_CHECKING, Any+from harbor.agents.agent_auth_resolution import require_credential_envfrom harbor.agents.installed.base import with_prompt_templatefrom harbor.agents.installed.hermes import Hermesfrom skillevaluator.tier3.harbor import HERMES_VERSIONif TYPE_CHECKING:+ from harbor.agents.agent_auth_resolution import ResolvedModelConnectionfrom harbor.environments.base import BaseEnvironmentfrom harbor.models.agent.context import AgentContextfrom harbor.models.trajectories import Trajectory@@ -27,6 +30,19 @@ _HOME = "/tmp/hermes"class SkillEvaluatorHermes(Hermes):"""Keep Harbor's session/ATIF contract without its OpenRouter fallback."""+ @classmethod+ def check_model_credentials(+ cls,+ connection: ResolvedModelConnection,+ *,+ kwargs: dict[str, Any],+ env: Mapping[str, str],+ explicit_env: Mapping[str, str] | None = None,+ ) -> None:+ """Validate the credential consumed by our fixed NVIDIA Build runtime."""+ del connection, kwargs, explicit_env+ require_credential_env(env, cls.name(), "NVIDIA_API_KEY")+def get_version_command(self) -> str:return 'export PATH="$HOME/.local/bin:/usr/local/bin:$PATH"; hermes --version'diff --git a/tests/test_hermes_agent.py b/tests/test_hermes_agent.pyindex 03c54bac..9b840f32 100644--- a/tests/test_hermes_agent.py+++ b/tests/test_hermes_agent.py@@ -134,6 +134,22 @@ def test_missing_credential_does_not_fall_back_to_openrouter(monkeypatch):assert "NVIDIA_API_KEY" in _validate_agent_provider_credentials(_provider(), ["hermes"], {})[0][email protected]("model", ["nvidia/nemotron-3.5-super-vl-120b-a12b", "moonshotai/kimi-k2.6"])+def test_harbor_preflight_accepts_only_selected_build_credential(monkeypatch, model):+ from harbor.agents.factory import AgentFactory++ for name in ("OPENAI_API_KEY", "OPENROUTER_API_KEY", "NVIDIA_API_KEY"):+ monkeypatch.delenv(name, raising=False)+ config = HarborAgentConfig(import_path=DOCKER_AGENT_IMPORT_PATHS["hermes"], model_name=model)+ monkeypatch.setenv("NVIDIA_API_KEY", "selected-build-key")+ AgentFactory.run_preflight(config)+ monkeypatch.delenv("NVIDIA_API_KEY")+ monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key")+ monkeypatch.setenv("OPENROUTER_API_KEY", "unrelated-openrouter-key")+ with pytest.raises(ValueError, match="NVIDIA_API_KEY"):+ AgentFactory.run_preflight(config)++def test_model_precedence_and_credential_isolation(monkeypatch):monkeypatch.setenv("OPENROUTER_API_KEY", "unrelated-key")monkeypatch.setenv("OPENAI_API_KEY", "another-key")