Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(hermes): validate the selected Build credential during preflight (#583)

Christopher Kevine7eb419b6e4b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3396a621..fb035442 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -332,6 +332,10 @@ All notable changes to SkillEvaluator are documented in this file.
side of the `@`, which still leaves a social handle such as
`Jane Doe (@jdoe)`, a bare `@jdoe` or `Platform Team @ Example` alone, and
now also catches a non-ASCII local part.
+- Hermes Docker runs with NVIDIA Build now validate `NVIDIA_API_KEY` during
+ Harbor preflight, matching the credential used by the runtime instead of
+ inheriting an OpenAI key requirement. Unrelated provider keys do not satisfy
+ the check.
- An advisory carried from a passing skill into a collection result now names
the skill it came from, in its location and in `metadata.collection_skill`.
The findings were preserved but attributed to nothing, so a collection run
diff --git a/src/skillevaluator/tier3/harbor/hermes_agent.py b/src/skillevaluator/tier3/harbor/hermes_agent.py
index 4a0d9aca..dfc99374 100644
--- a/src/skillevaluator/tier3/harbor/hermes_agent.py
+++ b/src/skillevaluator/tier3/harbor/hermes_agent.py
@@ -8,14 +8,17 @@ from __future__ import annotations
import json
import os
import shlex
-from typing import TYPE_CHECKING
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any
+from harbor.agents.agent_auth_resolution import require_credential_env
from harbor.agents.installed.base import with_prompt_template
from harbor.agents.installed.hermes import Hermes
from skillevaluator.tier3.harbor import HERMES_VERSION
if TYPE_CHECKING:
+ from harbor.agents.agent_auth_resolution import ResolvedModelConnection
from harbor.environments.base import BaseEnvironment
from harbor.models.agent.context import AgentContext
from harbor.models.trajectories import Trajectory
@@ -27,6 +30,19 @@ _HOME = "/tmp/hermes"
class SkillEvaluatorHermes(Hermes):
"""Keep Harbor's session/ATIF contract without its OpenRouter fallback."""
+ @classmethod
+ def check_model_credentials(
+ cls,
+ connection: ResolvedModelConnection,
+ *,
+ kwargs: dict[str, Any],
+ env: Mapping[str, str],
+ explicit_env: Mapping[str, str] | None = None,
+ ) -> None:
+ """Validate the credential consumed by our fixed NVIDIA Build runtime."""
+ del connection, kwargs, explicit_env
+ require_credential_env(env, cls.name(), "NVIDIA_API_KEY")
+
def get_version_command(self) -> str:
return 'export PATH="$HOME/.local/bin:/usr/local/bin:$PATH"; hermes --version'
diff --git a/tests/test_hermes_agent.py b/tests/test_hermes_agent.py
index 03c54bac..9b840f32 100644
--- a/tests/test_hermes_agent.py
+++ b/tests/test_hermes_agent.py
@@ -134,6 +134,22 @@ def test_missing_credential_does_not_fall_back_to_openrouter(monkeypatch):
assert "NVIDIA_API_KEY" in _validate_agent_provider_credentials(_provider(), ["hermes"], {})[0]
[email protected]("model", ["nvidia/nemotron-3.5-super-vl-120b-a12b", "moonshotai/kimi-k2.6"])
+def test_harbor_preflight_accepts_only_selected_build_credential(monkeypatch, model):
+ from harbor.agents.factory import AgentFactory
+
+ for name in ("OPENAI_API_KEY", "OPENROUTER_API_KEY", "NVIDIA_API_KEY"):
+ monkeypatch.delenv(name, raising=False)
+ config = HarborAgentConfig(import_path=DOCKER_AGENT_IMPORT_PATHS["hermes"], model_name=model)
+ monkeypatch.setenv("NVIDIA_API_KEY", "selected-build-key")
+ AgentFactory.run_preflight(config)
+ monkeypatch.delenv("NVIDIA_API_KEY")
+ monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key")
+ monkeypatch.setenv("OPENROUTER_API_KEY", "unrelated-openrouter-key")
+ with pytest.raises(ValueError, match="NVIDIA_API_KEY"):
+ AgentFactory.run_preflight(config)
+
+
def test_model_precedence_and_credential_isolation(monkeypatch):
monkeypatch.setenv("OPENROUTER_API_KEY", "unrelated-key")
monkeypatch.setenv("OPENAI_API_KEY", "another-key")