Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): require a model when a custom NeMo Gym policy URL is set (#643)

Christopher Kevin36f4e67af58a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 09b5517f..6f3ede7d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -328,6 +328,10 @@ All notable changes to SkillEvaluator are documented in this file.
- BENCHMARK cards explain the required with-skill goal and security case checks
- HTML reports explain the required with-skill goal and security case checks
that can fail quality even when all dimensions pass.
+- `--backend nemo-gym` with `SKILL_EVAL_NEMO_GYM_MODEL_URL` set now requires a
+ model (`--model`, `--agent-model` or `harbor.agents.<agent>.model`) before the
+ run starts. It no longer sends the evaluator provider's default model name to
+ that endpoint.
- The Tier 2 scale guide names the fixed 8 MiB selected-manifest input budget
and 1 MiB per-manifest limit, including in description-only embedding mode.
- Saved Tier 2 catalogs use compact JSON so indentation no longer rejects dense
diff --git a/docs/environment-variables.mdx b/docs/environment-variables.mdx
index fcf231cd..d25d1980 100644
--- a/docs/environment-variables.mdx
+++ b/docs/environment-variables.mdx
@@ -158,7 +158,7 @@ execution environments are covered in
| `SKILLEVALUATOR_OUTPUT_PROVENANCE_KEY_FILE` | macOS: `~/Library/Application Support/skillevaluator/output-provenance.key`; Linux: `$XDG_STATE_HOME/skillevaluator/output-provenance.key` (`~/.local/state` when `XDG_STATE_HOME` is unset); Windows: `%LOCALAPPDATA%\skillevaluator\output-provenance.key` | Local key that authenticates the markers SkillEvaluator writes into its generated output directories; a directory is replaced only when its marker verifies for that path. The key is created on first use. It must be outside every evaluated skill and generated output tree. On macOS and Linux, the file and its directory must be owned by the current user and not accessible to group or other users. |
| `SKILLEVALUATOR_LOCAL_SANDBOX` | `require` | Local-mode sandbox policy. `require` fails closed when no OS sandbox backend (Bubblewrap on Linux, Seatbelt on macOS) is usable; `prefer` degrades to advisory-only guardrails with a loud warning; `off` skips sandbox probing entirely — for skills you fully trust. Any other value is rejected. No value enables native Windows: local mode fails closed there before anything runs — use WSL2 or `--env-mode docker`. |
| `SKILL_EVAL_NEMO_GYM_BIN` | `gym` on `PATH` | Path to the experimental task-folder adapter's pinned Gym executable. Public `nemo-gym==0.7.0` is incompatible; the public source candidate still needs native runtime qualification. See [Execution backends](/skills/skillevaluator/tier3-live-evaluation#execution-backends). |
-| `SKILL_EVAL_NEMO_GYM_MODEL_URL` | provider base URL | OpenAI-compatible base URL (ending in `/v1`) of the policy model for `--backend nemo-gym`. |
+| `SKILL_EVAL_NEMO_GYM_MODEL_URL` | provider base URL | OpenAI-compatible base URL (ending in `/v1`) of the policy model for `--backend nemo-gym`. When set, also choose a model it serves with `--model`, `--agent-model` or `harbor.agents.<agent>.model`. |
| `SKILL_EVAL_NEMO_GYM_MODEL_API_KEY` | provider key, only when the URL is also the provider's | API key for that policy model. Required when `SKILL_EVAL_NEMO_GYM_MODEL_URL` is set; use `EMPTY` for a keyless server. Passed to Gym through the environment, never on the command line. |
| `SKILL_EVAL_NEMO_GYM_HOST` | — | Address of this host that the Gym sandbox can reach (for example the Docker bridge gateway). Needed when the host name resolves to a loopback address. |
| `SKILL_EVAL_NEMO_GYM_SANDBOX` | `docker` | NeMo Gym sandbox provider for `--backend nemo-gym`. |
diff --git a/src/skillevaluator/tier3/commands.py b/src/skillevaluator/tier3/commands.py
index 7a4629e4..df76b7ec 100644
--- a/src/skillevaluator/tier3/commands.py
+++ b/src/skillevaluator/tier3/commands.py
@@ -1224,7 +1224,7 @@ def _nemo_gym_doctor_rows(
checked_endpoints: set[str] = set()
for agent in agents:
try:
- _model, endpoint_url, _key = _model_endpoint((overrides.get(agent) or [None])[0])
+ _model, endpoint_url, _key = _model_endpoint((overrides.get(agent) or [None])[0], require_model=False)
except (ValueError, ProviderConfigurationError) as exc:
errors.append(f"{agent}: {exc}")
continue
diff --git a/src/skillevaluator/tier3/nemo_gym/runner.py b/src/skillevaluator/tier3/nemo_gym/runner.py
index c75c1c5e..26f072bc 100644
--- a/src/skillevaluator/tier3/nemo_gym/runner.py
+++ b/src/skillevaluator/tier3/nemo_gym/runner.py
@@ -306,7 +306,7 @@ def _task_case_ids(arm_dir: Path) -> dict[str, str]:
return mapping
-def _model_endpoint(model: str | None) -> tuple[str, str, str]:
+def _model_endpoint(model: str | None, *, require_model: bool = True) -> tuple[str, str, str]:
"""Return ``(model, base_url, api_key)`` for Gym's OpenAI-compatible policy model server."""
from skillevaluator.provider_config import resolve_llm_provider
@@ -315,6 +315,12 @@ def _model_endpoint(model: str | None) -> tuple[str, str, str]:
if url and not key:
# Never send the evaluator's provider key to a policy server it was not issued for.
raise ValueError(f"{MODEL_URL_ENV} is set, so also set {MODEL_API_KEY_ENV} (use EMPTY for a keyless server)")
+ if url and not model and require_model:
+ # The provider's default model name is rarely served by a separate policy endpoint.
+ raise ValueError(
+ f"{MODEL_URL_ENV} is set, so also choose a model it serves "
+ "(--model, --agent-model, or harbor.agents.<agent>.model)"
+ )
resolved_model = model
if not url or not key or not resolved_model:
provider = resolve_llm_provider()
@@ -1038,7 +1044,10 @@ def run_nemo_gym_eval(
for agent in agents:
cli_model = (agent_models or {}).get(agent) or model
config_model = (config_agents.get(agent) or {}).get("model")
- endpoints[agent] = _model_endpoint(cli_model or config_model)
+ try:
+ endpoints[agent] = _model_endpoint(cli_model or config_model)
+ except ValueError as exc:
+ return _config_failure(str(exc))
source = "CLI" if cli_model else "evals/config.yml" if config_model else "public provider default"
resolved_models[agent] = {"agent": agent, "model": endpoints[agent][0], "source": source}
if url_error := model_url_credential_error(endpoints[agent][1]):
diff --git a/src/skillevaluator/tier3/workflow.py b/src/skillevaluator/tier3/workflow.py
index 19086720..b8f8a62b 100644
--- a/src/skillevaluator/tier3/workflow.py
+++ b/src/skillevaluator/tier3/workflow.py
@@ -296,6 +296,26 @@ def nemo_gym_preflight_errors(skill_path: Path, params: dict[str, Any]) -> list[
for url in (os.environ.get(MODEL_URL_ENV, "").strip(), str(params.get("model") or "")):
if url and (url_error := model_url_credential_error(url)) and url_error not in errors:
errors.append(url_error)
+ if os.environ.get(MODEL_URL_ENV, "").strip() and not params.get("model") and params.get("agents"):
+ from skillevaluator.tier3.commands import parse_agent_model_overrides
+ from skillevaluator.tier3.evals_config import EvalsConfigError, load_evals_config
+
+ try:
+ overrides = parse_agent_model_overrides(tuple(params.get("agent_model") or ()))
+ config_agents = (load_evals_config(skill_path)[0].get("harbor") or {}).get("agents") or {}
+ except (ValueError, EvalsConfigError):
+ config_agents = None # Reported by the option and config checks.
+ if config_agents is not None and (
+ missing := [
+ agent
+ for agent in parse_agents(params["agents"])
+ if not overrides.get(agent) and not (config_agents.get(agent) or {}).get("model")
+ ]
+ ):
+ errors.append(
+ f"{MODEL_URL_ENV} is set, so also choose a model it serves "
+ f"(--model, --agent-model, or harbor.agents.<agent>.model) for: {', '.join(missing)}"
+ )
for name in ("override_cpus", "override_memory_mb"):
value = params.get(name)
if value is not None and value <= 0:
diff --git a/tests/cli/test_nemo_gym_autopilot_preflight.py b/tests/cli/test_nemo_gym_autopilot_preflight.py
index c1b54bf6..b1c49c6a 100644
--- a/tests/cli/test_nemo_gym_autopilot_preflight.py
+++ b/tests/cli/test_nemo_gym_autopilot_preflight.py
@@ -73,8 +73,21 @@ def test_keyed_gym_generates_with_its_endpoint_and_one_compatibility_probe(
lambda _self, options, **_kwargs: evaluated.append(options) or {"execution_status": "succeeded"},
)
+ # A custom policy URL needs a model that endpoint serves, so name one.
result = CliRunner().invoke(
- cli, [*command, str(skill), "--backend", "nemo-gym", "--agents", "claude-code", "--progress", "off"]
+ cli,
+ [
+ *command,
+ str(skill),
+ "--backend",
+ "nemo-gym",
+ "--agents",
+ "claude-code",
+ "--model",
+ "policy-model",
+ "--progress",
+ "off",
+ ],
)
assert result.exit_code == 0, result.output
diff --git a/tests/test_nemo_gym_backend.py b/tests/test_nemo_gym_backend.py
index 6b65ef2c..1cd42105 100644
--- a/tests/test_nemo_gym_backend.py
+++ b/tests/test_nemo_gym_backend.py
@@ -1133,6 +1133,19 @@ def test_custom_model_url_requires_its_own_key(monkeypatch: pytest.MonkeyPatch)
assert _model_endpoint("policy-model") == ("policy-model", "http://policy.example/v1", "EMPTY")
+def test_custom_model_url_requires_a_model_it_serves(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ from skillevaluator.tier3.nemo_gym.runner import _model_endpoint
+
+ monkeypatch.setenv("SKILL_EVAL_NEMO_GYM_MODEL_URL", "http://policy.example/v1")
+ monkeypatch.setenv(MODEL_API_KEY_ENV, "EMPTY")
+ with pytest.raises(ValueError, match="--model"):
+ _model_endpoint(None)
+ skill, _env_log = _fake_gym_skill(tmp_path, monkeypatch)
+ result = _run_fake_gym(skill, tmp_path, model=None, skip_baseline=True)
+ assert any("--model" in error for error in result["error"])
+ assert not list((tmp_path / "results").rglob("result.json"))
+
+
@pytest.mark.parametrize("value", ["abc", "0", "-5", "nan", "inf"])
def test_invalid_gym_timeout_is_rejected_before_launch(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, value: str
@@ -1682,7 +1695,9 @@ def test_doctor_reports_nemo_gym_readiness(tmp_path: Path, monkeypatch: pytest.M
monkeypatch.setattr(backend, "nemo_gym_version", lambda _bin: "0.7.0")
monkeypatch.setattr("shutil.which", lambda name: f"/usr/bin/{name}")
monkeypatch.setattr(commands, "_docker_daemon_error", lambda _env: None)
- monkeypatch.setattr(gym_runner, "_model_endpoint", lambda model: (model or "m", "http://127.0.0.1:9/v1", "k"))
+ monkeypatch.setattr(
+ gym_runner, "_model_endpoint", lambda model, **_kwargs: (model or "m", "http://127.0.0.1:9/v1", "k")
+ )
probes = []
def endpoint_probe(url):
@@ -2071,6 +2086,11 @@ def test_cli_checks_gym_settings_before_autopilot_writes_a_dataset(
[],
"username or password",
),
+ (
+ {"SKILL_EVAL_NEMO_GYM_MODEL_URL": "http://127.0.0.1:9/v1", MODEL_API_KEY_ENV: "EMPTY"},
+ ["--agents", "codex"],
+ "choose a model it serves",
+ ),
({}, ["--override-cpus", "0"], "--override-cpus"),
({}, ["--override-memory-mb", "0"], "--override-memory-mb"),
]
@@ -2340,7 +2360,7 @@ def test_doctor_fails_without_a_policy_endpoint_or_docker_daemon(
monkeypatch.setenv("SKILL_EVAL_NEMO_GYM_BIN", str(gym))
monkeypatch.setattr(backend, "nemo_gym_version", lambda _bin: "0.7.0")
- def no_endpoint(_model):
+ def no_endpoint(_model, **_kwargs):
raise ValueError("--backend nemo-gym needs a model, an OpenAI-compatible base URL and an API key")
monkeypatch.setattr(gym_runner, "_model_endpoint", no_endpoint)