Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): refresh an explicit Gym policy key that shares the Vertex ADC token (#613)
Christopher Kevind5cfecbdc56a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8e4c2696..eb412115 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -294,6 +294,9 @@ All notable changes to SkillEvaluator are documented in this file.- Copilot CLI `harness` results flag an empty or whitespace-only `description`(`CP-DESC-EMPTY`): Copilot lists the skill with a blank description, so themodel sees only its name.+- NeMo Gym runs refresh the policy model key between arms when+ `SKILL_EVAL_NEMO_GYM_MODEL_API_KEY` is set to the same Vertex AI ADC token as+ `OPENAI_API_KEY`, so later arms no longer start with an expired token.- NeMo Gym detached cleanup defers termination signals until teardown finishes,preserves cancellation over cleanup errors, and reaps adopted run-owned servers.- Native NeMo Gym project skill projections run after authored image instructionsdiff --git a/src/skillevaluator/tier3/nemo_gym/runner.py b/src/skillevaluator/tier3/nemo_gym/runner.pyindex d5a574d1..763c4afc 100644--- a/src/skillevaluator/tier3/nemo_gym/runner.py+++ b/src/skillevaluator/tier3/nemo_gym/runner.py@@ -767,8 +767,8 @@ def _refresh_adc_credentials("""Mint a fresh Vertex ADC token for one Gym launch and cap the launch to its lifetime.Mirrors the Harbor runner: the token is refreshed before every launch, reused as the- policy key when the policy has no key of its own, and the launch timeout is- capped by ``compute_adc_job_timeout``. Raises ``RuntimeError``/``ValueError`` on failure.+ policy key when the policy has no key of its own or its key is the provider token being+ replaced, and the launch timeout is capped by ``compute_adc_job_timeout``. Raises ``RuntimeError``/``ValueError`` on failure."""from skillevaluator.provider_config import (CREDENTIAL_EXPIRY_ENV,@@ -784,12 +784,14 @@ def _refresh_adc_credentials(from skillevaluator.tier3.harbor.runner import VERTEX_ADC_JOB_TIMEOUT_ENVrefreshed = dict(credentials)+ previous_provider_key = (credentials.get("OPENAI_API_KEY") or os.environ.get("OPENAI_API_KEY") or "").strip()fresh = refresh_host_vertex_adc_environment(refreshed, fallback_env=os.environ, fail_on_expired=True, token_getter=_get_google_access_token)if fresh:refreshed["OPENAI_API_KEY"] = fresh- if policy_uses_provider_key:+ policy_key = refreshed.get(MODEL_API_KEY_ENV, "").strip()+ if policy_uses_provider_key or (previous_provider_key and policy_key == previous_provider_key):refreshed[MODEL_API_KEY_ENV] = freshsource = refreshed.get(CREDENTIAL_SOURCE_ENV) or os.environ.get(CREDENTIAL_SOURCE_ENV)base_url = refreshed.get("OPENAI_BASE_URL") or os.environ.get("OPENAI_BASE_URL")diff --git a/tests/test_nemo_gym_backend.py b/tests/test_nemo_gym_backend.pyindex c8a62097..99a78fd1 100644--- a/tests/test_nemo_gym_backend.py+++ b/tests/test_nemo_gym_backend.py@@ -2183,6 +2183,30 @@ def test_adc_refresh_follows_the_policy_key_source(monkeypatch: pytest.MonkeyPatassert refreshed[MODEL_API_KEY_ENV] == "token-a"+def test_adc_refresh_renews_an_explicit_policy_key_that_is_the_provider_token(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from skillevaluator import provider_config+ from skillevaluator.tier3.nemo_gym import runner as gym_runner++ monkeypatch.setattr(provider_config, "refresh_host_vertex_adc_environment", lambda *_args, **_kwargs: "token-new")+ shared = {"OPENAI_API_KEY": "token-old", MODEL_API_KEY_ENV: "token-old"}+ refreshed, _timeout = gym_runner._refresh_adc_credentials(shared, policy_uses_provider_key=False, timeout_s=60)+ assert refreshed[MODEL_API_KEY_ENV] == "token-new" and refreshed["OPENAI_API_KEY"] == "token-new"+ assert shared[MODEL_API_KEY_ENV] == "token-old"++ monkeypatch.setenv("OPENAI_API_KEY", "token-old")+ refreshed, _timeout = gym_runner._refresh_adc_credentials(+ {MODEL_API_KEY_ENV: "token-old"}, policy_uses_provider_key=False, timeout_s=60+ )+ assert refreshed[MODEL_API_KEY_ENV] == "token-new"++ refreshed, _timeout = gym_runner._refresh_adc_credentials(+ {MODEL_API_KEY_ENV: "policy-own-key"}, policy_uses_provider_key=False, timeout_s=60+ )+ assert refreshed[MODEL_API_KEY_ENV] == "policy-own-key"++def test_second_interrupt_during_stop_still_kills_the_gym_group(tmp_path: Path) -> None:import sysimport time