Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): keep one [verifier.env] table when staging native Gym tasks (#639)
Christopher Kevinfa6b18ef9684
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9c7f2197..d294e6c2 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -308,6 +308,9 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- `--backend nemo-gym` no longer writes a second `[verifier.env]` table into a+ native task's `task.toml` when its header has a trailing comment, extra spaces+ or no final newline. The duplicate table made the file fail to parse.- The Tier 2 scale guide names the fixed 8 MiB selected-manifest input budgetand 1 MiB per-manifest limit, including in description-only embedding mode.- Saved Tier 2 catalogs use compact JSON so indentation no longer rejects densediff --git a/src/skillevaluator/tier3/nemo_gym/runner.py b/src/skillevaluator/tier3/nemo_gym/runner.pyindex 763c4afc..7de536e2 100644--- a/src/skillevaluator/tier3/nemo_gym/runner.py+++ b/src/skillevaluator/tier3/nemo_gym/runner.py@@ -108,9 +108,21 @@ def stage_for_gym(if name not in existing)if lines:- header = re.search(r"^\[verifier\.env\][ \t]*\n", staged, re.MULTILINE)- if header:- staged = staged[: header.end()] + lines + staged[header.end() :]+ if not staged.endswith("\n"):+ staged += "\n"+ added = set(verifier_env or {}) - existing+ for header in re.finditer(+ r"^[ \t]*\[[ \t]*verifier[ \t]*\.[ \t]*env[ \t]*\][ \t]*(?:#[^\n]*)?\n", staged, re.MULTILINE+ ):+ # A header-like line can also sit inside a multiline string: keep the real table.+ candidate = staged[: header.end()] + lines + staged[header.end() :]+ try:+ candidate_env = (tomllib.loads(candidate).get("verifier") or {}).get("env") or {}+ except tomllib.TOMLDecodeError:+ continue+ if added <= set(candidate_env):+ staged = candidate+ breakelse:staged = staged.rstrip("\n") + "\n\n[verifier.env]\n" + linesif timeout_multiplier != 1.0:diff --git a/tests/test_nemo_gym_backend.py b/tests/test_nemo_gym_backend.pyindex 99a78fd1..5f25c9ae 100644--- a/tests/test_nemo_gym_backend.py+++ b/tests/test_nemo_gym_backend.py@@ -1279,6 +1279,43 @@ def test_timeout_multiplier_scales_staged_task_timeouts(tmp_path: Path) -> None:assert "timeout_sec = 1\n" in data["instruction"]+ "header",+ ["[verifier.env] # judge\n", " [verifier.env]\n", "[ verifier . env ]\n", "[verifier.env]"],+)+def test_staging_adds_judge_env_to_an_existing_verifier_env_table(tmp_path: Path, header: str) -> None:+ import tomllib++ from skillevaluator.tier3.nemo_gym.runner import stage_for_gym++ task = tmp_path / "case-1"+ task.mkdir()+ body = 'FOO = "1"\n' if header.endswith("\n") else ""+ (task / "task.toml").write_text(f"[environment]\n\n{header}{body}")+ stage_for_gym(tmp_path, {"SKILL_EVAL_LLM_MODEL": "${SKILL_EVAL_LLM_MODEL}"}, agent="codex")+ env = tomllib.loads((task / "task.toml").read_text())["verifier"]["env"]+ assert env["SKILL_EVAL_LLM_MODEL"] == "${SKILL_EVAL_LLM_MODEL}"+ assert env.get("FOO", "1") == "1"++[email protected]("real_table", [True, False])+def test_staging_skips_a_verifier_env_header_inside_a_multiline_string(tmp_path: Path, real_table: bool) -> None:+ import tomllib++ from skillevaluator.tier3.nemo_gym.runner import stage_for_gym++ task = tmp_path / "case-1"+ task.mkdir()+ instruction = 'instruction = """\nExample:\n[verifier.env] # example\n"""\n'+ table = '\n[verifier.env]\nFOO = "1"\n' if real_table else ""+ (task / "task.toml").write_text(f"{instruction}\n[environment]\n{table}")+ stage_for_gym(tmp_path, {"SKILL_EVAL_LLM_MODEL": "${SKILL_EVAL_LLM_MODEL}"}, agent="codex")+ data = tomllib.loads((task / "task.toml").read_text())+ assert data["instruction"] == "Example:\n[verifier.env] # example\n"+ assert data["verifier"]["env"]["SKILL_EVAL_LLM_MODEL"] == "${SKILL_EVAL_LLM_MODEL}"+ assert data["verifier"]["env"].get("FOO") == ("1" if real_table else None)++def test_timeout_multiplier_is_validated_and_recorded(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, keep_staged_tasks: None) -> None: