Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): project Gym OpenCode skills into the workdir (#589)
Christopher Kevina7fadd7066ec
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 1336b5e2..8ebc5498 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -252,6 +252,11 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- Native NeMo Gym project skill projections run after authored image instructions+ and final workspace resets, even when an earlier layer already copied skills.+- Native NeMo Gym OpenCode evaluations project staged skills into the task workdir+ so skill discovery works with Gym's isolated home and configuration directories.+ Baseline tasks use the same layout without the target skill.- NeMo Gym doctor isolates its anonymous endpoint probe, honors proxy settingswithout forwarding credentials, and reports proxy authentication and gateway failures.- Native NeMo Gym doctor now checks policy endpoint reachability with a bounded,diff --git a/src/skillevaluator/tier3/nemo_gym/runner.py b/src/skillevaluator/tier3/nemo_gym/runner.pyindex 7e65806d..de370445 100644--- a/src/skillevaluator/tier3/nemo_gym/runner.py+++ b/src/skillevaluator/tier3/nemo_gym/runner.py@@ -76,8 +76,8 @@ def stage_for_gym("""Adapt Harbor-format tasks for Gym's agents and verifier.Drops Harbor's ``skills_dir`` task hook, which Gym's CLI agents reject. The task image- already carries the skills at each agent's native location. Claude also gets project- skills because Gym uses a private ``CLAUDE_CONFIG_DIR``. The baseline image still has+ already carries the skills at each agent's native location. Claude and OpenCode also get+ project skills because Gym isolates their user configuration directories. The baseline hasno target skill. Adds ``verifier_env`` (``${NAME}`` templates) to ``[verifier.env]``: Harborpasses the judge overrides at job level, and Gym has no job-level verifier env. Scales theagent, verifier and environment-build timeouts by ``timeout_multiplier``, as Harbor's@@ -90,13 +90,14 @@ def stage_for_gym(environment = tomllib.loads(text).get("environment") or {}if environment.get("mcp_servers"):raise ValueError(f"{task_toml.parent.name}: {NATIVE_MCP_SERVERS_ERROR}")- if agent == "claude-code":+ project_directory = {"opencode": ".agents", "claude-code": ".claude"}.get(agent or "")+ if project_directory:dockerfile = task_toml.parent / "environment" / "Dockerfile"if dockerfile.is_file() and (dockerfile.parent / "skills").is_dir():workdir = environment.get("workdir") or "/workspace"- projection = "COPY " + json.dumps(["skills/", f"{workdir.rstrip('/')}/.claude/skills/"])+ projection = "COPY " + json.dumps(["skills/", f"{workdir.rstrip('/')}/{project_directory}/skills/"])docker_text = dockerfile.read_text(encoding="utf-8")- if projection not in docker_text.splitlines():+ if not docker_text.rstrip().endswith("\n" + projection):dockerfile.write_text(docker_text.rstrip("\n") + "\n" + projection + "\n", encoding="utf-8")staged = _SKILLS_DIR_LINE.sub("", text)existing = set((tomllib.loads(staged).get("verifier") or {}).get("env") or {})diff --git a/tests/test_nemo_gym_backend.py b/tests/test_nemo_gym_backend.pyindex ba3ddf2c..7f49443d 100644--- a/tests/test_nemo_gym_backend.py+++ b/tests/test_nemo_gym_backend.py@@ -462,6 +462,59 @@ def test_stage_for_gym_drops_only_the_skills_dir_hook(tmp_path: Path) -> None:assert "cpus = 2" in text and 'A = "1"' in text[email protected]("workdir", [None, "/work/custom/", "/work with spaces"])[email protected]("with_skill", [True, False])+def test_opencode_project_skills_are_staged_in_both_arms(tmp_path: Path, workdir, with_skill: bool) -> None:+ task = tmp_path / "case"+ environment = task / "environment"+ skills = environment / "skills"+ skills.mkdir(parents=True)+ if with_skill:+ (skills / "calculator").mkdir()+ (skills / "calculator" / "SKILL.md").write_text("---\nname: calculator\n---\n")+ (task / "task.toml").write_text("[environment]\n" + (f"workdir = {json.dumps(workdir)}\n" if workdir else ""))+ dockerfile = environment / "Dockerfile"+ dockerfile.write_text("FROM python:3.13-slim\n")+ stage_for_gym(tmp_path, agent="opencode")+ staged = dockerfile.read_text()+ target = (workdir or "/workspace").rstrip("/") + "/.agents/skills/"+ assert "COPY " + json.dumps(["skills/", target]) in staged.splitlines()+ stage_for_gym(tmp_path, agent="opencode")+ assert dockerfile.read_text() == staged+ assert (skills / "calculator" / "SKILL.md").is_file() is with_skill++[email protected]("agent", [None, "codex", "claude-code", "opencode"])+def test_project_skill_projection_leaves_tasks_without_skills_unchanged(tmp_path: Path, agent) -> None:+ task = tmp_path / "case"+ environment = task / "environment"+ environment.mkdir(parents=True)+ (task / "task.toml").write_text("[environment]\n")+ dockerfile = environment / "Dockerfile"+ dockerfile.write_text("FROM python:3.13-slim\n")+ stage_for_gym(tmp_path, agent=agent)+ assert dockerfile.read_text() == "FROM python:3.13-slim\n"++[email protected]("agent, directory", [("opencode", ".agents"), ("claude-code", ".claude")])+def test_project_skill_projection_follows_final_authored_reset(tmp_path: Path, agent, directory) -> None:+ from skillevaluator.tier3.harbor.adapter import _append_task_input_projection++ task = tmp_path / "case"+ environment = task / "environment"+ (environment / "skills").mkdir(parents=True)+ (task / "task.toml").write_text("[environment]\n")+ projection = "COPY " + json.dumps(["skills/", f"/workspace/{directory}/skills/"])+ authored = f"FROM python:3.13-slim\n{projection}\nRUN rm -rf /workspace/{directory}\n"+ dockerfile = environment / "Dockerfile"+ dockerfile.write_text(_append_task_input_projection(authored, include_input=False, agent_config_lines=[]))+ stage_for_gym(tmp_path, agent=agent)+ assert dockerfile.read_text().splitlines()[-1] == projection+ staged = dockerfile.read_text()+ stage_for_gym(tmp_path, agent=agent)+ assert dockerfile.read_text() == staged++@pytest.mark.parametrize("workdir", [None, "/work", "/"])@pytest.mark.parametrize("with_skill", [True, False])def test_stage_for_gym_projects_claude_skills_into_task_workdir(@@ -488,14 +541,18 @@ def test_stage_for_gym_projects_claude_skills_into_task_workdir(@pytest.mark.parametrize("agent", [None, "codex", "opencode"])-def test_stage_for_gym_leaves_other_agent_images_unchanged(tmp_path: Path, agent: str | None) -> None:+def test_stage_for_gym_projects_only_supported_agent_images(tmp_path: Path, agent: str | None) -> None:task = tmp_path / "case"(task / "environment" / "skills").mkdir(parents=True)(task / "task.toml").write_text("[environment]\n")dockerfile = task / "environment" / "Dockerfile"dockerfile.write_text("FROM python:3.12-slim\n")stage_for_gym(tmp_path, agent=agent)- assert dockerfile.read_text() == "FROM python:3.12-slim\n"+ if agent == "opencode":+ projection = "COPY " + json.dumps(["skills/", "/workspace/.agents/skills/"])+ assert dockerfile.read_text().splitlines()[-1] == projection+ else:+ assert dockerfile.read_text() == "FROM python:3.12-slim\n"def test_stage_for_gym_claude_without_docker_skills(tmp_path: Path) -> None: