Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): turn off OpenCode snapshots to stop with-skill EPIPE exits (#763)

SUHAN RAMANI046290dd08fd
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 09aa6b76..0f10ad4a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -585,6 +585,11 @@ reviewed source revision.
checks for that script and subsequent files.
- GitLab JUnit collection includes nested skill folders in catalog reports.
+- Tier 3 OpenCode with-skill trials no longer exit 1 with `EPIPE: broken pipe, write` after a
+ finished answer (#757). OpenCode's snapshot passed the workspace file list to
+ `git check-ignore --stdin` for a `/workspace/.git` that does not exist, and git could exit before
+ reading it. The baseline's empty workspace skipped that call. OpenCode trials now run with
+ `"snapshot": false`; a job's `opencode_config` can turn it back on.
- API caller now percent-encodes scalar OpenAPI path values as individual segments and rejects `.` and `..` values with a JSON input error.
- The calculator reference skill converts negative values (`-40 celsius to fahrenheit`) and takes negative
percentages, and no longer answers `10% of 50 + 5` with `Result: 5` by ignoring the rest of the input.
diff --git a/src/skillevaluator/tier3/harbor/installed_agents.py b/src/skillevaluator/tier3/harbor/installed_agents.py
index 2fb1763e..873abae5 100644
--- a/src/skillevaluator/tier3/harbor/installed_agents.py
+++ b/src/skillevaluator/tier3/harbor/installed_agents.py
@@ -14,7 +14,7 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from time import monotonic
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ClassVar
from harbor.agents.installed.base import NonZeroAgentExitCodeError
from harbor.agents.installed.claude_code import ClaudeCode
@@ -179,6 +179,11 @@ class SkillEvaluatorClaudeCode(ClaudeCode):
class SkillEvaluatorOpenCode(OpenCode):
"""Avoid a redundant distribution Node/npm install before Harbor's NVM."""
+ # Headless trials never use OpenCode's undo, and its snapshot can EPIPE on a
+ # workspace without .git (#757). opencode_config can turn it back on.
+ # Newer OpenCode renames this key to "snapshots"; update it when bumping OPENCODE_VERSION.
+ _DEFAULT_CONFIG: ClassVar[dict[str, Any]] = {"snapshot": False}
+
def _convert_events_to_trajectory(self, events: list[dict[str, Any]]) -> Trajectory | None:
normalized = []
for event in events:
diff --git a/tests/tier3/test_opencode_snapshot_config.py b/tests/tier3/test_opencode_snapshot_config.py
new file mode 100644
index 00000000..7845f942
--- /dev/null
+++ b/tests/tier3/test_opencode_snapshot_config.py
@@ -0,0 +1,53 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""OpenCode trials run with snapshots off, so a git EPIPE cannot end a finished trial."""
+
+from __future__ import annotations
+
+import json
+import shlex
+from pathlib import Path
+
+import pytest
+
+pytest.importorskip("harbor")
+
+from harbor.agents.installed.opencode import OpenCode
+
+from skillevaluator.tier3.harbor.installed_agents import SkillEvaluatorOpenCode
+from skillevaluator.tier3.harbor.local_agents import SkillEvaluatorGatewayOpenCode, SkillEvaluatorLocalOpenCode
+
+_WRAPPERS = [SkillEvaluatorOpenCode, SkillEvaluatorLocalOpenCode, SkillEvaluatorGatewayOpenCode]
+
+
+def _written_config(command: str | None) -> dict:
+ assert command is not None
+ tokens = shlex.split(command)
+ return json.loads(tokens[tokens.index("echo") + 1])
+
+
[email protected]("agent_class", _WRAPPERS)
+def test_wrappers_write_config_with_snapshots_off(agent_class: type, tmp_path: Path) -> None:
+ agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")
+
+ assert _written_config(agent._build_register_config_command())["snapshot"] is False
+
+
+def test_nvidia_build_config_has_snapshots_off(tmp_path: Path) -> None:
+ agent = SkillEvaluatorLocalOpenCode(logs_dir=tmp_path, model_name="nvidia/test-model")
+
+ assert _written_config(agent._nvidia_provider_config_command("test-model"))["snapshot"] is False
+
+
[email protected]("agent_class", _WRAPPERS)
+def test_job_opencode_config_can_turn_snapshots_back_on(agent_class: type, tmp_path: Path) -> None:
+ agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model", opencode_config={"snapshot": True})
+
+ assert _written_config(agent._build_register_config_command())["snapshot"] is True
+
+
+def test_stock_harbor_opencode_is_unchanged(tmp_path: Path) -> None:
+ command = OpenCode(logs_dir=tmp_path, model_name="openai/test-model")._build_register_config_command()
+
+ assert command is None or "snapshot" not in _written_config(command)