Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): preserve OpenCode bash exit status in trajectories (#585)

Christopher Kevincaab02a90f64
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 98a3dd33..bdb747c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -397,6 +397,9 @@ All notable changes to SkillEvaluator are documented in this file.
user-request section when it is set below 2,000, and that a check budget too
small to hold the final response beside some tool history is logged as a
warning rather than quietly raised — lower the final-response limit with it.
+- OpenCode bash tool observations retain recorded shell exit codes in Harbor
+ trajectories. Declarative `command_succeeded` checks can now distinguish
+ successful commands from nonzero exits; missing exit codes remain not checked.
- `judge_attempts.json` now masks `Cookie` header values, credential flag values
(`--token`, `--password`, `-u user:password`, `--cookie`) and keys such as
`sk-proj-...` that start a line in JSON-escaped text, as in RAGAS prompts and
diff --git a/src/skillevaluator/tier3/harbor/installed_agents.py b/src/skillevaluator/tier3/harbor/installed_agents.py
index 2a9b6709..2fb1763e 100644
--- a/src/skillevaluator/tier3/harbor/installed_agents.py
+++ b/src/skillevaluator/tier3/harbor/installed_agents.py
@@ -14,7 +14,7 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from time import monotonic
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
from harbor.agents.installed.base import NonZeroAgentExitCodeError
from harbor.agents.installed.claude_code import ClaudeCode
@@ -26,6 +26,7 @@ from skillevaluator.utils.redaction import redact_sensitive_text
if TYPE_CHECKING:
from harbor.agents.installed.base import BaseInstalledAgent
from harbor.environments.base import BaseEnvironment
+ from harbor.models.trajectories import Trajectory
# Harbor's installers download over HTTPS but never request a CA bundle; apt
# images used to get one only as a dependency of the skipped npm package.
@@ -178,6 +179,24 @@ class SkillEvaluatorClaudeCode(ClaudeCode):
class SkillEvaluatorOpenCode(OpenCode):
"""Avoid a redundant distribution Node/npm install before Harbor's NVM."""
+ def _convert_events_to_trajectory(self, events: list[dict[str, Any]]) -> Trajectory | None:
+ normalized = []
+ for event in events:
+ part = event.get("part")
+ if event.get("type") == "tool_use" and isinstance(part, dict) and part.get("tool") == "bash":
+ state = part.get("state")
+ if isinstance(state, dict) and state.get("status") in {"completed", "error"}:
+ metadata = state.get("metadata")
+ exit_code = metadata.get("exit") if isinstance(metadata, dict) else None
+ if type(exit_code) is int:
+ # Harbor retains output but drops metadata.exit from its ATIF observation.
+ # "Exit code: N" is a header spelling the plugin failure scanner skips, so a
+ # leading error line in the output is still read as the first line.
+ output = f"Exit code: {exit_code}\n{state.get('output') or ''}"
+ event = {**event, "part": {**part, "state": {**state, "output": output}}}
+ normalized.append(event)
+ return super()._convert_events_to_trajectory(normalized)
+
async def ensure_system_dependencies(
self,
environment: BaseEnvironment,
diff --git a/src/skillevaluator/tier3/harbor/local_agents.py b/src/skillevaluator/tier3/harbor/local_agents.py
index 70b2b977..d357aada 100644
--- a/src/skillevaluator/tier3/harbor/local_agents.py
+++ b/src/skillevaluator/tier3/harbor/local_agents.py
@@ -23,7 +23,6 @@ from urllib.parse import urlsplit
from harbor.agents.agent_auth_resolution import ResolvedModelConnection, without_inferred_endpoint
from harbor.agents.installed.base import NetworkConnectionError, NonZeroAgentExitCodeError
-from harbor.agents.installed.opencode import OpenCode
from harbor.models.trial.paths import EnvironmentPaths
from harbor.utils.env import parse_bool_env_value
@@ -527,7 +526,7 @@ class SkillEvaluatorGatewayOpenCode(SkillEvaluatorOpenCode):
)
-class SkillEvaluatorLocalOpenCode(OpenCode):
+class SkillEvaluatorLocalOpenCode(SkillEvaluatorOpenCode):
"""OpenCode wrapper that skips nvm/npm bootstrap in local mode."""
_REMOTE_WORKSPACE = PurePosixPath("/workspace")
diff --git a/tests/tier3/fixtures/opencode-bash-exit.jsonl b/tests/tier3/fixtures/opencode-bash-exit.jsonl
new file mode 100644
index 00000000..8b8847d3
--- /dev/null
+++ b/tests/tier3/fixtures/opencode-bash-exit.jsonl
@@ -0,0 +1,3 @@
+{"type":"step_start","timestamp":1000,"sessionID":"session-test","part":{"id":"part-start","messageID":"message-tool","sessionID":"session-test","type":"step-start"}}
+{"type":"tool_use","timestamp":1100,"sessionID":"session-test","part":{"type":"tool","tool":"bash","callID":"call-validate","state":{"status":"completed","input":{"command":"python3 /root/.agents/skills/report-builder/scripts/validate_input.py"},"output":"ERROR: input data is missing required field 'owner'\n","metadata":{"output":"ERROR: input data is missing required field 'owner'\n","exit":2,"truncated":false},"title":"python3 /root/.agents/skills/report-builder/scripts/validate_input.py","time":{"start":1050,"end":1090}},"id":"part-tool","sessionID":"session-test","messageID":"message-tool"}}
+{"type":"step_finish","timestamp":1200,"sessionID":"session-test","part":{"id":"part-finish","reason":"tool-calls","messageID":"message-tool","sessionID":"session-test","type":"step-finish","tokens":{"total":100,"input":90,"output":10,"reasoning":0,"cache":{"write":0,"read":0}},"cost":0}}
diff --git a/tests/tier3/test_opencode_exit_status.py b/tests/tier3/test_opencode_exit_status.py
new file mode 100644
index 00000000..66ebadaa
--- /dev/null
+++ b/tests/tier3/test_opencode_exit_status.py
@@ -0,0 +1,160 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Native OpenCode bash events preserve exit evidence through Harbor's converter."""
+
+from __future__ import annotations
+
+import copy
+import json
+from pathlib import Path
+
+import pytest
+from harbor.models.agent.context import AgentContext
+
+from skillevaluator.tier3.eval_core.atif_helpers import extract_tool_calls_as_dicts
+from skillevaluator.tier3.eval_core.checks import check_script_execution, check_tool_efficiency
+from skillevaluator.tier3.eval_core.declarative_checks import evaluate_declarative_checks
+from skillevaluator.tier3.eval_core.log_converters import load_trajectory_with_fallback
+from skillevaluator.tier3.eval_core.plugin_signals import compute_plugin_signals
+from skillevaluator.tier3.harbor.installed_agents import SkillEvaluatorOpenCode
+from skillevaluator.tier3.harbor.local_agents import SkillEvaluatorGatewayOpenCode, SkillEvaluatorLocalOpenCode
+
+
+def native_events():
+ path = Path(__file__).parent / "fixtures/opencode-bash-exit.jsonl"
+ return [json.loads(line) for line in path.read_text().splitlines()]
+
+
+def _convert(agent_class, events, tmp_path):
+ agent = agent_class(logs_dir=tmp_path, model_name="openai/test-model")
+ trajectory = agent._convert_events_to_trajectory(events)
+ assert trajectory is not None
+ return extract_tool_calls_as_dicts(trajectory.to_json_dict())
+
+
+def _succeeded(calls):
+ return evaluate_declarative_checks(
+ [{"type": "command_succeeded", "script": "scripts/validate_input.py"}],
+ tool_calls=calls,
+ final_response="",
+ workspace_roots=[],
+ )[0]
+
+
+ "agent_class", [SkillEvaluatorOpenCode, SkillEvaluatorLocalOpenCode, SkillEvaluatorGatewayOpenCode]
+)
[email protected]("exit_code", [0, 2, 17, -9])
+def test_native_bash_exit_is_attributable(agent_class, exit_code, native_events, tmp_path):
+ native_events[1]["part"]["state"]["metadata"]["exit"] = exit_code
+ original = copy.deepcopy(native_events)
+ calls = _convert(agent_class, native_events, tmp_path)
+
+ assert native_events == original
+ assert len(calls) == 1
+ assert f"Exit code: {exit_code}\n" in calls[0]["observation"]
+ result = _succeeded(calls)
+ assert result["passed"] is (exit_code == 0)
+ if exit_code:
+ assert result["reason"].endswith(f"it was invoked but exited {exit_code}")
+ assert check_script_execution(calls, "validate_input.py")["score"] == 1.0
+ assert check_tool_efficiency(calls, expected_script="validate_input.py")["score"] == 1.0
+
+
[email protected]("metadata", [None, {}, {"exit": None}, {"exit": True}, {"exit": 0.5}, {"exit": "unknown"}])
+def test_unavailable_exit_stays_not_checked(metadata, native_events, tmp_path):
+ native_events[1]["part"]["state"]["metadata"] = metadata
+ calls = _convert(SkillEvaluatorOpenCode, native_events, tmp_path)
+ assert _succeeded(calls)["passed"] is None
+ assert "exit status was not recorded" in _succeeded(calls)["reason"]
+
+
[email protected]("tool,status", [("read", "completed"), ("bash", "running"), ("bash", "pending")])
+def test_only_finished_bash_calls_gain_exit_evidence(tool, status, native_events, tmp_path):
+ native_events[1]["part"]["tool"] = tool
+ native_events[1]["part"]["state"]["status"] = status
+ calls = _convert(SkillEvaluatorOpenCode, native_events, tmp_path)
+ assert "Exit code:" not in calls[0]["observation"]
+
+
+def test_empty_events_remain_empty(tmp_path):
+ agent = SkillEvaluatorOpenCode(logs_dir=tmp_path, model_name="openai/test-model")
+ assert agent._convert_events_to_trajectory([]) is None
+
+
[email protected]("status", ["completed", "error"])
[email protected]("output", [None, ""])
+def test_exit_without_output_is_retained(status, output, native_events, tmp_path):
+ state = native_events[1]["part"]["state"]
+ state.update(status=status, output=output)
+ calls = _convert(SkillEvaluatorOpenCode, native_events, tmp_path)
+ assert _succeeded(calls)["passed"] is False
+ assert "exited 2" in _succeeded(calls)["reason"]
+
+
+def test_grader_loads_generated_trajectory(native_events, tmp_path):
+ agent = SkillEvaluatorOpenCode(logs_dir=tmp_path, model_name="openai/test-model")
+ trajectory_path = tmp_path / "trajectory.json"
+ (tmp_path / "opencode.txt").write_text("\n".join(json.dumps(event) for event in native_events))
+ agent.populate_context_post_run(AgentContext())
+ loaded, metadata = load_trajectory_with_fallback(trajectory_path)
+ assert metadata["source"] == "trajectory.json"
+ result = _succeeded(extract_tool_calls_as_dicts(loaded))
+ assert result["passed"] is False
+ assert "exited 2" in result["reason"]
+
+
+def test_status_is_paired_per_call(native_events, tmp_path):
+ second = copy.deepcopy(native_events[1])
+ second["part"]["callID"] = "call-second"
+ second["part"]["state"]["metadata"]["exit"] = 0
+ native_events.insert(2, second)
+ calls = _convert(SkillEvaluatorOpenCode, native_events, tmp_path)
+ assert "Exit code: 2" in calls[0]["observation"]
+ assert "Exit code: 0" in calls[1]["observation"]
+
+
+def test_shell_tail_status_is_not_script_success(native_events, tmp_path):
+ state = native_events[1]["part"]["state"]
+ state["input"]["command"] += "; echo finished"
+ state["metadata"]["exit"] = 0
+ calls = _convert(SkillEvaluatorOpenCode, native_events, tmp_path)
+ result = _succeeded(calls)
+ assert result["passed"] is None
+ assert "could not be attributed" in result["reason"]
+
+
+ "agent_class", [SkillEvaluatorOpenCode, SkillEvaluatorLocalOpenCode, SkillEvaluatorGatewayOpenCode]
+)
+def test_failed_bash_with_leading_error_stays_failed_in_plugin_signals(agent_class, native_events, tmp_path):
+ state = native_events[1]["part"]["state"]
+ state["output"] = "bash: helper: command not found\nRun setup before retrying.\n"
+ state["metadata"].update(output=state["output"], exit=127)
+ trajectory = agent_class(logs_dir=tmp_path, model_name="openai/test-model")._convert_events_to_trajectory(
+ native_events
+ )
+ assert trajectory is not None
+
+ case = {"tool_arguments": [{"tool": "bash", "contains": {"command": "validate_input"}}]}
+ arguments = compute_plugin_signals(trajectory.to_json_dict(), case, declared={})["arguments"]
+
+ assert (arguments["checked"], arguments["passed"]) == (1, 0)
+ assert [failure["rule"] for failure in arguments["failures"]] == ["call_failed"]
+
+
+def test_zero_exit_bash_with_error_text_still_passes_in_plugin_signals(native_events, tmp_path):
+ state = native_events[1]["part"]["state"]
+ state["output"] = "bash: helper: command not found\nRun setup before retrying.\n"
+ state["metadata"].update(output=state["output"], exit=0)
+ trajectory = SkillEvaluatorOpenCode(
+ logs_dir=tmp_path, model_name="openai/test-model"
+ )._convert_events_to_trajectory(native_events)
+ assert trajectory is not None
+
+ case = {"tool_arguments": [{"tool": "bash", "contains": {"command": "validate_input"}}]}
+ arguments = compute_plugin_signals(trajectory.to_json_dict(), case, declared={})["arguments"]
+
+ assert (arguments["checked"], arguments["passed"]) == (1, 1)
diff --git a/tests/tier3/test_plugin_signals.py b/tests/tier3/test_plugin_signals.py
index 7988eec8..f2c33a50 100644
--- a/tests/tier3/test_plugin_signals.py
+++ b/tests/tier3/test_plugin_signals.py
@@ -514,6 +514,27 @@ class TestOutcomeTriState:
)
assert _activations(traj)[0]["succeeded"] is False
+ @pytest.mark.parametrize("header", ["Exit code: 127", "Process exited with code 127"])
+ def test_nonzero_exit_header_does_not_hide_a_leading_error(self, header: str) -> None:
+ # The status header is skipped, so the failing first line and the ordinary last line are what get read.
+ observation = f"{header}\nbash: helper: command not found\nRun setup before retrying.\n"
+ traj = _traj(_one("Bash", {"command": "helper --run"}, observation))
+ case = {"tool_arguments": [{"tool": "Bash", "contains": {"command": "helper"}}]}
+
+ block = _signals(traj, case)["arguments"]
+
+ assert (block["checked"], block["passed"]) == (1, 0)
+ assert [failure["rule"] for failure in block["failures"]] == ["call_failed"]
+
+ def test_zero_exit_header_still_outranks_a_leading_error_line(self) -> None:
+ observation = "Exit code: 0\nbash: helper: command not found\nRun setup before retrying.\n"
+ traj = _traj(_one("Bash", {"command": "helper --run"}, observation))
+ case = {"tool_arguments": [{"tool": "Bash", "contains": {"command": "helper"}}]}
+
+ block = _signals(traj, case)["arguments"]
+
+ assert (block["checked"], block["passed"]) == (1, 1)
+
def test_missing_sibling_file_does_not_fail_a_shell_manifest_read(self) -> None:
output = "cat: skills/alpha/REFERENCE.md: No such file or directory\n---\nname: alpha\n---\n"
traj = _traj(_one("exec_command", {"cmd": "cat skills/alpha/SKILL.md skills/alpha/REFERENCE.md"}, output))