Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): pin OpenCode 1.19.5 for managed local installs (#558)

Narendran Raghavan9944bfe07dd1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 35a33ab5..7f911a4b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -332,6 +332,9 @@ All notable changes to SkillEvaluator are documented in this file.
time limit, and on Linux the interrupt could be handled on a scan thread,
which did not wake that wait; the run then went on until a scan ended. The
runner now checks for the interrupt every 0.1 seconds.
+- In local mode, the offered OpenCode install now pins OpenCode 1.19.5, the
+ release Harbor and CI install, instead of whatever npm resolves as latest.
+ Claude Code and Codex installs were already pinned.
- The behavior evidence gate no longer keeps a score for an expectation that
names an observed artifact but also asks for something only the final answer
could show. "Writes `report.py` and explains why each column was dropped" was
diff --git a/ci/bootstrap_tiers.py b/ci/bootstrap_tiers.py
index 0ec3c395..4cee5e8a 100644
--- a/ci/bootstrap_tiers.py
+++ b/ci/bootstrap_tiers.py
@@ -25,12 +25,7 @@ from skillevaluator.tier3.harbor.local_runtime import managed_install_argv, mana
def install_agent(agent: str) -> None:
argv = managed_install_argv(agent)
- if agent == "opencode":
- # Unlike interactive local setup, CI never resolves an unpinned npm package.
- # Docker runs install the same release through Harbor's agent kwargs.
- from skillevaluator.tier3.harbor.runner import OPENCODE_VERSION
-
- argv[-1] = f"opencode-ai@{OPENCODE_VERSION}"
+ # Runs the same pinned install that local setup offers, at the release Harbor installs for Docker runs.
subprocess.run(argv, check=True, timeout=600)
print(f"Installed {agent} in {managed_runtime_prefix(agent)}")
diff --git a/docs/agents-and-sandboxes.mdx b/docs/agents-and-sandboxes.mdx
index 6426f5eb..068951c2 100644
--- a/docs/agents-and-sandboxes.mdx
+++ b/docs/agents-and-sandboxes.mdx
@@ -730,10 +730,12 @@ hard-unsupported and fails closed before agent-runtime discovery —
directs you to WSL2 or `--env-mode docker`. Only `claude-code`, `codex`,
and `opencode` can run in local mode.
-Local mode does not install the pinned CLI releases. Each trial checks the
-installed version against the selected model instead and fails before the agent
-starts when it is too old: Claude 5.5 and later models need Claude Code 2.1.280
-or newer, and GPT-6 models need Codex 0.159.1 or newer.
+When the selected agent's CLI is missing, a local-mode run offers to install it
+under `SKILLEVALUATOR_RUNTIME_DIR`, at the release Harbor installs; without a
+terminal it prints the `npm install` command instead. Each trial checks the
+installed version against the selected model and fails before the agent starts
+when it is too old: Claude 5.5 and later models need Claude Code 2.1.280 or
+newer, and GPT-6 models need Codex 0.159.1 or newer.
The sandbox policy is controlled by environment variables:
diff --git a/src/skillevaluator/tier3/harbor/local_runtime.py b/src/skillevaluator/tier3/harbor/local_runtime.py
index 7ccbaab1..cdee2b15 100644
--- a/src/skillevaluator/tier3/harbor/local_runtime.py
+++ b/src/skillevaluator/tier3/harbor/local_runtime.py
@@ -62,7 +62,9 @@ _AGENT_PACKAGES: dict[str, str] = {
"codex": "@openai/codex",
"opencode": "opencode-ai",
}
-PINNED_RUNTIME_VERSIONS: Mapping[str, str] = MappingProxyType({"claude-code": "2.1.292", "codex": "0.161.0"})
+PINNED_RUNTIME_VERSIONS: Mapping[str, str] = MappingProxyType(
+ {"claude-code": "2.1.292", "codex": "0.161.0", "opencode": "1.19.5"}
+)
RUNTIME_INSTALL_TIMEOUT_SECONDS = 600
# Vendor install hints shown when a CLI is missing. Local mode never runs these
diff --git a/src/skillevaluator/tier3/harbor/runner.py b/src/skillevaluator/tier3/harbor/runner.py
index d2f36891..b6122ded 100644
--- a/src/skillevaluator/tier3/harbor/runner.py
+++ b/src/skillevaluator/tier3/harbor/runner.py
@@ -1458,9 +1458,7 @@ _HARBOR_LAUNCH_ENV = MappingProxyType({"HARBOR_TELEMETRY": "0", "PYTHON_DOTENV_D
# wrappers check a minimum version for the selected model.
_CODEX_VERSION = PINNED_RUNTIME_VERSIONS["codex"]
_CLAUDE_CODE_VERSION = PINNED_RUNTIME_VERSIONS["claude-code"]
-# Harbor would otherwise install ``opencode-ai@latest``. CI's managed local
-# bootstrap installs this same release, so every CI mode shares one pin.
-OPENCODE_VERSION = "1.19.5"
+OPENCODE_VERSION = PINNED_RUNTIME_VERSIONS["opencode"]
# Harbor 0.22's Codex agent ran with ``model_reasoning_effort=high``; later
# releases defer to Codex's own default. Pin the effort SkillEvaluator was
# validated with so scores and cost stay comparable across Harbor upgrades.
diff --git a/tests/ci/test_all_tier_preflight.py b/tests/ci/test_all_tier_preflight.py
index 15672252..5a8ddfc3 100644
--- a/tests/ci/test_all_tier_preflight.py
+++ b/tests/ci/test_all_tier_preflight.py
@@ -98,6 +98,23 @@ def test_local_bootstrap_installs_the_opencode_release_docker_runs_pin(monkeypat
assert runner._harbor_agent_kwargs("opencode", None, "docker") == {"version": runner.OPENCODE_VERSION}
+def test_local_bootstrap_runs_the_pinned_install_that_local_setup_offers(monkeypatch):
+ from skillevaluator.tier3.harbor import local_runtime
+
+ def managed_install_argv(agent):
+ return ["npm", "install", f"{agent}[email protected]"]
+
+ monkeypatch.setattr(local_runtime, "managed_install_argv", managed_install_argv)
+ bootstrap = module("bootstrap_tiers")
+ calls = []
+ monkeypatch.setattr(
+ bootstrap.subprocess, "run", lambda args, **_kwargs: calls.append(args) or subprocess.CompletedProcess(args, 0)
+ )
+ for agent in local_runtime.LOCAL_RUNTIME_AGENTS:
+ bootstrap.install_agent(agent)
+ assert calls == [managed_install_argv(agent) for agent in local_runtime.LOCAL_RUNTIME_AGENTS]
+
+
def test_gitlab_admission_rejects_forks_and_stale_heads():
gate = module("eligible_gitlab")
env = {
diff --git a/tests/tier3/test_local_runtime_install.py b/tests/tier3/test_local_runtime_install.py
index 2270605a..4fd8dd98 100644
--- a/tests/tier3/test_local_runtime_install.py
+++ b/tests/tier3/test_local_runtime_install.py
@@ -16,7 +16,17 @@ def test_install_argv_pins_release_under_managed_root(tmp_path):
argv = managed_install_argv("claude-code", runtime_root=tmp_path)
assert argv[:5] == ["npm", "install", "--global", "--prefix", str(tmp_path.resolve() / "claude-code")]
assert argv[-1] == "@anthropic-ai/[email protected]"
- assert managed_install_argv("opencode", runtime_root=tmp_path)[-1] == "opencode-ai"
+ assert managed_install_argv("opencode", runtime_root=tmp_path)[-1] == "[email protected]"
+
+
+def test_every_managed_install_pins_the_release_harbor_installs(tmp_path):
+ # Both sides read PINNED_RUNTIME_VERSIONS, so this guards against runner.py
+ # handing Harbor a version literal of its own again.
+ from skillevaluator.tier3.harbor import runner
+
+ for agent in local_runtime.LOCAL_RUNTIME_AGENTS:
+ version = runner._harbor_agent_kwargs(agent, None, "docker")["version"]
+ assert managed_install_argv(agent, runtime_root=tmp_path)[-1].endswith(f"@{version}"), agent
def test_install_without_npm(monkeypatch, tmp_path):
@@ -54,6 +64,12 @@ def test_install_success_finds_cli(monkeypatch, tmp_path):
assert calls[0][0] == "/usr/bin/npm" and calls[0][-1] == "@openai/[email protected]"
+def test_install_runs_the_pinned_opencode_release(monkeypatch, tmp_path):
+ calls = _fake_npm(monkeypatch, 1, create_bin=False)
+ install_local_runtime("opencode", runtime_root=tmp_path)
+ assert calls[0][0] == "/usr/bin/npm" and calls[0][-1] == "[email protected]"
+
+
def test_install_failure_reports_npm_error(monkeypatch, tmp_path):
_fake_npm(monkeypatch, 1, create_bin=False)
assert install_local_runtime("codex", runtime_root=tmp_path) == "npm install failed (exit 1): E404 not found"