Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(benchmark): pre-bake hermes agent into dataset images for closed-book runs (#350)

GDRS0951d0ae8f79
diff --git a/benchmark/agent-versions.env b/benchmark/agent-versions.env
index 843959969..a064b2b37 100644
--- a/benchmark/agent-versions.env
+++ b/benchmark/agent-versions.env
@@ -5,3 +5,11 @@ CLAUDE_CODE_VERSION=2.1.211
CODEX_VERSION=0.144.5
OPENCODE_VERSION=1.18.3
NODE_VERSION=20.11.1
+# Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time.
+# Must be a full 40-character commit SHA; anything else is rejected at build time.
+# A tag would be enough to read but not to trust — it can be deleted or repointed,
+# so two builds could record the same string while installing different code, and
+# the run manifest records this value as a pin. The installer script is fetched
+# from this same commit, so the installer cannot drift either.
+# 3c27eb62 is the commit tagged v2026.8.3 (2026-08-03).
+HERMES_VERSION=3c27eb6234bf91b8ceee9e9071591b31e9b148cb
diff --git a/benchmark/patches/harbor-agent-patches.diff b/benchmark/patches/harbor-agent-patches.diff
index bc62838d1..54454cd68 100644
--- a/benchmark/patches/harbor-agent-patches.diff
+++ b/benchmark/patches/harbor-agent-patches.diff
@@ -426,3 +426,105 @@
+++ b/harbor/switchyard_patch_id.txt
@@ -0,0 +1 @@
+switchyard-harbor-patches-2026-05-22-v2
+
+--- a/harbor/agents/installed/hermes.py
++++ b/harbor/agents/installed/hermes.py
+@@ -59,6 +59,16 @@
+ return 'export PATH="$HOME/.local/bin:$PATH"; hermes version'
+
+ async def install(self, environment: BaseEnvironment) -> None:
++ # Skip if hermes is already installed (e.g. from the dataset
++ # agent-bake layer). hermes installs to $HOME/.local/bin, so export
++ # that path before probing. Required for closed-book runs where the
++ # task-time curl install cannot reach github/pypi.
++ probe = await self.exec_as_agent(
++ environment,
++ command='export PATH="$HOME/.local/bin:$PATH"; command -v hermes >/dev/null 2>&1 && echo present || true',
++ )
++ if "present" in (getattr(probe, "stdout", "") or ""):
++ return
+ await self.exec_as_root(
+ environment,
+ command="apt-get update && apt-get install -y curl git ripgrep xz-utils",
+@@ -92,11 +92,31 @@
+ # ------------------------------------------------------------------
+
+ @staticmethod
+- def _build_config_yaml(model: str) -> str:
+- """Generate a hermes config.yaml with full capabilities enabled."""
++ def _build_config_yaml(
++ model: str, base_url: str | None = None, api_key: str | None = None
++ ) -> str:
++ """Generate a hermes config.yaml with full capabilities enabled.
++
++ When ``base_url`` is given, route the model through a custom
++ OpenAI-compatible endpoint (e.g. a Switchyard gateway): hermes only
++ honors a custom endpoint via its config.yaml ``model.provider=custom`` +
++ ``model.base_url`` (the ``OPENAI_BASE_URL`` env var is ignored for the
++ chat model), so we nest the provider/base_url/api_key under ``model``.
++ """
++ model_field: Any
++ if base_url:
++ model_field = {
++ "default": model,
++ "provider": "custom",
++ "base_url": base_url,
++ }
++ if api_key:
++ model_field["api_key"] = api_key
++ else:
++ model_field = model
+ config: dict[str, Any] = {
+- "model": model,
+- "provider": "auto",
++ "model": model_field,
++ **({} if base_url else {"provider": "auto"}),
+ "toolsets": ["hermes-cli"],
+ "agent": {"max_turns": 90},
+ "memory": {
+@@ -357,6 +377,9 @@
+ # Try native provider key first, fall back to OpenRouter.
+ hermes_provider_flag: str | None = None
+ use_native = False
++ # Custom OpenAI-compatible endpoint (e.g. Switchyard) for the chat model.
++ custom_base_url: str | None = None
++ custom_api_key: str | None = None
+
+ if provider in _NATIVE_PROVIDERS:
+ native_flag, key_names = _NATIVE_PROVIDERS[provider]
+@@ -367,11 +390,15 @@
+ hermes_provider_flag = native_flag
+ use_native = True
+ break
+- # Forward OPENAI_BASE_URL when using native OpenAI key
++ # Forward OPENAI_BASE_URL when using native OpenAI key. Hermes ignores
++ # this env var for the chat model, so also thread it into config.yaml
++ # as a custom provider (see _build_config_yaml).
+ if use_native and provider == "openai":
+ base_url = os.environ.get("OPENAI_BASE_URL")
+ if base_url:
+ env["OPENAI_BASE_URL"] = base_url
++ custom_base_url = base_url
++ custom_api_key = os.environ.get("OPENAI_API_KEY")
+
+ if not use_native:
+ openrouter_key = os.environ.get("OPENROUTER_API_KEY")
+@@ -386,9 +413,15 @@
+ env["OPENROUTER_API_KEY"] = openrouter_key
+
+ # Native providers with --provider flag use just the model name;
+- # everything else (OpenRouter, openai direct) uses provider/model.
+- cli_model = model if hermes_provider_flag else self.model_name
+- config_yaml = self._build_config_yaml(cli_model)
++ # a custom OpenAI-compatible endpoint also uses the bare model (the
++ # base_url already targets the gateway); everything else uses provider/model.
++ if custom_base_url:
++ cli_model = model
++ else:
++ cli_model = model if hermes_provider_flag else self.model_name
++ config_yaml = self._build_config_yaml(
++ cli_model, base_url=custom_base_url, api_key=custom_api_key
++ )
+
+ # Pass instruction via env var (safe from shell escaping issues)
+ env["HARBOR_INSTRUCTION"] = instruction
diff --git a/benchmark/prepare_harbor_dataset.py b/benchmark/prepare_harbor_dataset.py
index a57e69447..79d5c1993 100644
--- a/benchmark/prepare_harbor_dataset.py
+++ b/benchmark/prepare_harbor_dataset.py
@@ -35,6 +35,7 @@ DEFAULT_OUTPUT_DIR = SCRIPT_DIR / "datasets" / "openthoughts-tblite-closed-book"
AGENT_VERSIONS_FILE = SCRIPT_DIR / "agent-versions.env"
PROXY_ASSET_DIR = SCRIPT_DIR / "closed_book_proxy" / "proxy"
AGENT_ENTRYPOINT = "switchyard-agent-entrypoint.sh"
+COMMIT_SHA_PATTERN = re.compile(r"[0-9a-f]{40}")
TERMINAL_BENCH_2_SOURCE_DATASET = "terminal-bench/terminal-bench-2"
TERMINAL_BENCH_2_1_SOURCE_DATASET = "terminal-bench/terminal-bench-2-1"
# Shared across the TB2 family (2.0 + the 2.1 verified iteration): 2.1 tweaks
@@ -199,10 +200,35 @@ def _install_layer(pins: dict[str, str]) -> str:
claude_version = pins["CLAUDE_CODE_VERSION"]
codex_version = pins["CODEX_VERSION"]
opencode_version = pins["OPENCODE_VERSION"]
+ # Hermes (NousResearch hermes-agent) is a per-user uv app installed from
+ # GitHub, not an npm package. Baking it here (build-time, with host network)
+ # means the runtime install() skip-guard short-circuits, so tasks need no
+ # egress for it — enabling closed-book Hermes runs.
+ #
+ # HERMES_VERSION must be a full commit SHA. A tag is not enough: tags can be
+ # deleted or repointed, so two builds could record the same version string while
+ # installing different code — the manifest would assert a reproducibility it does
+ # not have. Requiring one shape is complete by construction, where a deny-list of
+ # moving names ("main", "master", ...) can never be, since any branch name passes.
+ # The installer script is fetched from the same commit for the same reason: pinning
+ # the agent but running whatever installer main has today reintroduces the drift.
+ #
+ # The pin is applied with the installer's --commit, not --branch: --branch reaches
+ # `git clone --branch`, which accepts only branch and tag names and rejects a SHA.
+ # --force-commit is required with it. Without it the installer skips the pin when
+ # the commit is an ancestor of the freshly cloned HEAD, logging a warning and
+ # silently leaving the image on the tip of main — the drift this pin exists to
+ # prevent, arriving as a warning rather than a build failure.
+ hermes_version = pins["HERMES_VERSION"]
+ if not COMMIT_SHA_PATTERN.fullmatch(hermes_version):
+ raise SystemExit(
+ f"HERMES_VERSION={hermes_version!r} is not a full 40-character commit SHA; "
+ "a tag or branch can be repointed and cannot be recorded as a reproducible pin"
+ )
return f"""
# Switchyard benchmark prebaked coding agents.
-ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version}"
+ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version},hermes={hermes_version}"
RUN set -eux; \\
if command -v apt-get >/dev/null 2>&1; then \\
apt-get update; \\
@@ -243,6 +269,19 @@ RUN set -eux; \\
claude --version; \\
codex --version; \\
opencode --version
+RUN set -eux; \\
+ export HOME=/root; \\
+ export PATH="/root/.local/bin:$PATH"; \\
+ if command -v apt-get >/dev/null 2>&1; then \\
+ apt-get update; \\
+ apt-get install -y --no-install-recommends git ripgrep xz-utils; \\
+ rm -rf /var/lib/apt/lists/*; \\
+ elif command -v apk >/dev/null 2>&1; then \\
+ apk add --no-cache bash git ripgrep xz; \\
+ fi; \\
+ curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/{hermes_version}/scripts/install.sh \\
+ | bash -s -- --skip-setup --commit {hermes_version} --force-commit; \\
+ hermes version
"""
@@ -475,7 +514,13 @@ def prepare_dataset(
overwrite: bool,
) -> Path:
pins = _read_env_file(AGENT_VERSIONS_FILE)
- required = {"CLAUDE_CODE_VERSION", "CODEX_VERSION", "OPENCODE_VERSION", "NODE_VERSION"}
+ required = {
+ "CLAUDE_CODE_VERSION",
+ "CODEX_VERSION",
+ "HERMES_VERSION",
+ "NODE_VERSION",
+ "OPENCODE_VERSION",
+ }
missing = sorted(required - pins.keys())
if missing:
raise ValueError(f"missing pins in {AGENT_VERSIONS_FILE}: {', '.join(missing)}")
diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.py
index 1c69052ff..74dd620bf 100644
--- a/tests/test_prepare_harbor_dataset.py
+++ b/tests/test_prepare_harbor_dataset.py
@@ -8,6 +8,7 @@ import json
from pathlib import Path
from types import ModuleType
+import pytest
import yaml
REPO = Path(__file__).resolve().parents[1]
@@ -284,6 +285,7 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Pat
assert manifest["agent_versions"] == {
"CLAUDE_CODE_VERSION": "2.1.211",
"CODEX_VERSION": "0.144.5",
+ "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb",
"NODE_VERSION": "20.11.1",
"OPENCODE_VERSION": "1.18.3",
}
@@ -306,3 +308,110 @@ def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None:
proxy_env = "\n".join(compose["services"]["proxy"]["environment"])
assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_env
assert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_env
+
+def test_a_hermes_ref_that_is_not_a_commit_sha_is_rejected() -> None:
+ """Only a full commit SHA can be recorded as a pin.
+
+ The dataset manifest presents HERMES_VERSION as a reproducibility guarantee. Two
+ builds recording the same string while installing different Hermes code is worse
+ than recording nothing, so the build fails rather than asserting a pin it does not
+ have. Tags are rejected alongside branches: a tag can be deleted or repointed, so
+ it reads as immutable without being so.
+ """
+ base = {
+ "CLAUDE_CODE_VERSION": "1",
+ "CODEX_VERSION": "2",
+ "OPENCODE_VERSION": "3",
+ "NODE_VERSION": "4",
+ }
+ rejected = (
+ "main",
+ "master",
+ "HEAD",
+ "v2026.8.3",
+ "release/2026.8",
+ "3c27eb6",
+ "3C27EB6234BF91B8CEEE9E9071591B31E9B148CB",
+ "3c27eb6234bf91b8ceee9e9071591b31e9b148cbb",
+ )
+ for ref in rejected:
+ with pytest.raises(SystemExit, match="commit SHA"):
+ _load_generator_module()._install_layer({**base, "HERMES_VERSION": ref})
+
+
+def test_the_hermes_installer_is_fetched_at_the_pinned_commit() -> None:
+ """Pinning the agent but running main's installer reintroduces the same drift."""
+ sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb"
+ pins = {
+ "CLAUDE_CODE_VERSION": "1",
+ "CODEX_VERSION": "2",
+ "OPENCODE_VERSION": "3",
+ "NODE_VERSION": "4",
+ "HERMES_VERSION": sha,
+ }
+
+ layer = _load_generator_module()._install_layer(pins)
+
+ assert f"hermes-agent/{sha}/scripts/install.sh" in layer
+ assert "hermes-agent/main/" not in layer
+
+
+def test_the_hermes_pin_is_applied_by_commit_and_forced() -> None:
+ """`--branch` reaches `git clone --branch`, which rejects a SHA outright.
+
+ `--force-commit` is what makes the pin take effect. Without it the installer skips
+ the checkout whenever the commit is an ancestor of the freshly cloned HEAD, warns,
+ and leaves the image on the tip of main — the drift the pin exists to prevent,
+ arriving as a warning rather than a build failure.
+ """
+ sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb"
+ pins = {
+ "CLAUDE_CODE_VERSION": "1",
+ "CODEX_VERSION": "2",
+ "OPENCODE_VERSION": "3",
+ "NODE_VERSION": "4",
+ "HERMES_VERSION": sha,
+ }
+
+ layer = _load_generator_module()._install_layer(pins)
+
+ assert f"--commit {sha}" in layer
+ assert "--force-commit" in layer
+ assert "--branch" not in layer
+
+
+def test_the_alpine_branch_installs_the_shell_the_installer_needs() -> None:
+ """The installer is piped into bash, which Alpine does not ship by default."""
+ pins = {
+ "CLAUDE_CODE_VERSION": "1",
+ "CODEX_VERSION": "2",
+ "OPENCODE_VERSION": "3",
+ "NODE_VERSION": "4",
+ "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb",
+ }
+
+ layer = _load_generator_module()._install_layer(pins)
+
+ assert "apk add --no-cache bash git ripgrep xz" in layer
+
+
+def test_a_missing_hermes_pin_is_reported_with_the_other_pins(tmp_path: Path) -> None:
+ """Absent, it must fail the shared pin check rather than crash reading the layer."""
+ module = _load_generator_module()
+ versions = tmp_path / "agent-versions.env"
+ versions.write_text(
+ "CLAUDE_CODE_VERSION=1\nCODEX_VERSION=2\nOPENCODE_VERSION=3\nNODE_VERSION=4\n"
+ )
+ module.AGENT_VERSIONS_FILE = versions
+ source = tmp_path / "source"
+ _write_task(source, "task-a", "[environment]\n", "FROM ubuntu:22.04\n")
+
+ with pytest.raises(ValueError, match="missing pins.*HERMES_VERSION"):
+ module.prepare_dataset(
+ source_dataset="[email protected]",
+ source_dir=source,
+ output_dir=tmp_path / "prepared",
+ harbor_command="harbor",
+ overwrite=False,
+ )
+