Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(benchmark): pre-bake hermes agent into dataset images for closed-book runs (#350)
GDRS0951d0ae8f79
diff --git a/benchmark/agent-versions.env b/benchmark/agent-versions.envindex 843959969..a064b2b37 100644--- a/benchmark/agent-versions.env+++ b/benchmark/agent-versions.env@@ -5,3 +5,11 @@ CLAUDE_CODE_VERSION=2.1.211CODEX_VERSION=0.144.5OPENCODE_VERSION=1.18.3NODE_VERSION=20.11.1+# Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time.+# Must be a full 40-character commit SHA; anything else is rejected at build time.+# A tag would be enough to read but not to trust — it can be deleted or repointed,+# so two builds could record the same string while installing different code, and+# the run manifest records this value as a pin. The installer script is fetched+# from this same commit, so the installer cannot drift either.+# 3c27eb62 is the commit tagged v2026.8.3 (2026-08-03).+HERMES_VERSION=3c27eb6234bf91b8ceee9e9071591b31e9b148cbdiff --git a/benchmark/patches/harbor-agent-patches.diff b/benchmark/patches/harbor-agent-patches.diffindex bc62838d1..54454cd68 100644--- a/benchmark/patches/harbor-agent-patches.diff+++ b/benchmark/patches/harbor-agent-patches.diff@@ -426,3 +426,105 @@+++ b/harbor/switchyard_patch_id.txt@@ -0,0 +1 @@+switchyard-harbor-patches-2026-05-22-v2++--- a/harbor/agents/installed/hermes.py++++ b/harbor/agents/installed/hermes.py+@@ -59,6 +59,16 @@+ return 'export PATH="$HOME/.local/bin:$PATH"; hermes version'++ async def install(self, environment: BaseEnvironment) -> None:++ # Skip if hermes is already installed (e.g. from the dataset++ # agent-bake layer). hermes installs to $HOME/.local/bin, so export++ # that path before probing. Required for closed-book runs where the++ # task-time curl install cannot reach github/pypi.++ probe = await self.exec_as_agent(++ environment,++ command='export PATH="$HOME/.local/bin:$PATH"; command -v hermes >/dev/null 2>&1 && echo present || true',++ )++ if "present" in (getattr(probe, "stdout", "") or ""):++ return+ await self.exec_as_root(+ environment,+ command="apt-get update && apt-get install -y curl git ripgrep xz-utils",+@@ -92,11 +92,31 @@+ # ------------------------------------------------------------------++ @staticmethod+- def _build_config_yaml(model: str) -> str:+- """Generate a hermes config.yaml with full capabilities enabled."""++ def _build_config_yaml(++ model: str, base_url: str | None = None, api_key: str | None = None++ ) -> str:++ """Generate a hermes config.yaml with full capabilities enabled.++++ When ``base_url`` is given, route the model through a custom++ OpenAI-compatible endpoint (e.g. a Switchyard gateway): hermes only++ honors a custom endpoint via its config.yaml ``model.provider=custom`` +++ ``model.base_url`` (the ``OPENAI_BASE_URL`` env var is ignored for the++ chat model), so we nest the provider/base_url/api_key under ``model``.++ """++ model_field: Any++ if base_url:++ model_field = {++ "default": model,++ "provider": "custom",++ "base_url": base_url,++ }++ if api_key:++ model_field["api_key"] = api_key++ else:++ model_field = model+ config: dict[str, Any] = {+- "model": model,+- "provider": "auto",++ "model": model_field,++ **({} if base_url else {"provider": "auto"}),+ "toolsets": ["hermes-cli"],+ "agent": {"max_turns": 90},+ "memory": {+@@ -357,6 +377,9 @@+ # Try native provider key first, fall back to OpenRouter.+ hermes_provider_flag: str | None = None+ use_native = False++ # Custom OpenAI-compatible endpoint (e.g. Switchyard) for the chat model.++ custom_base_url: str | None = None++ custom_api_key: str | None = None++ if provider in _NATIVE_PROVIDERS:+ native_flag, key_names = _NATIVE_PROVIDERS[provider]+@@ -367,11 +390,15 @@+ hermes_provider_flag = native_flag+ use_native = True+ break+- # Forward OPENAI_BASE_URL when using native OpenAI key++ # Forward OPENAI_BASE_URL when using native OpenAI key. Hermes ignores++ # this env var for the chat model, so also thread it into config.yaml++ # as a custom provider (see _build_config_yaml).+ if use_native and provider == "openai":+ base_url = os.environ.get("OPENAI_BASE_URL")+ if base_url:+ env["OPENAI_BASE_URL"] = base_url++ custom_base_url = base_url++ custom_api_key = os.environ.get("OPENAI_API_KEY")++ if not use_native:+ openrouter_key = os.environ.get("OPENROUTER_API_KEY")+@@ -386,9 +413,15 @@+ env["OPENROUTER_API_KEY"] = openrouter_key++ # Native providers with --provider flag use just the model name;+- # everything else (OpenRouter, openai direct) uses provider/model.+- cli_model = model if hermes_provider_flag else self.model_name+- config_yaml = self._build_config_yaml(cli_model)++ # a custom OpenAI-compatible endpoint also uses the bare model (the++ # base_url already targets the gateway); everything else uses provider/model.++ if custom_base_url:++ cli_model = model++ else:++ cli_model = model if hermes_provider_flag else self.model_name++ config_yaml = self._build_config_yaml(++ cli_model, base_url=custom_base_url, api_key=custom_api_key++ )++ # Pass instruction via env var (safe from shell escaping issues)+ env["HARBOR_INSTRUCTION"] = instructiondiff --git a/benchmark/prepare_harbor_dataset.py b/benchmark/prepare_harbor_dataset.pyindex a57e69447..79d5c1993 100644--- a/benchmark/prepare_harbor_dataset.py+++ b/benchmark/prepare_harbor_dataset.py@@ -35,6 +35,7 @@ DEFAULT_OUTPUT_DIR = SCRIPT_DIR / "datasets" / "openthoughts-tblite-closed-book"AGENT_VERSIONS_FILE = SCRIPT_DIR / "agent-versions.env"PROXY_ASSET_DIR = SCRIPT_DIR / "closed_book_proxy" / "proxy"AGENT_ENTRYPOINT = "switchyard-agent-entrypoint.sh"+COMMIT_SHA_PATTERN = re.compile(r"[0-9a-f]{40}")TERMINAL_BENCH_2_SOURCE_DATASET = "terminal-bench/terminal-bench-2"TERMINAL_BENCH_2_1_SOURCE_DATASET = "terminal-bench/terminal-bench-2-1"# Shared across the TB2 family (2.0 + the 2.1 verified iteration): 2.1 tweaks@@ -199,10 +200,35 @@ def _install_layer(pins: dict[str, str]) -> str:claude_version = pins["CLAUDE_CODE_VERSION"]codex_version = pins["CODEX_VERSION"]opencode_version = pins["OPENCODE_VERSION"]+ # Hermes (NousResearch hermes-agent) is a per-user uv app installed from+ # GitHub, not an npm package. Baking it here (build-time, with host network)+ # means the runtime install() skip-guard short-circuits, so tasks need no+ # egress for it — enabling closed-book Hermes runs.+ #+ # HERMES_VERSION must be a full commit SHA. A tag is not enough: tags can be+ # deleted or repointed, so two builds could record the same version string while+ # installing different code — the manifest would assert a reproducibility it does+ # not have. Requiring one shape is complete by construction, where a deny-list of+ # moving names ("main", "master", ...) can never be, since any branch name passes.+ # The installer script is fetched from the same commit for the same reason: pinning+ # the agent but running whatever installer main has today reintroduces the drift.+ #+ # The pin is applied with the installer's --commit, not --branch: --branch reaches+ # `git clone --branch`, which accepts only branch and tag names and rejects a SHA.+ # --force-commit is required with it. Without it the installer skips the pin when+ # the commit is an ancestor of the freshly cloned HEAD, logging a warning and+ # silently leaving the image on the tip of main — the drift this pin exists to+ # prevent, arriving as a warning rather than a build failure.+ hermes_version = pins["HERMES_VERSION"]+ if not COMMIT_SHA_PATTERN.fullmatch(hermes_version):+ raise SystemExit(+ f"HERMES_VERSION={hermes_version!r} is not a full 40-character commit SHA; "+ "a tag or branch can be repointed and cannot be recorded as a reproducible pin"+ )return f"""# Switchyard benchmark prebaked coding agents.-ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version}"+ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version},hermes={hermes_version}"RUN set -eux; \\if command -v apt-get >/dev/null 2>&1; then \\apt-get update; \\@@ -243,6 +269,19 @@ RUN set -eux; \\claude --version; \\codex --version; \\opencode --version+RUN set -eux; \\+ export HOME=/root; \\+ export PATH="/root/.local/bin:$PATH"; \\+ if command -v apt-get >/dev/null 2>&1; then \\+ apt-get update; \\+ apt-get install -y --no-install-recommends git ripgrep xz-utils; \\+ rm -rf /var/lib/apt/lists/*; \\+ elif command -v apk >/dev/null 2>&1; then \\+ apk add --no-cache bash git ripgrep xz; \\+ fi; \\+ curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/{hermes_version}/scripts/install.sh \\+ | bash -s -- --skip-setup --commit {hermes_version} --force-commit; \\+ hermes version"""@@ -475,7 +514,13 @@ def prepare_dataset(overwrite: bool,) -> Path:pins = _read_env_file(AGENT_VERSIONS_FILE)- required = {"CLAUDE_CODE_VERSION", "CODEX_VERSION", "OPENCODE_VERSION", "NODE_VERSION"}+ required = {+ "CLAUDE_CODE_VERSION",+ "CODEX_VERSION",+ "HERMES_VERSION",+ "NODE_VERSION",+ "OPENCODE_VERSION",+ }missing = sorted(required - pins.keys())if missing:raise ValueError(f"missing pins in {AGENT_VERSIONS_FILE}: {', '.join(missing)}")diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.pyindex 1c69052ff..74dd620bf 100644--- a/tests/test_prepare_harbor_dataset.py+++ b/tests/test_prepare_harbor_dataset.py@@ -8,6 +8,7 @@ import jsonfrom pathlib import Pathfrom types import ModuleType+import pytestimport yamlREPO = Path(__file__).resolve().parents[1]@@ -284,6 +285,7 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Patassert manifest["agent_versions"] == {"CLAUDE_CODE_VERSION": "2.1.211","CODEX_VERSION": "0.144.5",+ "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb","NODE_VERSION": "20.11.1","OPENCODE_VERSION": "1.18.3",}@@ -306,3 +308,110 @@ def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None:proxy_env = "\n".join(compose["services"]["proxy"]["environment"])assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_envassert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_env++def test_a_hermes_ref_that_is_not_a_commit_sha_is_rejected() -> None:+ """Only a full commit SHA can be recorded as a pin.++ The dataset manifest presents HERMES_VERSION as a reproducibility guarantee. Two+ builds recording the same string while installing different Hermes code is worse+ than recording nothing, so the build fails rather than asserting a pin it does not+ have. Tags are rejected alongside branches: a tag can be deleted or repointed, so+ it reads as immutable without being so.+ """+ base = {+ "CLAUDE_CODE_VERSION": "1",+ "CODEX_VERSION": "2",+ "OPENCODE_VERSION": "3",+ "NODE_VERSION": "4",+ }+ rejected = (+ "main",+ "master",+ "HEAD",+ "v2026.8.3",+ "release/2026.8",+ "3c27eb6",+ "3C27EB6234BF91B8CEEE9E9071591B31E9B148CB",+ "3c27eb6234bf91b8ceee9e9071591b31e9b148cbb",+ )+ for ref in rejected:+ with pytest.raises(SystemExit, match="commit SHA"):+ _load_generator_module()._install_layer({**base, "HERMES_VERSION": ref})+++def test_the_hermes_installer_is_fetched_at_the_pinned_commit() -> None:+ """Pinning the agent but running main's installer reintroduces the same drift."""+ sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb"+ pins = {+ "CLAUDE_CODE_VERSION": "1",+ "CODEX_VERSION": "2",+ "OPENCODE_VERSION": "3",+ "NODE_VERSION": "4",+ "HERMES_VERSION": sha,+ }++ layer = _load_generator_module()._install_layer(pins)++ assert f"hermes-agent/{sha}/scripts/install.sh" in layer+ assert "hermes-agent/main/" not in layer+++def test_the_hermes_pin_is_applied_by_commit_and_forced() -> None:+ """`--branch` reaches `git clone --branch`, which rejects a SHA outright.++ `--force-commit` is what makes the pin take effect. Without it the installer skips+ the checkout whenever the commit is an ancestor of the freshly cloned HEAD, warns,+ and leaves the image on the tip of main — the drift the pin exists to prevent,+ arriving as a warning rather than a build failure.+ """+ sha = "3c27eb6234bf91b8ceee9e9071591b31e9b148cb"+ pins = {+ "CLAUDE_CODE_VERSION": "1",+ "CODEX_VERSION": "2",+ "OPENCODE_VERSION": "3",+ "NODE_VERSION": "4",+ "HERMES_VERSION": sha,+ }++ layer = _load_generator_module()._install_layer(pins)++ assert f"--commit {sha}" in layer+ assert "--force-commit" in layer+ assert "--branch" not in layer+++def test_the_alpine_branch_installs_the_shell_the_installer_needs() -> None:+ """The installer is piped into bash, which Alpine does not ship by default."""+ pins = {+ "CLAUDE_CODE_VERSION": "1",+ "CODEX_VERSION": "2",+ "OPENCODE_VERSION": "3",+ "NODE_VERSION": "4",+ "HERMES_VERSION": "3c27eb6234bf91b8ceee9e9071591b31e9b148cb",+ }++ layer = _load_generator_module()._install_layer(pins)++ assert "apk add --no-cache bash git ripgrep xz" in layer+++def test_a_missing_hermes_pin_is_reported_with_the_other_pins(tmp_path: Path) -> None:+ """Absent, it must fail the shared pin check rather than crash reading the layer."""+ module = _load_generator_module()+ versions = tmp_path / "agent-versions.env"+ versions.write_text(+ "CLAUDE_CODE_VERSION=1\nCODEX_VERSION=2\nOPENCODE_VERSION=3\nNODE_VERSION=4\n"+ )+ module.AGENT_VERSIONS_FILE = versions+ source = tmp_path / "source"+ _write_task(source, "task-a", "[environment]\n", "FROM ubuntu:22.04\n")++ with pytest.raises(ValueError, match="missing pins.*HERMES_VERSION"):+ module.prepare_dataset(+ source_dataset="[email protected]",+ source_dir=source,+ output_dir=tmp_path / "prepared",+ harbor_command="harbor",+ overwrite=False,+ )+