Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

chore: bump benchmark agent harness pins to current releases (#83)

ryan-lempkace7beadc625b
diff --git a/benchmark/agent-versions.env b/benchmark/agent-versions.env
index 8bf04c592..843959969 100644
--- a/benchmark/agent-versions.env
+++ b/benchmark/agent-versions.env
@@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-CLAUDE_CODE_VERSION=2.1.119
-CODEX_VERSION=0.125.0
-OPENCODE_VERSION=1.14.31
+CLAUDE_CODE_VERSION=2.1.211
+CODEX_VERSION=0.144.5
+OPENCODE_VERSION=1.18.3
NODE_VERSION=20.11.1
diff --git a/benchmark/run-baseline.sh b/benchmark/run-baseline.sh
index bd9d2748b..4a0a4952c 100755
--- a/benchmark/run-baseline.sh
+++ b/benchmark/run-baseline.sh
@@ -21,9 +21,9 @@ if [[ -f "${AGENT_VERSIONS_FILE}" ]]; then
source "${AGENT_VERSIONS_FILE}"
fi
-CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-2.1.119}"
-CODEX_VERSION="${CODEX_VERSION:-0.125.0}"
-OPENCODE_VERSION="${OPENCODE_VERSION:-1.14.31}"
+CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-2.1.211}"
+CODEX_VERSION="${CODEX_VERSION:-0.144.5}"
+OPENCODE_VERSION="${OPENCODE_VERSION:-1.18.3}"
NODE_VERSION="${NODE_VERSION:-20.11.1}"
DEFAULT_HARBOR_MODEL="openai/gpt-5.2"
diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.py
index e30048b33..899e9318a 100644
--- a/tests/test_prepare_harbor_dataset.py
+++ b/tests/test_prepare_harbor_dataset.py
@@ -187,9 +187,9 @@ def test_prebuilt_docker_image_task_becomes_derived_dockerfile(tmp_path: Path) -
assert "docker_image" not in (task / "task.toml").read_text()
dockerfile = (task / "environment" / "Dockerfile").read_text()
assert dockerfile.startswith("FROM python:3.12-slim\nUSER root\n")
- assert "@anthropic-ai/[email protected]" in dockerfile
- assert "@openai/[email protected]" in dockerfile
- assert "[email protected]" in dockerfile
+ assert "@anthropic-ai/[email protected]" in dockerfile
+ assert "@openai/[email protected]" in dockerfile
+ assert "[email protected]" in dockerfile
def test_dockerfile_only_task_gets_prebake_layer(tmp_path: Path) -> None:
@@ -282,10 +282,10 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Pat
assert manifest["source_dataset"] == "[email protected]"
assert manifest["task_count"] == 2
assert manifest["agent_versions"] == {
- "CLAUDE_CODE_VERSION": "2.1.119",
- "CODEX_VERSION": "0.125.0",
+ "CLAUDE_CODE_VERSION": "2.1.211",
+ "CODEX_VERSION": "0.144.5",
"NODE_VERSION": "20.11.1",
- "OPENCODE_VERSION": "1.14.31",
+ "OPENCODE_VERSION": "1.18.3",
}
assert manifest["closed_book"]["proxy_asset_digest"].startswith("sha256:")
assert manifest["closed_book"]["verifier_egress"] == "open-via-authenticated-proxy"
diff --git a/tests/test_run_baseline_script.py b/tests/test_run_baseline_script.py
index b7ad22a2b..85550f743 100644
--- a/tests/test_run_baseline_script.py
+++ b/tests/test_run_baseline_script.py
@@ -185,7 +185,7 @@ def _write_closed_book_dataset(path: Path) -> Path:
{
"source_dataset": "[email protected]",
"task_count": 1,
- "agent_versions": {"codex": "0.125.0"},
+ "agent_versions": {"codex": "0.144.5"},
}
)
)
@@ -297,7 +297,7 @@ def test_dry_run_claude_code_opus_defaults_high_reasoning(tmp_path: Path) -> Non
assert result.returncode == 0, result.stderr
harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
assert _option_value(harbor, "--model") == "tb-lite-single-opus-4-7"
- assert "version=2.1.119" in _option_values(harbor, "--ak")
+ assert "version=2.1.211" in _option_values(harbor, "--ak")
assert "reasoning_effort=high" in _option_values(harbor, "--ak")
assert "reasoning: high" in result.stdout
assert "harbor_server: http://switchyard:4000" in result.stdout
@@ -318,7 +318,7 @@ def test_dry_run_codex_gpt_defaults_high_reasoning(tmp_path: Path) -> None:
assert result.returncode == 0, result.stderr
harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
assert _option_value(harbor, "--model") == "tb-lite-single-gpt-5-5"
- assert "version=0.125.0" in _option_values(harbor, "--ak")
+ assert "version=0.144.5" in _option_values(harbor, "--ak")
assert "reasoning_effort=high" in _option_values(harbor, "--ak")
catalog_env = next(
value
@@ -550,7 +550,7 @@ def test_dry_run_harbor_path_uses_local_dataset_and_closed_book_artifact(
assert "--dataset" not in harbor
assert _option_value(harbor, "--path") == str(dataset)
assert _option_value(harbor, "--artifact") == "/etc/proxy-public/strip.jsonl"
- assert "version=0.125.0" in _option_values(harbor, "--ak")
+ assert "version=0.144.5" in _option_values(harbor, "--ak")
assert "CODEX_DISABLE_WEB_SEARCH=1" in _option_values(harbor, "--ae")
catalog_env = next(
value
@@ -626,7 +626,7 @@ def test_dry_run_claude_closed_book_merges_disallowed_tools(tmp_path: Path) -> N
assert result.returncode == 0, result.stderr
harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
- assert "version=2.1.119" in _option_values(harbor, "--ak")
+ assert "version=2.1.211" in _option_values(harbor, "--ak")
assert "disallowed_tools=Bash,WebFetch,WebSearch" in _option_values(harbor, "--ak")
@@ -651,7 +651,7 @@ def test_dry_run_opencode_closed_book_disables_webfetch(tmp_path: Path) -> None:
assert result.returncode == 0, result.stderr
harbor = _line_argv(result.stdout, "HARBOR_CMD: ")
assert _option_value(harbor, "--model") == "nvidia/tb-lite-random-routing"
- assert "version=1.14.31" in _option_values(harbor, "--ak")
+ assert "version=1.18.3" in _option_values(harbor, "--ak")
assert "OPENCODE_DISABLE_WEBFETCH=1" in _option_values(harbor, "--ae")
diff --git a/tests/test_run_manifest.py b/tests/test_run_manifest.py
index 72a8c563b..89a2b36b7 100644
--- a/tests/test_run_manifest.py
+++ b/tests/test_run_manifest.py
@@ -291,7 +291,7 @@ def test_cli_write_records_closed_book_local_dataset_snapshot(tmp_path: Path) ->
json.dumps(
{
"source_dataset": "[email protected]",
- "agent_versions": {"codex": "0.125.0"},
+ "agent_versions": {"codex": "0.144.5"},
}
)
)
@@ -324,7 +324,7 @@ def test_cli_write_records_closed_book_local_dataset_snapshot(tmp_path: Path) ->
"--closed-book-proxy-strip-artifact",
"/etc/proxy-public/strip.jsonl",
"--agent-versions-json",
- '{"codex":"0.125.0"}',
+ '{"codex":"0.144.5"}',
"--run-dir",
str(run_dir),
"--log-path",
@@ -344,7 +344,7 @@ def test_cli_write_records_closed_book_local_dataset_snapshot(tmp_path: Path) ->
assert manifest["closed_book"]["gateway_enforced"] is True
assert manifest["closed_book"]["hosted_tools_disabled"] is True
assert manifest["closed_book"]["verifier_egress"] == "open-via-authenticated-proxy"
- assert manifest["closed_book"]["agent_versions"] == {"codex": "0.125.0"}
+ assert manifest["closed_book"]["agent_versions"] == {"codex": "0.144.5"}
snapshot = run_dir / "dataset" / "switchyard_dataset_manifest.json"
assert manifest["closed_book"]["dataset_manifest_snapshot"] == str(snapshot)
assert snapshot.read_text() == source_manifest.read_text()