Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
chore: bump benchmark agent harness pins to current releases (#83)
ryan-lempkace7beadc625b
diff --git a/benchmark/agent-versions.env b/benchmark/agent-versions.envindex 8bf04c592..843959969 100644--- a/benchmark/agent-versions.env+++ b/benchmark/agent-versions.env@@ -1,7 +1,7 @@# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.# SPDX-License-Identifier: Apache-2.0-CLAUDE_CODE_VERSION=2.1.119-CODEX_VERSION=0.125.0-OPENCODE_VERSION=1.14.31+CLAUDE_CODE_VERSION=2.1.211+CODEX_VERSION=0.144.5+OPENCODE_VERSION=1.18.3NODE_VERSION=20.11.1diff --git a/benchmark/run-baseline.sh b/benchmark/run-baseline.shindex bd9d2748b..4a0a4952c 100755--- a/benchmark/run-baseline.sh+++ b/benchmark/run-baseline.sh@@ -21,9 +21,9 @@ if [[ -f "${AGENT_VERSIONS_FILE}" ]]; thensource "${AGENT_VERSIONS_FILE}"fi-CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-2.1.119}"-CODEX_VERSION="${CODEX_VERSION:-0.125.0}"-OPENCODE_VERSION="${OPENCODE_VERSION:-1.14.31}"+CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-2.1.211}"+CODEX_VERSION="${CODEX_VERSION:-0.144.5}"+OPENCODE_VERSION="${OPENCODE_VERSION:-1.18.3}"NODE_VERSION="${NODE_VERSION:-20.11.1}"DEFAULT_HARBOR_MODEL="openai/gpt-5.2"diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.pyindex e30048b33..899e9318a 100644--- a/tests/test_prepare_harbor_dataset.py+++ b/tests/test_prepare_harbor_dataset.py@@ -187,9 +187,9 @@ def test_prebuilt_docker_image_task_becomes_derived_dockerfile(tmp_path: Path) -assert "docker_image" not in (task / "task.toml").read_text()dockerfile = (task / "environment" / "Dockerfile").read_text()assert dockerfile.startswith("FROM python:3.12-slim\nUSER root\n")- assert "@anthropic-ai/[email protected]" in dockerfile- assert "@openai/[email protected]" in dockerfile- assert "[email protected]" in dockerfile+ assert "@anthropic-ai/[email protected]" in dockerfile+ assert "@openai/[email protected]" in dockerfile+ assert "[email protected]" in dockerfiledef test_dockerfile_only_task_gets_prebake_layer(tmp_path: Path) -> None:@@ -282,10 +282,10 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Patassert manifest["source_dataset"] == "[email protected]"assert manifest["task_count"] == 2assert manifest["agent_versions"] == {- "CLAUDE_CODE_VERSION": "2.1.119",- "CODEX_VERSION": "0.125.0",+ "CLAUDE_CODE_VERSION": "2.1.211",+ "CODEX_VERSION": "0.144.5","NODE_VERSION": "20.11.1",- "OPENCODE_VERSION": "1.14.31",+ "OPENCODE_VERSION": "1.18.3",}assert manifest["closed_book"]["proxy_asset_digest"].startswith("sha256:")assert manifest["closed_book"]["verifier_egress"] == "open-via-authenticated-proxy"diff --git a/tests/test_run_baseline_script.py b/tests/test_run_baseline_script.pyindex b7ad22a2b..85550f743 100644--- a/tests/test_run_baseline_script.py+++ b/tests/test_run_baseline_script.py@@ -185,7 +185,7 @@ def _write_closed_book_dataset(path: Path) -> Path:{"source_dataset": "[email protected]","task_count": 1,- "agent_versions": {"codex": "0.125.0"},+ "agent_versions": {"codex": "0.144.5"},}))@@ -297,7 +297,7 @@ def test_dry_run_claude_code_opus_defaults_high_reasoning(tmp_path: Path) -> Nonassert result.returncode == 0, result.stderrharbor = _line_argv(result.stdout, "HARBOR_CMD: ")assert _option_value(harbor, "--model") == "tb-lite-single-opus-4-7"- assert "version=2.1.119" in _option_values(harbor, "--ak")+ assert "version=2.1.211" in _option_values(harbor, "--ak")assert "reasoning_effort=high" in _option_values(harbor, "--ak")assert "reasoning: high" in result.stdoutassert "harbor_server: http://switchyard:4000" in result.stdout@@ -318,7 +318,7 @@ def test_dry_run_codex_gpt_defaults_high_reasoning(tmp_path: Path) -> None:assert result.returncode == 0, result.stderrharbor = _line_argv(result.stdout, "HARBOR_CMD: ")assert _option_value(harbor, "--model") == "tb-lite-single-gpt-5-5"- assert "version=0.125.0" in _option_values(harbor, "--ak")+ assert "version=0.144.5" in _option_values(harbor, "--ak")assert "reasoning_effort=high" in _option_values(harbor, "--ak")catalog_env = next(value@@ -550,7 +550,7 @@ def test_dry_run_harbor_path_uses_local_dataset_and_closed_book_artifact(assert "--dataset" not in harborassert _option_value(harbor, "--path") == str(dataset)assert _option_value(harbor, "--artifact") == "/etc/proxy-public/strip.jsonl"- assert "version=0.125.0" in _option_values(harbor, "--ak")+ assert "version=0.144.5" in _option_values(harbor, "--ak")assert "CODEX_DISABLE_WEB_SEARCH=1" in _option_values(harbor, "--ae")catalog_env = next(value@@ -626,7 +626,7 @@ def test_dry_run_claude_closed_book_merges_disallowed_tools(tmp_path: Path) -> Nassert result.returncode == 0, result.stderrharbor = _line_argv(result.stdout, "HARBOR_CMD: ")- assert "version=2.1.119" in _option_values(harbor, "--ak")+ assert "version=2.1.211" in _option_values(harbor, "--ak")assert "disallowed_tools=Bash,WebFetch,WebSearch" in _option_values(harbor, "--ak")@@ -651,7 +651,7 @@ def test_dry_run_opencode_closed_book_disables_webfetch(tmp_path: Path) -> None:assert result.returncode == 0, result.stderrharbor = _line_argv(result.stdout, "HARBOR_CMD: ")assert _option_value(harbor, "--model") == "nvidia/tb-lite-random-routing"- assert "version=1.14.31" in _option_values(harbor, "--ak")+ assert "version=1.18.3" in _option_values(harbor, "--ak")assert "OPENCODE_DISABLE_WEBFETCH=1" in _option_values(harbor, "--ae")diff --git a/tests/test_run_manifest.py b/tests/test_run_manifest.pyindex 72a8c563b..89a2b36b7 100644--- a/tests/test_run_manifest.py+++ b/tests/test_run_manifest.py@@ -291,7 +291,7 @@ def test_cli_write_records_closed_book_local_dataset_snapshot(tmp_path: Path) ->json.dumps({"source_dataset": "[email protected]",- "agent_versions": {"codex": "0.125.0"},+ "agent_versions": {"codex": "0.144.5"},}))@@ -324,7 +324,7 @@ def test_cli_write_records_closed_book_local_dataset_snapshot(tmp_path: Path) ->"--closed-book-proxy-strip-artifact","/etc/proxy-public/strip.jsonl","--agent-versions-json",- '{"codex":"0.125.0"}',+ '{"codex":"0.144.5"}',"--run-dir",str(run_dir),"--log-path",@@ -344,7 +344,7 @@ def test_cli_write_records_closed_book_local_dataset_snapshot(tmp_path: Path) ->assert manifest["closed_book"]["gateway_enforced"] is Trueassert manifest["closed_book"]["hosted_tools_disabled"] is Trueassert manifest["closed_book"]["verifier_egress"] == "open-via-authenticated-proxy"- assert manifest["closed_book"]["agent_versions"] == {"codex": "0.125.0"}+ assert manifest["closed_book"]["agent_versions"] == {"codex": "0.144.5"}snapshot = run_dir / "dataset" / "switchyard_dataset_manifest.json"assert manifest["closed_book"]["dataset_manifest_snapshot"] == str(snapshot)assert snapshot.read_text() == source_manifest.read_text()