Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Unified single-run Harbor baseline launcher for Switchyard evaluations.
#
# Writes run_manifest.json before Harbor starts, then runs the server + Harbor
# under a background nohup wrapper by default. The wrapper finalizes the
# manifest with Harbor exit status and artifact presence after the run ends.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
MANIFEST_HELPER="${SCRIPT_DIR}/run_manifest.py"
PATCH_FILE="${SWITCHYARD_HARBOR_PATCH_FILE:-${SCRIPT_DIR}/patches/harbor-agent-patches.diff}"
AGENT_VERSIONS_FILE="${SCRIPT_DIR}/agent-versions.env"
if [[ -f "${AGENT_VERSIONS_FILE}" ]]; then
# shellcheck source=/dev/null
source "${AGENT_VERSIONS_FILE}"
fi
CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-2.1.211}"
CODEX_VERSION="${CODEX_VERSION:-0.144.5}"
OPENCODE_VERSION="${OPENCODE_VERSION:-1.18.3}"
PI_VERSION="${PI_VERSION:-0.84.3}"
NODE_VERSION="${NODE_VERSION:-20.11.1}"
DEFAULT_HARBOR_MODEL="openai/gpt-5.2"
SERVER_PRESET="serve"
MODE=""
OUTPUT_DIR="${REPO_ROOT}/benchmark/tb_runs"
PORT=4000
SERVER_URL=""
HARBOR_SERVER_URL=""
SERVER_CONFIG=""
MODEL=""
ROUTE_MODEL=""
AGENT="terminus-2"
HARBOR_MODEL="${DEFAULT_HARBOR_MODEL}"
HARBOR_PATH=""
HARBOR_BIN="${HARBOR_BIN:-}"
HARBOR_PYTHON="${HARBOR_PYTHON:-}"
HARBOR_SOURCE=""
BOOK_MODE="closed"
PROXY_STRIP_ARTIFACT="/etc/proxy-ca/strip.jsonl"
VERIFIER_PROXY_TEMPLATE='${SWITCHYARD_VERIFIER_HTTP_PROXY}'
SWITCHYARD_DOCKER_IMAGE="${SWITCHYARD_DOCKER_IMAGE:-switchyard-baseline:local}"
SWITCHYARD_DOCKERFILE="${SWITCHYARD_DOCKERFILE:-Dockerfile}"
SWITCHYARD_DOCKER_SERVICE_NAME="${SWITCHYARD_DOCKER_SERVICE_NAME:-switchyard}"
SWITCHYARD_DOCKER_BUILD="${SWITCHYARD_DOCKER_BUILD:-1}"
SWITCHYARD_DOCKER_NETWORK="${SWITCHYARD_DOCKER_NETWORK:-}"
SWITCHYARD_DOCKER_CONTAINER="${SWITCHYARD_DOCKER_CONTAINER:-}"
UPSTREAM_BASE_URL="${UPSTREAM_BASE_URL:-}"
UPSTREAM_API_KEY_ENV="${UPSTREAM_API_KEY_ENV:-}"
REASONING_EFFORT=""
N_CONCURRENT=8
MAX_RETRIES=2
AGENT_TIMEOUT_MULTIPLIER="1.0"
N_TASKS=""
TASK_ID=""
TASK_LIST_FILE=""
DRY_RUN=0
FOREGROUND=0
SKIP_HEALTH_CHECK=0
HARBOR_MODEL_SET=0
REASONING_EFFORT_SET=0
SWITCHYARD_ENABLED=1
HARBOR_EXTRA=()
HARBOR_CMD_PREFIX=()
HARBOR_PYTHON_CMD=()
SERVER_EXTRA=()
SERVER_DOCKER_CMD=()
ORIGINAL_ARGV=("$@")
HARBOR_PATCH_JSON="{}"
usage() {
cat <<'EOF'
Run one pinned Harbor baseline and write a per-run manifest.
Required:
--harbor-path PATH Local generated Harbor dataset path.
Use benchmark/prepare_harbor_dataset.py to build it.
Main options:
--mode NAME Manifest server mode label (default: serve).
--output-dir PATH Root output dir (default: benchmark/tb_runs)
--port N Server port (default: 4000)
--server-url URL Host URL used for health and stats checks.
--harbor-server-url URL Server URL as seen from Harbor task containers.
Defaults to the Dockerized Switchyard service.
--server-config PATH Rust switchyard-server TOML configuration. When
omitted, Harbor connects directly upstream.
--model MODEL Model Harbor should request. With
--server-config this is a Switchyard route
key; without it, this is the upstream model.
Defaults --harbor-model to nvidia/MODEL for
opencode, switchyard/MODEL for pi, openai/MODEL
for other OpenAI-style agents, and MODEL for
claude-code/codex.
--route-model MODEL Deprecated alias for --model in Switchyard mode.
--agent NAME Harbor agent (default: terminus-2)
--harbor-model MODEL Explicit Harbor model label override.
--upstream-base-url URL Direct-upstream OpenAI-compatible base URL
(default: https://openrouter.ai/api/v1).
--upstream-api-key-env NAME Env var containing the direct upstream API key
(default: UPSTREAM_API_KEY for a custom
upstream URL, otherwise OPENROUTER_API_KEY).
--harbor-path PATH Local Harbor dataset path; runner passes --path.
--book-mode MODE closed or open (default: closed). Both modes use
the generated dataset proxy topology; open mode
allows broad egress through the proxy.
--reasoning-effort VALUE Forwarded as --ak reasoning_effort=VALUE
(--ak thinking=VALUE for pi).
Defaults by agent/model; pass an empty value to omit.
--harbor-bin PATH Optional Harbor executable override
(default: uv run --no-sync harbor)
--harbor-python PATH Optional Python that can import the active Harbor
(default: uv run --no-sync python)
--n-concurrent N Harbor concurrency (default: 8)
--max-retries N Harbor max retries (default: 2)
--agent-timeout-multiplier N Harbor agent timeout multiplier (default: 1.0)
--n-tasks N Optional Harbor task cap
--task-id NAME Optional single task filter
--task-list-file PATH File of task names, one per line, # comments allowed
--harbor-extra ARG Repeatable raw Harbor arg token
--server-extra ARG Repeatable raw server-launch arg token
SWITCHYARD_CLOSED_BOOK_PREFLIGHT=0 disables the benchmark Docker
reachability preflight in the background wrapper.
SWITCHYARD_DOCKER_IMAGE Closed-book Switchyard server image
(default: switchyard-baseline:local).
SWITCHYARD_DOCKER_BUILD=0 Reuse SWITCHYARD_DOCKER_IMAGE instead of building it.
SWITCHYARD_DOCKER_NETWORK Optional existing Docker network for closed-book runs.
--dry-run Print resolved commands and exit
Execution:
--foreground Run wrapper in the foreground instead of nohup
--skip-health-check Do not wait for /health before Harbor
EOF
}
die() {
echo "ERROR: $*" >&2
exit 1
}
q() {
printf "%q" "$1"
}
shell_join() {
printf "%q " "$@"
}
json_array() {
python3 - "$@" <<'PY'
import json
import sys
print(json.dumps(sys.argv[1:]))
PY
}
json_object_from_pairs() {
python3 - "$@" <<'PY'
import json
import sys
items = sys.argv[1:]
out = {}
for idx in range(0, len(items), 2):
key = items[idx]
value = items[idx + 1] if idx + 1 < len(items) else ""
if value != "":
out[key] = value
print(json.dumps(out, sort_keys=True))
PY
}
json_merge_object_pairs() {
python3 - "$@" <<'PY'
import json
import sys
out = json.loads(sys.argv[1] or "{}")
items = sys.argv[2:]
for idx in range(0, len(items), 2):
key = items[idx]
value = items[idx + 1] if idx + 1 < len(items) else ""
if value != "":
out[key] = value
print(json.dumps(out, sort_keys=True))
PY
}
lower_ascii() {
printf '%s' "$1" | tr '[:upper:]' '[:lower:]'
}
default_reasoning_effort_for() {
local agent model
agent="$(lower_ascii "$1")"
model="$(lower_ascii "$2")"
case "${agent}:${model}" in
claude-code:*opus*|claude-code:*sonnet*|claude-code:*haiku*) echo "high" ;;
codex:*gpt-5*|codex:*o3*|codex:*o4*) echo "high" ;;
*) echo "" ;;
esac
}
csv_append_unique() {
python3 - "$@" <<'PY'
import sys
items = [item.strip() for item in sys.argv[1].split(",") if item.strip()]
for item in sys.argv[2:]:
if item not in items:
items.append(item)
print(",".join(items))
PY
}
merge_claude_closed_book_tools() {
local i=0 token next value found=0
local merged=()
while [[ "${i}" -lt "${#HARBOR_EXTRA[@]}" ]]; do
token="${HARBOR_EXTRA[$i]}"
if [[ "${token}" == "--ak" || "${token}" == "--agent-kwarg" ]]; then
next="${HARBOR_EXTRA[$((i + 1))]:-}"
if [[ "${next}" == disallowed_tools=* ]]; then
value="${next#disallowed_tools=}"
merged+=("${token}" "disallowed_tools=$(csv_append_unique "${value}" WebFetch WebSearch)")
found=1
i=$((i + 2))
continue
fi
fi
merged+=("${token}")
i=$((i + 1))
done
HARBOR_EXTRA=()
if [[ "${#merged[@]}" -gt 0 ]]; then
HARBOR_EXTRA=("${merged[@]}")
fi
if [[ "${found}" -eq 0 ]]; then
HARBOR_CMD+=(--ak "disallowed_tools=WebFetch,WebSearch")
fi
}
harbor_extra_has_agent_version() {
harbor_extra_has_agent_kwarg version
}
harbor_extra_has_agent_kwarg() {
local key="$1"
local i=0 token next
while [[ "${i}" -lt "${#HARBOR_EXTRA[@]}" ]]; do
token="${HARBOR_EXTRA[$i]}"
if [[ "${token}" == "--ak" || "${token}" == "--agent-kwarg" ]]; then
next="${HARBOR_EXTRA[$((i + 1))]:-}"
[[ "${next}" == "${key}="* ]] && return 0
i=$((i + 2))
continue
fi
i=$((i + 1))
done
return 1
}
harbor_extra_has_agent_env() {
local key="$1" i=0 token next
while [[ "${i}" -lt "${#HARBOR_EXTRA[@]}" ]]; do
token="${HARBOR_EXTRA[$i]}"
if [[ "${token}" == "--ae" || "${token}" == "--agent-env" ]]; then
next="${HARBOR_EXTRA[$((i + 1))]:-}"
[[ "${next}" == "${key}="* ]] && return 0
i=$((i + 2))
continue
fi
i=$((i + 1))
done
return 1
}
add_agent_version_kwarg() {
harbor_extra_has_agent_version && return 0
case "${AGENT}" in
claude-code) HARBOR_CMD+=(--ak "version=${CLAUDE_CODE_VERSION}") ;;
codex) HARBOR_CMD+=(--ak "version=${CODEX_VERSION}") ;;
opencode) HARBOR_CMD+=(--ak "version=${OPENCODE_VERSION}") ;;
pi) HARBOR_CMD+=(--ak "version=${PI_VERSION}") ;;
esac
}
while [[ $# -gt 0 ]]; do
case "$1" in
--mode) MODE="$2"; shift 2 ;;
--output-dir) OUTPUT_DIR="$2"; shift 2 ;;
--port) PORT="$2"; shift 2 ;;
--server-url) SERVER_URL="$2"; shift 2 ;;
--harbor-server-url) HARBOR_SERVER_URL="$2"; shift 2 ;;
--server-config) SERVER_CONFIG="$2"; shift 2 ;;
--routing-profiles)
die "--routing-profiles is no longer supported by the benchmark runner; use --server-config with a Rust server TOML file"
;;
--model) MODEL="$2"; shift 2 ;;
--route-model) ROUTE_MODEL="$2"; shift 2 ;;
--agent) AGENT="$2"; shift 2 ;;
--harbor-model) HARBOR_MODEL="$2"; HARBOR_MODEL_SET=1; shift 2 ;;
--upstream-base-url) UPSTREAM_BASE_URL="$2"; shift 2 ;;
--upstream-api-key-env) UPSTREAM_API_KEY_ENV="$2"; shift 2 ;;
--harbor-path) HARBOR_PATH="$2"; shift 2 ;;
--book-mode) BOOK_MODE="$2"; shift 2 ;;
--harbor-bin) HARBOR_BIN="$2"; shift 2 ;;
--harbor-python) HARBOR_PYTHON="$2"; shift 2 ;;
--reasoning-effort) REASONING_EFFORT="$2"; REASONING_EFFORT_SET=1; shift 2 ;;
--n-concurrent) N_CONCURRENT="$2"; shift 2 ;;
--max-retries) MAX_RETRIES="$2"; shift 2 ;;
--agent-timeout-multiplier) AGENT_TIMEOUT_MULTIPLIER="$2"; shift 2 ;;
--n-tasks) N_TASKS="$2"; shift 2 ;;
--task-id) TASK_ID="$2"; shift 2 ;;
--task-list-file) TASK_LIST_FILE="$2"; shift 2 ;;
--harbor-extra) HARBOR_EXTRA+=("$2"); shift 2 ;;
--server-extra) SERVER_EXTRA+=("$2"); shift 2 ;;
--strong-model|--weak-model|--classifier-model|--profile|--strong-probability)
die "$1 belongs to the removed legacy benchmark launchers; encode server routing in --server-config instead"
;;
--dry-run) DRY_RUN=1; shift ;;
--foreground) FOREGROUND=1; shift ;;
--skip-health-check) SKIP_HEALTH_CHECK=1; shift ;;
-h|--help) usage; exit 0 ;;
*) die "unknown argument: $1" ;;
esac
done
cd "${REPO_ROOT}"
resolve_harbor() {
if [[ -n "${HARBOR_BIN}" ]]; then
[[ -x "${HARBOR_BIN}" ]] || die "--harbor-bin is not executable: ${HARBOR_BIN}"
HARBOR_BIN="$(cd "$(dirname "${HARBOR_BIN}")" && pwd)/$(basename "${HARBOR_BIN}")"
HARBOR_SOURCE="override"
HARBOR_CMD_PREFIX=("${HARBOR_BIN}")
else
HARBOR_SOURCE="uv-dev"
HARBOR_CMD_PREFIX=(uv run --no-sync harbor)
fi
if [[ -n "${HARBOR_PYTHON}" ]]; then
[[ -x "${HARBOR_PYTHON}" ]] || die "--harbor-python is not executable: ${HARBOR_PYTHON}"
HARBOR_PYTHON="$(cd "$(dirname "${HARBOR_PYTHON}")" && pwd)/$(basename "${HARBOR_PYTHON}")"
HARBOR_PYTHON_CMD=("${HARBOR_PYTHON}")
elif [[ "${HARBOR_SOURCE}" == "uv-dev" ]]; then
HARBOR_PYTHON_CMD=(uv run --no-sync python)
else
HARBOR_PYTHON_CMD=(python3)
fi
}
resolve_harbor
if [[ -n "${SERVER_CONFIG}" ]]; then
[[ -f "${SERVER_CONFIG}" ]] || die "--server-config not found: ${SERVER_CONFIG}"
SERVER_CONFIG="$(cd "$(dirname "${SERVER_CONFIG}")" && pwd)/$(basename "${SERVER_CONFIG}")"
fi
UPSTREAM_BASE_URL="${UPSTREAM_BASE_URL:-https://openrouter.ai/api/v1}"
if [[ -z "${UPSTREAM_API_KEY_ENV}" ]]; then
if [[ -n "${UPSTREAM_API_KEY:-}" && "${UPSTREAM_BASE_URL}" != "https://openrouter.ai/api/v1" ]]; then
UPSTREAM_API_KEY_ENV="UPSTREAM_API_KEY" # pragma: allowlist secret
else
UPSTREAM_API_KEY_ENV="OPENROUTER_API_KEY" # pragma: allowlist secret
fi
fi
[[ "${UPSTREAM_API_KEY_ENV}" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || die "--upstream-api-key-env must be a valid environment variable name"
if [[ -n "${SERVER_CONFIG}" ]]; then
if [[ -n "${MODEL}" && -n "${ROUTE_MODEL}" && "${MODEL}" != "${ROUTE_MODEL}" ]]; then
die "--model and --route-model must name the same model when both are provided"
fi
if [[ -z "${MODEL}" && -n "${ROUTE_MODEL}" ]]; then
MODEL="${ROUTE_MODEL}"
fi
SWITCHYARD_ENABLED=1
SERVER_PRESET="serve"
[[ -z "${MODE}" ]] && MODE="serve"
[[ -n "${MODEL}" ]] || die "--model is required when using --server-config"
ROUTE_MODEL="${MODEL}"
else
SWITCHYARD_ENABLED=0
SERVER_PRESET="direct"
[[ -z "${MODE}" ]] && MODE="direct"
[[ -z "${ROUTE_MODEL}" ]] || die "--route-model requires --server-config; use --model for direct upstream"
[[ -n "${MODEL}" ]] || die "--model is required when running direct upstream"
ROUTE_MODEL=""
[[ -z "${SERVER_URL}" ]] || die "--server-url requires --server-config"
[[ -z "${HARBOR_SERVER_URL}" ]] || die "--harbor-server-url requires --server-config"
[[ "${#SERVER_EXTRA[@]}" -eq 0 ]] || die "--server-extra requires --server-config"
[[ -n "${!UPSTREAM_API_KEY_ENV:-}" ]] || die "direct upstream requires \$${UPSTREAM_API_KEY_ENV} to be set"
fi
if [[ "${HARBOR_MODEL_SET}" -eq 0 ]]; then
if [[ "${SWITCHYARD_ENABLED}" -eq 1 ]]; then
case "${AGENT}" in
claude-code|codex) HARBOR_MODEL="${MODEL}" ;;
opencode) HARBOR_MODEL="nvidia/${MODEL}" ;;
pi) HARBOR_MODEL="switchyard/${MODEL}" ;;
*) HARBOR_MODEL="openai/${MODEL}" ;;
esac
else
HARBOR_MODEL="${MODEL}"
fi
fi
check_harbor_available() {
if ! "${HARBOR_CMD_PREFIX[@]}" --version >/dev/null 2>&1; then
echo "ERROR: Harbor is not available via:" >&2
printf ' ' >&2
shell_join "${HARBOR_CMD_PREFIX[@]}" >&2
echo >&2
echo "Run 'uv sync' first, or pass --harbor-bin." >&2
exit 1
fi
}
if [[ "${SWITCHYARD_ENABLED}" -eq 1 ]]; then
SERVER_CMD=(switchyard-server
--config "${SERVER_CONFIG}"
--host 0.0.0.0
--port "${PORT}")
SERVER_DOCKER_CMD=(--config "${SERVER_CONFIG}"
--host 0.0.0.0
--port "${PORT}")
SERVER_CONFIG_JSON="$(json_object_from_pairs \
mode "${MODE}" server_config "${SERVER_CONFIG}" route_model "${ROUTE_MODEL}")"
if [[ "${#SERVER_EXTRA[@]}" -gt 0 ]]; then
SERVER_CMD+=("${SERVER_EXTRA[@]}")
SERVER_DOCKER_CMD+=("${SERVER_EXTRA[@]}")
fi
else
UPSTREAM_BASE_URL="${UPSTREAM_BASE_URL%/}"
SERVER_CONFIG_JSON="$(json_object_from_pairs \
mode "${MODE}" upstream_base_url "${UPSTREAM_BASE_URL}" \
upstream_api_key_env "${UPSTREAM_API_KEY_ENV}" upstream_model "${MODEL}")"
fi
if [[ -n "${HARBOR_PATH}" ]]; then
[[ -d "${HARBOR_PATH}" ]] || die "--harbor-path must be a directory: ${HARBOR_PATH}"
HARBOR_PATH="$(cd "${HARBOR_PATH}" && pwd)"
fi
if [[ -z "${HARBOR_PATH}" ]]; then
die "--harbor-path is required. Prepare one with: uv run --no-sync python benchmark/prepare_harbor_dataset.py"
fi
case "${BOOK_MODE}" in
closed|open) ;;
*) die "--book-mode must be closed or open" ;;
esac
[[ -f "${HARBOR_PATH}/switchyard_dataset_manifest.json" ]] || die "--harbor-path must contain switchyard_dataset_manifest.json: ${HARBOR_PATH}"
DATASET="$(basename "${HARBOR_PATH}")"
if [[ "${REASONING_EFFORT_SET}" -eq 0 ]]; then
REASONING_EFFORT="$(default_reasoning_effort_for "${AGENT}" "${HARBOR_MODEL}")"
fi
if [[ -n "${TASK_LIST_FILE}" ]]; then
[[ -f "${TASK_LIST_FILE}" ]] || die "--task-list-file not found: ${TASK_LIST_FILE}"
TASK_LIST_FILE="$(cd "$(dirname "${TASK_LIST_FILE}")" && pwd)/$(basename "${TASK_LIST_FILE}")"
fi
if [[ "${SWITCHYARD_ENABLED}" -eq 1 ]]; then
SERVER_CONTROL_URL="${SERVER_URL:-http://127.0.0.1:${PORT}}"
SERVER_CONTROL_URL="${SERVER_CONTROL_URL%/}"
if [[ "${SERVER_CONTROL_URL}" == */v1 ]]; then
SERVER_ROOT_URL="${SERVER_CONTROL_URL%/v1}"
else
SERVER_ROOT_URL="${SERVER_CONTROL_URL}"
fi
SERVER_HEALTH_URL="${SERVER_ROOT_URL}/health"
SERVER_METRICS_URL="${SERVER_ROOT_URL}/metrics"
SERVER_STATS_URL="${SERVER_ROOT_URL}/v1/stats"
if [[ -z "${HARBOR_SERVER_URL}" ]]; then
HARBOR_SERVER_URL="http://${SWITCHYARD_DOCKER_SERVICE_NAME}:${PORT}"
fi
HARBOR_SERVER_URL="${HARBOR_SERVER_URL%/}"
if [[ "${HARBOR_SERVER_URL}" == */v1 ]]; then
HARBOR_SERVER_ROOT_URL="${HARBOR_SERVER_URL%/v1}"
HARBOR_BASE_URL="${HARBOR_SERVER_URL}"
else
HARBOR_SERVER_ROOT_URL="${HARBOR_SERVER_URL}"
HARBOR_BASE_URL="${HARBOR_SERVER_ROOT_URL}/v1"
fi
else
SERVER_ROOT_URL=""
SERVER_HEALTH_URL=""
SERVER_METRICS_URL=""
SERVER_STATS_URL=""
HARBOR_BASE_URL="${UPSTREAM_BASE_URL}"
if [[ "${HARBOR_BASE_URL}" == */v1 ]]; then
HARBOR_SERVER_ROOT_URL="${HARBOR_BASE_URL%/v1}"
else
HARBOR_SERVER_ROOT_URL="${HARBOR_BASE_URL}"
fi
fi
TS="$(date -u +%Y-%m-%d_%H-%M-%S)"
JOB_NAME="baseline-${SERVER_PRESET}-${MODE}-${TS}"
OUTPUT_DIR="$(mkdir -p "${OUTPUT_DIR}" && cd "${OUTPUT_DIR}" && pwd)"
RUN_DIR="${OUTPUT_DIR}/${JOB_NAME}"
MANIFEST_PATH="${RUN_DIR}/run_manifest.json"
LOG_PATH="${RUN_DIR}/${JOB_NAME}.log"
SERVER_LOG="${RUN_DIR}/server.log"
HARBOR_LOG="${RUN_DIR}/harbor.log"
HARBOR_RESULT_JSON="${RUN_DIR}/harbor_result.json"
SERVER_METRICS_PROM="${RUN_DIR}/server_metrics_final.prom"
ROUTING_STATS_JSON="${RUN_DIR}/routing_stats_final.json"
DOCKER_RUN_ID="$(printf '%s-%s' "${TS##*_}" "$$" | tr -c '[:alnum:]_.-' '-')"
SWITCHYARD_DOCKER_NETWORK="${SWITCHYARD_DOCKER_NETWORK:-switchyard-${DOCKER_RUN_ID}}"
SWITCHYARD_DOCKER_CONTAINER="${SWITCHYARD_DOCKER_CONTAINER:-switchyard-${DOCKER_RUN_ID}}"
CODEX_MODEL_CATALOG_HOST=""
if [[ "${AGENT}" == "codex" ]]; then
CODEX_MODEL_CATALOG_HOST="${RUN_DIR}/codex_model_catalog.json"
fi
if [[ "${BOOK_MODE}" == "closed" || "${BOOK_MODE}" == "open" ]]; then
if [[ "${SWITCHYARD_ENABLED}" -eq 1 ]]; then
SERVER_CONFIG_JSON="$(json_merge_object_pairs "${SERVER_CONFIG_JSON}" \
docker_image "${SWITCHYARD_DOCKER_IMAGE}" \
docker_network "${SWITCHYARD_DOCKER_NETWORK}" \
docker_container "${SWITCHYARD_DOCKER_CONTAINER}" \
docker_service "${SWITCHYARD_DOCKER_SERVICE_NAME}")"
else
SERVER_CONFIG_JSON="$(json_merge_object_pairs "${SERVER_CONFIG_JSON}" \
docker_network "${SWITCHYARD_DOCKER_NETWORK}")"
fi
fi
SERVER_CONFIG_DIR=""
if [[ -n "${SERVER_CONFIG}" ]]; then
SERVER_CONFIG_DIR="$(cd "$(dirname "${SERVER_CONFIG}")" && pwd)"
fi
WRAPPER="${RUN_DIR}/run-background.sh"
HARBOR_JOB_DIR="${RUN_DIR}/jobs/${JOB_NAME}"
HARBOR_CMD=("${HARBOR_CMD_PREFIX[@]}" run
--agent "${AGENT}"
--model "${HARBOR_MODEL}"
--jobs-dir "${RUN_DIR}/jobs"
--job-name "${JOB_NAME}"
-n "${N_CONCURRENT}"
--max-retries "${MAX_RETRIES}"
--agent-timeout-multiplier "${AGENT_TIMEOUT_MULTIPLIER}")
HARBOR_CMD+=(--path "${HARBOR_PATH}")
if [[ -n "${CODEX_MODEL_CATALOG_HOST}" ]]; then
if ! harbor_extra_has_agent_env CODEX_MODEL_CATALOG_JSON; then
HARBOR_CMD+=(--ae "CODEX_MODEL_CATALOG_JSON=${CODEX_MODEL_CATALOG_HOST}")
fi
fi
HARBOR_CMD+=(--artifact "${PROXY_STRIP_ARTIFACT}")
if [[ "${BOOK_MODE}" == "closed" ]]; then
HARBOR_CMD+=(--ve "HTTP_PROXY=${VERIFIER_PROXY_TEMPLATE}")
HARBOR_CMD+=(--ve "HTTPS_PROXY=${VERIFIER_PROXY_TEMPLATE}")
HARBOR_CMD+=(--ve "http_proxy=${VERIFIER_PROXY_TEMPLATE}")
HARBOR_CMD+=(--ve "https_proxy=${VERIFIER_PROXY_TEMPLATE}")
HARBOR_CMD+=(--ve "NO_PROXY=localhost,127.0.0.1,proxy")
HARBOR_CMD+=(--ve "no_proxy=localhost,127.0.0.1,proxy")
case "${AGENT}" in
codex) HARBOR_CMD+=(--ae "CODEX_DISABLE_WEB_SEARCH=1") ;;
claude-code) merge_claude_closed_book_tools ;;
opencode) HARBOR_CMD+=(--ae "OPENCODE_DISABLE_WEBFETCH=1") ;;
esac
fi
add_agent_version_kwarg
if [[ -n "${REASONING_EFFORT}" ]]; then
if [[ "${AGENT}" == "pi" ]]; then
# Harbor's pi agent takes --thinking (off|minimal|low|medium|high|xhigh), not reasoning_effort.
HARBOR_CMD+=(--ak "thinking=${REASONING_EFFORT}")
else
HARBOR_CMD+=(--ak "reasoning_effort=${REASONING_EFFORT}")
fi
# Claude code does not support thinking=true for now, so we use adaptive instead
if [[ "${AGENT}" == "claude-code" ]] && ! harbor_extra_has_agent_kwarg thinking; then
HARBOR_CMD+=(--ak "thinking=adaptive")
fi
fi
if [[ -n "${N_TASKS}" ]]; then
HARBOR_CMD+=(--n-tasks "${N_TASKS}")
fi
if [[ -n "${TASK_ID}" ]]; then
HARBOR_CMD+=(--include-task-name "${TASK_ID}")
fi
if [[ -n "${TASK_LIST_FILE}" ]]; then
while IFS= read -r raw_task || [[ -n "${raw_task}" ]]; do
task="${raw_task%%#*}"
task="${task//[[:space:]]/}"
[[ -n "${task}" ]] && HARBOR_CMD+=(--include-task-name "${task}")
done < "${TASK_LIST_FILE}"
fi
if [[ "${#HARBOR_EXTRA[@]}" -gt 0 ]]; then
HARBOR_CMD+=("${HARBOR_EXTRA[@]}")
fi
check_harbor_patch() {
local py_display site patch_root verify_method verify_status verify_error
py_display="$(shell_join "${HARBOR_PYTHON_CMD[@]}")"
site="$(
"${HARBOR_PYTHON_CMD[@]}" - <<'PY' 2>/dev/null || true
from pathlib import Path
import harbor
print(Path(harbor.__file__).resolve().parent)
PY
)"
if [[ -z "${site}" ]]; then
echo "ERROR: Harbor is not importable via ${py_display}; cannot verify benchmark patches." >&2
echo " Run 'uv sync' first, or pass --harbor-python for the Harbor install used by --harbor-bin." >&2
exit 1
fi
[[ -f "${PATCH_FILE}" ]] || die "Harbor patch file not found: ${PATCH_FILE}"
patch_root="$(dirname "${site}")"
verify_status="missing"
verify_error=""
if command -v git >/dev/null 2>&1; then
verify_method="git-apply-reverse-check"
if verify_error="$(cd "${patch_root}" && git apply --reverse --check -p1 "${PATCH_FILE}" 2>&1)"; then
verify_status="applied"
fi
else
verify_method="patch-reverse-dry-run"
if verify_error="$(cd "${patch_root}" && patch --dry-run --reverse --batch --fuzz=0 -p1 < "${PATCH_FILE}" 2>&1)"; then
verify_status="applied"
fi
fi
HARBOR_PATCH_JSON="$(
python3 - "${PATCH_FILE}" "${site}" "${patch_root}" "${verify_method}" "${verify_status}" <<'PY'
import hashlib
import json
import sys
from pathlib import Path
patch_file = Path(sys.argv[1])
site = Path(sys.argv[2])
patch_root = Path(sys.argv[3])
verify_method = sys.argv[4]
verify_status = sys.argv[5]
try:
digest = "sha256:" + hashlib.sha256(patch_file.read_bytes()).hexdigest()
except OSError:
digest = "sha256:missing"
print(json.dumps({
"required": True,
"status": verify_status,
"verification": verify_method,
"harbor_site": str(site),
"patch_root": str(patch_root),
"patch_file": str(patch_file.resolve()),
"patch_file_digest": digest,
}, sort_keys=True))
PY
)"
if [[ "${verify_status}" != "applied" ]]; then
echo "ERROR: The current Harbor patch is not applied cleanly in ${site}." >&2
echo " run-baseline.sh requires patched Harbor for reproducible evals." >&2
echo " Verification used: ${verify_method}" >&2
if [[ -n "${verify_error}" ]]; then
echo " Patch verification output:" >&2
echo "${verify_error}" | sed 's/^/ /' >&2
fi
echo " Reinstall/recreate Harbor, then apply the current patch before launching a baseline:" >&2
echo " cd $(q "$(dirname "${site}")") && patch -p1 < $(q "${PATCH_FILE}")" >&2
exit 1
fi
}
print_resolved() {
echo "Resolved baseline"
echo " run_dir: ${RUN_DIR}"
echo " manifest: ${MANIFEST_PATH}"
echo " server_preset: ${SERVER_PRESET}"
echo " server_mode: ${MODE}"
echo " book_mode: ${BOOK_MODE}"
echo " harbor_path: ${HARBOR_PATH}"
if [[ "${SWITCHYARD_ENABLED}" -eq 1 ]]; then
echo " server_url: ${SERVER_ROOT_URL}"
else
echo " upstream_url: ${HARBOR_BASE_URL}"
echo " api_key_env: ${UPSTREAM_API_KEY_ENV}"
fi
echo " harbor_server: ${HARBOR_SERVER_ROOT_URL}"
echo " harbor_url: ${HARBOR_BASE_URL}"
if [[ "${BOOK_MODE}" == "closed" || "${BOOK_MODE}" == "open" ]]; then
if [[ "${SWITCHYARD_ENABLED}" -eq 1 ]]; then
echo " docker_image: ${SWITCHYARD_DOCKER_IMAGE}"
fi
echo " docker_network: ${SWITCHYARD_DOCKER_NETWORK}"
echo " proxy_mode: ${BOOK_MODE}"
fi
if [[ "${BOOK_MODE}" == "closed" ]]; then
echo " verifier_net: authenticated proxy egress"
fi
if [[ -n "${SERVER_CONFIG}" ]]; then
echo " server_config: ${SERVER_CONFIG}"
echo " route_model: ${ROUTE_MODEL}"
fi
printf ' harbor_cmd: '
shell_join "${HARBOR_CMD_PREFIX[@]}"
echo "(${HARBOR_SOURCE})"
printf ' harbor_python: '
shell_join "${HARBOR_PYTHON_CMD[@]}"
echo
echo " harbor_patch: verified"
echo " agent/model: ${AGENT} / ${HARBOR_MODEL}"
echo " reasoning: ${REASONING_EFFORT:-unset}"
if [[ -n "${CODEX_MODEL_CATALOG_HOST}" ]]; then
echo " codex_catalog: ${CODEX_MODEL_CATALOG_HOST}"
fi
echo " concurrency: ${N_CONCURRENT}"
echo
if [[ "${SWITCHYARD_ENABLED}" -eq 1 ]]; then
printf 'SERVER_CMD: '
shell_join "${SERVER_CMD[@]}"
echo
else
echo "SERVER_CMD: <direct-upstream>"
fi
printf 'HARBOR_CMD: '
shell_join "${HARBOR_CMD[@]}"
echo
}
check_harbor_available
check_harbor_patch
print_resolved
if [[ "${DRY_RUN}" -eq 1 ]]; then
exit 0
fi
mkdir -p "${RUN_DIR}"
if [[ -n "${CODEX_MODEL_CATALOG_HOST}" ]]; then
uv run --no-sync python benchmark/codex_model_catalog.py \
--output "${CODEX_MODEL_CATALOG_HOST}" \
--model "${HARBOR_MODEL}"
fi
AGENT_VERSIONS_JSON="$(json_object_from_pairs \
claude_code "${CLAUDE_CODE_VERSION}" codex "${CODEX_VERSION}" \
opencode "${OPENCODE_VERSION}" pi "${PI_VERSION}" node "${NODE_VERSION}")"
HARBOR_EXTRA_JSON="$(json_array)"
if [[ "${#HARBOR_EXTRA[@]}" -gt 0 ]]; then
HARBOR_EXTRA_JSON="$(json_array "${HARBOR_EXTRA[@]}")"
fi
MANIFEST_CMD=(python3 "${MANIFEST_HELPER}" write
--output "${MANIFEST_PATH}"
--launcher-argv-json "$(json_array "${BASH_SOURCE[0]}" "${ORIGINAL_ARGV[@]}")"
--server-preset "${SERVER_PRESET}"
--server-mode "${MODE}"
--server-url "${SERVER_ROOT_URL}"
--server-port "${PORT}"
--server-argv-json "$(json_array "${SERVER_CMD[@]}")"
--server-config-json "${SERVER_CONFIG_JSON}"
--harbor-server-url "${HARBOR_SERVER_ROOT_URL}"
--harbor-base-url "${HARBOR_BASE_URL}"
--upstream-base-url "$([[ "${SWITCHYARD_ENABLED}" -eq 0 ]] && echo "${UPSTREAM_BASE_URL}")"
--upstream-api-key-env "$([[ "${SWITCHYARD_ENABLED}" -eq 0 ]] && echo "${UPSTREAM_API_KEY_ENV}")"
--route-model "${ROUTE_MODEL}"
--harbor-command-json "$(json_array "${HARBOR_CMD_PREFIX[@]}")"
--dataset-label "${DATASET}"
--agent "${AGENT}"
--harbor-model "${HARBOR_MODEL}"
--reasoning-effort "${REASONING_EFFORT}"
--n-concurrent "${N_CONCURRENT}"
--max-retries "${MAX_RETRIES}"
--agent-timeout-multiplier "${AGENT_TIMEOUT_MULTIPLIER}"
--n-tasks "${N_TASKS:--1}"
--task-id "${TASK_ID}"
--harbor-extra-json "${HARBOR_EXTRA_JSON}"
--closed-book-mode "${BOOK_MODE}"
--closed-book-gateway-enforced "1"
--closed-book-hosted-tools-disabled "$([[ "${BOOK_MODE}" == "closed" ]] && echo 1 || echo 0)"
--closed-book-proxy-strip-artifact "${PROXY_STRIP_ARTIFACT}"
--harbor-patch-json "${HARBOR_PATCH_JSON}"
--agent-versions-json "${AGENT_VERSIONS_JSON}"
--run-dir "${RUN_DIR}"
--log-path "${LOG_PATH}"
--harbor-result-json "${HARBOR_RESULT_JSON}"
--server-metrics-prom "${SERVER_METRICS_PROM}"
--server-metrics-status "$([[ "${SWITCHYARD_ENABLED}" -eq 1 ]] && echo predicted || echo not-requested)"
--routing-stats-json "${ROUTING_STATS_JSON}"
--routing-stats-status "$([[ "${SWITCHYARD_ENABLED}" -eq 1 ]] && echo predicted || echo not-requested)")
if [[ -n "${HARBOR_PATH}" ]]; then
MANIFEST_CMD+=(--harbor-path "${HARBOR_PATH}")
fi
if [[ -n "${TASK_LIST_FILE}" ]]; then
MANIFEST_CMD+=(--task-list-file "${TASK_LIST_FILE}")
fi
if [[ -n "${SERVER_CONFIG}" ]]; then
MANIFEST_CMD+=(--server-config "${SERVER_CONFIG}")
fi
if [[ -n "${CODEX_MODEL_CATALOG_HOST}" ]]; then
MANIFEST_CMD+=(--codex-model-catalog "${CODEX_MODEL_CATALOG_HOST}")
fi
"${MANIFEST_CMD[@]}"
cat > "${WRAPPER}" <<EOF
#!/usr/bin/env bash
set -euo pipefail
cd $(q "${REPO_ROOT}")
SERVER_PRESET=$(q "${SERVER_PRESET}")
SERVER_ENABLED=$(q "${SWITCHYARD_ENABLED}")
AGENT=$(q "${AGENT}")
HARBOR_MODEL=$(q "${HARBOR_MODEL}")
UPSTREAM_API_KEY_ENV=$(q "${UPSTREAM_API_KEY_ENV}")
SERVER_ROOT_URL=$(q "${SERVER_ROOT_URL}")
HARBOR_SERVER_ROOT_URL=$(q "${HARBOR_SERVER_ROOT_URL}")
SERVER_HEALTH_URL=$(q "${SERVER_HEALTH_URL}")
SERVER_METRICS_URL=$(q "${SERVER_METRICS_URL}")
SERVER_STATS_URL=$(q "${SERVER_STATS_URL}")
HARBOR_BASE_URL=$(q "${HARBOR_BASE_URL}")
MANIFEST_PATH=$(q "${MANIFEST_PATH}")
RUN_DIR=$(q "${RUN_DIR}")
REPO_ROOT=$(q "${REPO_ROOT}")
SERVER_LOG=$(q "${SERVER_LOG}")
HARBOR_LOG=$(q "${HARBOR_LOG}")
SERVER_METRICS_PROM=$(q "${SERVER_METRICS_PROM}")
ROUTING_STATS_JSON=$(q "${ROUTING_STATS_JSON}")
HARBOR_JOB_DIR=$(q "${HARBOR_JOB_DIR}")
SKIP_HEALTH_CHECK=$(q "${SKIP_HEALTH_CHECK}")
BOOK_MODE=$(q "${BOOK_MODE}")
SWITCHYARD_DOCKER_IMAGE=$(q "${SWITCHYARD_DOCKER_IMAGE}")
SWITCHYARD_DOCKERFILE=$(q "${SWITCHYARD_DOCKERFILE}")
SWITCHYARD_DOCKER_BUILD=$(q "${SWITCHYARD_DOCKER_BUILD}")
SWITCHYARD_DOCKER_NETWORK=$(q "${SWITCHYARD_DOCKER_NETWORK}")
SWITCHYARD_DOCKER_CONTAINER=$(q "${SWITCHYARD_DOCKER_CONTAINER}")
SWITCHYARD_DOCKER_SERVICE_NAME=$(q "${SWITCHYARD_DOCKER_SERVICE_NAME}")
SERVER_CONFIG_DIR=$(q "${SERVER_CONFIG_DIR}")
SERVER_CMD=($(shell_join "${SERVER_CMD[@]}"))
SERVER_DOCKER_CMD=($(shell_join "${SERVER_DOCKER_CMD[@]}"))
HARBOR_CMD=($(shell_join "${HARBOR_CMD[@]}"))
SERVER_LOG_PID=""
DOCKER_NETWORK_CREATED=0
cleanup() {
if [[ -n "\${SERVER_LOG_PID}" ]] && kill -0 "\${SERVER_LOG_PID}" >/dev/null 2>&1; then
kill "\${SERVER_LOG_PID}" >/dev/null 2>&1 || true
wait "\${SERVER_LOG_PID}" >/dev/null 2>&1 || true
fi
if [[ "\${SERVER_ENABLED}" == "1" ]]; then
docker rm -f "\${SWITCHYARD_DOCKER_CONTAINER}" >/dev/null 2>&1 || true
fi
if [[ "\${DOCKER_NETWORK_CREATED}" == "1" ]]; then
docker network rm "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT
export PYTHONHASHSEED=0
export LC_ALL=C.UTF-8
if [[ "\${SERVER_ENABLED}" == "1" ]]; then
{
echo "Starting Dockerized Switchyard service"
echo " image: \${SWITCHYARD_DOCKER_IMAGE}"
echo " network: \${SWITCHYARD_DOCKER_NETWORK}"
echo " command: switchyard-server \${SERVER_DOCKER_CMD[*]}"
if [[ "\${SWITCHYARD_DOCKER_BUILD}" != "0" ]]; then
docker build -f "\${SWITCHYARD_DOCKERFILE}" -t "\${SWITCHYARD_DOCKER_IMAGE}" .
fi
if ! docker network inspect "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1; then
docker network create "\${SWITCHYARD_DOCKER_NETWORK}"
DOCKER_NETWORK_CREATED=1
fi
docker rm -f "\${SWITCHYARD_DOCKER_CONTAINER}" >/dev/null 2>&1 || true
DOCKER_RUN_ARGS=(
-d --rm
--name "\${SWITCHYARD_DOCKER_CONTAINER}"
--network "\${SWITCHYARD_DOCKER_NETWORK}"
--network-alias "\${SWITCHYARD_DOCKER_SERVICE_NAME}"
-p "127.0.0.1:$(q "${PORT}"):$(q "${PORT}")"
-v "\${REPO_ROOT}:\${REPO_ROOT}:ro"
-v "\${RUN_DIR}:\${RUN_DIR}"
)
if [[ -n "\${SERVER_CONFIG_DIR}" && "\${SERVER_CONFIG_DIR}" != "\${REPO_ROOT}"* ]]; then
DOCKER_RUN_ARGS+=(-v "\${SERVER_CONFIG_DIR}:\${SERVER_CONFIG_DIR}:ro")
fi
if [[ -n "\${UPSTREAM_API_KEY_ENV}" ]]; then
DOCKER_RUN_ARGS+=(--env "\${UPSTREAM_API_KEY_ENV}")
fi
docker run "\${DOCKER_RUN_ARGS[@]}" \\
--workdir "\${REPO_ROOT}" \\
--env PYTHONHASHSEED=0 \\
--env LC_ALL=C.UTF-8 \\
--env OPENAI_API_KEY \\
--env NVIDIA_API_KEY \\
--env OPENROUTER_API_KEY \\
--env UPSTREAM_API_KEY \\
--env ANTHROPIC_API_KEY \\
--env ANTHROPIC_AUTH_TOKEN \\
--env SWITCHYARD_STRONG_API_KEY \\
--env SWITCHYARD_WEAK_API_KEY \\
--env SWITCHYARD_CLASSIFIER_API_KEY \\
--env AWS_ACCESS_KEY_ID \\
--env AWS_SECRET_ACCESS_KEY \\
--env AWS_SESSION_TOKEN \\
--env AWS_REGION \\
--env AWS_DEFAULT_REGION \\
"\${SWITCHYARD_DOCKER_IMAGE}" "\${SERVER_DOCKER_CMD[@]}"
} > "\${SERVER_LOG}" 2>&1
echo "\${SWITCHYARD_DOCKER_CONTAINER}" > "\${RUN_DIR}/server.pid"
docker logs -f "\${SWITCHYARD_DOCKER_CONTAINER}" >> "\${SERVER_LOG}" 2>&1 &
SERVER_LOG_PID="\$!"
else
{
echo "Switchyard disabled; using direct upstream"
echo " network: \${SWITCHYARD_DOCKER_NETWORK}"
echo " upstream: \${HARBOR_BASE_URL}"
if ! docker network inspect "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1; then
docker network create "\${SWITCHYARD_DOCKER_NETWORK}"
DOCKER_NETWORK_CREATED=1
fi
} > "\${SERVER_LOG}" 2>&1
echo "\${SWITCHYARD_DOCKER_NETWORK}" > "\${RUN_DIR}/docker_network"
fi
if [[ "\${SERVER_ENABLED}" == "1" && "\${SKIP_HEALTH_CHECK}" != "1" ]]; then
echo "Waiting for health: \${SERVER_HEALTH_URL}"
healthy=0
for _i in {1..240}; do
if curl -fsS "\${SERVER_HEALTH_URL}" >/dev/null 2>&1; then
healthy=1
break
fi
sleep 0.5
done
if [[ "\${healthy}" != "1" ]]; then
echo "ERROR: server did not become healthy: \${SERVER_HEALTH_URL}" >&2
python3 benchmark/run_manifest.py finalize \\
--manifest "\${MANIFEST_PATH}" \\
--harbor-rc 124 \\
--harbor-job-dir "\${HARBOR_JOB_DIR}" || true
exit 124
fi
fi
if [[ "\${SERVER_ENABLED}" == "1" && "\${SWITCHYARD_CLOSED_BOOK_PREFLIGHT:-1}" != "0" ]]; then
echo "Checking benchmark Docker network reachability: \${HARBOR_SERVER_ROOT_URL}/health"
if ! docker run --rm --network "\${SWITCHYARD_DOCKER_NETWORK}" \\
"\${SWITCHYARD_PREFLIGHT_IMAGE:-python:3.12-slim}" \\
python -c 'import sys, urllib.request; urllib.request.urlopen(sys.argv[1], timeout=5).read()' \\
"\${HARBOR_SERVER_ROOT_URL}/health" \\
>/dev/null 2>&1; then
echo "ERROR: benchmark Docker containers cannot reach Switchyard on network \${SWITCHYARD_DOCKER_NETWORK}." >&2
echo " Harbor would hang before reaching Switchyard; see \${SERVER_LOG}." >&2
python3 benchmark/run_manifest.py finalize \\
--manifest "\${MANIFEST_PATH}" \\
--harbor-rc 125 \\
--harbor-job-dir "\${HARBOR_JOB_DIR}" || true
exit 125
fi
fi
if [[ "\${SERVER_ENABLED}" == "1" ]]; then
export OPENAI_API_KEY="\${OPENAI_API_KEY:-switchyard-local}"
else
if [[ -z "\${!UPSTREAM_API_KEY_ENV:-}" ]]; then
echo "ERROR: direct upstream requires \${UPSTREAM_API_KEY_ENV} to be set." >&2
python3 benchmark/run_manifest.py finalize \\
--manifest "\${MANIFEST_PATH}" \\
--harbor-rc 126 \\
--harbor-job-dir "\${HARBOR_JOB_DIR}" || true
exit 126
fi
export OPENAI_API_KEY="\${!UPSTREAM_API_KEY_ENV}"
fi
export OPENAI_BASE_URL="\${HARBOR_BASE_URL}"
if [[ "\${BOOK_MODE}" == "closed" ]]; then
export CLOSED_BOOK_MODE=1
else
export CLOSED_BOOK_MODE=0
fi
if [[ "\${SERVER_ENABLED}" == "1" ]]; then
export ALLOWED_HOSTS="\${ALLOWED_HOSTS:-\${SWITCHYARD_DOCKER_SERVICE_NAME}}"
else
export ALLOWED_HOSTS="\${ALLOWED_HOSTS:-}"
fi
export SWITCHYARD_BASE_URL="\${SWITCHYARD_BASE_URL:-\${HARBOR_SERVER_ROOT_URL}}"
export SWITCHYARD_DOCKER_NETWORK
export SWITCHYARD_VERIFIER_PROXY_TOKEN="\${SWITCHYARD_VERIFIER_PROXY_TOKEN:-\$(python3 -c 'import secrets; print(secrets.token_urlsafe(32))')}"
export SWITCHYARD_VERIFIER_HTTP_PROXY="http://verifier:\${SWITCHYARD_VERIFIER_PROXY_TOKEN}@proxy:3129"
if [[ "\${BOOK_MODE}" == "closed" ]]; then
if [[ "\${AGENT}" == "codex" ]]; then
export CODEX_DISABLE_WEB_SEARCH="\${CODEX_DISABLE_WEB_SEARCH:-1}"
elif [[ "\${AGENT}" == "opencode" ]]; then
export OPENCODE_DISABLE_WEBFETCH="\${OPENCODE_DISABLE_WEBFETCH:-1}"
fi
fi
if [[ "\${AGENT}" == "claude-code" ]]; then
export ANTHROPIC_BASE_URL="\${HARBOR_SERVER_ROOT_URL}"
if [[ "\${SERVER_ENABLED}" == "1" ]]; then
export ANTHROPIC_AUTH_TOKEN="\${ANTHROPIC_AUTH_TOKEN:-switchyard}"
export ANTHROPIC_API_KEY="\${ANTHROPIC_API_KEY:-}"
else
export ANTHROPIC_AUTH_TOKEN="\${ANTHROPIC_AUTH_TOKEN:-\${!UPSTREAM_API_KEY_ENV}}"
export ANTHROPIC_API_KEY="\${ANTHROPIC_API_KEY:-\${!UPSTREAM_API_KEY_ENV}}"
fi
export ANTHROPIC_MODEL="\${ANTHROPIC_MODEL:-\${HARBOR_MODEL}}"
export ANTHROPIC_SMALL_FAST_MODEL="\${ANTHROPIC_SMALL_FAST_MODEL:-\${HARBOR_MODEL}}"
export ANTHROPIC_CUSTOM_MODEL_OPTION="\${ANTHROPIC_CUSTOM_MODEL_OPTION:-\${HARBOR_MODEL}}"
export CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY="\${CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY:-1}"
if [[ "\${SERVER_ENABLED}" == "1" ]]; then
export ANTHROPIC_CUSTOM_MODEL_OPTION_NAME="\${ANTHROPIC_CUSTOM_MODEL_OPTION_NAME:-\${HARBOR_MODEL##*/} (Switchyard)}"
export ANTHROPIC_CUSTOM_MODEL_OPTION_DESCRIPTION="\${ANTHROPIC_CUSTOM_MODEL_OPTION_DESCRIPTION:-Routed via switchyard -> \${HARBOR_MODEL}}"
else
export ANTHROPIC_CUSTOM_MODEL_OPTION_NAME="\${ANTHROPIC_CUSTOM_MODEL_OPTION_NAME:-\${HARBOR_MODEL##*/} (direct)}"
export ANTHROPIC_CUSTOM_MODEL_OPTION_DESCRIPTION="\${ANTHROPIC_CUSTOM_MODEL_OPTION_DESCRIPTION:-Direct upstream -> \${HARBOR_MODEL}}"
fi
fi
echo "Running Harbor: \${HARBOR_CMD[*]}"
set +e
"\${HARBOR_CMD[@]}" > "\${HARBOR_LOG}" 2>&1
HARBOR_RC="\$?"
set -e
echo "\${HARBOR_RC}" > "\${RUN_DIR}/harbor_exit"
FINALIZE_CMD=(python3 benchmark/run_manifest.py finalize \\
--manifest "\${MANIFEST_PATH}" \\
--harbor-rc "\${HARBOR_RC}" \\
--harbor-job-dir "\${HARBOR_JOB_DIR}")
if [[ "\${SERVER_ENABLED}" == "1" ]]; then
if curl -fsS "\${SERVER_METRICS_URL}" -o "\${SERVER_METRICS_PROM}" >/dev/null 2>&1; then
echo "Captured aggregate Rust server metrics"
else
echo "WARNING: Rust server metrics could not be collected from \${SERVER_METRICS_URL}" >&2
fi
FINALIZE_CMD+=(--server-metrics "\${SERVER_METRICS_PROM}")
if curl -fsS "\${SERVER_STATS_URL}" -o "\${ROUTING_STATS_JSON}" >/dev/null 2>&1; then
echo "Captured aggregate Rust server routing stats"
else
echo "WARNING: Rust server stats could not be collected from \${SERVER_STATS_URL}" >&2
fi
FINALIZE_CMD+=(--routing-stats "\${ROUTING_STATS_JSON}")
fi
"\${FINALIZE_CMD[@]}" || true
exit "\${HARBOR_RC}"
EOF
chmod +x "${WRAPPER}"
if [[ "${FOREGROUND}" -eq 1 ]]; then
"${WRAPPER}" 2>&1 | tee "${LOG_PATH}"
exit "${PIPESTATUS[0]}"
fi
nohup "${WRAPPER}" > "${LOG_PATH}" 2>&1 &
PID=$!
disown "${PID}" 2>/dev/null || true
echo
echo "Started PID ${PID}"
echo
echo "Monitor with:"
echo " tail -f ${LOG_PATH}"
echo
echo "Stop with:"
echo " kill ${PID}"