Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(benchmark): net out retried attempts by trial id in task-level routing stats (#120)
ryan-lempkaee5e96decf91
diff --git a/benchmark/README.md b/benchmark/README.mdindex 6ca511cb0..1dd64f6da 100644--- a/benchmark/README.md+++ b/benchmark/README.md@@ -286,10 +286,16 @@ deterministic LLM-classifier routes, `server.classifier_prompts` records each roprompt, prompt SHA-256, `max_request_chars`, and `recent_turn_window` for reproducibility. Directruns mark routing stats as `not-requested`.-Switchyard runs also write `routing_requests.jsonl` (one record per router request: task, session,-selected model, tier, token usage) and roll it up into `routing_stats_by_task.json` at finalize.-The task identity comes from the per-task proxy sidecar, which stamps `x-switchyard-intake-task`-and a per-trial `proxy_x_session_id` on model-bound requests.+Switchyard runs also write `routing_requests.jsonl` (one record per router request: task, trial,+session, served model, tier, and the six token fields from the global schema) and roll it up into+`routing_stats_by_task.json` at finalize. The per-task proxy sidecar stamps the identity headers on+model-bound requests: `x-switchyard-intake-task`, `x-switchyard-trial-id` (Harbor's trial name), and+a per-attempt `proxy_x_session_id`.++Each task in `routing_stats_by_task.json` splits into `final` and `retries`. Requests are grouped by+trial, and within a trial the latest attempt is the one Harbor kept (it retries sequentially and+discards earlier attempts), so tokens spent on retried-away attempts land in `retries` and can be+netted out of cost.## Docker Image Notesdiff --git a/benchmark/closed_book_proxy/proxy/rewriter.py b/benchmark/closed_book_proxy/proxy/rewriter.pyindex 86dfe559f..6f4e21448 100644--- a/benchmark/closed_book_proxy/proxy/rewriter.py+++ b/benchmark/closed_book_proxy/proxy/rewriter.py@@ -44,6 +44,14 @@ STRIP_PATH_SUFFIXES = ()INTAKE_TASK_HEADER = "x-switchyard-intake-task"SESSION_ID_HEADER = "proxy_x_session_id"+TRIAL_ID_HEADER = "x-switchyard-trial-id"+++def _trial_id_from_env() -> str:+ # Harbor sets HOST_AGENT_LOGS_PATH to <trials_dir>/<trial_name>/agent, passed+ # into the proxy as SWITCHYARD_TRIAL_DIR; the trial name is that dir's parent.+ trial_dir = os.environ.get("SWITCHYARD_TRIAL_DIR", "").strip()+ return Path(trial_dir).parent.name if trial_dir else ""def _truthy(value: str | None) -> bool:@@ -67,8 +75,9 @@ class ClosedBookProxy:)self.allowed_hosts = self._load_allowed_hosts()self.task_id = os.environ.get("SWITCHYARD_TASK_ID", "")- # One id per proxy boot; the sidecar boots once per trial, so this- # distinguishes trials of the same task.+ self.trial_id = _trial_id_from_env()+ # One id per proxy boot; the sidecar boots once per attempt, so this+ # distinguishes retries of the same trial.self.session_id = uuid.uuid4().hexdef _load_allowed_hosts(self) -> set[str]:@@ -128,6 +137,8 @@ class ClosedBookProxy:if self.task_id:flow.request.headers[INTAKE_TASK_HEADER] = self.task_idflow.request.headers[SESSION_ID_HEADER] = self.session_id+ if self.trial_id:+ flow.request.headers[TRIAL_ID_HEADER] = self.trial_idif not self.closed_book:returnif "application/json" not in flow.request.headers.get("content-type", ""):diff --git a/benchmark/prepare_harbor_dataset.py b/benchmark/prepare_harbor_dataset.pyindex f7c90afbc..fb30a8ce3 100644--- a/benchmark/prepare_harbor_dataset.py+++ b/benchmark/prepare_harbor_dataset.py@@ -412,6 +412,7 @@ def _merge_compose(task_dir: Path, proxy_allowlist_hosts: tuple[str, ...]) -> di"OPENAI_BASE_URL=${OPENAI_BASE_URL:-}","SWITCHYARD_BASE_URL=${SWITCHYARD_BASE_URL:-}",f"SWITCHYARD_TASK_ID={task_dir.name}",+ "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}","VERIFIER_PROXY_TOKEN=${SWITCHYARD_VERIFIER_PROXY_TOKEN:-}",],"healthcheck": {diff --git a/benchmark/run_manifest.py b/benchmark/run_manifest.pyindex 5f286530d..92879b5bf 100644--- a/benchmark/run_manifest.py+++ b/benchmark/run_manifest.py@@ -223,13 +223,67 @@ def _copy_if_present(source: Path | None, dest: Path | None) -> str:return "present"+_TOKEN_FIELDS = (+ "prompt_tokens",+ "cached_tokens",+ "cache_creation_tokens",+ "completion_tokens",+ "reasoning_tokens",+ "total_tokens",+)+++def _new_section() -> dict[str, Any]:+ return {"calls": 0, "totals": dict.fromkeys(_TOKEN_FIELDS, 0), "_buckets": {}}+++def _accumulate(section: dict[str, Any], record: dict[str, Any]) -> None:+ section["calls"] += 1+ model = record.get("model") or "unknown"+ tier = record.get("tier") or ""+ bucket = section["_buckets"].setdefault(+ (model, tier),+ {"model": model, "tier": tier, "calls": 0, **dict.fromkeys(_TOKEN_FIELDS, 0)},+ )+ bucket["calls"] += 1+ for field in _TOKEN_FIELDS:+ value = record.get(field)+ value = value if isinstance(value, int) else 0+ bucket[field] += value+ section["totals"][field] += value+++def _finalize_section(section: dict[str, Any]) -> None:+ buckets = section.pop("_buckets")+ section["models"] = [buckets[key] for key in sorted(buckets)]+++def _kept_sessions(records: list[dict[str, Any]]) -> set[str]:+ """The kept attempt of each trial is its latest-timestamped session.++ Harbor retries sequentially and keeps only the final attempt, so within a+ trial the session with the newest timestamp is the one that survived.+ Records without a ``trial_id`` are all treated as kept (no netting).+ """+ latest: dict[str, tuple[str, str]] = {}+ for record in records:+ trial = record.get("trial_id")+ session = record.get("session_id")+ if not trial or not session:+ continue+ ts = record.get("ts") or ""+ if trial not in latest or ts > latest[trial][0]:+ latest[trial] = (ts, session)+ return {session for _, session in latest.values()}++def summarize_routing_log(log_path: Path) -> dict[str, Any]:- """Aggregate a per-request routing JSONL log into per-task stats.+ """Roll the routing log into per-task ``final`` and ``retries`` sections.- Each task entry lists one bucket per (model, tier) pair so a model that- served more than one tier is not merged into a single row.+ Requests are grouped by trial; the latest attempt in each trial is ``final``+ and earlier attempts are ``retries`` (see :func:`_kept_sessions`)."""- tasks: dict[str, dict[str, Any]] = {}+ by_task: dict[str, list[dict[str, Any]]] = {}total_requests = 0for raw in log_path.read_text().splitlines():raw = raw.strip()@@ -242,32 +296,30 @@ def summarize_routing_log(log_path: Path) -> dict[str, Any]:if not isinstance(record, dict):continuetotal_requests += 1- task = record.get("task") or "unattributed"- entry = tasks.setdefault(task, {"requests": 0, "sessions": [], "_buckets": {}})- entry["requests"] += 1- session = record.get("session_id")- if session and session not in entry["sessions"]:- entry["sessions"].append(session)- model = record.get("model") or "unknown"- tier = record.get("tier") or ""- bucket = entry["_buckets"].setdefault(- (model, tier),- {- "model": model,- "tier": tier,- "calls": 0,- "prompt_tokens": 0,- "completion_tokens": 0,- "total_tokens": 0,- },- )- bucket["calls"] += 1- for key in ("prompt_tokens", "completion_tokens", "total_tokens"):- value = record.get(key)- bucket[key] += value if isinstance(value, int) else 0- for entry in tasks.values():- buckets = entry.pop("_buckets")- entry["models"] = [buckets[key] for key in sorted(buckets)]+ by_task.setdefault(record.get("task") or "unattributed", []).append(record)++ tasks: dict[str, dict[str, Any]] = {}+ for task, records in by_task.items():+ kept = _kept_sessions(records)+ entry = {+ "requests": len(records),+ "n_retries": 0,+ "final": _new_section(),+ "retries": _new_section(),+ }+ retry_sessions: set[str] = set()+ for record in records:+ session = record.get("session_id")+ # A record is a retry only when it has a trial_id and its session was+ # not the kept one; untagged records default to final.+ is_retry = bool(record.get("trial_id")) and session not in kept+ _accumulate(entry["retries" if is_retry else "final"], record)+ if is_retry and session:+ retry_sessions.add(session)+ entry["n_retries"] = len(retry_sessions)+ _finalize_section(entry["final"])+ _finalize_section(entry["retries"])+ tasks[task] = entryreturn {"total_requests": total_requests, "tasks": tasks}diff --git a/switchyard/lib/processors/routing_log_response_processor.py b/switchyard/lib/processors/routing_log_response_processor.pyindex 006f34c9b..89b2101c8 100644--- a/switchyard/lib/processors/routing_log_response_processor.py+++ b/switchyard/lib/processors/routing_log_response_processor.py@@ -18,7 +18,7 @@ from switchyard.lib.chat_response.streaming_response_accumulator import (attach_final_response_callback,)from switchyard.lib.proxy_context import CTX_PROXY_ACTUAL_MODEL, ProxyContext-from switchyard.lib.request_metadata import CTX_REQUEST_METADATA+from switchyard.lib.request_metadata import CTX_PROFILE_REQUEST_HEADERS, CTX_REQUEST_METADATAfrom switchyard_rust.core import ChatResponselogger = logging.getLogger(__name__)@@ -57,11 +57,16 @@ class RoutingLogResponseProcessor:def _write_record(self, ctx: ProxyContext, served_model: str, response: ChatResponse) -> None:metadata = ctx.metadata.get(CTX_REQUEST_METADATA)+ headers = ctx.metadata.get(CTX_PROFILE_REQUEST_HEADERS) or {}+ # Prefer the model id the backend actually served so buckets reconcile+ # with the global routing_stats schema, which keys on the served id.+ actual_model = _field(response.body, "model")record = {"ts": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z","task": getattr(getattr(metadata, "intake", None), "task", None),+ "trial_id": headers.get("x-switchyard-trial-id"),"session_id": getattr(metadata, "session_id", None),- "model": served_model,+ "model": actual_model if isinstance(actual_model, str) and actual_model else served_model,"tier": ctx.metadata.get("_random_routing_tier", "") or (ctx.selected_target or ""),**_usage_tokens(response.body),}@@ -74,20 +79,40 @@ class RoutingLogResponseProcessor:def _usage_tokens(body: object) -> dict[str, int]:- """Token counts from any native usage shape (OpenAI Chat/Responses, Anthropic)."""+ """Six-field token breakdown matching the global routing-stats schema.++ cached/cache_creation are subsets of prompt_tokens, reasoning a subset of+ completion_tokens, total = prompt + completion.+ """usage = _field(body, "usage")prompt = _int_field(usage, "prompt_tokens")completion = _int_field(usage, "completion_tokens")- if not prompt and not completion:- prompt = (- _int_field(usage, "input_tokens")- + _int_field(usage, "cache_creation_input_tokens")- + _int_field(usage, "cache_read_input_tokens")- )+ cached = 0+ cache_creation = 0+ reasoning = 0+ if prompt or completion:+ # OpenAI Chat Completions.+ cached = _int_field(_field(usage, "prompt_tokens_details"), "cached_tokens")+ reasoning = _int_field(_field(usage, "completion_tokens_details"), "reasoning_tokens")+ else:completion = _int_field(usage, "output_tokens")+ reasoning = _int_field(_field(usage, "output_tokens_details"), "reasoning_tokens")+ input_details = _field(usage, "input_tokens_details")+ if input_details is not None:+ # OpenAI Responses API: prompt_tokens already includes cached.+ prompt = _int_field(usage, "input_tokens")+ cached = _int_field(input_details, "cached_tokens")+ else:+ # Anthropic Messages: cache tokens are prompt siblings, so fold in.+ cached = _int_field(usage, "cache_read_input_tokens")+ cache_creation = _int_field(usage, "cache_creation_input_tokens")+ prompt = _int_field(usage, "input_tokens") + cached + cache_creationreturn {"prompt_tokens": prompt,+ "cached_tokens": cached,+ "cache_creation_tokens": cache_creation,"completion_tokens": completion,+ "reasoning_tokens": reasoning,"total_tokens": prompt + completion,}diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.pyindex 37f6a55be..1c69052ff 100644--- a/tests/test_prepare_harbor_dataset.py+++ b/tests/test_prepare_harbor_dataset.py@@ -305,3 +305,4 @@ def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None:proxy_env = "\n".join(compose["services"]["proxy"]["environment"])assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_env+ assert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_envdiff --git a/tests/test_routing_log_response_processor.py b/tests/test_routing_log_response_processor.pyindex c404a84fb..62af541be 100644--- a/tests/test_routing_log_response_processor.py+++ b/tests/test_routing_log_response_processor.py@@ -26,6 +26,7 @@ from switchyard_rust.core import ChatResponse, ProxyContextTASK_HEADERS = {"x-switchyard-intake-task": "hello-world-abc1",+ "x-switchyard-trial-id": "hello-world-abc1-Xy7","proxy_x_session_id": "trial-session-1",}@@ -49,7 +50,13 @@ def _openai_completion() -> ChatResponse:"message": {"role": "assistant", "content": "hi"},"finish_reason": "stop",}],- "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},+ "usage": {+ "prompt_tokens": 8,+ "completion_tokens": 3,+ "total_tokens": 11,+ "prompt_tokens_details": {"cached_tokens": 6},+ "completion_tokens_details": {"reasoning_tokens": 2},+ },})@@ -87,7 +94,13 @@ def _responses_completion() -> ChatResponse:"parallel_tool_calls": False,"tool_choice": "auto","tools": [],- "usage": {"input_tokens": 6, "output_tokens": 2, "total_tokens": 8},+ "usage": {+ "input_tokens": 6,+ "output_tokens": 2,+ "total_tokens": 8,+ "input_tokens_details": {"cached_tokens": 3},+ "output_tokens_details": {"reasoning_tokens": 1},+ },})@@ -103,12 +116,16 @@ async def test_openai_completion_record_carries_task_and_usage(tmp_path: Path) -(record,) = _records(log_file)assert record["task"] == "hello-world-abc1"+ assert record["trial_id"] == "hello-world-abc1-Xy7"assert record["session_id"] == "trial-session-1"assert record["model"] == "gpt-test"assert record["tier"] == "weak"- assert record["prompt_tokens"] == 5+ assert record["prompt_tokens"] == 8+ assert record["cached_tokens"] == 6+ assert record["cache_creation_tokens"] == 0assert record["completion_tokens"] == 3- assert record["total_tokens"] == 8+ assert record["reasoning_tokens"] == 2+ assert record["total_tokens"] == 11async def test_anthropic_usage_sums_cache_siblings(tmp_path: Path) -> None:@@ -119,7 +136,10 @@ async def test_anthropic_usage_sums_cache_siblings(tmp_path: Path) -> None:(record,) = _records(log_file)assert record["prompt_tokens"] == 13 # input + cache_creation + cache_read+ assert record["cached_tokens"] == 4+ assert record["cache_creation_tokens"] == 2assert record["completion_tokens"] == 3+ assert record["reasoning_tokens"] == 0async def test_responses_completion_uses_input_output_tokens(tmp_path: Path) -> None:@@ -130,7 +150,10 @@ async def test_responses_completion_uses_input_output_tokens(tmp_path: Path) ->(record,) = _records(log_file)assert record["prompt_tokens"] == 6+ assert record["cached_tokens"] == 3+ assert record["cache_creation_tokens"] == 0assert record["completion_tokens"] == 2+ assert record["reasoning_tokens"] == 1async def test_missing_headers_log_null_task_and_session(tmp_path: Path) -> None:@@ -139,6 +162,7 @@ async def test_missing_headers_log_null_task_and_session(tmp_path: Path) -> None(record,) = _records(log_file)assert record["task"] is None+ assert record["trial_id"] is Noneassert record["session_id"] is Nonediff --git a/tests/test_run_manifest.py b/tests/test_run_manifest.pyindex a26d50215..18a6a5513 100644--- a/tests/test_run_manifest.py+++ b/tests/test_run_manifest.py@@ -469,59 +469,107 @@ def test_finalize_copies_proxy_strip_log_artifact(tmp_path: Path) -> None:assert (run_dir / "proxy_strip_log.jsonl").read_text() == '{"removed":["web_search"]}\n'+def _record(task: str, ts: str, model: str, tier: str, *, trial: str = "t", session: str = "s",+ **tokens: int) -> str:+ base = {"prompt_tokens": 0, "cached_tokens": 0, "cache_creation_tokens": 0,+ "completion_tokens": 0, "reasoning_tokens": 0, "total_tokens": 0}+ base.update(tokens)+ return json.dumps({"task": task, "ts": ts, "trial_id": trial, "session_id": session,+ "model": model, "tier": tier, **base})++def test_summarize_routing_log_groups_by_task_and_model(tmp_path: Path) -> None:module = _load_manifest_module()log = tmp_path / "routing_requests.jsonl"log.write_text("\n".join([- json.dumps({"task": "task-a", "session_id": "s1", "model": "opus",- "tier": "strong", "prompt_tokens": 10, "completion_tokens": 2,- "total_tokens": 12}),- json.dumps({"task": "task-a", "session_id": "s1", "model": "kimi",- "tier": "weak", "prompt_tokens": 4, "completion_tokens": 1,- "total_tokens": 5}),- json.dumps({"task": "task-b", "session_id": "s2", "model": "opus",- "tier": "strong", "prompt_tokens": 7, "completion_tokens": 3,- "total_tokens": 10}),+ _record("task-a", "2026-01-01T00:00:01Z", "opus", "strong",+ prompt_tokens=10, cached_tokens=6, completion_tokens=2,+ reasoning_tokens=1, total_tokens=12),+ _record("task-a", "2026-01-01T00:00:02Z", "kimi", "weak",+ prompt_tokens=4, completion_tokens=1, total_tokens=5),"not json",- json.dumps({"model": "opus", "prompt_tokens": 1, "completion_tokens": 1,- "total_tokens": 2}),+ json.dumps(["not", "a", "dict"]),]) + "\n")summary = module.summarize_routing_log(log)- assert summary["total_requests"] == 4+ assert summary["total_requests"] == 2task_a = summary["tasks"]["task-a"]assert task_a["requests"] == 2- assert task_a["sessions"] == ["s1"]- assert task_a["models"] == [- {"model": "kimi", "tier": "weak", "calls": 1, "prompt_tokens": 4,- "completion_tokens": 1, "total_tokens": 5},- {"model": "opus", "tier": "strong", "calls": 1, "prompt_tokens": 10,- "completion_tokens": 2, "total_tokens": 12},+ assert task_a["n_retries"] == 0+ assert task_a["retries"]["calls"] == 0+ assert task_a["final"]["calls"] == 2+ assert task_a["final"]["totals"] == {+ "prompt_tokens": 14, "cached_tokens": 6, "cache_creation_tokens": 0,+ "completion_tokens": 3, "reasoning_tokens": 1, "total_tokens": 17,+ }+ assert [(b["model"], b["tier"], b["calls"]) for b in task_a["final"]["models"]] == [+ ("kimi", "weak", 1), ("opus", "strong", 1),]- assert summary["tasks"]["task-b"]["models"][0]["total_tokens"] == 10- assert summary["tasks"]["unattributed"]["requests"] == 1-def test_summarize_routing_log_keeps_mixed_tiers_separate(tmp_path: Path) -> None:+def test_summarize_routing_log_nets_out_retried_attempt(tmp_path: Path) -> None:module = _load_manifest_module()log = tmp_path / "routing_requests.jsonl"+ # Same trial, two attempts: the earlier session was retried away.log.write_text("\n".join([- json.dumps({"task": "task-a", "model": "opus", "tier": "strong",- "prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}),- json.dumps({"task": "task-a", "model": "opus", "tier": "weak",- "prompt_tokens": 4, "completion_tokens": 1, "total_tokens": 5}),+ _record("task-a", "2026-01-01T00:00:01Z", "opus", "strong",+ trial="trial-1", session="retry-sess",+ prompt_tokens=100, completion_tokens=10, total_tokens=110),+ _record("task-a", "2026-01-01T00:05:01Z", "opus", "strong",+ trial="trial-1", session="final-sess",+ prompt_tokens=20, completion_tokens=3, total_tokens=23),]) + "\n")- buckets = module.summarize_routing_log(log)["tasks"]["task-a"]["models"]+ task_a = module.summarize_routing_log(log)["tasks"]["task-a"]- assert [(b["model"], b["tier"], b["calls"]) for b in buckets] == [- ("opus", "strong", 1), ("opus", "weak", 1),- ]+ assert task_a["n_retries"] == 1+ assert task_a["final"]["totals"]["total_tokens"] == 23+ assert task_a["retries"]["totals"]["total_tokens"] == 110+ assert task_a["final"]["calls"] == 1+ assert task_a["retries"]["calls"] == 1+++def test_summarize_routing_log_keeps_all_concurrent_trials(tmp_path: Path) -> None:+ module = _load_manifest_module()+ log = tmp_path / "routing_requests.jsonl"+ # Two trials of the same task (k>1). Neither retried, so both are final even+ # though their calls interleave in time.+ log.write_text(+ "\n".join([+ _record("task-a", "2026-01-01T00:00:01Z", "opus", "strong",+ trial="trial-1", session="s1", total_tokens=10),+ _record("task-a", "2026-01-01T00:00:02Z", "opus", "strong",+ trial="trial-2", session="s2", total_tokens=20),+ ]) + "\n"+ )++ task_a = module.summarize_routing_log(log)["tasks"]["task-a"]++ assert task_a["n_retries"] == 0+ assert task_a["final"]["calls"] == 2+ assert task_a["final"]["totals"]["total_tokens"] == 30+ assert task_a["retries"]["calls"] == 0+++def test_summarize_routing_log_untagged_records_all_final(tmp_path: Path) -> None:+ module = _load_manifest_module()+ log = tmp_path / "routing_requests.jsonl"+ # No trial_id -> no netting, everything counts as final.+ log.write_text(json.dumps({+ "task": "task-a", "ts": "2026-01-01T00:00:01Z", "session_id": "s1",+ "model": "opus", "tier": "strong", "total_tokens": 5,+ }) + "\n")++ task_a = module.summarize_routing_log(log)["tasks"]["task-a"]++ assert task_a["final"]["calls"] == 1+ assert task_a["retries"]["calls"] == 0+ assert task_a["n_retries"] == 0def test_finalize_writes_routing_stats_by_task(tmp_path: Path) -> None:@@ -543,7 +591,7 @@ def test_finalize_writes_routing_stats_by_task(tmp_path: Path) -> None:assert rc == 0summary = json.loads(by_task.read_text())- assert summary["tasks"]["task-a"]["models"][0]["calls"] == 1+ assert summary["tasks"]["task-a"]["final"]["models"][0]["calls"] == 1manifest = json.loads(out.read_text())assert manifest["outcomes"]["routing_stats_by_task_json_status"] == "present"