Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(serve): route-log per-request routing stats for profile-config serve (#118)
Brian Newsomd0a924c597b1
diff --git a/docs/cli_reference.md b/docs/cli_reference.mdindex 864225a0d..b08ebc714 100644--- a/docs/cli_reference.md+++ b/docs/cli_reference.md@@ -214,6 +214,7 @@ The server exposes the OpenAI Chat Completions (`/v1/chat/completions`), Anthropswitchyard [--routing-profiles PATH] serve[--host HOST] [--port PORT] [--workers N][--reload] [--inbound FORMAT]+ [--routing-log-file PATH][--intake-enabled|--enable-intake [INTAKE OVERRIDES]]```@@ -225,7 +226,7 @@ switchyard [--routing-profiles PATH] serve| `--host`, `--port`/`-p`, `--reload` | [Transport](#transport-server-verbs). || `--inbound FORMAT` | Backwards-compatible no-op; all request APIs are always registered. || `--workers` / `-w` | uvicorn worker count. |-| `--routing-log-file PATH` | Append one JSONL routing record per request (task, session, selected model, tier, token usage) to PATH. |+| `--routing-log-file PATH` | Append one JSONL routing record per request (task, session, actual model, tier, token and cache usage) to PATH. Also exposes `GET /v1/routing/session-stats?session_id=...`. || `--intake-enabled` / `--enable-intake`, `--intake-base-url`, `--intake-workspace`, `--intake-api-key`, `--intake-target-url` | [Intake sink](#intake-sink-serve-and-launchers). |**Notes**diff --git a/switchyard/lib/endpoints/__init__.py b/switchyard/lib/endpoints/__init__.pyindex 8cb676cb8..fa6c61bd1 100644--- a/switchyard/lib/endpoints/__init__.py+++ b/switchyard/lib/endpoints/__init__.py@@ -18,6 +18,9 @@ if TYPE_CHECKING:OpenAIChatEndpoint,)from switchyard.lib.endpoints.responses_endpoint import ResponsesEndpoint+ from switchyard.lib.endpoints.routing_log_stats_endpoint import (+ RoutingLogStatsEndpoint,+ )from switchyard.lib.endpoints.stats_endpoint import StatsEndpoint__all__ = [@@ -26,6 +29,7 @@ __all__ = ["ModelsEndpoint","OpenAIChatEndpoint","ResponsesEndpoint",+ "RoutingLogStatsEndpoint",]@@ -52,4 +56,9 @@ def __getattr__(name: str) -> Any:ResponsesEndpoint,)return ResponsesEndpoint+ elif name == "RoutingLogStatsEndpoint":+ from switchyard.lib.endpoints.routing_log_stats_endpoint import (+ RoutingLogStatsEndpoint,+ )+ return RoutingLogStatsEndpointraise AttributeError(f"module {__name__!r} has no attribute {name!r}")diff --git a/switchyard/lib/endpoints/routing_log_stats_endpoint.py b/switchyard/lib/endpoints/routing_log_stats_endpoint.pynew file mode 100644index 000000000..4491f877b--- /dev/null+++ b/switchyard/lib/endpoints/routing_log_stats_endpoint.py@@ -0,0 +1,45 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""HTTP access to session-scoped aggregates from the durable routing log."""++from __future__ import annotations++import asyncio+from typing import TYPE_CHECKING++from fastapi import APIRouter, HTTPException++from switchyard.lib.endpoints.base import Endpoint++if TYPE_CHECKING:+ from fastapi import FastAPI++ from switchyard.lib.processors.routing_log_response_processor import (+ RoutingLogResponseProcessor,+ )+++class RoutingLogStatsEndpoint(Endpoint):+ """Expose one trial session's model and token aggregates."""++ register_once = True++ def __init__(self, processor: RoutingLogResponseProcessor) -> None:+ self._processor = processor++ def register(self, app: FastAPI) -> None:+ routes = APIRouter()+ processor = self._processor++ async def get_session_stats(session_id: str) -> dict[str, object]:+ snapshot = await asyncio.to_thread(processor.snapshot_session, session_id)+ if snapshot is None:+ raise HTTPException(status_code=404, detail="routing session not found")+ return snapshot++ routes.get("/v1/routing/session-stats")(get_session_stats)+ app.include_router(routes, tags=["Routing log"])+++__all__ = ["RoutingLogStatsEndpoint"]diff --git a/switchyard/lib/processors/routing_log_response_processor.py b/switchyard/lib/processors/routing_log_response_processor.pyindex 89b2101c8..b378d39c8 100644--- a/switchyard/lib/processors/routing_log_response_processor.py+++ b/switchyard/lib/processors/routing_log_response_processor.py@@ -12,7 +12,7 @@ import threadingfrom collections.abc import Mappingfrom datetime import UTC, datetimefrom pathlib import Path-from typing import Any+from typing import TYPE_CHECKING, Anyfrom switchyard.lib.chat_response.streaming_response_accumulator import (attach_final_response_callback,@@ -21,6 +21,9 @@ from switchyard.lib.proxy_context import CTX_PROXY_ACTUAL_MODEL, ProxyContextfrom switchyard.lib.request_metadata import CTX_PROFILE_REQUEST_HEADERS, CTX_REQUEST_METADATAfrom switchyard_rust.core import ChatResponse+if TYPE_CHECKING:+ from switchyard.lib.endpoints.base import Endpoint+logger = logging.getLogger(__name__)@@ -41,8 +44,8 @@ class RoutingLogResponseProcessor:async def process(self, ctx: ProxyContext, response: ChatResponse) -> ChatResponse:"""Log one routing record for the completed request; return the response unchanged."""- served_model: str = ctx.selected_model or ctx.metadata.get(- CTX_PROXY_ACTUAL_MODEL, "unknown",+ served_model: str = (+ ctx.metadata.get(CTX_PROXY_ACTUAL_MODEL) or ctx.selected_model or "unknown")async def _emit(final: ChatResponse) -> None:@@ -77,6 +80,77 @@ class RoutingLogResponseProcessor:except OSError as exc:logger.warning("Routing log: failed to append to %s: %s", self._log_file, exc)+ def snapshot_session(self, session_id: str) -> dict[str, object] | None:+ """Aggregate the durable request log for one exact trial session.++ This re-reads and re-parses the whole log on every call. It is sized for+ benchmark runs (thousands of records), not a long-lived production server+ with millions of sessions, where it would be O(records) per query.+ """+ models: dict[str, dict[str, int]] = {}+ totals = {+ "total_calls": 0,+ "total_prompt_tokens": 0,+ "total_cached_tokens": 0,+ "total_cache_creation_tokens": 0,+ "total_completion_tokens": 0,+ }+ try:+ if not self._log_file.is_file():+ return None+ lines = self._log_file.read_text(encoding="utf-8").splitlines()+ except OSError as exc:+ logger.warning("Routing log: failed to read %s: %s", self._log_file, exc)+ return None++ for line in lines:+ try:+ record = json.loads(line)+ except json.JSONDecodeError:+ continue+ if not isinstance(record, Mapping) or record.get("session_id") != session_id:+ continue+ model_value = record.get("model")+ model = model_value if isinstance(model_value, str) and model_value else "unknown"+ bucket = models.setdefault(+ model,+ {+ "calls": 0,+ "prompt_tokens": 0,+ "cached_tokens": 0,+ "cache_creation_tokens": 0,+ "completion_tokens": 0,+ },+ )+ bucket["calls"] += 1+ totals["total_calls"] += 1+ for record_key, bucket_key, total_key in (+ ("prompt_tokens", "prompt_tokens", "total_prompt_tokens"),+ ("cached_tokens", "cached_tokens", "total_cached_tokens"),+ (+ "cache_creation_tokens",+ "cache_creation_tokens",+ "total_cache_creation_tokens",+ ),+ ("completion_tokens", "completion_tokens", "total_completion_tokens"),+ ):+ value = record.get(record_key)+ if isinstance(value, int) and not isinstance(value, bool) and value >= 0:+ bucket[bucket_key] += value+ totals[total_key] += value++ if not totals["total_calls"]:+ return None+ return {"session_id": session_id, **totals, "models": models}++ def get_endpoint(self) -> Endpoint:+ """Contribute the session-scoped routing-stat snapshot endpoint."""+ from switchyard.lib.endpoints.routing_log_stats_endpoint import (+ RoutingLogStatsEndpoint,+ )++ return RoutingLogStatsEndpoint(self)+def _usage_tokens(body: object) -> dict[str, int]:"""Six-field token breakdown matching the global routing-stats schema.@@ -92,7 +166,9 @@ def _usage_tokens(body: object) -> dict[str, int]:reasoning = 0if prompt or completion:# OpenAI Chat Completions.- cached = _int_field(_field(usage, "prompt_tokens_details"), "cached_tokens")+ prompt_details = _field(usage, "prompt_tokens_details")+ cached = _int_field(prompt_details, "cached_tokens")+ cache_creation = _int_field(prompt_details, "cache_creation_tokens")reasoning = _int_field(_field(usage, "completion_tokens_details"), "reasoning_tokens")else:completion = _int_field(usage, "output_tokens")@@ -102,6 +178,7 @@ def _usage_tokens(body: object) -> dict[str, int]:# OpenAI Responses API: prompt_tokens already includes cached.prompt = _int_field(usage, "input_tokens")cached = _int_field(input_details, "cached_tokens")+ cache_creation = _int_field(input_details, "cache_creation_tokens")else:# Anthropic Messages: cache tokens are prompt siblings, so fold in.cached = _int_field(usage, "cache_read_input_tokens")diff --git a/switchyard/lib/profiles/switchyard_adapter.py b/switchyard/lib/profiles/switchyard_adapter.pyindex d33699650..6ae96dcc9 100644--- a/switchyard/lib/profiles/switchyard_adapter.py+++ b/switchyard/lib/profiles/switchyard_adapter.py@@ -3,14 +3,19 @@"""Compatibility adapter that serves a Profile through the legacy endpoint contract."""-from collections.abc import Mapping-from typing import Any+from collections.abc import Mapping, Sequence+from typing import Any, castfrom switchyard.lib.profiles.protocols import ContextAwareProfile, Profile, ProfileLifecyclefrom switchyard.lib.proxy_context import ProxyContextfrom switchyard.lib.request_metadata import CTX_PROFILE_REQUEST_HEADERSfrom switchyard.lib.roles import TranslatedResponse-from switchyard_rust.core import ChatRequest, request_type_value+from switchyard_rust.core import (+ ChatRequest,+ ChatResponse,+ SwitchyardProcessorError,+ request_type_value,+)from switchyard_rust.profiles import ProfileInput, ProfileRequestMetadatafrom switchyard_rust.translation import TranslationEngine@@ -31,10 +36,12 @@ class ProfileSwitchyard:self,profile: Profile[Any],translator: TranslationEngine | None = None,+ response_processors: Sequence[Any] = (),) -> None:"""Create an adapter around one concrete Profile runtime."""self._profile = profileself._translator = translator or TranslationEngine()+ self._response_processors = tuple(response_processors)def iter_components(self) -> list[object]:"""Return lifecycle components in startup order."""@@ -42,14 +49,19 @@ class ProfileSwitchyard:components = self._profile.iter_components()else:components = [self._profile]- return [*components, self._translator]+ return [*components, *self._response_processors, self._translator]async def call(self,request: ChatRequest,ctx: ProxyContext | None = None,) -> TranslatedResponse:- """Run the wrapped Profile and translate its response for the caller."""+ """Run the wrapped Profile and translate its response for the caller.++ Raises:+ SwitchyardProcessorError: A response processor raises or returns a value that is not+ a ``ChatResponse``.+ """context = ctx if ctx is not None else ProxyContext()profile_input = ProfileInput(request,@@ -59,7 +71,22 @@ class ProfileSwitchyard:response = await self._profile.run_with_context(profile_input, context)else:response = await self._profile.run(profile_input)- return await self._translator.translate(context, request, response)+ current: Any = response+ for processor in self._response_processors:+ try:+ current = await processor.process(context, current)+ except Exception as error:+ raise SwitchyardProcessorError(str(error)) from error+ if not isinstance(current, ChatResponse):+ actual = type(current).__name__+ raise SwitchyardProcessorError(+ f"Response processor returned {actual}, expected ChatResponse"+ )+ return await self._translator.translate(+ context,+ request,+ cast(ChatResponse, current),+ )def _profile_metadata_from_context(context: ProxyContext) -> ProfileRequestMetadata:diff --git a/tests/test_routing_log_response_processor.py b/tests/test_routing_log_response_processor.pyindex 62af541be..736ea5520 100644--- a/tests/test_routing_log_response_processor.py+++ b/tests/test_routing_log_response_processor.py@@ -9,6 +9,8 @@ import jsonfrom collections.abc import AsyncIteratorfrom pathlib import Path+from fastapi import FastAPI+from fastapi.testclient import TestClientfrom openai.types.chat import ChatCompletionChunkfrom openai.types.chat.chat_completion_chunk import Choice as ChunkChoicefrom openai.types.chat.chat_completion_chunk import ChoiceDelta@@ -16,6 +18,7 @@ from openai.types.completion_usage import CompletionUsagefrom switchyard.cli.switchyard_cli import _build_parserfrom switchyard.lib.chat_response import ResponseStream+from switchyard.lib.endpoints import RoutingLogStatsEndpointfrom switchyard.lib.processors.routing_log_response_processor import (RoutingLogResponseProcessor,)@@ -54,7 +57,10 @@ def _openai_completion() -> ChatResponse:"prompt_tokens": 8,"completion_tokens": 3,"total_tokens": 11,- "prompt_tokens_details": {"cached_tokens": 6},+ "prompt_tokens_details": {+ "cached_tokens": 6,+ "cache_creation_tokens": 1,+ },"completion_tokens_details": {"reasoning_tokens": 2},},})@@ -111,6 +117,7 @@ def _records(log_file: Path) -> list[dict]:async def test_openai_completion_record_carries_task_and_usage(tmp_path: Path) -> None:log_file = tmp_path / "routing_requests.jsonl"ctx = _ctx(headers=TASK_HEADERS)+ ctx.selected_model = "configured-profile"ctx.metadata["_random_routing_tier"] = "weak"await RoutingLogResponseProcessor(log_file).process(ctx, _openai_completion())@@ -122,7 +129,7 @@ async def test_openai_completion_record_carries_task_and_usage(tmp_path: Path) -assert record["tier"] == "weak"assert record["prompt_tokens"] == 8assert record["cached_tokens"] == 6- assert record["cache_creation_tokens"] == 0+ assert record["cache_creation_tokens"] == 1assert record["completion_tokens"] == 3assert record["reasoning_tokens"] == 2assert record["total_tokens"] == 11@@ -205,7 +212,79 @@ async def test_appends_one_line_per_request(tmp_path: Path) -> None:assert len(_records(log_file)) == 3+def test_snapshot_session_aggregates_models_and_ignores_bad_records(tmp_path: Path) -> None:+ log_file = tmp_path / "routing_requests.jsonl"+ log_file.write_text(+ "\n".join([+ json.dumps({+ "session_id": "trial-session-1", "model": "model-a",+ "prompt_tokens": 8, "cached_tokens": 2,+ "cache_creation_tokens": 1, "completion_tokens": 3,+ }),+ json.dumps({+ "session_id": "trial-session-1", "model": "model-b",+ "prompt_tokens": 13, "cached_tokens": 4,+ "cache_creation_tokens": 2, "completion_tokens": 5,+ }),+ json.dumps({"session_id": "other", "model": "model-a"}),+ json.dumps(["not", "an", "object"]),+ "not json",+ ]) + "\n",+ encoding="utf-8",+ )++ snapshot = RoutingLogResponseProcessor(log_file).snapshot_session("trial-session-1")++ assert snapshot == {+ "session_id": "trial-session-1",+ "total_calls": 2,+ "total_prompt_tokens": 21,+ "total_cached_tokens": 6,+ "total_cache_creation_tokens": 3,+ "total_completion_tokens": 8,+ "models": {+ "model-a": {+ "calls": 1, "prompt_tokens": 8, "cached_tokens": 2,+ "cache_creation_tokens": 1, "completion_tokens": 3,+ },+ "model-b": {+ "calls": 1, "prompt_tokens": 13, "cached_tokens": 4,+ "cache_creation_tokens": 2, "completion_tokens": 5,+ },+ },+ }+++def test_session_stats_endpoint_returns_snapshot_and_404(tmp_path: Path) -> None:+ log_file = tmp_path / "routing_requests.jsonl"+ processor = RoutingLogResponseProcessor(log_file)+ app = FastAPI()+ endpoint = processor.get_endpoint()+ assert isinstance(endpoint, RoutingLogStatsEndpoint)+ endpoint.register(app)++ with TestClient(app) as client:+ assert client.get(+ "/v1/routing/session-stats", params={"session_id": "missing"}+ ).status_code == 404++ log_file.write_text(+ json.dumps({+ "session_id": "trial-session-1", "model": "model-a",+ "prompt_tokens": 8, "cached_tokens": 2,+ "cache_creation_tokens": 1, "completion_tokens": 3,+ }) + "\n",+ encoding="utf-8",+ )+ with TestClient(app) as client:+ response = client.get(+ "/v1/routing/session-stats", params={"session_id": "trial-session-1"}+ )+ assert response.status_code == 200+ assert response.json()["models"]["model-a"]["cached_tokens"] == 2++def test_serve_parser_accepts_routing_log_file() -> None:parser = _build_parser()- args = parser.parse_args(["serve", "--routing-log-file", "/tmp/routing.jsonl"])- assert args.routing_log_file == "/tmp/routing.jsonl"+ args = parser.parse_args(["serve", "--routing-log-file", "tmp/routing.jsonl"])+ assert args.routing_log_file == "tmp/routing.jsonl"diff --git a/tests/test_run_manifest.py b/tests/test_run_manifest.pyindex 18a6a5513..a94ec9dce 100644--- a/tests/test_run_manifest.py+++ b/tests/test_run_manifest.py@@ -580,7 +580,8 @@ def test_finalize_writes_routing_stats_by_task(tmp_path: Path) -> None:log = run_dir / "routing_requests.jsonl"log.write_text(json.dumps({"task": "task-a", "session_id": "s1", "model": "opus", "tier": "strong",- "prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12,+ "prompt_tokens": 10, "cached_tokens": 3, "cache_creation_tokens": 1,+ "completion_tokens": 2, "total_tokens": 12,}) + "\n")by_task = run_dir / "routing_stats_by_task.json"@@ -592,6 +593,7 @@ def test_finalize_writes_routing_stats_by_task(tmp_path: Path) -> None:assert rc == 0summary = json.loads(by_task.read_text())assert summary["tasks"]["task-a"]["final"]["models"][0]["calls"] == 1+ assert summary["tasks"]["task-a"]["final"]["models"][0]["cached_tokens"] == 3manifest = json.loads(out.read_text())assert manifest["outcomes"]["routing_stats_by_task_json_status"] == "present"