Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(serve): route-log per-request routing stats for profile-config serve (#118)

Brian Newsomd0a924c597b1
diff --git a/docs/cli_reference.md b/docs/cli_reference.md
index 864225a0d..b08ebc714 100644
--- a/docs/cli_reference.md
+++ b/docs/cli_reference.md
@@ -214,6 +214,7 @@ The server exposes the OpenAI Chat Completions (`/v1/chat/completions`), Anthrop
switchyard [--routing-profiles PATH] serve
[--host HOST] [--port PORT] [--workers N]
[--reload] [--inbound FORMAT]
+ [--routing-log-file PATH]
[--intake-enabled|--enable-intake [INTAKE OVERRIDES]]
```
@@ -225,7 +226,7 @@ switchyard [--routing-profiles PATH] serve
| `--host`, `--port`/`-p`, `--reload` | [Transport](#transport-server-verbs). |
| `--inbound FORMAT` | Backwards-compatible no-op; all request APIs are always registered. |
| `--workers` / `-w` | uvicorn worker count. |
-| `--routing-log-file PATH` | Append one JSONL routing record per request (task, session, selected model, tier, token usage) to PATH. |
+| `--routing-log-file PATH` | Append one JSONL routing record per request (task, session, actual model, tier, token and cache usage) to PATH. Also exposes `GET /v1/routing/session-stats?session_id=...`. |
| `--intake-enabled` / `--enable-intake`, `--intake-base-url`, `--intake-workspace`, `--intake-api-key`, `--intake-target-url` | [Intake sink](#intake-sink-serve-and-launchers). |
**Notes**
diff --git a/switchyard/lib/endpoints/__init__.py b/switchyard/lib/endpoints/__init__.py
index 8cb676cb8..fa6c61bd1 100644
--- a/switchyard/lib/endpoints/__init__.py
+++ b/switchyard/lib/endpoints/__init__.py
@@ -18,6 +18,9 @@ if TYPE_CHECKING:
OpenAIChatEndpoint,
)
from switchyard.lib.endpoints.responses_endpoint import ResponsesEndpoint
+ from switchyard.lib.endpoints.routing_log_stats_endpoint import (
+ RoutingLogStatsEndpoint,
+ )
from switchyard.lib.endpoints.stats_endpoint import StatsEndpoint
__all__ = [
@@ -26,6 +29,7 @@ __all__ = [
"ModelsEndpoint",
"OpenAIChatEndpoint",
"ResponsesEndpoint",
+ "RoutingLogStatsEndpoint",
]
@@ -52,4 +56,9 @@ def __getattr__(name: str) -> Any:
ResponsesEndpoint,
)
return ResponsesEndpoint
+ elif name == "RoutingLogStatsEndpoint":
+ from switchyard.lib.endpoints.routing_log_stats_endpoint import (
+ RoutingLogStatsEndpoint,
+ )
+ return RoutingLogStatsEndpoint
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/switchyard/lib/endpoints/routing_log_stats_endpoint.py b/switchyard/lib/endpoints/routing_log_stats_endpoint.py
new file mode 100644
index 000000000..4491f877b
--- /dev/null
+++ b/switchyard/lib/endpoints/routing_log_stats_endpoint.py
@@ -0,0 +1,45 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""HTTP access to session-scoped aggregates from the durable routing log."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING
+
+from fastapi import APIRouter, HTTPException
+
+from switchyard.lib.endpoints.base import Endpoint
+
+if TYPE_CHECKING:
+ from fastapi import FastAPI
+
+ from switchyard.lib.processors.routing_log_response_processor import (
+ RoutingLogResponseProcessor,
+ )
+
+
+class RoutingLogStatsEndpoint(Endpoint):
+ """Expose one trial session's model and token aggregates."""
+
+ register_once = True
+
+ def __init__(self, processor: RoutingLogResponseProcessor) -> None:
+ self._processor = processor
+
+ def register(self, app: FastAPI) -> None:
+ routes = APIRouter()
+ processor = self._processor
+
+ async def get_session_stats(session_id: str) -> dict[str, object]:
+ snapshot = await asyncio.to_thread(processor.snapshot_session, session_id)
+ if snapshot is None:
+ raise HTTPException(status_code=404, detail="routing session not found")
+ return snapshot
+
+ routes.get("/v1/routing/session-stats")(get_session_stats)
+ app.include_router(routes, tags=["Routing log"])
+
+
+__all__ = ["RoutingLogStatsEndpoint"]
diff --git a/switchyard/lib/processors/routing_log_response_processor.py b/switchyard/lib/processors/routing_log_response_processor.py
index 89b2101c8..b378d39c8 100644
--- a/switchyard/lib/processors/routing_log_response_processor.py
+++ b/switchyard/lib/processors/routing_log_response_processor.py
@@ -12,7 +12,7 @@ import threading
from collections.abc import Mapping
from datetime import UTC, datetime
from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
from switchyard.lib.chat_response.streaming_response_accumulator import (
attach_final_response_callback,
@@ -21,6 +21,9 @@ from switchyard.lib.proxy_context import CTX_PROXY_ACTUAL_MODEL, ProxyContext
from switchyard.lib.request_metadata import CTX_PROFILE_REQUEST_HEADERS, CTX_REQUEST_METADATA
from switchyard_rust.core import ChatResponse
+if TYPE_CHECKING:
+ from switchyard.lib.endpoints.base import Endpoint
+
logger = logging.getLogger(__name__)
@@ -41,8 +44,8 @@ class RoutingLogResponseProcessor:
async def process(self, ctx: ProxyContext, response: ChatResponse) -> ChatResponse:
"""Log one routing record for the completed request; return the response unchanged."""
- served_model: str = ctx.selected_model or ctx.metadata.get(
- CTX_PROXY_ACTUAL_MODEL, "unknown",
+ served_model: str = (
+ ctx.metadata.get(CTX_PROXY_ACTUAL_MODEL) or ctx.selected_model or "unknown"
)
async def _emit(final: ChatResponse) -> None:
@@ -77,6 +80,77 @@ class RoutingLogResponseProcessor:
except OSError as exc:
logger.warning("Routing log: failed to append to %s: %s", self._log_file, exc)
+ def snapshot_session(self, session_id: str) -> dict[str, object] | None:
+ """Aggregate the durable request log for one exact trial session.
+
+ This re-reads and re-parses the whole log on every call. It is sized for
+ benchmark runs (thousands of records), not a long-lived production server
+ with millions of sessions, where it would be O(records) per query.
+ """
+ models: dict[str, dict[str, int]] = {}
+ totals = {
+ "total_calls": 0,
+ "total_prompt_tokens": 0,
+ "total_cached_tokens": 0,
+ "total_cache_creation_tokens": 0,
+ "total_completion_tokens": 0,
+ }
+ try:
+ if not self._log_file.is_file():
+ return None
+ lines = self._log_file.read_text(encoding="utf-8").splitlines()
+ except OSError as exc:
+ logger.warning("Routing log: failed to read %s: %s", self._log_file, exc)
+ return None
+
+ for line in lines:
+ try:
+ record = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if not isinstance(record, Mapping) or record.get("session_id") != session_id:
+ continue
+ model_value = record.get("model")
+ model = model_value if isinstance(model_value, str) and model_value else "unknown"
+ bucket = models.setdefault(
+ model,
+ {
+ "calls": 0,
+ "prompt_tokens": 0,
+ "cached_tokens": 0,
+ "cache_creation_tokens": 0,
+ "completion_tokens": 0,
+ },
+ )
+ bucket["calls"] += 1
+ totals["total_calls"] += 1
+ for record_key, bucket_key, total_key in (
+ ("prompt_tokens", "prompt_tokens", "total_prompt_tokens"),
+ ("cached_tokens", "cached_tokens", "total_cached_tokens"),
+ (
+ "cache_creation_tokens",
+ "cache_creation_tokens",
+ "total_cache_creation_tokens",
+ ),
+ ("completion_tokens", "completion_tokens", "total_completion_tokens"),
+ ):
+ value = record.get(record_key)
+ if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
+ bucket[bucket_key] += value
+ totals[total_key] += value
+
+ if not totals["total_calls"]:
+ return None
+ return {"session_id": session_id, **totals, "models": models}
+
+ def get_endpoint(self) -> Endpoint:
+ """Contribute the session-scoped routing-stat snapshot endpoint."""
+ from switchyard.lib.endpoints.routing_log_stats_endpoint import (
+ RoutingLogStatsEndpoint,
+ )
+
+ return RoutingLogStatsEndpoint(self)
+
def _usage_tokens(body: object) -> dict[str, int]:
"""Six-field token breakdown matching the global routing-stats schema.
@@ -92,7 +166,9 @@ def _usage_tokens(body: object) -> dict[str, int]:
reasoning = 0
if prompt or completion:
# OpenAI Chat Completions.
- cached = _int_field(_field(usage, "prompt_tokens_details"), "cached_tokens")
+ prompt_details = _field(usage, "prompt_tokens_details")
+ cached = _int_field(prompt_details, "cached_tokens")
+ cache_creation = _int_field(prompt_details, "cache_creation_tokens")
reasoning = _int_field(_field(usage, "completion_tokens_details"), "reasoning_tokens")
else:
completion = _int_field(usage, "output_tokens")
@@ -102,6 +178,7 @@ def _usage_tokens(body: object) -> dict[str, int]:
# OpenAI Responses API: prompt_tokens already includes cached.
prompt = _int_field(usage, "input_tokens")
cached = _int_field(input_details, "cached_tokens")
+ cache_creation = _int_field(input_details, "cache_creation_tokens")
else:
# Anthropic Messages: cache tokens are prompt siblings, so fold in.
cached = _int_field(usage, "cache_read_input_tokens")
diff --git a/switchyard/lib/profiles/switchyard_adapter.py b/switchyard/lib/profiles/switchyard_adapter.py
index d33699650..6ae96dcc9 100644
--- a/switchyard/lib/profiles/switchyard_adapter.py
+++ b/switchyard/lib/profiles/switchyard_adapter.py
@@ -3,14 +3,19 @@
"""Compatibility adapter that serves a Profile through the legacy endpoint contract."""
-from collections.abc import Mapping
-from typing import Any
+from collections.abc import Mapping, Sequence
+from typing import Any, cast
from switchyard.lib.profiles.protocols import ContextAwareProfile, Profile, ProfileLifecycle
from switchyard.lib.proxy_context import ProxyContext
from switchyard.lib.request_metadata import CTX_PROFILE_REQUEST_HEADERS
from switchyard.lib.roles import TranslatedResponse
-from switchyard_rust.core import ChatRequest, request_type_value
+from switchyard_rust.core import (
+ ChatRequest,
+ ChatResponse,
+ SwitchyardProcessorError,
+ request_type_value,
+)
from switchyard_rust.profiles import ProfileInput, ProfileRequestMetadata
from switchyard_rust.translation import TranslationEngine
@@ -31,10 +36,12 @@ class ProfileSwitchyard:
self,
profile: Profile[Any],
translator: TranslationEngine | None = None,
+ response_processors: Sequence[Any] = (),
) -> None:
"""Create an adapter around one concrete Profile runtime."""
self._profile = profile
self._translator = translator or TranslationEngine()
+ self._response_processors = tuple(response_processors)
def iter_components(self) -> list[object]:
"""Return lifecycle components in startup order."""
@@ -42,14 +49,19 @@ class ProfileSwitchyard:
components = self._profile.iter_components()
else:
components = [self._profile]
- return [*components, self._translator]
+ return [*components, *self._response_processors, self._translator]
async def call(
self,
request: ChatRequest,
ctx: ProxyContext | None = None,
) -> TranslatedResponse:
- """Run the wrapped Profile and translate its response for the caller."""
+ """Run the wrapped Profile and translate its response for the caller.
+
+ Raises:
+ SwitchyardProcessorError: A response processor raises or returns a value that is not
+ a ``ChatResponse``.
+ """
context = ctx if ctx is not None else ProxyContext()
profile_input = ProfileInput(
request,
@@ -59,7 +71,22 @@ class ProfileSwitchyard:
response = await self._profile.run_with_context(profile_input, context)
else:
response = await self._profile.run(profile_input)
- return await self._translator.translate(context, request, response)
+ current: Any = response
+ for processor in self._response_processors:
+ try:
+ current = await processor.process(context, current)
+ except Exception as error:
+ raise SwitchyardProcessorError(str(error)) from error
+ if not isinstance(current, ChatResponse):
+ actual = type(current).__name__
+ raise SwitchyardProcessorError(
+ f"Response processor returned {actual}, expected ChatResponse"
+ )
+ return await self._translator.translate(
+ context,
+ request,
+ cast(ChatResponse, current),
+ )
def _profile_metadata_from_context(context: ProxyContext) -> ProfileRequestMetadata:
diff --git a/tests/test_routing_log_response_processor.py b/tests/test_routing_log_response_processor.py
index 62af541be..736ea5520 100644
--- a/tests/test_routing_log_response_processor.py
+++ b/tests/test_routing_log_response_processor.py
@@ -9,6 +9,8 @@ import json
from collections.abc import AsyncIterator
from pathlib import Path
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
from openai.types.chat import ChatCompletionChunk
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
from openai.types.chat.chat_completion_chunk import ChoiceDelta
@@ -16,6 +18,7 @@ from openai.types.completion_usage import CompletionUsage
from switchyard.cli.switchyard_cli import _build_parser
from switchyard.lib.chat_response import ResponseStream
+from switchyard.lib.endpoints import RoutingLogStatsEndpoint
from switchyard.lib.processors.routing_log_response_processor import (
RoutingLogResponseProcessor,
)
@@ -54,7 +57,10 @@ def _openai_completion() -> ChatResponse:
"prompt_tokens": 8,
"completion_tokens": 3,
"total_tokens": 11,
- "prompt_tokens_details": {"cached_tokens": 6},
+ "prompt_tokens_details": {
+ "cached_tokens": 6,
+ "cache_creation_tokens": 1,
+ },
"completion_tokens_details": {"reasoning_tokens": 2},
},
})
@@ -111,6 +117,7 @@ def _records(log_file: Path) -> list[dict]:
async def test_openai_completion_record_carries_task_and_usage(tmp_path: Path) -> None:
log_file = tmp_path / "routing_requests.jsonl"
ctx = _ctx(headers=TASK_HEADERS)
+ ctx.selected_model = "configured-profile"
ctx.metadata["_random_routing_tier"] = "weak"
await RoutingLogResponseProcessor(log_file).process(ctx, _openai_completion())
@@ -122,7 +129,7 @@ async def test_openai_completion_record_carries_task_and_usage(tmp_path: Path) -
assert record["tier"] == "weak"
assert record["prompt_tokens"] == 8
assert record["cached_tokens"] == 6
- assert record["cache_creation_tokens"] == 0
+ assert record["cache_creation_tokens"] == 1
assert record["completion_tokens"] == 3
assert record["reasoning_tokens"] == 2
assert record["total_tokens"] == 11
@@ -205,7 +212,79 @@ async def test_appends_one_line_per_request(tmp_path: Path) -> None:
assert len(_records(log_file)) == 3
+def test_snapshot_session_aggregates_models_and_ignores_bad_records(tmp_path: Path) -> None:
+ log_file = tmp_path / "routing_requests.jsonl"
+ log_file.write_text(
+ "\n".join([
+ json.dumps({
+ "session_id": "trial-session-1", "model": "model-a",
+ "prompt_tokens": 8, "cached_tokens": 2,
+ "cache_creation_tokens": 1, "completion_tokens": 3,
+ }),
+ json.dumps({
+ "session_id": "trial-session-1", "model": "model-b",
+ "prompt_tokens": 13, "cached_tokens": 4,
+ "cache_creation_tokens": 2, "completion_tokens": 5,
+ }),
+ json.dumps({"session_id": "other", "model": "model-a"}),
+ json.dumps(["not", "an", "object"]),
+ "not json",
+ ]) + "\n",
+ encoding="utf-8",
+ )
+
+ snapshot = RoutingLogResponseProcessor(log_file).snapshot_session("trial-session-1")
+
+ assert snapshot == {
+ "session_id": "trial-session-1",
+ "total_calls": 2,
+ "total_prompt_tokens": 21,
+ "total_cached_tokens": 6,
+ "total_cache_creation_tokens": 3,
+ "total_completion_tokens": 8,
+ "models": {
+ "model-a": {
+ "calls": 1, "prompt_tokens": 8, "cached_tokens": 2,
+ "cache_creation_tokens": 1, "completion_tokens": 3,
+ },
+ "model-b": {
+ "calls": 1, "prompt_tokens": 13, "cached_tokens": 4,
+ "cache_creation_tokens": 2, "completion_tokens": 5,
+ },
+ },
+ }
+
+
+def test_session_stats_endpoint_returns_snapshot_and_404(tmp_path: Path) -> None:
+ log_file = tmp_path / "routing_requests.jsonl"
+ processor = RoutingLogResponseProcessor(log_file)
+ app = FastAPI()
+ endpoint = processor.get_endpoint()
+ assert isinstance(endpoint, RoutingLogStatsEndpoint)
+ endpoint.register(app)
+
+ with TestClient(app) as client:
+ assert client.get(
+ "/v1/routing/session-stats", params={"session_id": "missing"}
+ ).status_code == 404
+
+ log_file.write_text(
+ json.dumps({
+ "session_id": "trial-session-1", "model": "model-a",
+ "prompt_tokens": 8, "cached_tokens": 2,
+ "cache_creation_tokens": 1, "completion_tokens": 3,
+ }) + "\n",
+ encoding="utf-8",
+ )
+ with TestClient(app) as client:
+ response = client.get(
+ "/v1/routing/session-stats", params={"session_id": "trial-session-1"}
+ )
+ assert response.status_code == 200
+ assert response.json()["models"]["model-a"]["cached_tokens"] == 2
+
+
def test_serve_parser_accepts_routing_log_file() -> None:
parser = _build_parser()
- args = parser.parse_args(["serve", "--routing-log-file", "/tmp/routing.jsonl"])
- assert args.routing_log_file == "/tmp/routing.jsonl"
+ args = parser.parse_args(["serve", "--routing-log-file", "tmp/routing.jsonl"])
+ assert args.routing_log_file == "tmp/routing.jsonl"
diff --git a/tests/test_run_manifest.py b/tests/test_run_manifest.py
index 18a6a5513..a94ec9dce 100644
--- a/tests/test_run_manifest.py
+++ b/tests/test_run_manifest.py
@@ -580,7 +580,8 @@ def test_finalize_writes_routing_stats_by_task(tmp_path: Path) -> None:
log = run_dir / "routing_requests.jsonl"
log.write_text(json.dumps({
"task": "task-a", "session_id": "s1", "model": "opus", "tier": "strong",
- "prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12,
+ "prompt_tokens": 10, "cached_tokens": 3, "cache_creation_tokens": 1,
+ "completion_tokens": 2, "total_tokens": 12,
}) + "\n")
by_task = run_dir / "routing_stats_by_task.json"
@@ -592,6 +593,7 @@ def test_finalize_writes_routing_stats_by_task(tmp_path: Path) -> None:
assert rc == 0
summary = json.loads(by_task.read_text())
assert summary["tasks"]["task-a"]["final"]["models"][0]["calls"] == 1
+ assert summary["tasks"]["task-a"]["final"]["models"][0]["cached_tokens"] == 3
manifest = json.loads(out.read_text())
assert manifest["outcomes"]["routing_stats_by_task_json_status"] == "present"