Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(python): remove dead chain processors (#127)

nachiketb-nvidia7315625e64e8
diff --git a/AGENTS.md b/AGENTS.md
index 73dba19e1..5a97c7ba5 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -155,7 +155,6 @@ switchyard/
│ ├── route_table_builders.py # Shared profile-backed table builders
│ ├── llm_client.py # OpenAILLMClient
│ ├── cost_estimator.py # Token-cost bookkeeping
-│ ├── live_stats_collector.py # Live request stats
│ ├── stats_accumulator.py # Stats accumulation helpers
│ ├── request_metadata.py # RequestMetadata
│ ├── chat_response/ # Rust-backed response re-exports + stream adapters
@@ -178,7 +177,6 @@ switchyard/
│ │ ├── random_routing_request_processor.py
│ │ ├── stats_request_processor.py
│ │ ├── stats_response_processor_accumulator.py
-│ │ ├── stats_response_processor_live_collector.py
│ │ ├── intake_request_processor.py
│ │ ├── intake_response_processor.py
│ │ ├── intake_payload_builder.py
diff --git a/switchyard/lib/backends/anthropic_cache_breakpoint_backend.py b/switchyard/lib/backends/anthropic_cache_breakpoint_backend.py
index 563bcb88b..cbab05005 100644
--- a/switchyard/lib/backends/anthropic_cache_breakpoint_backend.py
+++ b/switchyard/lib/backends/anthropic_cache_breakpoint_backend.py
@@ -22,10 +22,9 @@ honors prompt caching:
request and passes the body through verbatim (preserving the
markers all the way to the wire).
-Why this lives in a wrapper instead of in
-:class:`switchyard.lib.processors.turn_based_router_request_processor.TurnBasedRouterRequestProcessor`:
-mutating the chain-level :attr:`ChatRequest.request_type` at the
-request-processor layer propagates through the rest of the chain.
+Why this lives in a wrapper instead of a request processor: mutating the
+chain-level :attr:`ChatRequest.request_type` before the backend call propagates
+through the rest of the chain.
The terminal :class:`TranslationEngine` step then reads the modified
type, decides "the client wanted Anthropic," and skips translation
of the Anthropic response back to the client's original wire format.
diff --git a/switchyard/lib/endpoints/stats_endpoint.py b/switchyard/lib/endpoints/stats_endpoint.py
index e381363cc..31d96f94b 100644
--- a/switchyard/lib/endpoints/stats_endpoint.py
+++ b/switchyard/lib/endpoints/stats_endpoint.py
@@ -21,7 +21,6 @@ from fastapi.responses import Response
from switchyard.lib.endpoints import outcome_metrics
from switchyard.lib.endpoints.base import Endpoint as NemoSwitchyardEndpoint
from switchyard.lib.endpoints.prometheus_emitter import render as render_extra_metrics
-from switchyard.lib.live_stats_collector import LiveStatsCollector
from switchyard.lib.prometheus_exposition import render_prometheus
from switchyard.lib.stats_accumulator import StatsAccumulator
@@ -29,8 +28,6 @@ if TYPE_CHECKING:
from fastapi import FastAPI
-StatsSource = StatsAccumulator | LiveStatsCollector
-
# Prometheus text exposition format 0.0.4 content-type.
PROMETHEUS_CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
@@ -52,7 +49,7 @@ class StatsEndpoint(NemoSwitchyardEndpoint):
register_once = True
- def __init__(self, stats: StatsSource) -> None:
+ def __init__(self, stats: StatsAccumulator) -> None:
self._stats = stats
def register(self, app: FastAPI) -> None:
@@ -61,16 +58,11 @@ class StatsEndpoint(NemoSwitchyardEndpoint):
async def get_stats() -> dict[str, Any]:
"""Snapshot of per-model request / token / latency / cost stats."""
- if isinstance(stats, LiveStatsCollector):
- return stats.to_dict()
return await stats.snapshot()
async def reset_stats() -> dict[str, str]:
"""Zero all stats counters."""
- if isinstance(stats, LiveStatsCollector):
- stats.reset()
- else:
- await stats.reset()
+ await stats.reset()
return {"status": "reset"}
async def get_metrics() -> Response:
@@ -81,11 +73,7 @@ class StatsEndpoint(NemoSwitchyardEndpoint):
:mod:`switchyard.lib.endpoints.prometheus_emitter` so a single
``/metrics`` scrape carries both surfaces.
"""
- snapshot = (
- stats.to_dict()
- if isinstance(stats, LiveStatsCollector)
- else await stats.snapshot()
- )
+ snapshot = await stats.snapshot()
outcome_block = "\n".join(outcome_metrics.render_lines()) + "\n"
return Response(
content=(
diff --git a/switchyard/lib/live_stats_collector.py b/switchyard/lib/live_stats_collector.py
deleted file mode 100644
index afb865087..000000000
--- a/switchyard/lib/live_stats_collector.py
+++ /dev/null
@@ -1,277 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Thread-safe per-model LLM usage statistics collector.
-
-Tracks token usage bucketed by model name, with an optional tier label
-(e.g. ``"strong"`` / ``"weak"`` from random routing). Used by:
-
-* :class:`StatsResponseProcessor` — records after every LLM response.
-* ``launch_claude`` TUI footer — reads the aggregate via ``snapshot()``.
-* ``GET /v1/routing/stats`` HTTP endpoint — reads ``to_dict()``.
-
-Cost estimation is best-effort: ``to_dict()`` includes an
-``estimated_cost_usd`` field per model when the model name appears in
-the built-in price table; ``None`` otherwise. Prices are per 1 M
-tokens (input / output) as of 2025 Q3 and do **not** account for cache
-read / write premiums.
-"""
-
-from __future__ import annotations
-
-import dataclasses
-import re
-import threading
-from dataclasses import dataclass
-
-# ---------------------------------------------------------------------------
-# Price table — USD per 1 M tokens (input, output).
-# Add entries as needed; unknown models get estimated_cost_usd = None.
-# Keys use the base name without date suffixes (see _normalize_model_name).
-# ---------------------------------------------------------------------------
-
-_PRICE_PER_1M: dict[str, tuple[float, float]] = {
- # Anthropic
- "claude-opus-4-7": (15.00, 75.00),
- "claude-sonnet-4-6": ( 3.00, 15.00),
- "claude-sonnet-4-5": ( 3.00, 15.00),
- "claude-haiku-4-5": ( 0.80, 4.00),
- # OpenAI
- "gpt-4o": ( 2.50, 10.00),
- "gpt-4o-mini": ( 0.15, 0.60),
- "o1": (15.00, 60.00),
- "o3": (10.00, 40.00),
- "o4-mini": ( 1.10, 4.40),
-}
-
-_DATE_SUFFIX_RE = re.compile(r"-\d{8}$")
-
-
-def _normalize_model_name(model: str) -> str:
- """Strip trailing ISO-8601 date suffixes used by some providers.
-
- ``"claude-sonnet-4-6-20251022"`` → ``"claude-sonnet-4-6"``
- """
- return _DATE_SUFFIX_RE.sub("", model)
-
-
-def _estimate_cost(
- model: str,
- prompt_tokens: int,
- completion_tokens: int,
-) -> float | None:
- prices = _PRICE_PER_1M.get(model) or _PRICE_PER_1M.get(_normalize_model_name(model))
- if prices is None:
- return None
- input_rate, output_rate = prices
- return (prompt_tokens * input_rate + completion_tokens * output_rate) / 1_000_000
-
-
-# ---------------------------------------------------------------------------
-# Data classes
-# ---------------------------------------------------------------------------
-
-
-@dataclass
-class RequestStats:
- """Aggregate snapshot for the TUI footer."""
-
- request_count: int = 0
- prompt_tokens: int = 0
- completion_tokens: int = 0
- cache_read_tokens: int = 0
- cache_creation_tokens: int = 0
-
-
-@dataclass
-class ModelStats:
- """Accumulated token usage for a single model across all requests.
-
- Field semantics:
-
- * ``prompt_tokens`` — total input billed. For Anthropic this is the
- sum ``input_tokens + cache_creation_input_tokens +
- cache_read_input_tokens`` (Anthropic reports them as siblings, not
- parent/child). For OpenAI it is ``usage.prompt_tokens``.
- * ``max_observed_context_tokens`` — largest prompt-plus-completion
- token count observed in one completed response.
- * ``completion_tokens`` — total output tokens.
- * ``total_tokens`` — ``prompt_tokens + completion_tokens``.
- * ``reasoning_tokens`` — subset of ``completion_tokens`` for chain-of-
- thought / thinking output (OpenAI o-series / Responses API). 0 for
- Anthropic (not reported on the Messages API).
- * ``cache_read_tokens`` — tokens served from the provider cache
- (``cache_read_input_tokens`` on Anthropic, ``cached_tokens`` on
- OpenAI). Priced at 0.1× on Anthropic.
- * ``cache_creation_tokens`` — tokens written to the provider cache
- (Anthropic only, priced at 1.25×). Always 0 for OpenAI.
- """
-
- tier: str = ""
- calls: int = 0
- prompt_tokens: int = 0
- max_observed_context_tokens: int = 0
- completion_tokens: int = 0
- total_tokens: int = 0
- reasoning_tokens: int = 0
- cache_read_tokens: int = 0
- cache_creation_tokens: int = 0
-
-
-# ---------------------------------------------------------------------------
-# Collector
-# ---------------------------------------------------------------------------
-
-
-class LiveStatsCollector:
- """Thread-safe per-model token usage collector.
-
- ``record()`` is called from the async request path (response
- processors / stream taps); ``snapshot()`` is called from the
- footer-repaint thread; ``to_dict()`` is called by the HTTP stats
- endpoint. All methods acquire a single :class:`threading.Lock`
- so they are safe across both threads and async tasks.
- """
-
- def __init__(self) -> None:
- self._lock = threading.Lock()
- self._models: dict[str, ModelStats] = {}
-
- # ------------------------------------------------------------------
- # Write path
- # ------------------------------------------------------------------
-
- def record(
- self,
- model: str = "unknown",
- tier: str = "",
- *,
- prompt_tokens: int = 0,
- completion_tokens: int = 0,
- reasoning_tokens: int = 0,
- cache_read_tokens: int = 0,
- cache_creation_tokens: int = 0,
- ) -> None:
- """Accumulate one response's token counts into *model*'s bucket."""
- with self._lock:
- bucket = self._models.get(model)
- if bucket is None:
- bucket = ModelStats(tier=tier)
- self._models[model] = bucket
- bucket.calls += 1
- bucket.prompt_tokens += prompt_tokens
- bucket.max_observed_context_tokens = max(
- bucket.max_observed_context_tokens,
- prompt_tokens + completion_tokens,
- )
- bucket.completion_tokens += completion_tokens
- bucket.total_tokens += prompt_tokens + completion_tokens
- bucket.reasoning_tokens += reasoning_tokens
- bucket.cache_read_tokens += cache_read_tokens
- bucket.cache_creation_tokens += cache_creation_tokens
-
- def reset(self) -> None:
- """Reset all counters to zero."""
- with self._lock:
- self._models.clear()
-
- # ------------------------------------------------------------------
- # Read path
- # ------------------------------------------------------------------
-
- def snapshot(self) -> RequestStats:
- """Aggregate totals for the TUI footer."""
- with self._lock:
- buckets = list(self._models.values())
- return RequestStats(
- request_count=sum(b.calls for b in buckets),
- prompt_tokens=sum(b.prompt_tokens for b in buckets),
- completion_tokens=sum(b.completion_tokens for b in buckets),
- cache_read_tokens=sum(b.cache_read_tokens for b in buckets),
- cache_creation_tokens=sum(b.cache_creation_tokens for b in buckets),
- )
-
- def tier_breakdown(self) -> list[tuple[str, ModelStats]]:
- """Per-model rows ordered ``strong → weak → others``.
-
- Returned for the random-routing footer so the user can see, at
- a glance, how requests / tokens have split across tiers. The
- ordering is stable across paints (sort key = ``(tier_rank,
- model_name)``) so the footer doesn't flicker between rows.
- """
- _ORDER = {"strong": 0, "weak": 1}
- with self._lock:
- items = [(name, dataclasses.replace(b)) for name, b in self._models.items()]
- items.sort(key=lambda kv: (_ORDER.get(kv[1].tier, 99), kv[0]))
- return items
-
- def to_dict(self) -> dict[str, object]:
- """Full per-model breakdown for the HTTP stats endpoint.
-
- Wire shape::
-
- {
- "total_requests": int,
- "total_tokens": {
- "prompt": int, "completion": int, "total": int,
- "reasoning": int, "cache_read": int, "cache_creation": int,
- },
- "models": {
- "<model-name>": {
- "tier": str,
- "calls": int,
- "prompt_tokens": int,
- "max_observed_context_tokens": int,
- "completion_tokens": int,
- "total_tokens": int,
- "reasoning_tokens": int,
- "cache_read_tokens": int,
- "cache_creation_tokens": int,
- "token_pct": float,
- "estimated_cost_usd": float | null,
- },
- ...
- },
- }
- """
- with self._lock:
- snapshot = {k: dataclasses.replace(v) for k, v in self._models.items()}
-
- total_prompt = sum(b.prompt_tokens for b in snapshot.values())
- total_completion = sum(b.completion_tokens for b in snapshot.values())
- total_tokens = total_prompt + total_completion
- total_requests = sum(b.calls for b in snapshot.values())
-
- def _pct(part: int, whole: int) -> float:
- return round(part / whole * 100, 2) if whole else 0.0
-
- models_out: dict[str, object] = {}
- for name, b in snapshot.items():
- models_out[name] = {
- "tier": b.tier,
- "calls": b.calls,
- "prompt_tokens": b.prompt_tokens,
- "max_observed_context_tokens": b.max_observed_context_tokens,
- "completion_tokens": b.completion_tokens,
- "total_tokens": b.total_tokens,
- "reasoning_tokens": b.reasoning_tokens,
- "cache_read_tokens": b.cache_read_tokens,
- "cache_creation_tokens": b.cache_creation_tokens,
- "token_pct": _pct(b.total_tokens, total_tokens),
- "estimated_cost_usd": _estimate_cost(
- name, b.prompt_tokens, b.completion_tokens,
- ),
- }
-
- return {
- "total_requests": total_requests,
- "total_tokens": {
- "prompt": total_prompt,
- "completion": total_completion,
- "total": total_tokens,
- "reasoning": sum(b.reasoning_tokens for b in snapshot.values()),
- "cache_read": sum(b.cache_read_tokens for b in snapshot.values()),
- "cache_creation": sum(b.cache_creation_tokens for b in snapshot.values()),
- },
- "models": models_out,
- }
diff --git a/switchyard/lib/processors/__init__.py b/switchyard/lib/processors/__init__.py
index 00ab52cf3..17f8a468b 100644
--- a/switchyard/lib/processors/__init__.py
+++ b/switchyard/lib/processors/__init__.py
@@ -14,9 +14,6 @@ from switchyard.lib.processors.stats_request_processor import (
from switchyard.lib.processors.stats_response_processor_accumulator import (
StatsResponseProcessor,
)
-from switchyard.lib.processors.stats_response_processor_live_collector import (
- StatsResponseProcessor as StatsResponseProcessorLiveCollector,
-)
if TYPE_CHECKING:
from switchyard.lib.processors.intake_client import IntakeClient
@@ -32,7 +29,6 @@ __all__ = [
"ModelRewriteRequestProcessor",
"StatsRequestProcessor",
"StatsResponseProcessor",
- "StatsResponseProcessorLiveCollector",
]
diff --git a/switchyard/lib/processors/empty_tool_content_normalizer.py b/switchyard/lib/processors/empty_tool_content_normalizer.py
deleted file mode 100644
index 5ce955569..000000000
--- a/switchyard/lib/processors/empty_tool_content_normalizer.py
+++ /dev/null
@@ -1,85 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Replace empty message content that NIH's Nemotron Nano adapter rejects.
-
-NVIDIA Inference Hub's LiteLLM adapter for ``Nemotron-3-Nano-30B-A3B``
-rejects any message whose ``content`` is empty (or whitespace-only)
-with HTTP 400 ``message content cannot be empty``. Direct probes on
-2026-05-15 mapped exactly which roles trip the guard:
-
-* ``role: "tool"`` with ``content == ""`` or ``None`` — rejected.
-* ``role: "assistant"`` with ``content == ""`` or ``None`` **and no
- ``tool_calls``** — rejected. (Assistant + ``tool_calls`` + empty
- content is accepted, matching the OpenAI Chat Completions spec.)
-* ``role: "system" / "user"`` with empty/null content — accepted (NIH
- doesn't enforce there).
-* Whitespace-only strings (``" "``, ``"\\n"``) are *also* rejected —
- the adapter strips before checking. Placeholder must contain a
- non-whitespace character.
-
-Harbor + terminus-2 routinely produces both shapes (empty shell
-output, retried assistant turns with no content). This processor
-normalises them to ``"(no output)"`` so the upstream never sees an
-empty payload.
-"""
-
-from __future__ import annotations
-
-import logging
-
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.core import ChatRequest
-
-log = logging.getLogger(__name__)
-
-_PLACEHOLDER = "(no output)"
-
-
-def _is_empty(content: object) -> bool:
- """True iff ``content`` is None, empty string, or whitespace-only string."""
- if content is None:
- return True
- if isinstance(content, str) and content.strip() == "":
- return True
- return False
-
-
-class EmptyToolContentNormalizer:
- """Replace empty content on ``tool`` and bare-``assistant`` messages.
-
- Mutates only the shapes NIH's Nemotron adapter rejects (see module
- docstring); leaves everything else untouched.
- """
-
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: # noqa: ARG002
- body = request.body
- if not isinstance(body, dict):
- return request
- messages = body.get("messages")
- if not isinstance(messages, list):
- return request
-
- mutated = False
- for msg in messages:
- if not isinstance(msg, dict):
- continue
- role = msg.get("role")
- if role == "tool":
- if _is_empty(msg.get("content")):
- msg["content"] = _PLACEHOLDER
- mutated = True
- elif role == "assistant":
- # Assistant with tool_calls is allowed to have empty
- # content per the OpenAI spec — and the Nano adapter
- # accepts it. Only normalise the bare case.
- if not msg.get("tool_calls") and _is_empty(msg.get("content")):
- msg["content"] = _PLACEHOLDER
- mutated = True
-
- if mutated:
- request.replace_body(body)
- return request
-
-
-__all__ = ["EmptyToolContentNormalizer"]
diff --git a/switchyard/lib/processors/fixed_tier_request_processor.py b/switchyard/lib/processors/fixed_tier_request_processor.py
deleted file mode 100644
index 98259b6ef..000000000
--- a/switchyard/lib/processors/fixed_tier_request_processor.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Force every request to a single pre-chosen tier — no LLM call.
-
-Drop-in replacement for the
-:class:`LLMClassifierRequestProcessor` + :class:`SignalTierSelectorRequestProcessor`
-pair when you want an all-strong or all-weak baseline run. Skipping
-the classifier round-trip eliminates the per-request token cost and
-the ~3–17 s wall-time tax of a real classification call, so the
-forced-tier baseline measures only the upstream backend's behaviour
-(solve rate, cost, latency) without any classifier contamination.
-
-Stamps :data:`CTX_DETERMINISTIC_ROUTING_TIER` directly so the
-downstream :class:`DeterministicRoutingLLMBackend` dispatches without
-needing a synthetic :class:`RouteDecision` to materialise.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- CTX_DETERMINISTIC_ROUTING_TIER,
-)
-
-if TYPE_CHECKING:
- from switchyard.lib.proxy_context import ProxyContext
- from switchyard_rust.core import ChatRequest
-
-
-class FixedTierRequestProcessor:
- """Stamp a fixed tier label onto every request.
-
- Args:
- tier: Tier label to stamp. Must match one of the labels the
- downstream :class:`DeterministicRoutingLLMBackend` knows
- about (typically ``"strong"`` or ``"weak"``).
- """
-
- def __init__(self, tier: str) -> None:
- if not isinstance(tier, str) or not tier:
- raise ValueError(f"tier must be a non-empty string, got {tier!r}")
- self._tier = tier
-
- @property
- def tier(self) -> str:
- return self._tier
-
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = self._tier
- return request
-
-
-__all__ = ["FixedTierRequestProcessor"]
diff --git a/switchyard/lib/processors/stats_response_processor_live_collector.py b/switchyard/lib/processors/stats_response_processor_live_collector.py
deleted file mode 100644
index 2f140f53e..000000000
--- a/switchyard/lib/processors/stats_response_processor_live_collector.py
+++ /dev/null
@@ -1,337 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Response-side processor that records per-model token usage into a LiveStatsCollector.
-
-This is the single stats-recording mechanism for the chain when using
-LiveStatsCollector (vs StatsAccumulator). It is format-agnostic (handles
-Anthropic Messages, OpenAI Chat Completions, and OpenAI Responses API)
-and routing-policy-agnostic (works with any LLMBackend).
-
-Model attribution uses Rust-owned ``ProxyContext.selected_model`` first,
-falling back to the legacy ``ctx.metadata["_proxy_actual_model"]`` key.
-Tier labelling uses ``ctx.metadata["_random_routing_tier"]`` when present,
-or Rust-owned ``ProxyContext.selected_target`` for strong/weak routing.
-
-The processor exposes ``get_routing_stats()`` / ``reset_routing_stats()``
-and ``get_endpoint()`` so the chain's app factory can mount a live stats
-endpoint at ``GET /v1/routing/stats`` with no extra wiring.
-"""
-
-from __future__ import annotations
-
-from collections.abc import Mapping
-from typing import TYPE_CHECKING
-
-from anthropic.types import RawMessageStreamEvent
-
-from switchyard.lib.live_stats_collector import LiveStatsCollector
-from switchyard.lib.proxy_context import CTX_PROXY_ACTUAL_MODEL, ProxyContext
-from switchyard_rust.core import (
- ChatResponse,
- ChatResponseType,
- response_type_matches,
-)
-from switchyard_rust.translation import sse_frame_payloads
-
-if TYPE_CHECKING:
- from switchyard.lib.endpoints.base import Endpoint
-
-
-class StatsResponseProcessor:
- """Records token usage after each LLM call into a :class:`LiveStatsCollector`.
-
- Non-streaming responses: reads usage directly from the response body.
-
- Streaming responses: attaches a lightweight tap to the stream so
- usage-carrying events update the collector as they flow through —
- no buffering, zero impact on streaming latency.
-
- Supported response shapes are identified by Rust-owned ``ChatResponseType``.
- """
-
- def __init__(
- self,
- collector: LiveStatsCollector,
- *,
- expose_endpoint: bool = True,
- ) -> None:
- self._collector = collector
- self._expose_endpoint = expose_endpoint
-
- async def process(self, ctx: ProxyContext, response: ChatResponse) -> ChatResponse:
- model: str = ctx.selected_model or ctx.metadata.get(CTX_PROXY_ACTUAL_MODEL, "unknown")
- tier = _stats_tier(ctx)
-
- if response_type_matches(response, ChatResponseType.ANTHROPIC_COMPLETION):
- _record_anthropic(response, model, tier, self._collector)
- elif response_type_matches(response, ChatResponseType.ANTHROPIC_STREAM):
- _attach_anthropic_tap(response, model, tier, self._collector)
- elif response_type_matches(response, ChatResponseType.OPENAI_COMPLETION):
- _record_openai_chat(response, model, tier, self._collector)
- elif response_type_matches(response, ChatResponseType.OPENAI_STREAM):
- _attach_openai_chat_tap(response, model, tier, self._collector)
- elif response_type_matches(response, ChatResponseType.OPENAI_RESPONSES_COMPLETION):
- _record_openai_responses(response, model, tier, self._collector)
- elif response_type_matches(response, ChatResponseType.OPENAI_RESPONSES_STREAM):
- _attach_openai_responses_tap(response, model, tier, self._collector)
- return response
-
- # ------------------------------------------------------------------
- # HTTP stats endpoint integration
- # ------------------------------------------------------------------
-
- def get_routing_stats(self) -> dict[str, object]:
- """Return a snapshot of accumulated per-model statistics."""
- return self._collector.to_dict()
-
- def reset_routing_stats(self) -> None:
- """Reset all accumulated statistics to zero."""
- self._collector.reset()
-
- def get_endpoint(self) -> Endpoint | None:
- """Contribute ``GET /v1/routing/stats`` to the server."""
- if not self._expose_endpoint:
- return None
- from switchyard.lib.endpoints.stats_endpoint import (
- StatsEndpoint,
- )
-
- return StatsEndpoint(self._collector)
-
-
-def _stats_tier(ctx: ProxyContext) -> str:
- tier: str = ctx.metadata.get("_random_routing_tier", "")
- if tier:
- return tier
- selected_target = ctx.selected_target
- return selected_target if selected_target in {"strong", "weak"} else ""
-
-
-# ---------------------------------------------------------------------------
-# Non-streaming extractors
-# ---------------------------------------------------------------------------
-
-
-def _record_anthropic(
- response: ChatResponse,
- model: str,
- tier: str,
- collector: LiveStatsCollector,
-) -> None:
- u = _field(response.body, "usage")
- if not u:
- return
- input_tok = _int_field(u, "input_tokens")
- cache_create = _int_field(u, "cache_creation_input_tokens")
- cache_read = _int_field(u, "cache_read_input_tokens")
- completion = _int_field(u, "output_tokens")
- collector.record(
- model, tier,
- prompt_tokens=input_tok + cache_create + cache_read,
- completion_tokens=completion,
- cache_read_tokens=cache_read,
- cache_creation_tokens=cache_create,
- )
-
-
-def _record_openai_chat(
- response: ChatResponse,
- model: str,
- tier: str,
- collector: LiveStatsCollector,
-) -> None:
- u = _field(response.body, "usage")
- if u is None:
- return
- prompt = _int_field(u, "prompt_tokens")
- completion = _int_field(u, "completion_tokens")
- reasoning = 0
- cached = 0
- if (details := _field(u, "completion_tokens_details")) is not None:
- reasoning = _int_field(details, "reasoning_tokens")
- if (prompt_details := _field(u, "prompt_tokens_details")) is not None:
- cached = _int_field(prompt_details, "cached_tokens")
- collector.record(
- model, tier,
- prompt_tokens=prompt,
- completion_tokens=completion,
- reasoning_tokens=reasoning,
- cache_read_tokens=cached,
- )
-
-
-def _record_openai_responses(
- response: ChatResponse,
- model: str,
- tier: str,
- collector: LiveStatsCollector,
-) -> None:
- u = _field(response.body, "usage")
- if u is None:
- return
- prompt = _int_field(u, "input_tokens")
- completion = _int_field(u, "output_tokens")
- reasoning = 0
- cached = 0
- if (out_details := _field(u, "output_tokens_details")) is not None:
- reasoning = _int_field(out_details, "reasoning_tokens")
- if (in_details := _field(u, "input_tokens_details")) is not None:
- cached = _int_field(in_details, "cached_tokens")
- collector.record(
- model, tier,
- prompt_tokens=prompt,
- completion_tokens=completion,
- reasoning_tokens=reasoning,
- cache_read_tokens=cached,
- )
-
-
-# ---------------------------------------------------------------------------
-# Streaming tap installers
-# ---------------------------------------------------------------------------
-
-
-def _attach_anthropic_tap(
- response: ChatResponse,
- model: str,
- tier: str,
- collector: LiveStatsCollector,
-) -> None:
- """Register a tap that accumulates Anthropic SSE usage."""
- acc: dict[str, int] = {
- "input_tokens": 0,
- "output_tokens": 0,
- "cache_creation_input_tokens": 0,
- "cache_read_input_tokens": 0,
- }
- committed = False
-
- def _merge(usage: object) -> None:
- for key in acc:
- value = _field(usage, key)
- if isinstance(value, int):
- acc[key] = value
-
- async def _tap(event: RawMessageStreamEvent) -> None:
- nonlocal committed
- if committed:
- return
- event_type = getattr(event, "type", None)
- if event_type == "message_start":
- msg = getattr(event, "message", None)
- if msg is not None:
- usage = _field(msg, "usage")
- if usage is not None:
- _merge(usage)
- elif event_type == "message_delta":
- usage = _field(event, "usage")
- if usage is not None:
- _merge(usage)
- elif event_type == "message_stop":
- input_tok = acc["input_tokens"]
- cache_create = acc["cache_creation_input_tokens"]
- cache_read = acc["cache_read_input_tokens"]
- completion = acc["output_tokens"]
- collector.record(
- model, tier,
- prompt_tokens=input_tok + cache_create + cache_read,
- completion_tokens=completion,
- cache_read_tokens=cache_read,
- cache_creation_tokens=cache_create,
- )
- committed = True
-
- response.stream.tap(_tap)
-
-
-def _attach_openai_chat_tap(
- response: ChatResponse,
- model: str,
- tier: str,
- collector: LiveStatsCollector,
-) -> None:
- """Register a tap for OpenAI Chat streaming usage."""
- seen = False
-
- async def _tap(chunk) -> None: # type: ignore[no-untyped-def]
- nonlocal seen
- if seen:
- return
- u = _field(chunk, "usage")
- if u is None:
- return
- prompt = _int_field(u, "prompt_tokens")
- completion = _int_field(u, "completion_tokens")
- reasoning = 0
- cached = 0
- if (details := _field(u, "completion_tokens_details")) is not None:
- reasoning = _int_field(details, "reasoning_tokens")
- if (prompt_details := _field(u, "prompt_tokens_details")) is not None:
- cached = _int_field(prompt_details, "cached_tokens")
- collector.record(
- model, tier,
- prompt_tokens=prompt,
- completion_tokens=completion,
- reasoning_tokens=reasoning,
- cache_read_tokens=cached,
- )
- seen = True
-
- response.stream.tap(_tap)
-
-
-def _attach_openai_responses_tap(
- response: ChatResponse,
- model: str,
- tier: str,
- collector: LiveStatsCollector,
-) -> None:
- """Register a tap for OpenAI Responses API streaming usage."""
- seen = False
-
- async def _tap(event) -> None: # type: ignore[no-untyped-def]
- nonlocal seen
- if seen:
- return
- # Verbatim-passthrough backends yield raw SSE frame strings; parse the
- # frame's data payload(s) to find the usage-carrying event.
- candidates = sse_frame_payloads(event) if isinstance(event, str) else [event]
- inner = next(
- (found for c in candidates if (found := _field(c, "response")) is not None),
- None,
- )
- if inner is None:
- return
- u = _field(inner, "usage")
- if u is None:
- return
- prompt = _int_field(u, "input_tokens")
- completion = _int_field(u, "output_tokens")
- reasoning = 0
- cached = 0
- if (out_details := _field(u, "output_tokens_details")) is not None:
- reasoning = _int_field(out_details, "reasoning_tokens")
- if (in_details := _field(u, "input_tokens_details")) is not None:
- cached = _int_field(in_details, "cached_tokens")
- collector.record(
- model, tier,
- prompt_tokens=prompt,
- completion_tokens=completion,
- reasoning_tokens=reasoning,
- cache_read_tokens=cached,
- )
- seen = True
-
- response.stream.tap(_tap)
-
-
-def _field(value: object, name: str) -> object | None:
- if isinstance(value, Mapping):
- return value.get(name)
- return getattr(value, name, None)
-
-
-def _int_field(value: object, name: str) -> int:
- field = _field(value, name)
- return field if isinstance(field, int) else 0
diff --git a/switchyard/lib/processors/turn_based_router_request_processor.py b/switchyard/lib/processors/turn_based_router_request_processor.py
deleted file mode 100644
index adf96123f..000000000
--- a/switchyard/lib/processors/turn_based_router_request_processor.py
+++ /dev/null
@@ -1,238 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Deterministic turn-based router — cadenced strong/weak dispatch.
-
-Sibling routing primitive to :mod:`switchyard.lib.processors.llm_classifier`
-and :mod:`switchyard.lib.processors.plan_execute`. Where the classifier
-inspects the prompt and routes by semantic signals, and the random
-router flips a weighted coin, this router routes by **cadence**: turn
-1 of any task is always strong; subsequent turns are weak by default,
-with strong "anchor" turns scheduled at a fixed long-run fraction.
-
-The hypothesis: strong models earn their cost at *anchor* turns —
-turn 1 for initial framing, periodic later turns for course correction
-— while routine execution turns can use a cheaper executor without
-hurting solve rate. Random routing tests this with a coin flip;
-turn-based routing tests it deterministically so the cadence is
-predictable, transparent, and the same on every benchmark replay.
-
-Mirrors :class:`switchyard.lib.profiles.random_routing.RandomRoutingConfig`'s
-API exactly: same ``strong_probability`` knob, same long-run
-distribution, just no randomness. Drop-in replacement for benchmark
-sweeps where reproducibility matters.
-
-Tier selection formula
-----------------------
-
-For a target strong fraction ``p ∈ [0, 1]`` and 1-indexed turn ``t``::
-
- select strong iff ceil(t * p) > ceil((t - 1) * p)
-
-This produces:
-
-* Turn 1 always strong whenever ``p > 0`` (``ceil(p) ≥ 1 > 0``).
-* Long-run strong rate exactly ``p`` (Bresenham line algorithm).
-* Stateless — the decision depends only on the current turn number,
- which is computed from the inbound conversation history.
-
-Distribution sanity check::
-
- p=1.0: S S S S S S S S S S always strong
- p=0.7: S S S W S S W S S W ... ~70% strong
- p=0.5: S W S W S W S W S W ... every other turn = strong
- p=0.3: S W W S W W S W W W ... ~30% strong (every 3-4 turns)
- p=0.2: S W W W W S W W W W S ... 1 in 5 turns strong
- p=0.0: W W W W W W W W W W always weak
-
-Turn counting
--------------
-
-The "turn number" is the 1-indexed count of LLM invocations for this
-task — i.e. *how deep in the agent loop are we?* Computed from the
-inbound conversation history as ``count(LLM-side prior items) + 1``.
-
-Across the three inbound formats switchyard accepts:
-
-* **OpenAI Chat Completions** — count ``messages[*].role == "assistant"``.
- Each LLM round produces exactly one assistant message (text and / or
- ``tool_calls``); tool results live in separate ``role: "tool"``
- messages so they don't inflate the count.
-* **Anthropic Messages** — same: count ``messages[*].role == "assistant"``.
- Note that Anthropic puts ``tool_result`` blocks inside ``role: "user"``
- messages (not ``role: "tool"`` like OpenAI), but we don't count
- ``user``, so the asymmetry doesn't matter.
-* **OpenAI Responses** — coarser fallback: with ``input: <string>`` the
- count is 0 (always turn 1); with ``input: <list>`` we count items
- that are agent-side (``role == "user"`` or ``type ==
- "function_call_output"``) and use that as an approximation of
- *completed agent acks*, which is one-less than completed LLM rounds.
- Adjusted by 1 if the trailing item is itself agent-side. This is
- approximate — exact block-counting would require segmentation logic.
- Agent loops using the Responses API for tool calls may see slightly
- off cadence; for benchmark workloads we run (terminus-2 over OpenAI
- Chat Completions) this branch is never hit.
-
-Unknown formats default to turn 1 (strong on first call). Better to
-have a known-safe baseline behavior than to fail.
-
-What gets stamped
------------------
-
-* :data:`CTX_DETERMINISTIC_ROUTING_TIER` — the chosen tier label
- (``"strong"`` or ``"weak"`` by default, configurable via
- :attr:`TurnBasedRoutingConfig.strong_tier` /
- :attr:`TurnBasedRoutingConfig.weak_tier`). Same key
- :class:`switchyard.lib.backends.deterministic_routing_llm_backend.DeterministicRoutingLLMBackend` reads, so
- the existing per-tier backend dispatch plugs in unchanged.
-
-Observability
--------------
-
-One ``turn_based_decision={...}`` JSON line per request to stderr
-(mirrors the classifier's ``classifier_signals=…`` and the planner's
-``planner_decision=…`` pattern). Lands in captured server logs
-regardless of uvicorn log level. Fields::
-
- {
- "turn": 6,
- "strong_probability": 0.5,
- "tier": "strong",
- "request_type": "openai_chat",
- }
-
-Grep ``turn_based_decision=`` on the server log to tally cadence
-distribution + verify the routing matched the expected pattern.
-"""
-
-from __future__ import annotations
-
-import json
-import math
-import sys
-from typing import TYPE_CHECKING
-
-from pydantic import BaseModel, ConfigDict, Field
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- CTX_DETERMINISTIC_ROUTING_TIER,
-)
-from switchyard.lib.conversation_turn import conversation_turn_number
-
-if TYPE_CHECKING:
- from switchyard.lib.proxy_context import ProxyContext
- from switchyard_rust.core import ChatRequest
-
-
-#: ``ProxyContext.metadata`` key for the turn number this request was
-#: routed at. Always stamped; useful for downstream audit / cost
-#: attribution that wants to slice executor spend by turn position.
-CTX_TURN_BASED_TURN = "_turn_based_turn"
-
-
-class TurnBasedRoutingConfig(BaseModel):
- """Configuration for :class:`TurnBasedRouterRequestProcessor`.
-
- Mirrors
- :class:`switchyard.lib.profiles.random_routing.RandomRoutingConfig`'s
- surface so the two router primitives are drop-in interchangeable
- for benchmark sweeps that vary the routing *strategy* against the
- same upstream tier targets.
- """
-
- model_config = ConfigDict(frozen=True)
-
- strong_tier: str = Field(default="strong", min_length=1)
- """Tier label stamped for strong-routed turns. Must match one of
- the labels the downstream
- :class:`switchyard.lib.backends.deterministic_routing_llm_backend.DeterministicRoutingLLMBackend` knows
- about."""
-
- weak_tier: str = Field(default="weak", min_length=1)
- """Tier label stamped for weak-routed turns."""
-
- strong_probability: float = Field(default=0.5, ge=0.0, le=1.0)
- """Target long-run fraction of turns routed to the strong tier.
- ``1.0`` = every turn strong (equivalent to ``FixedTierRequestProcessor("strong")``);
- ``0.0`` = every turn weak (equivalent to
- ``FixedTierRequestProcessor("weak")``); intermediate values
- produce a deterministic cadence whose long-run rate matches.
- Default ``0.5`` (every other turn = strong) mirrors the
- ``RandomRoutingConfig`` default."""
-
-
-class TurnBasedRouterRequestProcessor:
- """Stamp a strong/weak tier label based on a deterministic per-turn cadence.
-
- Reads the inbound conversation history to compute the current
- turn number, applies the Bresenham ceil formula to decide whether
- this turn should hit the strong or weak tier, and stamps the
- label so the downstream
- :class:`DeterministicRoutingLLMBackend` dispatches accordingly.
-
- Emits one ``turn_based_decision={...}`` JSON audit line per request
- to stderr for observability.
- """
-
- def __init__(self, config: TurnBasedRoutingConfig) -> None:
- self._config = config
-
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- turn = conversation_turn_number(request)
- tier = _select_tier(
- turn=turn,
- strong_probability=self._config.strong_probability,
- strong_tier=self._config.strong_tier,
- weak_tier=self._config.weak_tier,
- )
- ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = tier
- ctx.metadata[CTX_TURN_BASED_TURN] = turn
-
- # Audit line — mirrors classifier_signals=... and
- # planner_decision=... so the three routers share a grep
- # convention. Written directly to stderr so the line lands in
- # captured server logs regardless of uvicorn's logger config.
- request_type = request.request_type.value
- audit_payload = {
- "turn": turn,
- "strong_probability": self._config.strong_probability,
- "tier": tier,
- "request_type": request_type,
- }
- sys.stderr.write(
- f"turn_based_decision={json.dumps(audit_payload, sort_keys=True)}\n"
- )
- sys.stderr.flush()
- return request
-
-
-def _select_tier(
- *,
- turn: int,
- strong_probability: float,
- strong_tier: str,
- weak_tier: str,
-) -> str:
- """Return the tier label for ``turn`` under the given ``strong_probability``.
-
- Bresenham ceil formula: strong iff ``ceil(t * p) > ceil((t-1) * p)``.
-
- The two extremes are short-circuited because Python's ``math.ceil``
- is finicky at ``p == 0`` (would give weak for every turn including
- turn 1 — fine but wasteful to call) and ``p == 1`` would always
- pass (correct, but explicit is clearer).
- """
- if strong_probability >= 1.0:
- return strong_tier
- if strong_probability <= 0.0:
- return weak_tier
- cur = math.ceil(turn * strong_probability)
- prev = math.ceil((turn - 1) * strong_probability)
- return strong_tier if cur > prev else weak_tier
-
-
-__all__ = [
- "CTX_TURN_BASED_TURN",
- "TurnBasedRouterRequestProcessor",
- "TurnBasedRoutingConfig",
-]
diff --git a/tests/e2e/test_turn_based_router_chain_e2e.py b/tests/e2e/test_turn_based_router_chain_e2e.py
deleted file mode 100644
index 24162f17e..000000000
--- a/tests/e2e/test_turn_based_router_chain_e2e.py
+++ /dev/null
@@ -1,344 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Real-backend e2e test for the **full** turn-based router chain.
-
-Sibling to :mod:`tests.e2e.test_planning_request_processor_e2e` and
-:mod:`tests.e2e.test_classifier_planner_chain_e2e`. This file walks
-the turn-based router end-to-end against a live backend::
-
- inbound ChatRequest
- -> TurnBasedRouterRequestProcessor (no LLM call; counts
- assistant messages,
- applies Bresenham
- ceil schedule)
- -> DeterministicRoutingLLMBackend (dispatches to per-tier
- backend based on the
- tier label stamped on
- ctx by the router)
- -> strong tier: real LLM call (gpt-5.2)
- -> weak tier: real LLM call (V4 Pro)
- -> ChatResponse
-
-The router itself does no LLM call (just compute), so the cost on
-this test comes entirely from the executor tier the router picked.
-At ``strong_probability=0.5`` over three crafted requests the
-expected pattern is **strong → weak → strong**, with each request
-landing on the correct tier model.
-
-Strong vs weak distinct models is the critical thing this test
-exercises that the unit tests cannot: it verifies the
-``ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER]`` stamp actually
-drives the multi-tier dispatcher's per-tier dispatch. The unit
-tests stub the backend entirely; this test wires
-:class:`DeterministicRoutingLLMBackend` with two real backends
-pointed at two real models, and verifies the response's ``model``
-field matches the tier the router picked.
-
-Prerequisites:
- - ``OPENROUTER_API_KEY`` or ``NVIDIA_API_KEY`` env var (skips otherwise).
-
-Run with::
-
- OPENROUTER_API_KEY=sk-or-... uv run pytest \\
- tests/e2e/test_turn_based_router_chain_e2e.py -v
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-from typing import Any
-
-import pytest
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- CTX_DETERMINISTIC_ROUTING_TIER,
- DeterministicRoutingLLMBackend,
-)
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
-from switchyard.lib.chat_response.base import ChatResponse
-from switchyard.lib.processors.turn_based_router_request_processor import (
- CTX_TURN_BASED_TURN,
- TurnBasedRouterRequestProcessor,
- TurnBasedRoutingConfig,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.roles import LLMBackend
-from switchyard_rust.core import ChatRequest
-
-from .conftest import get_nvidia_config
-
-# Three real LLM calls per cadence test (one per crafted turn). 120s
-# headroom is generous for the small-prompt smoke calls these tests
-# do (each request is ~10 tokens out).
-pytestmark = [
- pytest.mark.integration,
- pytest.mark.timeout(120),
-]
-
-logger = logging.getLogger("e2e.turn_based_router_chain")
-
-HTTP_TIMEOUT_S = 60.0
-
-_nvidia = get_nvidia_config()
-_skip_reason = "OPENROUTER_API_KEY or NVIDIA_API_KEY not set"
-
-BACKEND_BASE_URL = _nvidia["base_url"]
-
-# Distinct strong / weak model ids so the test can verify routing by
-# inspecting ``response.body["model"]``. Both are OpenAI-Chat-Completions
-# compatible on the selected backend — no Anthropic-native translation
-# required, keeping the test focused on routing rather than wire-format
-# wrangling.
-if _nvidia["provider"] == "openrouter":
- STRONG_MODEL = _nvidia["model"]
- WEAK_MODEL = os.environ.get("OPENROUTER_WEAK_MODEL") or "moonshotai/kimi-k2.6"
-else:
- STRONG_MODEL = "openai/openai/gpt-5.2"
- WEAK_MODEL = "nvidia/deepseek-ai/evals-deepseek-v4-pro"
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _build_full_chain(
- *,
- strong_probability: float,
-) -> tuple[list[Any], LLMBackend]:
- """Build (request_processors, multi-tier backend) for the turn-based chain.
-
- Mirrors the turn-based routing benchmark chain in-process with direct
- OpenAiPassthrough-equivalent tier
- backends (via :meth:`DeterministicRoutingLLMBackend.from_tiers`,
- which builds standard OpenAI / Anthropic backends from
- :class:`LlmTarget` config). No stats wrapping — the router itself
- doesn't record stats anyway, and tier-side stats are out of scope
- for this routing-correctness test.
- """
- router = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(
- strong_tier="strong",
- weak_tier="weak",
- strong_probability=strong_probability,
- ),
- )
- backend = DeterministicRoutingLLMBackend.from_tiers(
- tiers={
- "strong": LlmTarget(
- model=STRONG_MODEL,
- format=BackendFormat.OPENAI,
- api_key=_nvidia["api_key"],
- base_url=BACKEND_BASE_URL,
- timeout=HTTP_TIMEOUT_S,
- ),
- "weak": LlmTarget(
- model=WEAK_MODEL,
- format=BackendFormat.OPENAI,
- api_key=_nvidia["api_key"],
- base_url=BACKEND_BASE_URL,
- timeout=HTTP_TIMEOUT_S,
- # DeepSeek V4 needs enable_thinking=False or its JSON
- # output gets misrouted into reasoning_content. This
- # call doesn't request structured output so the risk
- # is mostly latency (V4 with thinking on is 10-15s
- # slower per call) — set anyway for parity with the
- # benchmark server's auto-detect.
- extra_body={"chat_template_kwargs": {"enable_thinking": False}},
- extra_headers={"X-Inference-Priority": "batch"},
- ),
- },
- default_tier="strong",
- )
- return [router], backend
-
-
-async def _call_through_chain(
- request_processors: list[Any],
- backend: LLMBackend,
- request: ChatRequest,
-) -> tuple[ProxyContext, ChatResponse]:
- """Run a request through router + multi-tier backend in-process."""
- ctx = ProxyContext()
- for proc in request_processors:
- request = await proc.process(ctx, request)
- response = await backend.call(ctx, request)
- return ctx, response
-
-
-def _chat_request_with_n_prior_assistants(
- *,
- n_prior_assistants: int,
- user_message: str = "Reply with: ok",
-) -> ChatRequest:
- """Build an OpenAI Chat Completions ``ChatRequest`` whose history
- contains exactly ``n_prior_assistants`` assistant messages.
-
- Used to drive the router's turn counter — the router computes
- ``turn = count(assistant) + 1``, so ``n_prior_assistants=0`` →
- turn 1, ``n_prior_assistants=1`` → turn 2, etc.
- """
- messages: list[dict[str, Any]] = [
- {"role": "user", "content": "initial task"},
- ]
- for i in range(n_prior_assistants):
- messages.append({"role": "assistant", "content": f"r{i + 1}"})
- messages.append({"role": "user", "content": f"continue ({i + 1})"})
- # Replace the trailing "continue" with the actual prompt for the
- # current turn so the model has something concrete to answer.
- if n_prior_assistants > 0:
- messages[-1] = {"role": "user", "content": user_message}
- else:
- messages[-1] = {"role": "user", "content": user_message}
- return ChatRequest.openai_chat({
- "model": "client-placeholder",
- "messages": messages,
- "max_tokens": 10,
- })
-
-
-def _model_from_response(response: ChatResponse) -> str:
- """The upstream-reported model in the response body."""
- return str(response.body.get("model") or "")
-
-
-def _content_from_response(response: ChatResponse) -> str:
- choice = response.body["choices"][0]
- message = choice.get("message", {})
- return str(message.get("content") or "")
-
-
-# ---------------------------------------------------------------------------
-# Tests
-# ---------------------------------------------------------------------------
-
-
[email protected](not _nvidia["api_key"], reason=_skip_reason)
-class TestTurnBasedRouterChainLiveE2E:
- """Full turn-based router chain (router + multi-tier backend) against the live hub."""
-
- async def test_cadence_routes_through_both_tiers_at_half_probability(self) -> None:
- """At ``strong_probability=0.5`` three crafted turns yield S → W → S.
-
- The router stamps ``CTX_DETERMINISTIC_ROUTING_TIER`` on each
- request; :class:`DeterministicRoutingLLMBackend` dispatches to
- the matching tier; the response's ``model`` field reflects the
- upstream model that was actually called. This test wires
- distinct strong / weak models specifically so we can verify
- the dispatch end-to-end — the response model identifying which
- tier handled the request.
-
- The cadence assertion is the user-facing invariant: turn 1
- always strong, turn 2 weak at p=0.5, turn 3 strong again.
- """
- req_processors, backend = _build_full_chain(strong_probability=0.5)
-
- # --- Turn 1 (no prior assistants) → strong ------------------
- req1 = _chat_request_with_n_prior_assistants(
- n_prior_assistants=0,
- user_message="Reply with one word: hi",
- )
- ctx1, resp1 = await _call_through_chain(req_processors, backend, req1)
-
- assert ctx1.metadata[CTX_TURN_BASED_TURN] == 1
- assert ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- model1 = _model_from_response(resp1)
- assert model1 == STRONG_MODEL, (
- f"turn 1 should hit the strong tier ({STRONG_MODEL!r}); "
- f"response reported model={model1!r}"
- )
- content1 = _content_from_response(resp1)
- assert content1, "expected non-empty content from strong-tier executor"
-
- # --- Turn 2 (1 prior assistant) → weak ----------------------
- req2 = _chat_request_with_n_prior_assistants(
- n_prior_assistants=1,
- user_message="Reply with one word: ok",
- )
- ctx2, resp2 = await _call_through_chain(req_processors, backend, req2)
-
- assert ctx2.metadata[CTX_TURN_BASED_TURN] == 2
- assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- model2 = _model_from_response(resp2)
- assert model2 == WEAK_MODEL, (
- f"turn 2 at p=0.5 should hit the weak tier ({WEAK_MODEL!r}); "
- f"response reported model={model2!r}"
- )
- content2 = _content_from_response(resp2)
- assert content2, "expected non-empty content from weak-tier executor"
-
- # --- Turn 3 (2 prior assistants) → strong --------------------
- req3 = _chat_request_with_n_prior_assistants(
- n_prior_assistants=2,
- user_message="Reply with one word: done",
- )
- ctx3, resp3 = await _call_through_chain(req_processors, backend, req3)
-
- assert ctx3.metadata[CTX_TURN_BASED_TURN] == 3
- assert ctx3.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- model3 = _model_from_response(resp3)
- assert model3 == STRONG_MODEL, (
- f"turn 3 at p=0.5 should hit the strong tier ({STRONG_MODEL!r}); "
- f"response reported model={model3!r}"
- )
-
- logger.info(
- " [turn-based-e2e][p=0.5] cadence=[t1=%s/%s, t2=%s/%s, t3=%s/%s]",
- ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER], model1,
- ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER], model2,
- ctx3.metadata[CTX_DETERMINISTIC_ROUTING_TIER], model3,
- )
-
- async def test_low_probability_anchors_first_turn_then_routes_weak(self) -> None:
- """At ``strong_probability=0.2`` turn 1 routes strong, turns 2-5 route weak.
-
- Verifies the "anchor + sparse strong" cadence the turn-based
- router was designed for: cheap executor handles the bulk of
- the loop while turn 1 (and every Nth turn after) gets a
- strong-model anchor. At p=0.2 the next strong turn after
- turn 1 is turn 6.
-
- Sweeps turn 1 + a sample of subsequent turns rather than the
- full 1-6 to keep the test under a few seconds of LLM calls;
- the unit tests cover the exact cadence pattern over a longer
- window.
- """
- req_processors, backend = _build_full_chain(strong_probability=0.2)
-
- # Turn 1: always strong (the anchor).
- req1 = _chat_request_with_n_prior_assistants(
- n_prior_assistants=0,
- user_message="Reply with: anchor",
- )
- ctx1, resp1 = await _call_through_chain(req_processors, backend, req1)
- assert ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- assert _model_from_response(resp1) == STRONG_MODEL
-
- # Turn 2: weak (cadence at p=0.2 only strong at turns 1, 6, 11, …).
- req2 = _chat_request_with_n_prior_assistants(
- n_prior_assistants=1,
- user_message="Reply with: step2",
- )
- ctx2, resp2 = await _call_through_chain(req_processors, backend, req2)
- assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert _model_from_response(resp2) == WEAK_MODEL
-
- # Turn 5: still weak (next strong is turn 6).
- req5 = _chat_request_with_n_prior_assistants(
- n_prior_assistants=4,
- user_message="Reply with: step5",
- )
- ctx5, resp5 = await _call_through_chain(req_processors, backend, req5)
- assert ctx5.metadata[CTX_TURN_BASED_TURN] == 5
- assert ctx5.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert _model_from_response(resp5) == WEAK_MODEL
-
- logger.info(
- " [turn-based-e2e][p=0.2] sparse-strong: t1=strong/%s, "
- "t2=weak/%s, t5=weak/%s",
- _model_from_response(resp1),
- _model_from_response(resp2),
- _model_from_response(resp5),
- )
diff --git a/tests/test_fixed_tier_request_processor.py b/tests/test_fixed_tier_request_processor.py
deleted file mode 100644
index b48acde5a..000000000
--- a/tests/test_fixed_tier_request_processor.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for :class:`FixedTierRequestProcessor`."""
-
-from __future__ import annotations
-
-from typing import Any, cast
-
-import pytest
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- CTX_DETERMINISTIC_ROUTING_TIER,
-)
-from switchyard.lib.processors.fixed_tier_request_processor import (
- FixedTierRequestProcessor,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.core import ChatRequest
-
-
-def _request() -> ChatRequest:
- return ChatRequest.openai_chat(
- cast(Any, {"model": "client-model", "messages": [{"role": "user", "content": "hi"}]}),
- )
-
-
-async def test_fixed_tier_stamps_tier_on_ctx() -> None:
- proc = FixedTierRequestProcessor("strong")
- ctx = ProxyContext()
-
- await proc.process(ctx, _request())
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
-
-
-async def test_fixed_tier_weak() -> None:
- proc = FixedTierRequestProcessor("weak")
- ctx = ProxyContext()
-
- await proc.process(ctx, _request())
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
-
-
-async def test_fixed_tier_request_passes_through_unchanged() -> None:
- """The processor must not mutate the request body — pure tier stamp."""
- proc = FixedTierRequestProcessor("strong")
- req = _request()
- body_before = dict(req.body)
-
- returned = await proc.process(ProxyContext(), req)
-
- assert returned is req
- assert dict(req.body) == body_before
-
-
-async def test_fixed_tier_rejects_empty_label() -> None:
- with pytest.raises(ValueError, match="non-empty string"):
- FixedTierRequestProcessor("")
-
-
-async def test_fixed_tier_rejects_non_string() -> None:
- with pytest.raises(ValueError, match="non-empty string"):
- FixedTierRequestProcessor(cast(Any, None))
-
-
-async def test_fixed_tier_overwrites_existing_tier() -> None:
- """If an upstream processor stamped a tier, the fixed processor still wins.
-
- Force-mode is meant to override whatever else might be in the chain;
- this guards against an accidental compose with a real classifier.
- """
- proc = FixedTierRequestProcessor("strong")
- ctx = ProxyContext()
- ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = "weak"
-
- await proc.process(ctx, _request())
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
diff --git a/tests/test_live_stats_collector.py b/tests/test_live_stats_collector.py
deleted file mode 100644
index e8de312ff..000000000
--- a/tests/test_live_stats_collector.py
+++ /dev/null
@@ -1,437 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for LiveStatsCollector and StatsResponseProcessor."""
-
-from __future__ import annotations
-
-import threading
-from unittest.mock import MagicMock
-
-import pytest
-
-from switchyard.lib.live_stats_collector import (
- LiveStatsCollector,
- RequestStats,
-)
-from switchyard.lib.processors.stats_response_processor_live_collector import (
- StatsResponseProcessor,
-)
-from switchyard_rust.core import ChatResponse
-
-# ---------------------------------------------------------------------------
-# LiveStatsCollector unit tests
-# ---------------------------------------------------------------------------
-
-
-def test_initial_snapshot_is_zero():
- c = LiveStatsCollector()
- s = c.snapshot()
- assert s == RequestStats()
-
-
-def test_record_increments_all_fields():
- c = LiveStatsCollector()
- c.record(
- prompt_tokens=100,
- completion_tokens=50,
- cache_read_tokens=20,
- cache_creation_tokens=5,
- )
- s = c.snapshot()
- assert s.request_count == 1
- assert s.prompt_tokens == 100
- assert s.completion_tokens == 50
- assert s.cache_read_tokens == 20
- assert s.cache_creation_tokens == 5
-
-
-def test_record_accumulates_across_calls():
- c = LiveStatsCollector()
- c.record(prompt_tokens=10, completion_tokens=5)
- c.record(prompt_tokens=20, completion_tokens=10)
- s = c.snapshot()
- assert s.request_count == 2
- assert s.prompt_tokens == 30
- assert s.completion_tokens == 15
- assert c.to_dict()["models"]["unknown"]["max_observed_context_tokens"] == 30
-
-
-def test_snapshot_returns_copy():
- c = LiveStatsCollector()
- c.record(prompt_tokens=10)
- snap1 = c.snapshot()
- c.record(prompt_tokens=10)
- snap2 = c.snapshot()
- assert snap1.request_count == 1
- assert snap2.request_count == 2
-
-
-def test_record_defaults_to_zero_tokens():
- c = LiveStatsCollector()
- c.record()
- s = c.snapshot()
- assert s.request_count == 1
- assert s.prompt_tokens == 0
- assert s.completion_tokens == 0
-
-
-def test_thread_safety():
- c = LiveStatsCollector()
- n_threads = 50
- calls_per_thread = 100
-
- def _worker():
- for _ in range(calls_per_thread):
- c.record(prompt_tokens=1, completion_tokens=1)
-
- threads = [threading.Thread(target=_worker) for _ in range(n_threads)]
- for t in threads:
- t.start()
- for t in threads:
- t.join()
-
- s = c.snapshot()
- assert s.request_count == n_threads * calls_per_thread
- assert s.prompt_tokens == n_threads * calls_per_thread
-
-
-# ---------------------------------------------------------------------------
-# tier_breakdown — ordering + isolation
-# ---------------------------------------------------------------------------
-
-
-def test_tier_breakdown_empty_when_no_records():
- c = LiveStatsCollector()
- assert c.tier_breakdown() == []
-
-
-def test_tier_breakdown_orders_strong_before_weak():
- """Insertion order must NOT determine UI order — the random-routing
- footer relies on a stable ``strong → weak`` layout regardless of
- which tier rolled first.
- """
- c = LiveStatsCollector()
- # Record weak first to confirm sort, not insertion order, drives output.
- c.record(model="nemotron", tier="weak", prompt_tokens=10, completion_tokens=5)
- c.record(model="kimi-k2.5", tier="strong", prompt_tokens=20, completion_tokens=8)
-
- rows = c.tier_breakdown()
- assert [name for name, _ in rows] == ["kimi-k2.5", "nemotron"]
- assert [b.tier for _, b in rows] == ["strong", "weak"]
-
-
-def test_tier_breakdown_unknown_tier_sorts_last():
- """Models recorded without a tier (or with an unrecognised label)
- sink to the bottom — the strong/weak rows are always on top.
- """
- c = LiveStatsCollector()
- c.record(model="other-model", tier="", prompt_tokens=1)
- c.record(model="kimi", tier="strong", prompt_tokens=1)
- c.record(model="nemotron", tier="weak", prompt_tokens=1)
-
- rows = c.tier_breakdown()
- assert [name for name, _ in rows] == ["kimi", "nemotron", "other-model"]
-
-
-def test_tier_breakdown_returns_copies():
- """Mutating the returned ``ModelStats`` must not affect the
- collector's internal state — the footer paints from this list and
- rendering work shouldn't be able to corrupt request accounting.
- """
- c = LiveStatsCollector()
- c.record(model="kimi", tier="strong", prompt_tokens=10, completion_tokens=5)
-
- rows = c.tier_breakdown()
- rows[0][1].calls = 999
-
- # Re-fetch; collector's truth is unchanged.
- again = c.tier_breakdown()
- assert again[0][1].calls == 1
-
-
-# ---------------------------------------------------------------------------
-# StatsResponseProcessor — non-streaming Anthropic
-# ---------------------------------------------------------------------------
-
-
-def _make_ctx():
- from switchyard.lib.proxy_context import ProxyContext
- return ProxyContext()
-
-
-def _make_anthropic_usage(input_tokens=10, output_tokens=5, cache_read=0, cache_creation=0):
- return {
- "input_tokens": input_tokens,
- "output_tokens": output_tokens,
- "cache_read_input_tokens": cache_read,
- "cache_creation_input_tokens": cache_creation,
- }
-
-
-async def test_records_anthropic_completion():
- collector = LiveStatsCollector()
- proc = StatsResponseProcessor(collector)
-
- body = {"usage": _make_anthropic_usage(input_tokens=100, output_tokens=40, cache_read=30)}
- response = ChatResponse.anthropic_completion(body)
-
- result = await proc.process(_make_ctx(), response)
- assert result is response
-
- s = collector.snapshot()
- assert s.request_count == 1
- # prompt_tokens sums input + cache_read + cache_creation for Anthropic
- assert s.prompt_tokens == 130
- assert s.completion_tokens == 40
- assert s.cache_read_tokens == 30
-
-
-# ---------------------------------------------------------------------------
-# StatsResponseProcessor — non-streaming OpenAI
-# ---------------------------------------------------------------------------
-
-
-async def test_records_openai_completion():
- collector = LiveStatsCollector()
- proc = StatsResponseProcessor(collector)
-
- usage = {
- "prompt_tokens": 80,
- "completion_tokens": 30,
- "prompt_tokens_details": {"cached_tokens": 15},
- }
-
- body = {"usage": usage}
- response = ChatResponse.openai_completion(body)
-
- await proc.process(_make_ctx(), response)
-
- s = collector.snapshot()
- assert s.request_count == 1
- assert s.prompt_tokens == 80
- assert s.completion_tokens == 30
- assert s.cache_read_tokens == 15
-
-
-async def test_records_rust_context_selected_model_and_target():
- collector = LiveStatsCollector()
- proc = StatsResponseProcessor(collector)
- ctx = _make_ctx()
- ctx.selected_model = "served-model"
- ctx.selected_target = "weak"
-
- await proc.process(
- ctx,
- ChatResponse.openai_completion({
- "usage": {
- "prompt_tokens": 9,
- "completion_tokens": 4,
- },
- }),
- )
-
- rows = collector.tier_breakdown()
- assert [(model, stats.tier, stats.prompt_tokens) for model, stats in rows] == [
- ("served-model", "weak", 9),
- ]
-
-
-async def test_handles_missing_openai_usage():
- collector = LiveStatsCollector()
- proc = StatsResponseProcessor(collector)
-
- body = {"usage": None}
- response = ChatResponse.openai_completion(body)
-
- await proc.process(_make_ctx(), response)
- assert collector.snapshot().request_count == 0
-
-
-# ---------------------------------------------------------------------------
-# StatsResponseProcessor — streaming Anthropic tap
-# ---------------------------------------------------------------------------
-
-
-async def test_attaches_anthropic_stream_tap():
- from switchyard.lib.chat_response.anthropic import AnthropicResponseStream
-
- collector = LiveStatsCollector()
- proc = StatsResponseProcessor(collector)
-
- async def _events():
- start = MagicMock()
- start.type = "message_start"
- start.message.usage.input_tokens = 50
- start.message.usage.cache_read_input_tokens = 10
- start.message.usage.cache_creation_input_tokens = 0
- yield start
-
- delta = MagicMock()
- delta.type = "message_delta"
- delta.usage.output_tokens = 20
- yield delta
-
- stop = MagicMock()
- stop.type = "message_stop"
- yield stop
-
- stream = AnthropicResponseStream(_events())
- response = ChatResponse.anthropic_stream(stream)
-
- await proc.process(_make_ctx(), response)
-
- events = [e async for e in response.stream]
- assert len(events) == 3
-
- s = collector.snapshot()
- assert s.request_count == 1
- # prompt_tokens sums input(50) + cache_read(10) + cache_creation(0)
- assert s.prompt_tokens == 60
- assert s.completion_tokens == 20
- assert s.cache_read_tokens == 10
-
-
-async def test_anthropic_tap_delta_input_tokens_fallback():
- """Backends that report input_tokens in message_delta instead of message_start.
-
- Some Anthropic-compatible APIs (e.g. NVIDIA Inference Hub) set
- input_tokens=0 in message_start and report the real count in
- message_delta.usage.input_tokens (Optional[int]). The tap must use
- the delta value as a fallback so the ``in`` counter is non-zero.
- """
- from switchyard.lib.chat_response.anthropic import AnthropicResponseStream
-
- collector = LiveStatsCollector()
- proc = StatsResponseProcessor(collector)
-
- async def _events():
- start = MagicMock()
- start.type = "message_start"
- start.message.usage.input_tokens = 0 # backend defers counting
- start.message.usage.cache_read_input_tokens = 0
- start.message.usage.cache_creation_input_tokens = 0
- yield start
-
- delta = MagicMock()
- delta.type = "message_delta"
- delta.usage.output_tokens = 30
- # Backend reports actual input tokens here
- delta.usage.input_tokens = 75
- delta.usage.cache_read_input_tokens = None
- delta.usage.cache_creation_input_tokens = None
- yield delta
-
- stop = MagicMock()
- stop.type = "message_stop"
- yield stop
-
- stream = AnthropicResponseStream(_events())
- response = ChatResponse.anthropic_stream(stream)
- await proc.process(_make_ctx(), response)
- [_ async for _ in response.stream]
-
- s = collector.snapshot()
- assert s.request_count == 1
- assert s.prompt_tokens == 75
- assert s.completion_tokens == 30
-
-
-# ---------------------------------------------------------------------------
-# StatsResponseProcessor — streaming OpenAI Responses API tap
-# ---------------------------------------------------------------------------
-
-
-async def test_attaches_openai_responses_stream_tap():
- from openai.types.responses import Response as OpenAIResponse
- from openai.types.responses import ResponseCompletedEvent
- from openai.types.responses.response_usage import (
- InputTokensDetails,
- OutputTokensDetails,
- ResponseUsage,
- )
-
- from switchyard.lib.chat_response.openai_responses import ResponsesApiStream
-
- collector = LiveStatsCollector()
- proc = StatsResponseProcessor(collector)
-
- response_obj = OpenAIResponse.model_construct(
- id="resp-test", object="response", created_at=1_700_000_000,
- model="gpt-4o", output=[], status="completed",
- usage=ResponseUsage(
- input_tokens=300, output_tokens=120, total_tokens=420,
- input_tokens_details=InputTokensDetails(cached_tokens=60),
- output_tokens_details=OutputTokensDetails(reasoning_tokens=40),
- ),
- )
-
- async def _events():
- yield ResponseCompletedEvent(
- type="response.completed", response=response_obj, sequence_number=1,
- )
-
- response = ChatResponse.openai_responses_stream(ResponsesApiStream(_events()))
- await proc.process(_make_ctx(), response)
- [_ async for _ in response.stream]
-
- s = collector.snapshot()
- assert s.request_count == 1
- assert s.prompt_tokens == 300
- assert s.completion_tokens == 120
- assert s.cache_read_tokens == 60
-
-
-async def test_openai_responses_tap_reads_usage_from_raw_sse_frames():
- """Verbatim-passthrough streams yield raw SSE frame strings; the tap must
- still find the usage in the ``response.completed`` frame."""
- from switchyard.lib.chat_response.openai_responses import ResponsesApiStream
-
- collector = LiveStatsCollector()
- proc = StatsResponseProcessor(collector)
-
- async def _frames():
- yield (
- 'event: response.output_text.delta\n'
- 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n'
- )
- yield ": keep-alive\n\n"
- yield (
- 'event: response.completed\n'
- 'data: {"type":"response.completed","response":{"id":"resp-raw","usage":'
- '{"input_tokens":300,"output_tokens":120,'
- '"input_tokens_details":{"cached_tokens":60},'
- '"output_tokens_details":{"reasoning_tokens":40}}}}\n\n'
- )
-
- response = ChatResponse.openai_responses_stream(ResponsesApiStream(_frames()))
- await proc.process(_make_ctx(), response)
- [_ async for _ in response.stream]
-
- s = collector.snapshot()
- assert s.request_count == 1
- assert s.prompt_tokens == 300
- assert s.completion_tokens == 120
- assert s.cache_read_tokens == 60
-
-
-# ---------------------------------------------------------------------------
-# Cost estimation — model name normalization
-# ---------------------------------------------------------------------------
-
-
-def test_cost_estimate_strips_date_suffix():
- """Versioned model names like 'claude-sonnet-4-6-20251022' should match
- the base name in the price table."""
- collector = LiveStatsCollector()
- collector.record("claude-sonnet-4-6-20251022", prompt_tokens=1_000_000, completion_tokens=0)
- m = collector.to_dict()["models"]["claude-sonnet-4-6-20251022"]
- # Should resolve to claude-sonnet-4-6 price: $3.00 / 1M input
- assert m["estimated_cost_usd"] == pytest.approx(3.00)
-
-
-def test_cost_estimate_none_for_unknown_model():
- collector = LiveStatsCollector()
- collector.record("some-unknown-model", prompt_tokens=100, completion_tokens=50)
- m = collector.to_dict()["models"]["some-unknown-model"]
- assert m["estimated_cost_usd"] is None
diff --git a/tests/test_turn_based_router_request_processor.py b/tests/test_turn_based_router_request_processor.py
deleted file mode 100644
index 4471cf445..000000000
--- a/tests/test_turn_based_router_request_processor.py
+++ /dev/null
@@ -1,503 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for :class:`TurnBasedRouterRequestProcessor`.
-
-Pure compute, no LLM, no network — covers:
-
-* The Bresenham ceil formula's long-run distribution and first-turn-strong
- invariant at edges (p=0.0, 1.0) and intermediate values (0.3, 0.5, 0.7).
-* OpenAI Chat Completions turn counting (role=='assistant').
-* Anthropic Messages turn counting (verifies the role=='user' tool-result
- asymmetry doesn't inflate the counter).
-* OpenAI Responses turn counting (string input and list input coarse path).
-* Edge cases: empty / missing messages, malformed entries.
-* ctx-metadata stamping for both the tier label and the turn number.
-* The ``turn_based_decision={...}`` audit line on stderr.
-"""
-
-from __future__ import annotations
-
-import json
-import math
-from typing import Any, cast
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- CTX_DETERMINISTIC_ROUTING_TIER,
-)
-from switchyard.lib.processors.turn_based_router_request_processor import (
- CTX_TURN_BASED_TURN,
- TurnBasedRouterRequestProcessor,
- TurnBasedRoutingConfig,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.core import ChatRequest
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _build_openai_chat_request(messages: list[dict[str, Any]]) -> ChatRequest:
- """Wrap a messages list in a minimal OpenAI Chat Completions request."""
- return ChatRequest.openai_chat(cast(Any, {
- "model": "placeholder",
- "messages": messages,
- }))
-
-
-def _build_anthropic_request(messages: list[dict[str, Any]]) -> ChatRequest:
- """Wrap messages in a minimal Anthropic Messages request."""
- return ChatRequest.anthropic(cast(Any, {
- "model": "claude-test",
- "max_tokens": 128,
- "messages": messages,
- }))
-
-
-def _build_responses_request(input_value: Any) -> ChatRequest:
- """Wrap input in a minimal OpenAI Responses request."""
- return ChatRequest.openai_responses(cast(Any, {
- "model": "placeholder",
- "input": input_value,
- }))
-
-
-def _conversation_with_n_assistant_turns(n: int) -> list[dict[str, Any]]:
- """Synthesize an OpenAI Chat history with ``n`` prior assistant turns.
-
- Layout: initial user prompt, then alternating assistant/tool pairs
- matching the terminus-2 pattern. Sufficient to drive turn counting
- without needing a live agent.
- """
- msgs: list[dict[str, Any]] = [
- {"role": "user", "content": "initial task"},
- ]
- for i in range(n):
- msgs.append({
- "role": "assistant",
- "content": f"step {i + 1}",
- "tool_calls": [{
- "id": f"c{i}",
- "type": "function",
- "function": {"name": "bash", "arguments": "{}"},
- }],
- })
- msgs.append({
- "role": "tool",
- "tool_call_id": f"c{i}",
- "content": f"output {i + 1}",
- })
- return msgs
-
-
-# ---------------------------------------------------------------------------
-# Cadence formula
-# ---------------------------------------------------------------------------
-
-
-class TestCadenceFormula:
- """Direct tests on _select_tier / the Bresenham ceil schedule."""
-
- async def test_first_turn_always_strong_when_probability_positive(self) -> None:
- """Turn 1 must always route strong when strong_probability > 0.
-
- This is the user-facing invariant — "first turn always goes
- to strong". Verified at multiple probabilities.
- """
- for p in (0.01, 0.1, 0.3, 0.5, 0.7, 0.9, 1.0):
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=p),
- )
- req = _build_openai_chat_request(
- [{"role": "user", "content": "first turn"}],
- )
- ctx = ProxyContext()
- await processor.process(ctx, req)
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong", (
- f"turn 1 must be strong at p={p}; got "
- f"{ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER]!r}"
- )
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1
-
- async def test_zero_probability_routes_every_turn_weak(self) -> None:
- """strong_probability=0.0 routes every turn — including turn 1 — to weak.
-
- This is the documented degenerate behavior: p=0 is equivalent
- to FixedTierRequestProcessor("weak").
- """
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.0),
- )
- for n_prior in range(5):
- ctx = ProxyContext()
- req = _build_openai_chat_request(
- _conversation_with_n_assistant_turns(n_prior),
- )
- await processor.process(ctx, req)
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert ctx.metadata[CTX_TURN_BASED_TURN] == n_prior + 1
-
- async def test_one_probability_routes_every_turn_strong(self) -> None:
- """strong_probability=1.0 routes every turn to strong."""
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=1.0),
- )
- for n_prior in range(5):
- ctx = ProxyContext()
- req = _build_openai_chat_request(
- _conversation_with_n_assistant_turns(n_prior),
- )
- await processor.process(ctx, req)
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
-
- async def test_half_probability_alternates_strong_weak(self) -> None:
- """strong_probability=0.5 should produce S W S W S W... cadence.
-
- Verifies the ceil(t*0.5) > ceil((t-1)*0.5) formula gives the
- expected alternation: odd turns strong, even turns weak.
- """
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- expected = ["strong", "weak"] * 10 # 20 turns
- actual: list[str] = []
- for n_prior in range(20):
- ctx = ProxyContext()
- req = _build_openai_chat_request(
- _conversation_with_n_assistant_turns(n_prior),
- )
- await processor.process(ctx, req)
- actual.append(ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER])
-
- assert actual == expected, (
- f"expected strict alternation at p=0.5; got: {actual}"
- )
-
- async def test_thirty_percent_probability_hits_roughly_thirty_percent_strong(self) -> None:
- """strong_probability=0.3 over 100 turns should produce exactly 30 strong.
-
- The Bresenham formula gives an *exact* long-run rate (not just
- approximate) at any rational p with denominator dividing the
- turn count. At p=0.3 over 100 turns, count(strong) == 30.
- """
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.3),
- )
- strong_count = 0
- for n_prior in range(100):
- ctx = ProxyContext()
- req = _build_openai_chat_request(
- _conversation_with_n_assistant_turns(n_prior),
- )
- await processor.process(ctx, req)
- if ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong":
- strong_count += 1
- assert strong_count == 30, (
- f"at p=0.3, expected exactly 30 strong over 100 turns; got {strong_count}"
- )
-
- async def test_seventy_percent_probability_hits_roughly_seventy_percent_strong(self) -> None:
- """Symmetric: p=0.7 over 100 turns yields exactly 70 strong."""
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.7),
- )
- strong_count = 0
- for n_prior in range(100):
- ctx = ProxyContext()
- req = _build_openai_chat_request(
- _conversation_with_n_assistant_turns(n_prior),
- )
- await processor.process(ctx, req)
- if ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong":
- strong_count += 1
- assert strong_count == 70, (
- f"at p=0.7, expected exactly 70 strong over 100 turns; got {strong_count}"
- )
-
- async def test_low_probability_strong_turns_are_spaced(self) -> None:
- """At p=0.2, strong turns should appear at indices 1, 6, 11, 16, ...
-
- Bresenham formula at p=0.2 yields ``ceil(t*0.2) > ceil((t-1)*0.2)``
- when ``t * 0.2`` crosses an integer — that's t=1, 5, 10, 15...
- but offset by 1 because turn 1 is strong (ceil(0.2)=1 > ceil(0)=0).
-
- Let me compute directly to be sure:
- - t=1: ceil(0.2)=1, ceil(0)=0, 1>0 → strong
- - t=2: ceil(0.4)=1, ceil(0.2)=1, NO → weak
- - t=3: ceil(0.6)=1, ceil(0.4)=1, NO → weak
- - t=4: ceil(0.8)=1, ceil(0.6)=1, NO → weak
- - t=5: ceil(1.0)=1, ceil(0.8)=1, NO → weak
- - t=6: ceil(1.2)=2, ceil(1.0)=1, YES → strong
- - t=7-10: weak
- - t=11: strong
- """
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.2),
- )
- strong_indices: list[int] = []
- for n_prior in range(20):
- turn = n_prior + 1
- ctx = ProxyContext()
- req = _build_openai_chat_request(
- _conversation_with_n_assistant_turns(n_prior),
- )
- await processor.process(ctx, req)
- if ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong":
- strong_indices.append(turn)
-
- assert strong_indices == [1, 6, 11, 16], (
- f"expected strong turns at [1, 6, 11, 16] at p=0.2 over 20 turns; "
- f"got: {strong_indices}"
- )
-
-
-# ---------------------------------------------------------------------------
-# Per-format turn counting
-# ---------------------------------------------------------------------------
-
-
-class TestOpenAIChatTurnCounting:
- """OpenAI Chat Completions: count role=='assistant', add 1."""
-
- async def test_no_messages_treated_as_turn_one(self) -> None:
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- req = _build_openai_chat_request([])
- ctx = ProxyContext()
- await processor.process(ctx, req)
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1
-
- async def test_user_only_history_is_turn_one(self) -> None:
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- req = _build_openai_chat_request([
- {"role": "system", "content": "..."},
- {"role": "user", "content": "first turn"},
- ])
- ctx = ProxyContext()
- await processor.process(ctx, req)
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1
-
- async def test_three_prior_assistants_is_turn_four(self) -> None:
- """3 prior assistant turns → this is the 4th LLM invocation."""
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- req = _build_openai_chat_request(
- _conversation_with_n_assistant_turns(3),
- )
- ctx = ProxyContext()
- await processor.process(ctx, req)
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 4
-
- async def test_tool_messages_do_not_inflate_count(self) -> None:
- """role=='tool' messages must not be counted as turns."""
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- messages = [
- {"role": "user", "content": "task"},
- {"role": "assistant", "content": "step 1"},
- # 5 tool messages — pure agent-side, shouldn't count.
- *[{"role": "tool", "tool_call_id": f"c{i}", "content": "..."} for i in range(5)],
- ]
- req = _build_openai_chat_request(messages)
- ctx = ProxyContext()
- await processor.process(ctx, req)
- # One assistant → turn 2.
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 2
-
- async def test_system_messages_do_not_inflate_count(self) -> None:
- """role=='system' messages must not be counted as turns."""
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- messages = [
- {"role": "system", "content": "agent system prompt"},
- {"role": "system", "content": "another system note"},
- {"role": "user", "content": "task"},
- ]
- req = _build_openai_chat_request(messages)
- ctx = ProxyContext()
- await processor.process(ctx, req)
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1
-
-
-class TestAnthropicTurnCounting:
- """Anthropic Messages: count role=='assistant' — verify role=='user'
- tool_result messages don't accidentally inflate the count."""
-
- async def test_initial_anthropic_request_is_turn_one(self) -> None:
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- req = _build_anthropic_request([
- {"role": "user", "content": "initial task"},
- ])
- ctx = ProxyContext()
- await processor.process(ctx, req)
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1
-
- async def test_anthropic_user_tool_result_messages_not_counted(self) -> None:
- """In Anthropic Messages, tool_result is inside role=='user' messages.
-
- The counter is on role=='assistant', so these don't inflate.
- Verifies the format-asymmetry trap doesn't bite — we picked
- ``role == "assistant"`` as the discriminator precisely because
- it's invariant across the two message formats.
- """
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- messages = [
- {"role": "user", "content": "task"},
- {
- "role": "assistant",
- "content": [
- {"type": "text", "text": "looking..."},
- {"type": "tool_use", "id": "tu1", "name": "bash", "input": {}},
- ],
- },
- # The Anthropic-shape tool_result: comes back as role=='user'.
- # Counting role=='user' would now wrongly say 2 turns.
- # We count role=='assistant', so this should still be turn 2.
- {
- "role": "user",
- "content": [
- {"type": "tool_result", "tool_use_id": "tu1", "content": "stdout"},
- ],
- },
- ]
- req = _build_anthropic_request(messages)
- ctx = ProxyContext()
- await processor.process(ctx, req)
- # One assistant → turn 2.
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 2
-
- async def test_multi_round_anthropic_conversation_counts_correctly(self) -> None:
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- messages = [
- {"role": "user", "content": "task"},
- {"role": "assistant", "content": [{"type": "text", "text": "r1"}]},
- {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "1", "content": "out1"}]},
- {"role": "assistant", "content": [{"type": "text", "text": "r2"}]},
- {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "2", "content": "out2"}]},
- ]
- req = _build_anthropic_request(messages)
- ctx = ProxyContext()
- await processor.process(ctx, req)
- # Two assistants → turn 3.
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 3
-
-
-class TestOpenAIResponsesTurnCounting:
- """OpenAI Responses: string input → turn 1; list input → coarse count."""
-
- async def test_string_input_is_turn_one(self) -> None:
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- req = _build_responses_request("initial task")
- ctx = ProxyContext()
- await processor.process(ctx, req)
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1
-
- async def test_list_input_with_only_user_message_is_turn_one(self) -> None:
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- req = _build_responses_request([
- {"role": "user", "content": "initial task"},
- ])
- ctx = ProxyContext()
- await processor.process(ctx, req)
- # One agent-side ack (the user prompt) → coarse turn 1.
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1
-
- async def test_list_input_with_function_call_outputs_advances_turn(self) -> None:
- """function_call_output items mark prior LLM-round acks."""
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- req = _build_responses_request([
- {"role": "user", "content": "task"},
- {"type": "function_call", "id": "fc1", "name": "bash", "arguments": "{}"},
- {"type": "function_call_output", "call_id": "fc1", "output": "stdout"},
- {"type": "function_call", "id": "fc2", "name": "bash", "arguments": "{}"},
- {"type": "function_call_output", "call_id": "fc2", "output": "stdout"},
- ])
- ctx = ProxyContext()
- await processor.process(ctx, req)
- # 1 user ack + 2 function_call_output acks = 3.
- # Coarse approximation: turn 3 (we're 2 rounds in).
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 3
-
-
-# ---------------------------------------------------------------------------
-# Stamping + audit line
-# ---------------------------------------------------------------------------
-
-
-class TestStampingAndAudit:
- """ctx-metadata stamping + stderr audit-line shape."""
-
- async def test_stamps_both_tier_and_turn_metadata_keys(self) -> None:
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- req = _build_openai_chat_request(
- _conversation_with_n_assistant_turns(2),
- )
- ctx = ProxyContext()
- await processor.process(ctx, req)
- # Two prior assistants → turn 3 → odd → strong at p=0.5.
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- assert ctx.metadata[CTX_TURN_BASED_TURN] == 3
-
- async def test_custom_tier_labels_are_stamped(self) -> None:
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(
- strong_tier="premium",
- weak_tier="economy",
- strong_probability=0.5,
- ),
- )
- req = _build_openai_chat_request(
- [{"role": "user", "content": "task"}],
- )
- ctx = ProxyContext()
- await processor.process(ctx, req)
- # Turn 1 always strong.
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "premium"
-
- async def test_audit_line_lands_on_stderr_with_expected_fields(self, capsys: Any) -> None:
- """One ``turn_based_decision=...`` line per invocation on stderr."""
- processor = TurnBasedRouterRequestProcessor(
- TurnBasedRoutingConfig(strong_probability=0.5),
- )
- req = _build_openai_chat_request(
- [{"role": "user", "content": "task"}],
- )
- ctx = ProxyContext()
- await processor.process(ctx, req)
-
- captured = capsys.readouterr()
- # The audit line is on stderr (mirrors classifier + planner).
- lines = [
- ln for ln in captured.err.splitlines()
- if ln.startswith("turn_based_decision=")
- ]
- assert len(lines) == 1, (
- f"expected exactly 1 turn_based_decision line on stderr; "
- f"got {len(lines)}. Stderr was: {captured.err!r}"
- )
- payload = json.loads(lines[0][len("turn_based_decision="):])
- assert payload["turn"] == 1
- assert payload["tier"] == "strong"
- assert math.isclose(payload["strong_probability"], 0.5)
- assert payload["request_type"] == "openai_chat"