Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
refactor(python): remove dead chain processors (#127)
nachiketb-nvidia7315625e64e8
diff --git a/AGENTS.md b/AGENTS.mdindex 73dba19e1..5a97c7ba5 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -155,7 +155,6 @@ switchyard/│ ├── route_table_builders.py # Shared profile-backed table builders│ ├── llm_client.py # OpenAILLMClient│ ├── cost_estimator.py # Token-cost bookkeeping-│ ├── live_stats_collector.py # Live request stats│ ├── stats_accumulator.py # Stats accumulation helpers│ ├── request_metadata.py # RequestMetadata│ ├── chat_response/ # Rust-backed response re-exports + stream adapters@@ -178,7 +177,6 @@ switchyard/│ │ ├── random_routing_request_processor.py│ │ ├── stats_request_processor.py│ │ ├── stats_response_processor_accumulator.py-│ │ ├── stats_response_processor_live_collector.py│ │ ├── intake_request_processor.py│ │ ├── intake_response_processor.py│ │ ├── intake_payload_builder.pydiff --git a/switchyard/lib/backends/anthropic_cache_breakpoint_backend.py b/switchyard/lib/backends/anthropic_cache_breakpoint_backend.pyindex 563bcb88b..cbab05005 100644--- a/switchyard/lib/backends/anthropic_cache_breakpoint_backend.py+++ b/switchyard/lib/backends/anthropic_cache_breakpoint_backend.py@@ -22,10 +22,9 @@ honors prompt caching:request and passes the body through verbatim (preserving themarkers all the way to the wire).-Why this lives in a wrapper instead of in-:class:`switchyard.lib.processors.turn_based_router_request_processor.TurnBasedRouterRequestProcessor`:-mutating the chain-level :attr:`ChatRequest.request_type` at the-request-processor layer propagates through the rest of the chain.+Why this lives in a wrapper instead of a request processor: mutating the+chain-level :attr:`ChatRequest.request_type` before the backend call propagates+through the rest of the chain.The terminal :class:`TranslationEngine` step then reads the modifiedtype, decides "the client wanted Anthropic," and skips translationof the Anthropic response back to the client's original wire format.diff --git a/switchyard/lib/endpoints/stats_endpoint.py b/switchyard/lib/endpoints/stats_endpoint.pyindex e381363cc..31d96f94b 100644--- a/switchyard/lib/endpoints/stats_endpoint.py+++ b/switchyard/lib/endpoints/stats_endpoint.py@@ -21,7 +21,6 @@ from fastapi.responses import Responsefrom switchyard.lib.endpoints import outcome_metricsfrom switchyard.lib.endpoints.base import Endpoint as NemoSwitchyardEndpointfrom switchyard.lib.endpoints.prometheus_emitter import render as render_extra_metrics-from switchyard.lib.live_stats_collector import LiveStatsCollectorfrom switchyard.lib.prometheus_exposition import render_prometheusfrom switchyard.lib.stats_accumulator import StatsAccumulator@@ -29,8 +28,6 @@ if TYPE_CHECKING:from fastapi import FastAPI-StatsSource = StatsAccumulator | LiveStatsCollector-# Prometheus text exposition format 0.0.4 content-type.PROMETHEUS_CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"@@ -52,7 +49,7 @@ class StatsEndpoint(NemoSwitchyardEndpoint):register_once = True- def __init__(self, stats: StatsSource) -> None:+ def __init__(self, stats: StatsAccumulator) -> None:self._stats = statsdef register(self, app: FastAPI) -> None:@@ -61,16 +58,11 @@ class StatsEndpoint(NemoSwitchyardEndpoint):async def get_stats() -> dict[str, Any]:"""Snapshot of per-model request / token / latency / cost stats."""- if isinstance(stats, LiveStatsCollector):- return stats.to_dict()return await stats.snapshot()async def reset_stats() -> dict[str, str]:"""Zero all stats counters."""- if isinstance(stats, LiveStatsCollector):- stats.reset()- else:- await stats.reset()+ await stats.reset()return {"status": "reset"}async def get_metrics() -> Response:@@ -81,11 +73,7 @@ class StatsEndpoint(NemoSwitchyardEndpoint)::mod:`switchyard.lib.endpoints.prometheus_emitter` so a single``/metrics`` scrape carries both surfaces."""- snapshot = (- stats.to_dict()- if isinstance(stats, LiveStatsCollector)- else await stats.snapshot()- )+ snapshot = await stats.snapshot()outcome_block = "\n".join(outcome_metrics.render_lines()) + "\n"return Response(content=(diff --git a/switchyard/lib/live_stats_collector.py b/switchyard/lib/live_stats_collector.pydeleted file mode 100644index afb865087..000000000--- a/switchyard/lib/live_stats_collector.py+++ /dev/null@@ -1,277 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Thread-safe per-model LLM usage statistics collector.--Tracks token usage bucketed by model name, with an optional tier label-(e.g. ``"strong"`` / ``"weak"`` from random routing). Used by:--* :class:`StatsResponseProcessor` — records after every LLM response.-* ``launch_claude`` TUI footer — reads the aggregate via ``snapshot()``.-* ``GET /v1/routing/stats`` HTTP endpoint — reads ``to_dict()``.--Cost estimation is best-effort: ``to_dict()`` includes an-``estimated_cost_usd`` field per model when the model name appears in-the built-in price table; ``None`` otherwise. Prices are per 1 M-tokens (input / output) as of 2025 Q3 and do **not** account for cache-read / write premiums.-"""--from __future__ import annotations--import dataclasses-import re-import threading-from dataclasses import dataclass--# ----------------------------------------------------------------------------# Price table — USD per 1 M tokens (input, output).-# Add entries as needed; unknown models get estimated_cost_usd = None.-# Keys use the base name without date suffixes (see _normalize_model_name).-# -----------------------------------------------------------------------------_PRICE_PER_1M: dict[str, tuple[float, float]] = {- # Anthropic- "claude-opus-4-7": (15.00, 75.00),- "claude-sonnet-4-6": ( 3.00, 15.00),- "claude-sonnet-4-5": ( 3.00, 15.00),- "claude-haiku-4-5": ( 0.80, 4.00),- # OpenAI- "gpt-4o": ( 2.50, 10.00),- "gpt-4o-mini": ( 0.15, 0.60),- "o1": (15.00, 60.00),- "o3": (10.00, 40.00),- "o4-mini": ( 1.10, 4.40),-}--_DATE_SUFFIX_RE = re.compile(r"-\d{8}$")---def _normalize_model_name(model: str) -> str:- """Strip trailing ISO-8601 date suffixes used by some providers.-- ``"claude-sonnet-4-6-20251022"`` → ``"claude-sonnet-4-6"``- """- return _DATE_SUFFIX_RE.sub("", model)---def _estimate_cost(- model: str,- prompt_tokens: int,- completion_tokens: int,-) -> float | None:- prices = _PRICE_PER_1M.get(model) or _PRICE_PER_1M.get(_normalize_model_name(model))- if prices is None:- return None- input_rate, output_rate = prices- return (prompt_tokens * input_rate + completion_tokens * output_rate) / 1_000_000---# ----------------------------------------------------------------------------# Data classes-# ------------------------------------------------------------------------------@dataclass-class RequestStats:- """Aggregate snapshot for the TUI footer."""-- request_count: int = 0- prompt_tokens: int = 0- completion_tokens: int = 0- cache_read_tokens: int = 0- cache_creation_tokens: int = 0---@dataclass-class ModelStats:- """Accumulated token usage for a single model across all requests.-- Field semantics:-- * ``prompt_tokens`` — total input billed. For Anthropic this is the- sum ``input_tokens + cache_creation_input_tokens +- cache_read_input_tokens`` (Anthropic reports them as siblings, not- parent/child). For OpenAI it is ``usage.prompt_tokens``.- * ``max_observed_context_tokens`` — largest prompt-plus-completion- token count observed in one completed response.- * ``completion_tokens`` — total output tokens.- * ``total_tokens`` — ``prompt_tokens + completion_tokens``.- * ``reasoning_tokens`` — subset of ``completion_tokens`` for chain-of-- thought / thinking output (OpenAI o-series / Responses API). 0 for- Anthropic (not reported on the Messages API).- * ``cache_read_tokens`` — tokens served from the provider cache- (``cache_read_input_tokens`` on Anthropic, ``cached_tokens`` on- OpenAI). Priced at 0.1× on Anthropic.- * ``cache_creation_tokens`` — tokens written to the provider cache- (Anthropic only, priced at 1.25×). Always 0 for OpenAI.- """-- tier: str = ""- calls: int = 0- prompt_tokens: int = 0- max_observed_context_tokens: int = 0- completion_tokens: int = 0- total_tokens: int = 0- reasoning_tokens: int = 0- cache_read_tokens: int = 0- cache_creation_tokens: int = 0---# ----------------------------------------------------------------------------# Collector-# ------------------------------------------------------------------------------class LiveStatsCollector:- """Thread-safe per-model token usage collector.-- ``record()`` is called from the async request path (response- processors / stream taps); ``snapshot()`` is called from the- footer-repaint thread; ``to_dict()`` is called by the HTTP stats- endpoint. All methods acquire a single :class:`threading.Lock`- so they are safe across both threads and async tasks.- """-- def __init__(self) -> None:- self._lock = threading.Lock()- self._models: dict[str, ModelStats] = {}-- # ------------------------------------------------------------------- # Write path- # -------------------------------------------------------------------- def record(- self,- model: str = "unknown",- tier: str = "",- *,- prompt_tokens: int = 0,- completion_tokens: int = 0,- reasoning_tokens: int = 0,- cache_read_tokens: int = 0,- cache_creation_tokens: int = 0,- ) -> None:- """Accumulate one response's token counts into *model*'s bucket."""- with self._lock:- bucket = self._models.get(model)- if bucket is None:- bucket = ModelStats(tier=tier)- self._models[model] = bucket- bucket.calls += 1- bucket.prompt_tokens += prompt_tokens- bucket.max_observed_context_tokens = max(- bucket.max_observed_context_tokens,- prompt_tokens + completion_tokens,- )- bucket.completion_tokens += completion_tokens- bucket.total_tokens += prompt_tokens + completion_tokens- bucket.reasoning_tokens += reasoning_tokens- bucket.cache_read_tokens += cache_read_tokens- bucket.cache_creation_tokens += cache_creation_tokens-- def reset(self) -> None:- """Reset all counters to zero."""- with self._lock:- self._models.clear()-- # ------------------------------------------------------------------- # Read path- # -------------------------------------------------------------------- def snapshot(self) -> RequestStats:- """Aggregate totals for the TUI footer."""- with self._lock:- buckets = list(self._models.values())- return RequestStats(- request_count=sum(b.calls for b in buckets),- prompt_tokens=sum(b.prompt_tokens for b in buckets),- completion_tokens=sum(b.completion_tokens for b in buckets),- cache_read_tokens=sum(b.cache_read_tokens for b in buckets),- cache_creation_tokens=sum(b.cache_creation_tokens for b in buckets),- )-- def tier_breakdown(self) -> list[tuple[str, ModelStats]]:- """Per-model rows ordered ``strong → weak → others``.-- Returned for the random-routing footer so the user can see, at- a glance, how requests / tokens have split across tiers. The- ordering is stable across paints (sort key = ``(tier_rank,- model_name)``) so the footer doesn't flicker between rows.- """- _ORDER = {"strong": 0, "weak": 1}- with self._lock:- items = [(name, dataclasses.replace(b)) for name, b in self._models.items()]- items.sort(key=lambda kv: (_ORDER.get(kv[1].tier, 99), kv[0]))- return items-- def to_dict(self) -> dict[str, object]:- """Full per-model breakdown for the HTTP stats endpoint.-- Wire shape::-- {- "total_requests": int,- "total_tokens": {- "prompt": int, "completion": int, "total": int,- "reasoning": int, "cache_read": int, "cache_creation": int,- },- "models": {- "<model-name>": {- "tier": str,- "calls": int,- "prompt_tokens": int,- "max_observed_context_tokens": int,- "completion_tokens": int,- "total_tokens": int,- "reasoning_tokens": int,- "cache_read_tokens": int,- "cache_creation_tokens": int,- "token_pct": float,- "estimated_cost_usd": float | null,- },- ...- },- }- """- with self._lock:- snapshot = {k: dataclasses.replace(v) for k, v in self._models.items()}-- total_prompt = sum(b.prompt_tokens for b in snapshot.values())- total_completion = sum(b.completion_tokens for b in snapshot.values())- total_tokens = total_prompt + total_completion- total_requests = sum(b.calls for b in snapshot.values())-- def _pct(part: int, whole: int) -> float:- return round(part / whole * 100, 2) if whole else 0.0-- models_out: dict[str, object] = {}- for name, b in snapshot.items():- models_out[name] = {- "tier": b.tier,- "calls": b.calls,- "prompt_tokens": b.prompt_tokens,- "max_observed_context_tokens": b.max_observed_context_tokens,- "completion_tokens": b.completion_tokens,- "total_tokens": b.total_tokens,- "reasoning_tokens": b.reasoning_tokens,- "cache_read_tokens": b.cache_read_tokens,- "cache_creation_tokens": b.cache_creation_tokens,- "token_pct": _pct(b.total_tokens, total_tokens),- "estimated_cost_usd": _estimate_cost(- name, b.prompt_tokens, b.completion_tokens,- ),- }-- return {- "total_requests": total_requests,- "total_tokens": {- "prompt": total_prompt,- "completion": total_completion,- "total": total_tokens,- "reasoning": sum(b.reasoning_tokens for b in snapshot.values()),- "cache_read": sum(b.cache_read_tokens for b in snapshot.values()),- "cache_creation": sum(b.cache_creation_tokens for b in snapshot.values()),- },- "models": models_out,- }diff --git a/switchyard/lib/processors/__init__.py b/switchyard/lib/processors/__init__.pyindex 00ab52cf3..17f8a468b 100644--- a/switchyard/lib/processors/__init__.py+++ b/switchyard/lib/processors/__init__.py@@ -14,9 +14,6 @@ from switchyard.lib.processors.stats_request_processor import (from switchyard.lib.processors.stats_response_processor_accumulator import (StatsResponseProcessor,)-from switchyard.lib.processors.stats_response_processor_live_collector import (- StatsResponseProcessor as StatsResponseProcessorLiveCollector,-)if TYPE_CHECKING:from switchyard.lib.processors.intake_client import IntakeClient@@ -32,7 +29,6 @@ __all__ = ["ModelRewriteRequestProcessor","StatsRequestProcessor","StatsResponseProcessor",- "StatsResponseProcessorLiveCollector",]diff --git a/switchyard/lib/processors/empty_tool_content_normalizer.py b/switchyard/lib/processors/empty_tool_content_normalizer.pydeleted file mode 100644index 5ce955569..000000000--- a/switchyard/lib/processors/empty_tool_content_normalizer.py+++ /dev/null@@ -1,85 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Replace empty message content that NIH's Nemotron Nano adapter rejects.--NVIDIA Inference Hub's LiteLLM adapter for ``Nemotron-3-Nano-30B-A3B``-rejects any message whose ``content`` is empty (or whitespace-only)-with HTTP 400 ``message content cannot be empty``. Direct probes on-2026-05-15 mapped exactly which roles trip the guard:--* ``role: "tool"`` with ``content == ""`` or ``None`` — rejected.-* ``role: "assistant"`` with ``content == ""`` or ``None`` **and no- ``tool_calls``** — rejected. (Assistant + ``tool_calls`` + empty- content is accepted, matching the OpenAI Chat Completions spec.)-* ``role: "system" / "user"`` with empty/null content — accepted (NIH- doesn't enforce there).-* Whitespace-only strings (``" "``, ``"\\n"``) are *also* rejected —- the adapter strips before checking. Placeholder must contain a- non-whitespace character.--Harbor + terminus-2 routinely produces both shapes (empty shell-output, retried assistant turns with no content). This processor-normalises them to ``"(no output)"`` so the upstream never sees an-empty payload.-"""--from __future__ import annotations--import logging--from switchyard.lib.proxy_context import ProxyContext-from switchyard_rust.core import ChatRequest--log = logging.getLogger(__name__)--_PLACEHOLDER = "(no output)"---def _is_empty(content: object) -> bool:- """True iff ``content`` is None, empty string, or whitespace-only string."""- if content is None:- return True- if isinstance(content, str) and content.strip() == "":- return True- return False---class EmptyToolContentNormalizer:- """Replace empty content on ``tool`` and bare-``assistant`` messages.-- Mutates only the shapes NIH's Nemotron adapter rejects (see module- docstring); leaves everything else untouched.- """-- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: # noqa: ARG002- body = request.body- if not isinstance(body, dict):- return request- messages = body.get("messages")- if not isinstance(messages, list):- return request-- mutated = False- for msg in messages:- if not isinstance(msg, dict):- continue- role = msg.get("role")- if role == "tool":- if _is_empty(msg.get("content")):- msg["content"] = _PLACEHOLDER- mutated = True- elif role == "assistant":- # Assistant with tool_calls is allowed to have empty- # content per the OpenAI spec — and the Nano adapter- # accepts it. Only normalise the bare case.- if not msg.get("tool_calls") and _is_empty(msg.get("content")):- msg["content"] = _PLACEHOLDER- mutated = True-- if mutated:- request.replace_body(body)- return request---__all__ = ["EmptyToolContentNormalizer"]diff --git a/switchyard/lib/processors/fixed_tier_request_processor.py b/switchyard/lib/processors/fixed_tier_request_processor.pydeleted file mode 100644index 98259b6ef..000000000--- a/switchyard/lib/processors/fixed_tier_request_processor.py+++ /dev/null@@ -1,55 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Force every request to a single pre-chosen tier — no LLM call.--Drop-in replacement for the-:class:`LLMClassifierRequestProcessor` + :class:`SignalTierSelectorRequestProcessor`-pair when you want an all-strong or all-weak baseline run. Skipping-the classifier round-trip eliminates the per-request token cost and-the ~3–17 s wall-time tax of a real classification call, so the-forced-tier baseline measures only the upstream backend's behaviour-(solve rate, cost, latency) without any classifier contamination.--Stamps :data:`CTX_DETERMINISTIC_ROUTING_TIER` directly so the-downstream :class:`DeterministicRoutingLLMBackend` dispatches without-needing a synthetic :class:`RouteDecision` to materialise.-"""--from __future__ import annotations--from typing import TYPE_CHECKING--from switchyard.lib.backends.deterministic_routing_llm_backend import (- CTX_DETERMINISTIC_ROUTING_TIER,-)--if TYPE_CHECKING:- from switchyard.lib.proxy_context import ProxyContext- from switchyard_rust.core import ChatRequest---class FixedTierRequestProcessor:- """Stamp a fixed tier label onto every request.-- Args:- tier: Tier label to stamp. Must match one of the labels the- downstream :class:`DeterministicRoutingLLMBackend` knows- about (typically ``"strong"`` or ``"weak"``).- """-- def __init__(self, tier: str) -> None:- if not isinstance(tier, str) or not tier:- raise ValueError(f"tier must be a non-empty string, got {tier!r}")- self._tier = tier-- @property- def tier(self) -> str:- return self._tier-- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:- ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = self._tier- return request---__all__ = ["FixedTierRequestProcessor"]diff --git a/switchyard/lib/processors/stats_response_processor_live_collector.py b/switchyard/lib/processors/stats_response_processor_live_collector.pydeleted file mode 100644index 2f140f53e..000000000--- a/switchyard/lib/processors/stats_response_processor_live_collector.py+++ /dev/null@@ -1,337 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Response-side processor that records per-model token usage into a LiveStatsCollector.--This is the single stats-recording mechanism for the chain when using-LiveStatsCollector (vs StatsAccumulator). It is format-agnostic (handles-Anthropic Messages, OpenAI Chat Completions, and OpenAI Responses API)-and routing-policy-agnostic (works with any LLMBackend).--Model attribution uses Rust-owned ``ProxyContext.selected_model`` first,-falling back to the legacy ``ctx.metadata["_proxy_actual_model"]`` key.-Tier labelling uses ``ctx.metadata["_random_routing_tier"]`` when present,-or Rust-owned ``ProxyContext.selected_target`` for strong/weak routing.--The processor exposes ``get_routing_stats()`` / ``reset_routing_stats()``-and ``get_endpoint()`` so the chain's app factory can mount a live stats-endpoint at ``GET /v1/routing/stats`` with no extra wiring.-"""--from __future__ import annotations--from collections.abc import Mapping-from typing import TYPE_CHECKING--from anthropic.types import RawMessageStreamEvent--from switchyard.lib.live_stats_collector import LiveStatsCollector-from switchyard.lib.proxy_context import CTX_PROXY_ACTUAL_MODEL, ProxyContext-from switchyard_rust.core import (- ChatResponse,- ChatResponseType,- response_type_matches,-)-from switchyard_rust.translation import sse_frame_payloads--if TYPE_CHECKING:- from switchyard.lib.endpoints.base import Endpoint---class StatsResponseProcessor:- """Records token usage after each LLM call into a :class:`LiveStatsCollector`.-- Non-streaming responses: reads usage directly from the response body.-- Streaming responses: attaches a lightweight tap to the stream so- usage-carrying events update the collector as they flow through —- no buffering, zero impact on streaming latency.-- Supported response shapes are identified by Rust-owned ``ChatResponseType``.- """-- def __init__(- self,- collector: LiveStatsCollector,- *,- expose_endpoint: bool = True,- ) -> None:- self._collector = collector- self._expose_endpoint = expose_endpoint-- async def process(self, ctx: ProxyContext, response: ChatResponse) -> ChatResponse:- model: str = ctx.selected_model or ctx.metadata.get(CTX_PROXY_ACTUAL_MODEL, "unknown")- tier = _stats_tier(ctx)-- if response_type_matches(response, ChatResponseType.ANTHROPIC_COMPLETION):- _record_anthropic(response, model, tier, self._collector)- elif response_type_matches(response, ChatResponseType.ANTHROPIC_STREAM):- _attach_anthropic_tap(response, model, tier, self._collector)- elif response_type_matches(response, ChatResponseType.OPENAI_COMPLETION):- _record_openai_chat(response, model, tier, self._collector)- elif response_type_matches(response, ChatResponseType.OPENAI_STREAM):- _attach_openai_chat_tap(response, model, tier, self._collector)- elif response_type_matches(response, ChatResponseType.OPENAI_RESPONSES_COMPLETION):- _record_openai_responses(response, model, tier, self._collector)- elif response_type_matches(response, ChatResponseType.OPENAI_RESPONSES_STREAM):- _attach_openai_responses_tap(response, model, tier, self._collector)- return response-- # ------------------------------------------------------------------- # HTTP stats endpoint integration- # -------------------------------------------------------------------- def get_routing_stats(self) -> dict[str, object]:- """Return a snapshot of accumulated per-model statistics."""- return self._collector.to_dict()-- def reset_routing_stats(self) -> None:- """Reset all accumulated statistics to zero."""- self._collector.reset()-- def get_endpoint(self) -> Endpoint | None:- """Contribute ``GET /v1/routing/stats`` to the server."""- if not self._expose_endpoint:- return None- from switchyard.lib.endpoints.stats_endpoint import (- StatsEndpoint,- )-- return StatsEndpoint(self._collector)---def _stats_tier(ctx: ProxyContext) -> str:- tier: str = ctx.metadata.get("_random_routing_tier", "")- if tier:- return tier- selected_target = ctx.selected_target- return selected_target if selected_target in {"strong", "weak"} else ""---# ----------------------------------------------------------------------------# Non-streaming extractors-# ------------------------------------------------------------------------------def _record_anthropic(- response: ChatResponse,- model: str,- tier: str,- collector: LiveStatsCollector,-) -> None:- u = _field(response.body, "usage")- if not u:- return- input_tok = _int_field(u, "input_tokens")- cache_create = _int_field(u, "cache_creation_input_tokens")- cache_read = _int_field(u, "cache_read_input_tokens")- completion = _int_field(u, "output_tokens")- collector.record(- model, tier,- prompt_tokens=input_tok + cache_create + cache_read,- completion_tokens=completion,- cache_read_tokens=cache_read,- cache_creation_tokens=cache_create,- )---def _record_openai_chat(- response: ChatResponse,- model: str,- tier: str,- collector: LiveStatsCollector,-) -> None:- u = _field(response.body, "usage")- if u is None:- return- prompt = _int_field(u, "prompt_tokens")- completion = _int_field(u, "completion_tokens")- reasoning = 0- cached = 0- if (details := _field(u, "completion_tokens_details")) is not None:- reasoning = _int_field(details, "reasoning_tokens")- if (prompt_details := _field(u, "prompt_tokens_details")) is not None:- cached = _int_field(prompt_details, "cached_tokens")- collector.record(- model, tier,- prompt_tokens=prompt,- completion_tokens=completion,- reasoning_tokens=reasoning,- cache_read_tokens=cached,- )---def _record_openai_responses(- response: ChatResponse,- model: str,- tier: str,- collector: LiveStatsCollector,-) -> None:- u = _field(response.body, "usage")- if u is None:- return- prompt = _int_field(u, "input_tokens")- completion = _int_field(u, "output_tokens")- reasoning = 0- cached = 0- if (out_details := _field(u, "output_tokens_details")) is not None:- reasoning = _int_field(out_details, "reasoning_tokens")- if (in_details := _field(u, "input_tokens_details")) is not None:- cached = _int_field(in_details, "cached_tokens")- collector.record(- model, tier,- prompt_tokens=prompt,- completion_tokens=completion,- reasoning_tokens=reasoning,- cache_read_tokens=cached,- )---# ----------------------------------------------------------------------------# Streaming tap installers-# ------------------------------------------------------------------------------def _attach_anthropic_tap(- response: ChatResponse,- model: str,- tier: str,- collector: LiveStatsCollector,-) -> None:- """Register a tap that accumulates Anthropic SSE usage."""- acc: dict[str, int] = {- "input_tokens": 0,- "output_tokens": 0,- "cache_creation_input_tokens": 0,- "cache_read_input_tokens": 0,- }- committed = False-- def _merge(usage: object) -> None:- for key in acc:- value = _field(usage, key)- if isinstance(value, int):- acc[key] = value-- async def _tap(event: RawMessageStreamEvent) -> None:- nonlocal committed- if committed:- return- event_type = getattr(event, "type", None)- if event_type == "message_start":- msg = getattr(event, "message", None)- if msg is not None:- usage = _field(msg, "usage")- if usage is not None:- _merge(usage)- elif event_type == "message_delta":- usage = _field(event, "usage")- if usage is not None:- _merge(usage)- elif event_type == "message_stop":- input_tok = acc["input_tokens"]- cache_create = acc["cache_creation_input_tokens"]- cache_read = acc["cache_read_input_tokens"]- completion = acc["output_tokens"]- collector.record(- model, tier,- prompt_tokens=input_tok + cache_create + cache_read,- completion_tokens=completion,- cache_read_tokens=cache_read,- cache_creation_tokens=cache_create,- )- committed = True-- response.stream.tap(_tap)---def _attach_openai_chat_tap(- response: ChatResponse,- model: str,- tier: str,- collector: LiveStatsCollector,-) -> None:- """Register a tap for OpenAI Chat streaming usage."""- seen = False-- async def _tap(chunk) -> None: # type: ignore[no-untyped-def]- nonlocal seen- if seen:- return- u = _field(chunk, "usage")- if u is None:- return- prompt = _int_field(u, "prompt_tokens")- completion = _int_field(u, "completion_tokens")- reasoning = 0- cached = 0- if (details := _field(u, "completion_tokens_details")) is not None:- reasoning = _int_field(details, "reasoning_tokens")- if (prompt_details := _field(u, "prompt_tokens_details")) is not None:- cached = _int_field(prompt_details, "cached_tokens")- collector.record(- model, tier,- prompt_tokens=prompt,- completion_tokens=completion,- reasoning_tokens=reasoning,- cache_read_tokens=cached,- )- seen = True-- response.stream.tap(_tap)---def _attach_openai_responses_tap(- response: ChatResponse,- model: str,- tier: str,- collector: LiveStatsCollector,-) -> None:- """Register a tap for OpenAI Responses API streaming usage."""- seen = False-- async def _tap(event) -> None: # type: ignore[no-untyped-def]- nonlocal seen- if seen:- return- # Verbatim-passthrough backends yield raw SSE frame strings; parse the- # frame's data payload(s) to find the usage-carrying event.- candidates = sse_frame_payloads(event) if isinstance(event, str) else [event]- inner = next(- (found for c in candidates if (found := _field(c, "response")) is not None),- None,- )- if inner is None:- return- u = _field(inner, "usage")- if u is None:- return- prompt = _int_field(u, "input_tokens")- completion = _int_field(u, "output_tokens")- reasoning = 0- cached = 0- if (out_details := _field(u, "output_tokens_details")) is not None:- reasoning = _int_field(out_details, "reasoning_tokens")- if (in_details := _field(u, "input_tokens_details")) is not None:- cached = _int_field(in_details, "cached_tokens")- collector.record(- model, tier,- prompt_tokens=prompt,- completion_tokens=completion,- reasoning_tokens=reasoning,- cache_read_tokens=cached,- )- seen = True-- response.stream.tap(_tap)---def _field(value: object, name: str) -> object | None:- if isinstance(value, Mapping):- return value.get(name)- return getattr(value, name, None)---def _int_field(value: object, name: str) -> int:- field = _field(value, name)- return field if isinstance(field, int) else 0diff --git a/switchyard/lib/processors/turn_based_router_request_processor.py b/switchyard/lib/processors/turn_based_router_request_processor.pydeleted file mode 100644index adf96123f..000000000--- a/switchyard/lib/processors/turn_based_router_request_processor.py+++ /dev/null@@ -1,238 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Deterministic turn-based router — cadenced strong/weak dispatch.--Sibling routing primitive to :mod:`switchyard.lib.processors.llm_classifier`-and :mod:`switchyard.lib.processors.plan_execute`. Where the classifier-inspects the prompt and routes by semantic signals, and the random-router flips a weighted coin, this router routes by **cadence**: turn-1 of any task is always strong; subsequent turns are weak by default,-with strong "anchor" turns scheduled at a fixed long-run fraction.--The hypothesis: strong models earn their cost at *anchor* turns —-turn 1 for initial framing, periodic later turns for course correction-— while routine execution turns can use a cheaper executor without-hurting solve rate. Random routing tests this with a coin flip;-turn-based routing tests it deterministically so the cadence is-predictable, transparent, and the same on every benchmark replay.--Mirrors :class:`switchyard.lib.profiles.random_routing.RandomRoutingConfig`'s-API exactly: same ``strong_probability`` knob, same long-run-distribution, just no randomness. Drop-in replacement for benchmark-sweeps where reproducibility matters.--Tier selection formula-------------------------For a target strong fraction ``p ∈ [0, 1]`` and 1-indexed turn ``t``::-- select strong iff ceil(t * p) > ceil((t - 1) * p)--This produces:--* Turn 1 always strong whenever ``p > 0`` (``ceil(p) ≥ 1 > 0``).-* Long-run strong rate exactly ``p`` (Bresenham line algorithm).-* Stateless — the decision depends only on the current turn number,- which is computed from the inbound conversation history.--Distribution sanity check::-- p=1.0: S S S S S S S S S S always strong- p=0.7: S S S W S S W S S W ... ~70% strong- p=0.5: S W S W S W S W S W ... every other turn = strong- p=0.3: S W W S W W S W W W ... ~30% strong (every 3-4 turns)- p=0.2: S W W W W S W W W W S ... 1 in 5 turns strong- p=0.0: W W W W W W W W W W always weak--Turn counting----------------The "turn number" is the 1-indexed count of LLM invocations for this-task — i.e. *how deep in the agent loop are we?* Computed from the-inbound conversation history as ``count(LLM-side prior items) + 1``.--Across the three inbound formats switchyard accepts:--* **OpenAI Chat Completions** — count ``messages[*].role == "assistant"``.- Each LLM round produces exactly one assistant message (text and / or- ``tool_calls``); tool results live in separate ``role: "tool"``- messages so they don't inflate the count.-* **Anthropic Messages** — same: count ``messages[*].role == "assistant"``.- Note that Anthropic puts ``tool_result`` blocks inside ``role: "user"``- messages (not ``role: "tool"`` like OpenAI), but we don't count- ``user``, so the asymmetry doesn't matter.-* **OpenAI Responses** — coarser fallback: with ``input: <string>`` the- count is 0 (always turn 1); with ``input: <list>`` we count items- that are agent-side (``role == "user"`` or ``type ==- "function_call_output"``) and use that as an approximation of- *completed agent acks*, which is one-less than completed LLM rounds.- Adjusted by 1 if the trailing item is itself agent-side. This is- approximate — exact block-counting would require segmentation logic.- Agent loops using the Responses API for tool calls may see slightly- off cadence; for benchmark workloads we run (terminus-2 over OpenAI- Chat Completions) this branch is never hit.--Unknown formats default to turn 1 (strong on first call). Better to-have a known-safe baseline behavior than to fail.--What gets stamped--------------------* :data:`CTX_DETERMINISTIC_ROUTING_TIER` — the chosen tier label- (``"strong"`` or ``"weak"`` by default, configurable via- :attr:`TurnBasedRoutingConfig.strong_tier` /- :attr:`TurnBasedRoutingConfig.weak_tier`). Same key- :class:`switchyard.lib.backends.deterministic_routing_llm_backend.DeterministicRoutingLLMBackend` reads, so- the existing per-tier backend dispatch plugs in unchanged.--Observability----------------One ``turn_based_decision={...}`` JSON line per request to stderr-(mirrors the classifier's ``classifier_signals=…`` and the planner's-``planner_decision=…`` pattern). Lands in captured server logs-regardless of uvicorn log level. Fields::-- {- "turn": 6,- "strong_probability": 0.5,- "tier": "strong",- "request_type": "openai_chat",- }--Grep ``turn_based_decision=`` on the server log to tally cadence-distribution + verify the routing matched the expected pattern.-"""--from __future__ import annotations--import json-import math-import sys-from typing import TYPE_CHECKING--from pydantic import BaseModel, ConfigDict, Field--from switchyard.lib.backends.deterministic_routing_llm_backend import (- CTX_DETERMINISTIC_ROUTING_TIER,-)-from switchyard.lib.conversation_turn import conversation_turn_number--if TYPE_CHECKING:- from switchyard.lib.proxy_context import ProxyContext- from switchyard_rust.core import ChatRequest---#: ``ProxyContext.metadata`` key for the turn number this request was-#: routed at. Always stamped; useful for downstream audit / cost-#: attribution that wants to slice executor spend by turn position.-CTX_TURN_BASED_TURN = "_turn_based_turn"---class TurnBasedRoutingConfig(BaseModel):- """Configuration for :class:`TurnBasedRouterRequestProcessor`.-- Mirrors- :class:`switchyard.lib.profiles.random_routing.RandomRoutingConfig`'s- surface so the two router primitives are drop-in interchangeable- for benchmark sweeps that vary the routing *strategy* against the- same upstream tier targets.- """-- model_config = ConfigDict(frozen=True)-- strong_tier: str = Field(default="strong", min_length=1)- """Tier label stamped for strong-routed turns. Must match one of- the labels the downstream- :class:`switchyard.lib.backends.deterministic_routing_llm_backend.DeterministicRoutingLLMBackend` knows- about."""-- weak_tier: str = Field(default="weak", min_length=1)- """Tier label stamped for weak-routed turns."""-- strong_probability: float = Field(default=0.5, ge=0.0, le=1.0)- """Target long-run fraction of turns routed to the strong tier.- ``1.0`` = every turn strong (equivalent to ``FixedTierRequestProcessor("strong")``);- ``0.0`` = every turn weak (equivalent to- ``FixedTierRequestProcessor("weak")``); intermediate values- produce a deterministic cadence whose long-run rate matches.- Default ``0.5`` (every other turn = strong) mirrors the- ``RandomRoutingConfig`` default."""---class TurnBasedRouterRequestProcessor:- """Stamp a strong/weak tier label based on a deterministic per-turn cadence.-- Reads the inbound conversation history to compute the current- turn number, applies the Bresenham ceil formula to decide whether- this turn should hit the strong or weak tier, and stamps the- label so the downstream- :class:`DeterministicRoutingLLMBackend` dispatches accordingly.-- Emits one ``turn_based_decision={...}`` JSON audit line per request- to stderr for observability.- """-- def __init__(self, config: TurnBasedRoutingConfig) -> None:- self._config = config-- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:- turn = conversation_turn_number(request)- tier = _select_tier(- turn=turn,- strong_probability=self._config.strong_probability,- strong_tier=self._config.strong_tier,- weak_tier=self._config.weak_tier,- )- ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = tier- ctx.metadata[CTX_TURN_BASED_TURN] = turn-- # Audit line — mirrors classifier_signals=... and- # planner_decision=... so the three routers share a grep- # convention. Written directly to stderr so the line lands in- # captured server logs regardless of uvicorn's logger config.- request_type = request.request_type.value- audit_payload = {- "turn": turn,- "strong_probability": self._config.strong_probability,- "tier": tier,- "request_type": request_type,- }- sys.stderr.write(- f"turn_based_decision={json.dumps(audit_payload, sort_keys=True)}\n"- )- sys.stderr.flush()- return request---def _select_tier(- *,- turn: int,- strong_probability: float,- strong_tier: str,- weak_tier: str,-) -> str:- """Return the tier label for ``turn`` under the given ``strong_probability``.-- Bresenham ceil formula: strong iff ``ceil(t * p) > ceil((t-1) * p)``.-- The two extremes are short-circuited because Python's ``math.ceil``- is finicky at ``p == 0`` (would give weak for every turn including- turn 1 — fine but wasteful to call) and ``p == 1`` would always- pass (correct, but explicit is clearer).- """- if strong_probability >= 1.0:- return strong_tier- if strong_probability <= 0.0:- return weak_tier- cur = math.ceil(turn * strong_probability)- prev = math.ceil((turn - 1) * strong_probability)- return strong_tier if cur > prev else weak_tier---__all__ = [- "CTX_TURN_BASED_TURN",- "TurnBasedRouterRequestProcessor",- "TurnBasedRoutingConfig",-]diff --git a/tests/e2e/test_turn_based_router_chain_e2e.py b/tests/e2e/test_turn_based_router_chain_e2e.pydeleted file mode 100644index 24162f17e..000000000--- a/tests/e2e/test_turn_based_router_chain_e2e.py+++ /dev/null@@ -1,344 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Real-backend e2e test for the **full** turn-based router chain.--Sibling to :mod:`tests.e2e.test_planning_request_processor_e2e` and-:mod:`tests.e2e.test_classifier_planner_chain_e2e`. This file walks-the turn-based router end-to-end against a live backend::-- inbound ChatRequest- -> TurnBasedRouterRequestProcessor (no LLM call; counts- assistant messages,- applies Bresenham- ceil schedule)- -> DeterministicRoutingLLMBackend (dispatches to per-tier- backend based on the- tier label stamped on- ctx by the router)- -> strong tier: real LLM call (gpt-5.2)- -> weak tier: real LLM call (V4 Pro)- -> ChatResponse--The router itself does no LLM call (just compute), so the cost on-this test comes entirely from the executor tier the router picked.-At ``strong_probability=0.5`` over three crafted requests the-expected pattern is **strong → weak → strong**, with each request-landing on the correct tier model.--Strong vs weak distinct models is the critical thing this test-exercises that the unit tests cannot: it verifies the-``ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER]`` stamp actually-drives the multi-tier dispatcher's per-tier dispatch. The unit-tests stub the backend entirely; this test wires-:class:`DeterministicRoutingLLMBackend` with two real backends-pointed at two real models, and verifies the response's ``model``-field matches the tier the router picked.--Prerequisites:- - ``OPENROUTER_API_KEY`` or ``NVIDIA_API_KEY`` env var (skips otherwise).--Run with::-- OPENROUTER_API_KEY=sk-or-... uv run pytest \\- tests/e2e/test_turn_based_router_chain_e2e.py -v-"""--from __future__ import annotations--import logging-import os-from typing import Any--import pytest--from switchyard.lib.backends.deterministic_routing_llm_backend import (- CTX_DETERMINISTIC_ROUTING_TIER,- DeterministicRoutingLLMBackend,-)-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget-from switchyard.lib.chat_response.base import ChatResponse-from switchyard.lib.processors.turn_based_router_request_processor import (- CTX_TURN_BASED_TURN,- TurnBasedRouterRequestProcessor,- TurnBasedRoutingConfig,-)-from switchyard.lib.proxy_context import ProxyContext-from switchyard.lib.roles import LLMBackend-from switchyard_rust.core import ChatRequest--from .conftest import get_nvidia_config--# Three real LLM calls per cadence test (one per crafted turn). 120s-# headroom is generous for the small-prompt smoke calls these tests-# do (each request is ~10 tokens out).-pytestmark = [- pytest.mark.integration,- pytest.mark.timeout(120),-]--logger = logging.getLogger("e2e.turn_based_router_chain")--HTTP_TIMEOUT_S = 60.0--_nvidia = get_nvidia_config()-_skip_reason = "OPENROUTER_API_KEY or NVIDIA_API_KEY not set"--BACKEND_BASE_URL = _nvidia["base_url"]--# Distinct strong / weak model ids so the test can verify routing by-# inspecting ``response.body["model"]``. Both are OpenAI-Chat-Completions-# compatible on the selected backend — no Anthropic-native translation-# required, keeping the test focused on routing rather than wire-format-# wrangling.-if _nvidia["provider"] == "openrouter":- STRONG_MODEL = _nvidia["model"]- WEAK_MODEL = os.environ.get("OPENROUTER_WEAK_MODEL") or "moonshotai/kimi-k2.6"-else:- STRONG_MODEL = "openai/openai/gpt-5.2"- WEAK_MODEL = "nvidia/deepseek-ai/evals-deepseek-v4-pro"---# ----------------------------------------------------------------------------# Helpers-# ------------------------------------------------------------------------------def _build_full_chain(- *,- strong_probability: float,-) -> tuple[list[Any], LLMBackend]:- """Build (request_processors, multi-tier backend) for the turn-based chain.-- Mirrors the turn-based routing benchmark chain in-process with direct- OpenAiPassthrough-equivalent tier- backends (via :meth:`DeterministicRoutingLLMBackend.from_tiers`,- which builds standard OpenAI / Anthropic backends from- :class:`LlmTarget` config). No stats wrapping — the router itself- doesn't record stats anyway, and tier-side stats are out of scope- for this routing-correctness test.- """- router = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(- strong_tier="strong",- weak_tier="weak",- strong_probability=strong_probability,- ),- )- backend = DeterministicRoutingLLMBackend.from_tiers(- tiers={- "strong": LlmTarget(- model=STRONG_MODEL,- format=BackendFormat.OPENAI,- api_key=_nvidia["api_key"],- base_url=BACKEND_BASE_URL,- timeout=HTTP_TIMEOUT_S,- ),- "weak": LlmTarget(- model=WEAK_MODEL,- format=BackendFormat.OPENAI,- api_key=_nvidia["api_key"],- base_url=BACKEND_BASE_URL,- timeout=HTTP_TIMEOUT_S,- # DeepSeek V4 needs enable_thinking=False or its JSON- # output gets misrouted into reasoning_content. This- # call doesn't request structured output so the risk- # is mostly latency (V4 with thinking on is 10-15s- # slower per call) — set anyway for parity with the- # benchmark server's auto-detect.- extra_body={"chat_template_kwargs": {"enable_thinking": False}},- extra_headers={"X-Inference-Priority": "batch"},- ),- },- default_tier="strong",- )- return [router], backend---async def _call_through_chain(- request_processors: list[Any],- backend: LLMBackend,- request: ChatRequest,-) -> tuple[ProxyContext, ChatResponse]:- """Run a request through router + multi-tier backend in-process."""- ctx = ProxyContext()- for proc in request_processors:- request = await proc.process(ctx, request)- response = await backend.call(ctx, request)- return ctx, response---def _chat_request_with_n_prior_assistants(- *,- n_prior_assistants: int,- user_message: str = "Reply with: ok",-) -> ChatRequest:- """Build an OpenAI Chat Completions ``ChatRequest`` whose history- contains exactly ``n_prior_assistants`` assistant messages.-- Used to drive the router's turn counter — the router computes- ``turn = count(assistant) + 1``, so ``n_prior_assistants=0`` →- turn 1, ``n_prior_assistants=1`` → turn 2, etc.- """- messages: list[dict[str, Any]] = [- {"role": "user", "content": "initial task"},- ]- for i in range(n_prior_assistants):- messages.append({"role": "assistant", "content": f"r{i + 1}"})- messages.append({"role": "user", "content": f"continue ({i + 1})"})- # Replace the trailing "continue" with the actual prompt for the- # current turn so the model has something concrete to answer.- if n_prior_assistants > 0:- messages[-1] = {"role": "user", "content": user_message}- else:- messages[-1] = {"role": "user", "content": user_message}- return ChatRequest.openai_chat({- "model": "client-placeholder",- "messages": messages,- "max_tokens": 10,- })---def _model_from_response(response: ChatResponse) -> str:- """The upstream-reported model in the response body."""- return str(response.body.get("model") or "")---def _content_from_response(response: ChatResponse) -> str:- choice = response.body["choices"][0]- message = choice.get("message", {})- return str(message.get("content") or "")---# ----------------------------------------------------------------------------# Tests-# -----------------------------------------------------------------------------[email protected](not _nvidia["api_key"], reason=_skip_reason)-class TestTurnBasedRouterChainLiveE2E:- """Full turn-based router chain (router + multi-tier backend) against the live hub."""-- async def test_cadence_routes_through_both_tiers_at_half_probability(self) -> None:- """At ``strong_probability=0.5`` three crafted turns yield S → W → S.-- The router stamps ``CTX_DETERMINISTIC_ROUTING_TIER`` on each- request; :class:`DeterministicRoutingLLMBackend` dispatches to- the matching tier; the response's ``model`` field reflects the- upstream model that was actually called. This test wires- distinct strong / weak models specifically so we can verify- the dispatch end-to-end — the response model identifying which- tier handled the request.-- The cadence assertion is the user-facing invariant: turn 1- always strong, turn 2 weak at p=0.5, turn 3 strong again.- """- req_processors, backend = _build_full_chain(strong_probability=0.5)-- # --- Turn 1 (no prior assistants) → strong ------------------- req1 = _chat_request_with_n_prior_assistants(- n_prior_assistants=0,- user_message="Reply with one word: hi",- )- ctx1, resp1 = await _call_through_chain(req_processors, backend, req1)-- assert ctx1.metadata[CTX_TURN_BASED_TURN] == 1- assert ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"- model1 = _model_from_response(resp1)- assert model1 == STRONG_MODEL, (- f"turn 1 should hit the strong tier ({STRONG_MODEL!r}); "- f"response reported model={model1!r}"- )- content1 = _content_from_response(resp1)- assert content1, "expected non-empty content from strong-tier executor"-- # --- Turn 2 (1 prior assistant) → weak ----------------------- req2 = _chat_request_with_n_prior_assistants(- n_prior_assistants=1,- user_message="Reply with one word: ok",- )- ctx2, resp2 = await _call_through_chain(req_processors, backend, req2)-- assert ctx2.metadata[CTX_TURN_BASED_TURN] == 2- assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"- model2 = _model_from_response(resp2)- assert model2 == WEAK_MODEL, (- f"turn 2 at p=0.5 should hit the weak tier ({WEAK_MODEL!r}); "- f"response reported model={model2!r}"- )- content2 = _content_from_response(resp2)- assert content2, "expected non-empty content from weak-tier executor"-- # --- Turn 3 (2 prior assistants) → strong --------------------- req3 = _chat_request_with_n_prior_assistants(- n_prior_assistants=2,- user_message="Reply with one word: done",- )- ctx3, resp3 = await _call_through_chain(req_processors, backend, req3)-- assert ctx3.metadata[CTX_TURN_BASED_TURN] == 3- assert ctx3.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"- model3 = _model_from_response(resp3)- assert model3 == STRONG_MODEL, (- f"turn 3 at p=0.5 should hit the strong tier ({STRONG_MODEL!r}); "- f"response reported model={model3!r}"- )-- logger.info(- " [turn-based-e2e][p=0.5] cadence=[t1=%s/%s, t2=%s/%s, t3=%s/%s]",- ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER], model1,- ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER], model2,- ctx3.metadata[CTX_DETERMINISTIC_ROUTING_TIER], model3,- )-- async def test_low_probability_anchors_first_turn_then_routes_weak(self) -> None:- """At ``strong_probability=0.2`` turn 1 routes strong, turns 2-5 route weak.-- Verifies the "anchor + sparse strong" cadence the turn-based- router was designed for: cheap executor handles the bulk of- the loop while turn 1 (and every Nth turn after) gets a- strong-model anchor. At p=0.2 the next strong turn after- turn 1 is turn 6.-- Sweeps turn 1 + a sample of subsequent turns rather than the- full 1-6 to keep the test under a few seconds of LLM calls;- the unit tests cover the exact cadence pattern over a longer- window.- """- req_processors, backend = _build_full_chain(strong_probability=0.2)-- # Turn 1: always strong (the anchor).- req1 = _chat_request_with_n_prior_assistants(- n_prior_assistants=0,- user_message="Reply with: anchor",- )- ctx1, resp1 = await _call_through_chain(req_processors, backend, req1)- assert ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"- assert _model_from_response(resp1) == STRONG_MODEL-- # Turn 2: weak (cadence at p=0.2 only strong at turns 1, 6, 11, …).- req2 = _chat_request_with_n_prior_assistants(- n_prior_assistants=1,- user_message="Reply with: step2",- )- ctx2, resp2 = await _call_through_chain(req_processors, backend, req2)- assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"- assert _model_from_response(resp2) == WEAK_MODEL-- # Turn 5: still weak (next strong is turn 6).- req5 = _chat_request_with_n_prior_assistants(- n_prior_assistants=4,- user_message="Reply with: step5",- )- ctx5, resp5 = await _call_through_chain(req_processors, backend, req5)- assert ctx5.metadata[CTX_TURN_BASED_TURN] == 5- assert ctx5.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"- assert _model_from_response(resp5) == WEAK_MODEL-- logger.info(- " [turn-based-e2e][p=0.2] sparse-strong: t1=strong/%s, "- "t2=weak/%s, t5=weak/%s",- _model_from_response(resp1),- _model_from_response(resp2),- _model_from_response(resp5),- )diff --git a/tests/test_fixed_tier_request_processor.py b/tests/test_fixed_tier_request_processor.pydeleted file mode 100644index b48acde5a..000000000--- a/tests/test_fixed_tier_request_processor.py+++ /dev/null@@ -1,80 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Tests for :class:`FixedTierRequestProcessor`."""--from __future__ import annotations--from typing import Any, cast--import pytest--from switchyard.lib.backends.deterministic_routing_llm_backend import (- CTX_DETERMINISTIC_ROUTING_TIER,-)-from switchyard.lib.processors.fixed_tier_request_processor import (- FixedTierRequestProcessor,-)-from switchyard.lib.proxy_context import ProxyContext-from switchyard_rust.core import ChatRequest---def _request() -> ChatRequest:- return ChatRequest.openai_chat(- cast(Any, {"model": "client-model", "messages": [{"role": "user", "content": "hi"}]}),- )---async def test_fixed_tier_stamps_tier_on_ctx() -> None:- proc = FixedTierRequestProcessor("strong")- ctx = ProxyContext()-- await proc.process(ctx, _request())-- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"---async def test_fixed_tier_weak() -> None:- proc = FixedTierRequestProcessor("weak")- ctx = ProxyContext()-- await proc.process(ctx, _request())-- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"---async def test_fixed_tier_request_passes_through_unchanged() -> None:- """The processor must not mutate the request body — pure tier stamp."""- proc = FixedTierRequestProcessor("strong")- req = _request()- body_before = dict(req.body)-- returned = await proc.process(ProxyContext(), req)-- assert returned is req- assert dict(req.body) == body_before---async def test_fixed_tier_rejects_empty_label() -> None:- with pytest.raises(ValueError, match="non-empty string"):- FixedTierRequestProcessor("")---async def test_fixed_tier_rejects_non_string() -> None:- with pytest.raises(ValueError, match="non-empty string"):- FixedTierRequestProcessor(cast(Any, None))---async def test_fixed_tier_overwrites_existing_tier() -> None:- """If an upstream processor stamped a tier, the fixed processor still wins.-- Force-mode is meant to override whatever else might be in the chain;- this guards against an accidental compose with a real classifier.- """- proc = FixedTierRequestProcessor("strong")- ctx = ProxyContext()- ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = "weak"-- await proc.process(ctx, _request())-- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"diff --git a/tests/test_live_stats_collector.py b/tests/test_live_stats_collector.pydeleted file mode 100644index e8de312ff..000000000--- a/tests/test_live_stats_collector.py+++ /dev/null@@ -1,437 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Tests for LiveStatsCollector and StatsResponseProcessor."""--from __future__ import annotations--import threading-from unittest.mock import MagicMock--import pytest--from switchyard.lib.live_stats_collector import (- LiveStatsCollector,- RequestStats,-)-from switchyard.lib.processors.stats_response_processor_live_collector import (- StatsResponseProcessor,-)-from switchyard_rust.core import ChatResponse--# ----------------------------------------------------------------------------# LiveStatsCollector unit tests-# ------------------------------------------------------------------------------def test_initial_snapshot_is_zero():- c = LiveStatsCollector()- s = c.snapshot()- assert s == RequestStats()---def test_record_increments_all_fields():- c = LiveStatsCollector()- c.record(- prompt_tokens=100,- completion_tokens=50,- cache_read_tokens=20,- cache_creation_tokens=5,- )- s = c.snapshot()- assert s.request_count == 1- assert s.prompt_tokens == 100- assert s.completion_tokens == 50- assert s.cache_read_tokens == 20- assert s.cache_creation_tokens == 5---def test_record_accumulates_across_calls():- c = LiveStatsCollector()- c.record(prompt_tokens=10, completion_tokens=5)- c.record(prompt_tokens=20, completion_tokens=10)- s = c.snapshot()- assert s.request_count == 2- assert s.prompt_tokens == 30- assert s.completion_tokens == 15- assert c.to_dict()["models"]["unknown"]["max_observed_context_tokens"] == 30---def test_snapshot_returns_copy():- c = LiveStatsCollector()- c.record(prompt_tokens=10)- snap1 = c.snapshot()- c.record(prompt_tokens=10)- snap2 = c.snapshot()- assert snap1.request_count == 1- assert snap2.request_count == 2---def test_record_defaults_to_zero_tokens():- c = LiveStatsCollector()- c.record()- s = c.snapshot()- assert s.request_count == 1- assert s.prompt_tokens == 0- assert s.completion_tokens == 0---def test_thread_safety():- c = LiveStatsCollector()- n_threads = 50- calls_per_thread = 100-- def _worker():- for _ in range(calls_per_thread):- c.record(prompt_tokens=1, completion_tokens=1)-- threads = [threading.Thread(target=_worker) for _ in range(n_threads)]- for t in threads:- t.start()- for t in threads:- t.join()-- s = c.snapshot()- assert s.request_count == n_threads * calls_per_thread- assert s.prompt_tokens == n_threads * calls_per_thread---# ----------------------------------------------------------------------------# tier_breakdown — ordering + isolation-# ------------------------------------------------------------------------------def test_tier_breakdown_empty_when_no_records():- c = LiveStatsCollector()- assert c.tier_breakdown() == []---def test_tier_breakdown_orders_strong_before_weak():- """Insertion order must NOT determine UI order — the random-routing- footer relies on a stable ``strong → weak`` layout regardless of- which tier rolled first.- """- c = LiveStatsCollector()- # Record weak first to confirm sort, not insertion order, drives output.- c.record(model="nemotron", tier="weak", prompt_tokens=10, completion_tokens=5)- c.record(model="kimi-k2.5", tier="strong", prompt_tokens=20, completion_tokens=8)-- rows = c.tier_breakdown()- assert [name for name, _ in rows] == ["kimi-k2.5", "nemotron"]- assert [b.tier for _, b in rows] == ["strong", "weak"]---def test_tier_breakdown_unknown_tier_sorts_last():- """Models recorded without a tier (or with an unrecognised label)- sink to the bottom — the strong/weak rows are always on top.- """- c = LiveStatsCollector()- c.record(model="other-model", tier="", prompt_tokens=1)- c.record(model="kimi", tier="strong", prompt_tokens=1)- c.record(model="nemotron", tier="weak", prompt_tokens=1)-- rows = c.tier_breakdown()- assert [name for name, _ in rows] == ["kimi", "nemotron", "other-model"]---def test_tier_breakdown_returns_copies():- """Mutating the returned ``ModelStats`` must not affect the- collector's internal state — the footer paints from this list and- rendering work shouldn't be able to corrupt request accounting.- """- c = LiveStatsCollector()- c.record(model="kimi", tier="strong", prompt_tokens=10, completion_tokens=5)-- rows = c.tier_breakdown()- rows[0][1].calls = 999-- # Re-fetch; collector's truth is unchanged.- again = c.tier_breakdown()- assert again[0][1].calls == 1---# ----------------------------------------------------------------------------# StatsResponseProcessor — non-streaming Anthropic-# ------------------------------------------------------------------------------def _make_ctx():- from switchyard.lib.proxy_context import ProxyContext- return ProxyContext()---def _make_anthropic_usage(input_tokens=10, output_tokens=5, cache_read=0, cache_creation=0):- return {- "input_tokens": input_tokens,- "output_tokens": output_tokens,- "cache_read_input_tokens": cache_read,- "cache_creation_input_tokens": cache_creation,- }---async def test_records_anthropic_completion():- collector = LiveStatsCollector()- proc = StatsResponseProcessor(collector)-- body = {"usage": _make_anthropic_usage(input_tokens=100, output_tokens=40, cache_read=30)}- response = ChatResponse.anthropic_completion(body)-- result = await proc.process(_make_ctx(), response)- assert result is response-- s = collector.snapshot()- assert s.request_count == 1- # prompt_tokens sums input + cache_read + cache_creation for Anthropic- assert s.prompt_tokens == 130- assert s.completion_tokens == 40- assert s.cache_read_tokens == 30---# ----------------------------------------------------------------------------# StatsResponseProcessor — non-streaming OpenAI-# ------------------------------------------------------------------------------async def test_records_openai_completion():- collector = LiveStatsCollector()- proc = StatsResponseProcessor(collector)-- usage = {- "prompt_tokens": 80,- "completion_tokens": 30,- "prompt_tokens_details": {"cached_tokens": 15},- }-- body = {"usage": usage}- response = ChatResponse.openai_completion(body)-- await proc.process(_make_ctx(), response)-- s = collector.snapshot()- assert s.request_count == 1- assert s.prompt_tokens == 80- assert s.completion_tokens == 30- assert s.cache_read_tokens == 15---async def test_records_rust_context_selected_model_and_target():- collector = LiveStatsCollector()- proc = StatsResponseProcessor(collector)- ctx = _make_ctx()- ctx.selected_model = "served-model"- ctx.selected_target = "weak"-- await proc.process(- ctx,- ChatResponse.openai_completion({- "usage": {- "prompt_tokens": 9,- "completion_tokens": 4,- },- }),- )-- rows = collector.tier_breakdown()- assert [(model, stats.tier, stats.prompt_tokens) for model, stats in rows] == [- ("served-model", "weak", 9),- ]---async def test_handles_missing_openai_usage():- collector = LiveStatsCollector()- proc = StatsResponseProcessor(collector)-- body = {"usage": None}- response = ChatResponse.openai_completion(body)-- await proc.process(_make_ctx(), response)- assert collector.snapshot().request_count == 0---# ----------------------------------------------------------------------------# StatsResponseProcessor — streaming Anthropic tap-# ------------------------------------------------------------------------------async def test_attaches_anthropic_stream_tap():- from switchyard.lib.chat_response.anthropic import AnthropicResponseStream-- collector = LiveStatsCollector()- proc = StatsResponseProcessor(collector)-- async def _events():- start = MagicMock()- start.type = "message_start"- start.message.usage.input_tokens = 50- start.message.usage.cache_read_input_tokens = 10- start.message.usage.cache_creation_input_tokens = 0- yield start-- delta = MagicMock()- delta.type = "message_delta"- delta.usage.output_tokens = 20- yield delta-- stop = MagicMock()- stop.type = "message_stop"- yield stop-- stream = AnthropicResponseStream(_events())- response = ChatResponse.anthropic_stream(stream)-- await proc.process(_make_ctx(), response)-- events = [e async for e in response.stream]- assert len(events) == 3-- s = collector.snapshot()- assert s.request_count == 1- # prompt_tokens sums input(50) + cache_read(10) + cache_creation(0)- assert s.prompt_tokens == 60- assert s.completion_tokens == 20- assert s.cache_read_tokens == 10---async def test_anthropic_tap_delta_input_tokens_fallback():- """Backends that report input_tokens in message_delta instead of message_start.-- Some Anthropic-compatible APIs (e.g. NVIDIA Inference Hub) set- input_tokens=0 in message_start and report the real count in- message_delta.usage.input_tokens (Optional[int]). The tap must use- the delta value as a fallback so the ``in`` counter is non-zero.- """- from switchyard.lib.chat_response.anthropic import AnthropicResponseStream-- collector = LiveStatsCollector()- proc = StatsResponseProcessor(collector)-- async def _events():- start = MagicMock()- start.type = "message_start"- start.message.usage.input_tokens = 0 # backend defers counting- start.message.usage.cache_read_input_tokens = 0- start.message.usage.cache_creation_input_tokens = 0- yield start-- delta = MagicMock()- delta.type = "message_delta"- delta.usage.output_tokens = 30- # Backend reports actual input tokens here- delta.usage.input_tokens = 75- delta.usage.cache_read_input_tokens = None- delta.usage.cache_creation_input_tokens = None- yield delta-- stop = MagicMock()- stop.type = "message_stop"- yield stop-- stream = AnthropicResponseStream(_events())- response = ChatResponse.anthropic_stream(stream)- await proc.process(_make_ctx(), response)- [_ async for _ in response.stream]-- s = collector.snapshot()- assert s.request_count == 1- assert s.prompt_tokens == 75- assert s.completion_tokens == 30---# ----------------------------------------------------------------------------# StatsResponseProcessor — streaming OpenAI Responses API tap-# ------------------------------------------------------------------------------async def test_attaches_openai_responses_stream_tap():- from openai.types.responses import Response as OpenAIResponse- from openai.types.responses import ResponseCompletedEvent- from openai.types.responses.response_usage import (- InputTokensDetails,- OutputTokensDetails,- ResponseUsage,- )-- from switchyard.lib.chat_response.openai_responses import ResponsesApiStream-- collector = LiveStatsCollector()- proc = StatsResponseProcessor(collector)-- response_obj = OpenAIResponse.model_construct(- id="resp-test", object="response", created_at=1_700_000_000,- model="gpt-4o", output=[], status="completed",- usage=ResponseUsage(- input_tokens=300, output_tokens=120, total_tokens=420,- input_tokens_details=InputTokensDetails(cached_tokens=60),- output_tokens_details=OutputTokensDetails(reasoning_tokens=40),- ),- )-- async def _events():- yield ResponseCompletedEvent(- type="response.completed", response=response_obj, sequence_number=1,- )-- response = ChatResponse.openai_responses_stream(ResponsesApiStream(_events()))- await proc.process(_make_ctx(), response)- [_ async for _ in response.stream]-- s = collector.snapshot()- assert s.request_count == 1- assert s.prompt_tokens == 300- assert s.completion_tokens == 120- assert s.cache_read_tokens == 60---async def test_openai_responses_tap_reads_usage_from_raw_sse_frames():- """Verbatim-passthrough streams yield raw SSE frame strings; the tap must- still find the usage in the ``response.completed`` frame."""- from switchyard.lib.chat_response.openai_responses import ResponsesApiStream-- collector = LiveStatsCollector()- proc = StatsResponseProcessor(collector)-- async def _frames():- yield (- 'event: response.output_text.delta\n'- 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n'- )- yield ": keep-alive\n\n"- yield (- 'event: response.completed\n'- 'data: {"type":"response.completed","response":{"id":"resp-raw","usage":'- '{"input_tokens":300,"output_tokens":120,'- '"input_tokens_details":{"cached_tokens":60},'- '"output_tokens_details":{"reasoning_tokens":40}}}}\n\n'- )-- response = ChatResponse.openai_responses_stream(ResponsesApiStream(_frames()))- await proc.process(_make_ctx(), response)- [_ async for _ in response.stream]-- s = collector.snapshot()- assert s.request_count == 1- assert s.prompt_tokens == 300- assert s.completion_tokens == 120- assert s.cache_read_tokens == 60---# ----------------------------------------------------------------------------# Cost estimation — model name normalization-# ------------------------------------------------------------------------------def test_cost_estimate_strips_date_suffix():- """Versioned model names like 'claude-sonnet-4-6-20251022' should match- the base name in the price table."""- collector = LiveStatsCollector()- collector.record("claude-sonnet-4-6-20251022", prompt_tokens=1_000_000, completion_tokens=0)- m = collector.to_dict()["models"]["claude-sonnet-4-6-20251022"]- # Should resolve to claude-sonnet-4-6 price: $3.00 / 1M input- assert m["estimated_cost_usd"] == pytest.approx(3.00)---def test_cost_estimate_none_for_unknown_model():- collector = LiveStatsCollector()- collector.record("some-unknown-model", prompt_tokens=100, completion_tokens=50)- m = collector.to_dict()["models"]["some-unknown-model"]- assert m["estimated_cost_usd"] is Nonediff --git a/tests/test_turn_based_router_request_processor.py b/tests/test_turn_based_router_request_processor.pydeleted file mode 100644index 4471cf445..000000000--- a/tests/test_turn_based_router_request_processor.py+++ /dev/null@@ -1,503 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Unit tests for :class:`TurnBasedRouterRequestProcessor`.--Pure compute, no LLM, no network — covers:--* The Bresenham ceil formula's long-run distribution and first-turn-strong- invariant at edges (p=0.0, 1.0) and intermediate values (0.3, 0.5, 0.7).-* OpenAI Chat Completions turn counting (role=='assistant').-* Anthropic Messages turn counting (verifies the role=='user' tool-result- asymmetry doesn't inflate the counter).-* OpenAI Responses turn counting (string input and list input coarse path).-* Edge cases: empty / missing messages, malformed entries.-* ctx-metadata stamping for both the tier label and the turn number.-* The ``turn_based_decision={...}`` audit line on stderr.-"""--from __future__ import annotations--import json-import math-from typing import Any, cast--from switchyard.lib.backends.deterministic_routing_llm_backend import (- CTX_DETERMINISTIC_ROUTING_TIER,-)-from switchyard.lib.processors.turn_based_router_request_processor import (- CTX_TURN_BASED_TURN,- TurnBasedRouterRequestProcessor,- TurnBasedRoutingConfig,-)-from switchyard.lib.proxy_context import ProxyContext-from switchyard_rust.core import ChatRequest--# ----------------------------------------------------------------------------# Helpers-# ------------------------------------------------------------------------------def _build_openai_chat_request(messages: list[dict[str, Any]]) -> ChatRequest:- """Wrap a messages list in a minimal OpenAI Chat Completions request."""- return ChatRequest.openai_chat(cast(Any, {- "model": "placeholder",- "messages": messages,- }))---def _build_anthropic_request(messages: list[dict[str, Any]]) -> ChatRequest:- """Wrap messages in a minimal Anthropic Messages request."""- return ChatRequest.anthropic(cast(Any, {- "model": "claude-test",- "max_tokens": 128,- "messages": messages,- }))---def _build_responses_request(input_value: Any) -> ChatRequest:- """Wrap input in a minimal OpenAI Responses request."""- return ChatRequest.openai_responses(cast(Any, {- "model": "placeholder",- "input": input_value,- }))---def _conversation_with_n_assistant_turns(n: int) -> list[dict[str, Any]]:- """Synthesize an OpenAI Chat history with ``n`` prior assistant turns.-- Layout: initial user prompt, then alternating assistant/tool pairs- matching the terminus-2 pattern. Sufficient to drive turn counting- without needing a live agent.- """- msgs: list[dict[str, Any]] = [- {"role": "user", "content": "initial task"},- ]- for i in range(n):- msgs.append({- "role": "assistant",- "content": f"step {i + 1}",- "tool_calls": [{- "id": f"c{i}",- "type": "function",- "function": {"name": "bash", "arguments": "{}"},- }],- })- msgs.append({- "role": "tool",- "tool_call_id": f"c{i}",- "content": f"output {i + 1}",- })- return msgs---# ----------------------------------------------------------------------------# Cadence formula-# ------------------------------------------------------------------------------class TestCadenceFormula:- """Direct tests on _select_tier / the Bresenham ceil schedule."""-- async def test_first_turn_always_strong_when_probability_positive(self) -> None:- """Turn 1 must always route strong when strong_probability > 0.-- This is the user-facing invariant — "first turn always goes- to strong". Verified at multiple probabilities.- """- for p in (0.01, 0.1, 0.3, 0.5, 0.7, 0.9, 1.0):- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=p),- )- req = _build_openai_chat_request(- [{"role": "user", "content": "first turn"}],- )- ctx = ProxyContext()- await processor.process(ctx, req)- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong", (- f"turn 1 must be strong at p={p}; got "- f"{ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER]!r}"- )- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1-- async def test_zero_probability_routes_every_turn_weak(self) -> None:- """strong_probability=0.0 routes every turn — including turn 1 — to weak.-- This is the documented degenerate behavior: p=0 is equivalent- to FixedTierRequestProcessor("weak").- """- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.0),- )- for n_prior in range(5):- ctx = ProxyContext()- req = _build_openai_chat_request(- _conversation_with_n_assistant_turns(n_prior),- )- await processor.process(ctx, req)- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"- assert ctx.metadata[CTX_TURN_BASED_TURN] == n_prior + 1-- async def test_one_probability_routes_every_turn_strong(self) -> None:- """strong_probability=1.0 routes every turn to strong."""- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=1.0),- )- for n_prior in range(5):- ctx = ProxyContext()- req = _build_openai_chat_request(- _conversation_with_n_assistant_turns(n_prior),- )- await processor.process(ctx, req)- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"-- async def test_half_probability_alternates_strong_weak(self) -> None:- """strong_probability=0.5 should produce S W S W S W... cadence.-- Verifies the ceil(t*0.5) > ceil((t-1)*0.5) formula gives the- expected alternation: odd turns strong, even turns weak.- """- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- expected = ["strong", "weak"] * 10 # 20 turns- actual: list[str] = []- for n_prior in range(20):- ctx = ProxyContext()- req = _build_openai_chat_request(- _conversation_with_n_assistant_turns(n_prior),- )- await processor.process(ctx, req)- actual.append(ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER])-- assert actual == expected, (- f"expected strict alternation at p=0.5; got: {actual}"- )-- async def test_thirty_percent_probability_hits_roughly_thirty_percent_strong(self) -> None:- """strong_probability=0.3 over 100 turns should produce exactly 30 strong.-- The Bresenham formula gives an *exact* long-run rate (not just- approximate) at any rational p with denominator dividing the- turn count. At p=0.3 over 100 turns, count(strong) == 30.- """- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.3),- )- strong_count = 0- for n_prior in range(100):- ctx = ProxyContext()- req = _build_openai_chat_request(- _conversation_with_n_assistant_turns(n_prior),- )- await processor.process(ctx, req)- if ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong":- strong_count += 1- assert strong_count == 30, (- f"at p=0.3, expected exactly 30 strong over 100 turns; got {strong_count}"- )-- async def test_seventy_percent_probability_hits_roughly_seventy_percent_strong(self) -> None:- """Symmetric: p=0.7 over 100 turns yields exactly 70 strong."""- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.7),- )- strong_count = 0- for n_prior in range(100):- ctx = ProxyContext()- req = _build_openai_chat_request(- _conversation_with_n_assistant_turns(n_prior),- )- await processor.process(ctx, req)- if ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong":- strong_count += 1- assert strong_count == 70, (- f"at p=0.7, expected exactly 70 strong over 100 turns; got {strong_count}"- )-- async def test_low_probability_strong_turns_are_spaced(self) -> None:- """At p=0.2, strong turns should appear at indices 1, 6, 11, 16, ...-- Bresenham formula at p=0.2 yields ``ceil(t*0.2) > ceil((t-1)*0.2)``- when ``t * 0.2`` crosses an integer — that's t=1, 5, 10, 15...- but offset by 1 because turn 1 is strong (ceil(0.2)=1 > ceil(0)=0).-- Let me compute directly to be sure:- - t=1: ceil(0.2)=1, ceil(0)=0, 1>0 → strong- - t=2: ceil(0.4)=1, ceil(0.2)=1, NO → weak- - t=3: ceil(0.6)=1, ceil(0.4)=1, NO → weak- - t=4: ceil(0.8)=1, ceil(0.6)=1, NO → weak- - t=5: ceil(1.0)=1, ceil(0.8)=1, NO → weak- - t=6: ceil(1.2)=2, ceil(1.0)=1, YES → strong- - t=7-10: weak- - t=11: strong- """- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.2),- )- strong_indices: list[int] = []- for n_prior in range(20):- turn = n_prior + 1- ctx = ProxyContext()- req = _build_openai_chat_request(- _conversation_with_n_assistant_turns(n_prior),- )- await processor.process(ctx, req)- if ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong":- strong_indices.append(turn)-- assert strong_indices == [1, 6, 11, 16], (- f"expected strong turns at [1, 6, 11, 16] at p=0.2 over 20 turns; "- f"got: {strong_indices}"- )---# ----------------------------------------------------------------------------# Per-format turn counting-# ------------------------------------------------------------------------------class TestOpenAIChatTurnCounting:- """OpenAI Chat Completions: count role=='assistant', add 1."""-- async def test_no_messages_treated_as_turn_one(self) -> None:- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- req = _build_openai_chat_request([])- ctx = ProxyContext()- await processor.process(ctx, req)- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1-- async def test_user_only_history_is_turn_one(self) -> None:- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- req = _build_openai_chat_request([- {"role": "system", "content": "..."},- {"role": "user", "content": "first turn"},- ])- ctx = ProxyContext()- await processor.process(ctx, req)- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1-- async def test_three_prior_assistants_is_turn_four(self) -> None:- """3 prior assistant turns → this is the 4th LLM invocation."""- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- req = _build_openai_chat_request(- _conversation_with_n_assistant_turns(3),- )- ctx = ProxyContext()- await processor.process(ctx, req)- assert ctx.metadata[CTX_TURN_BASED_TURN] == 4-- async def test_tool_messages_do_not_inflate_count(self) -> None:- """role=='tool' messages must not be counted as turns."""- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- messages = [- {"role": "user", "content": "task"},- {"role": "assistant", "content": "step 1"},- # 5 tool messages — pure agent-side, shouldn't count.- *[{"role": "tool", "tool_call_id": f"c{i}", "content": "..."} for i in range(5)],- ]- req = _build_openai_chat_request(messages)- ctx = ProxyContext()- await processor.process(ctx, req)- # One assistant → turn 2.- assert ctx.metadata[CTX_TURN_BASED_TURN] == 2-- async def test_system_messages_do_not_inflate_count(self) -> None:- """role=='system' messages must not be counted as turns."""- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- messages = [- {"role": "system", "content": "agent system prompt"},- {"role": "system", "content": "another system note"},- {"role": "user", "content": "task"},- ]- req = _build_openai_chat_request(messages)- ctx = ProxyContext()- await processor.process(ctx, req)- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1---class TestAnthropicTurnCounting:- """Anthropic Messages: count role=='assistant' — verify role=='user'- tool_result messages don't accidentally inflate the count."""-- async def test_initial_anthropic_request_is_turn_one(self) -> None:- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- req = _build_anthropic_request([- {"role": "user", "content": "initial task"},- ])- ctx = ProxyContext()- await processor.process(ctx, req)- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1-- async def test_anthropic_user_tool_result_messages_not_counted(self) -> None:- """In Anthropic Messages, tool_result is inside role=='user' messages.-- The counter is on role=='assistant', so these don't inflate.- Verifies the format-asymmetry trap doesn't bite — we picked- ``role == "assistant"`` as the discriminator precisely because- it's invariant across the two message formats.- """- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- messages = [- {"role": "user", "content": "task"},- {- "role": "assistant",- "content": [- {"type": "text", "text": "looking..."},- {"type": "tool_use", "id": "tu1", "name": "bash", "input": {}},- ],- },- # The Anthropic-shape tool_result: comes back as role=='user'.- # Counting role=='user' would now wrongly say 2 turns.- # We count role=='assistant', so this should still be turn 2.- {- "role": "user",- "content": [- {"type": "tool_result", "tool_use_id": "tu1", "content": "stdout"},- ],- },- ]- req = _build_anthropic_request(messages)- ctx = ProxyContext()- await processor.process(ctx, req)- # One assistant → turn 2.- assert ctx.metadata[CTX_TURN_BASED_TURN] == 2-- async def test_multi_round_anthropic_conversation_counts_correctly(self) -> None:- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- messages = [- {"role": "user", "content": "task"},- {"role": "assistant", "content": [{"type": "text", "text": "r1"}]},- {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "1", "content": "out1"}]},- {"role": "assistant", "content": [{"type": "text", "text": "r2"}]},- {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "2", "content": "out2"}]},- ]- req = _build_anthropic_request(messages)- ctx = ProxyContext()- await processor.process(ctx, req)- # Two assistants → turn 3.- assert ctx.metadata[CTX_TURN_BASED_TURN] == 3---class TestOpenAIResponsesTurnCounting:- """OpenAI Responses: string input → turn 1; list input → coarse count."""-- async def test_string_input_is_turn_one(self) -> None:- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- req = _build_responses_request("initial task")- ctx = ProxyContext()- await processor.process(ctx, req)- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1-- async def test_list_input_with_only_user_message_is_turn_one(self) -> None:- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- req = _build_responses_request([- {"role": "user", "content": "initial task"},- ])- ctx = ProxyContext()- await processor.process(ctx, req)- # One agent-side ack (the user prompt) → coarse turn 1.- assert ctx.metadata[CTX_TURN_BASED_TURN] == 1-- async def test_list_input_with_function_call_outputs_advances_turn(self) -> None:- """function_call_output items mark prior LLM-round acks."""- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- req = _build_responses_request([- {"role": "user", "content": "task"},- {"type": "function_call", "id": "fc1", "name": "bash", "arguments": "{}"},- {"type": "function_call_output", "call_id": "fc1", "output": "stdout"},- {"type": "function_call", "id": "fc2", "name": "bash", "arguments": "{}"},- {"type": "function_call_output", "call_id": "fc2", "output": "stdout"},- ])- ctx = ProxyContext()- await processor.process(ctx, req)- # 1 user ack + 2 function_call_output acks = 3.- # Coarse approximation: turn 3 (we're 2 rounds in).- assert ctx.metadata[CTX_TURN_BASED_TURN] == 3---# ----------------------------------------------------------------------------# Stamping + audit line-# ------------------------------------------------------------------------------class TestStampingAndAudit:- """ctx-metadata stamping + stderr audit-line shape."""-- async def test_stamps_both_tier_and_turn_metadata_keys(self) -> None:- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- req = _build_openai_chat_request(- _conversation_with_n_assistant_turns(2),- )- ctx = ProxyContext()- await processor.process(ctx, req)- # Two prior assistants → turn 3 → odd → strong at p=0.5.- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"- assert ctx.metadata[CTX_TURN_BASED_TURN] == 3-- async def test_custom_tier_labels_are_stamped(self) -> None:- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(- strong_tier="premium",- weak_tier="economy",- strong_probability=0.5,- ),- )- req = _build_openai_chat_request(- [{"role": "user", "content": "task"}],- )- ctx = ProxyContext()- await processor.process(ctx, req)- # Turn 1 always strong.- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "premium"-- async def test_audit_line_lands_on_stderr_with_expected_fields(self, capsys: Any) -> None:- """One ``turn_based_decision=...`` line per invocation on stderr."""- processor = TurnBasedRouterRequestProcessor(- TurnBasedRoutingConfig(strong_probability=0.5),- )- req = _build_openai_chat_request(- [{"role": "user", "content": "task"}],- )- ctx = ProxyContext()- await processor.process(ctx, req)-- captured = capsys.readouterr()- # The audit line is on stderr (mirrors classifier + planner).- lines = [- ln for ln in captured.err.splitlines()- if ln.startswith("turn_based_decision=")- ]- assert len(lines) == 1, (- f"expected exactly 1 turn_based_decision line on stderr; "- f"got {len(lines)}. Stderr was: {captured.err!r}"- )- payload = json.loads(lines[0][len("turn_based_decision="):])- assert payload["turn"] == 1- assert payload["tier"] == "strong"- assert math.isclose(payload["strong_probability"], 0.5)- assert payload["request_type"] == "openai_chat"