Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
refactor(routing)!: remove latency-aware router (#162)
eric-liu-nvidiac6a850f2f19e
diff --git a/AGENTS.md b/AGENTS.mdindex 0a3117ad4..e214dc9b9 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -182,12 +182,10 @@ switchyard/│ │ ├── openai_llm_backend.py # OpenAiPassthroughBackend│ │ ├── openai_native_backend.py # OpenAiNativeBackend│ │ ├── anthropic_native_llm_backend.py # AnthropicNativeBackend-│ │ ├── latency_service_llm_backend.py # LatencyServiceLLMBackend│ │ ├── llm_target.py # LlmTarget, BackendFormat│ │ ├── multi_llm_backend.py # MultiLlmBackend helpers│ │ ├── stats_llm_backend.py # StatsLlmBackend-│ │ ├── backend_format_resolver.py # BackendFormatResolver-│ │ └── health_poller.py # HealthPoller, EndpointHealthStatus+│ │ └── backend_format_resolver.py # BackendFormatResolver│ ├── processors/ # Request-side / response-side component implementations│ │ ├── format_translate.py│ │ ├── random_routing_request_processor.py@@ -205,8 +203,7 @@ switchyard/│ │ ├── sse_helpers.py│ │ └── base.py│ └── config/-│ ├── intake_sink_config.py-│ └── latency_service_backend_config.py+│ └── intake_sink_config.py├── cli/ # CLI (requires `nemo-switchyard[cli]`)│ ├── switchyard_cli.py # `switchyard` entry point│ ├── launch_command.py # `switchyard launch claude/codex`diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a711afd12..bd9d04b45 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,6 +4,20 @@ All notable changes to Switchyard are documented here. The format is based on[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this projectadheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## [Unreleased]++### Removed++- **Latency-aware router** — the `latency_service` route type and its+ `LatencyServiceLLMBackend`, `LatencyServiceBackendConfig`,+ `LatencyServiceEndpoint`, and `LatencyServiceProfileConfig` public API are+ removed. It depended on NVIDIA Inference Hub's latency endpoint and schema.+ Deployments that need multi-endpoint, load- or latency-aware routing should+ migrate to [Dynamo](https://github.com/ai-dynamo/dynamo) (backend-load /+ KV-cache-aware routing with request failover) or an external load balancer+ such as [Traefik](https://doc.traefik.io/traefik/reference/routing-configuration/http/load-balancing/service/)+ or HAProxy.+## [0.1.0] — Initial releaseFirst public release of Switchyard — a typed, composable control plane for LLMdiff --git a/crates/switchyard-components/tests/stats_accumulator.rs b/crates/switchyard-components/tests/stats_accumulator.rsindex d6cd37401..1f2e5134d 100644--- a/crates/switchyard-components/tests/stats_accumulator.rs+++ b/crates/switchyard-components/tests/stats_accumulator.rs@@ -751,7 +751,7 @@ fn routing_decision_counts_are_grouped_by_profile_type() -> Result<()> {accumulator.record_routing_decision("stage_router", "dimensions")?;accumulator.record_routing_decision("stage_router", "dimensions")?;accumulator.record_routing_decision("stage_router", "llm-classifier")?;- accumulator.record_routing_decision("latency-service", "health")?;+ accumulator.record_routing_decision("escalation_router", "judge")?;let snapshot = accumulator.snapshot()?;assert_eq!(@@ -771,8 +771,8 @@ fn routing_decision_counts_are_grouped_by_profile_type() -> Result<()> {assert_eq!(snapshot.routing_decisions- .get("latency-service")- .and_then(|sources| sources.get("health")),+ .get("escalation_router")+ .and_then(|sources| sources.get("judge")),Some(&1));diff --git a/crates/switchyard-py/src/core_bindings/context.rs b/crates/switchyard-py/src/core_bindings/context.rsindex 57c8fd4b5..1b361f819 100644--- a/crates/switchyard-py/src/core_bindings/context.rs+++ b/crates/switchyard-py/src/core_bindings/context.rs@@ -614,11 +614,10 @@ impl PyProxyContext {/// Records the measured backend-call latency in ms.////// The Rust ``StatsLlmBackend`` wraps native backends and writes this- /// slot automatically. Python-only backends (e.g. ``LatencyServiceLLMBackend``)- /// that can't be wrapped record their measurement here so the- /// downstream ``StatsResponseProcessor`` can compute- /// ``routing_overhead_ms = total_latency - backend_latency`` and emit- /// it on ``/metrics``. Setting ``None`` clears the slot.+ /// slot automatically. Python-only backends that can't be wrapped record+ /// their measurement here so the downstream ``StatsResponseProcessor`` can+ /// compute ``routing_overhead_ms = total_latency - backend_latency`` and+ /// emit it on ``/metrics``. Setting ``None`` clears the slot.#[setter]fn set_backend_call_latency_ms(&self, value: Option<f64>) -> PyResult<()> {let mut inner = self.lock()?;diff --git a/crates/switchyard-py/src/core_bindings/response.rs b/crates/switchyard-py/src/core_bindings/response.rsindex 85fcb4442..e9a02eded 100644--- a/crates/switchyard-py/src/core_bindings/response.rs+++ b/crates/switchyard-py/src/core_bindings/response.rs@@ -492,9 +492,9 @@ impl PyResponseStream {/// ``Switchyard.call`` round trip (``source`` is ``None``): the source was/// moved onto the core stream by ``take_core_stream`` as a/// ``SourceClosingStream``; dropping the inner adapter here drops that- /// wrapper, which closes the source on drop. This is the latency-router- /// production path, where response processors re-wrap the core stream and- /// ``source`` cannot be recovered on this object.+ /// wrapper, which closes the source on drop. This is the production path+ /// where response processors re-wrap the core stream and ``source``+ /// cannot be recovered on this object.fn aclose<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {self.consumed.store(true, Ordering::Release);let stream = Arc::clone(&self.stream);diff --git a/docs/internal/latency_service_routing.md b/docs/internal/latency_service_routing.mddeleted file mode 100644index d57b731b3..000000000--- a/docs/internal/latency_service_routing.md+++ /dev/null@@ -1,609 +0,0 @@-# Latency Service Routing--The **latency-aware router** (`type: latency_service`) routes each request to one of-several equivalent endpoints, biasing traffic toward whichever endpoint a central-**Latency Service** most recently observed as healthy and fast. It is the production-routing policy for OpenRouter-backed deployments, where a separate Latency Service-owns heartbeat probing and statistical latency profiling, and Switchyard consumes its-verdicts to make per-request steering decisions.--This document explains how the router works, the design choices behind it, and how to-configure and operate it.--## When to use it--Use `latency_service` when you have **multiple interchangeable endpoints** serving the-same logical model (e.g. the same model behind different providers or regions) and you-want Switchyard to:--- bias traffic toward the lowest-latency healthy endpoint,-- automatically steer away from degraded endpoints, and-- retry on a different endpoint when one fails, without the client ever seeing it.--If you only have a single backend, use `passthrough`. If you want a fixed weighted-strong/weak split for benchmarking, use-[Random Routing](../routing_algorithms/random_routing.md).--## Architecture--The router is implemented as a single custom `LLMBackend`,-`LatencyServiceLLMBackend` (`switchyard/lib/backends/latency_service_llm_backend.py`),-rather than a monolithic routing strategy. It slots into the standard four-role chain:--```-[RequestProcessor*] → LatencyServiceLLMBackend → [ResponseProcessor*] → TranslationEngine-```--The backend owns four things:--1. **A pool of `OpenAILLMClient` instances**, one per configured endpoint, keyed by the- endpoint's `model` ID.-2. **A thread-safe in-memory health cache** (`dict[str, EndpointHealth]` guarded by a- `threading.Lock`).-3. **A background `HealthPoller` daemon thread**- (`switchyard/lib/backends/health_poller.py`) that refreshes the cache from the- Latency Service.-4. **Endpoint-selection + retry logic** on the request hot path.--Inbound format translation (Anthropic Messages / OpenAI Responses → OpenAI Chat) is-delegated to the Rust `TranslationEngine`, so the backend transparently accepts any-inbound wire format and only ever speaks OpenAI Chat Completions upstream.--### Two-clock design: poll loop vs. hot path--The defining property of this router is that **health polling and request serving run on-separate clocks**:--- The **`HealthPoller` daemon thread** is the only thing that ever talks to the Latency- Service. It polls on a fixed interval and *writes* verdicts into the cache.-- The **request hot path** only ever *reads* the cache (under the lock). It never makes a- network call to the Latency Service.--Health awareness adds **zero per-request latency**. Selection is a dictionary-read and a weighted random pick. The poller is a plain daemon thread (not an asyncio-task), so its lifecycle is trivial. It starts when the backend is constructed and dies-with the process. Its synchronous `httpx.Client` never contends with the event loop-serving requests.--## The health poller--`HealthPoller` polls `{latency_service_url}/v1/endpoints/health` every `poll_interval_s`-seconds (default 10s), passing the configured endpoint IDs as `endpoint_ids` query-params. The Latency Service responds with a verdict and a latency sample per endpoint:--```json-{- "endpoint_health": {- "openai/gpt-5.5": {"status": "healthy", "last_latency_ms": 420.0},- "azure_openai/gpt-5.5": {"status": "degraded", "last_latency_ms": 980.0}- }-}-```--Each cache entry is an `EndpointHealth` snapshot carrying the discrete `status` and the-most recent `last_latency_ms`. The status enum is the **shared contract** between-Switchyard and the Latency Service: both sides must agree on these three string values:--| Status | Meaning |-|---|---|-| `healthy` | Endpoint is up and within latency expectations. |-| `degraded` | Endpoint is reachable but slow or partially failing. |-| `unknown` | No current verdict (warming up, or fallback after a poll failure). |--### Failure handling: fall back to UNKNOWN, not stale data--If a poll fails for any reason (DNS, network, timeout, non-2xx, or a schema mismatch on-the response), the poller **resets every endpoint in the cache to `UNKNOWN`** rather than-leaving stale verdicts in place. With every endpoint at `UNKNOWN`, the router degrades to-uniform random routing across the full pool. This is the safe default when health-information is unavailable. Acting on stale "healthy" data after the Latency Service has-gone dark would be strictly worse.--`last_latency_ms` is `None` before the first successful poll and whenever the Latency-Service reports it as null.--## Endpoint selection--On each request, `_select_endpoint()` picks a `model` ID from the cache in two steps:--### 1. Tier preference: `HEALTHY > UNKNOWN > DEGRADED`--Endpoints are bucketed by status, and the router picks from the **best non-empty tier**.-`UNKNOWN` is deliberately preferred over `DEGRADED`: an endpoint we have no verdict for is-a better bet than one the Latency Service explicitly flagged as degraded.--### 2. Within a tier: inverse-latency weighted random--Within the chosen tier, `_pick_by_latency()` selects an endpoint with probability-proportional to `1 / last_latency_ms`, so an endpoint the Latency Service recently saw-at 200 ms receives roughly twice the traffic of one at 400 ms. This is **weighted random,-not always-pick-the-fastest**, which spreads load and avoids stampeding a single endpoint.--The picker falls back to **uniform random** within the tier when:--- there is only one candidate, or-- any candidate's `last_latency_ms` is unknown (`None`), or-- any sample is non-positive (defensive: the Latency Service should never report ≤ 0).--This keeps behavior predictable while the poller is warming up or when the upstream-reports nulls.--### 3. Optional: session affinity (sticky routing)--By default the router picks per **turn**. Every request runs the tier + latency steps-above independently. For a multi-turn conversation this interleaves endpoints, which lets-each endpoint's upstream prompt/KV cache lapse and forces expensive cache *re-writes* on-the next turn that lands there.--Set `session_affinity: true` to route per **conversation** instead. The latency-aware-picker decides only the **first** turn, then pins that conversation to the endpoint-that served it. Every later turn reuses that endpoint (the affinity fast-path bypasses-tier + latency selection entirely), so the upstream cache stays warm for the life of-the conversation.--- **Conversation identity.** A streaming proxy holds no session ID. The client re-sends- the full history each turn. `session_key_from_body()`- (`switchyard/lib/session_key.py`) derives a stable key by hashing the stable harness- prefix made from the system prompt + the first user message. Later turns only append, so- every turn of one conversation hashes the same; distinct conversations differ.-- **Health overrides the pin.** A pin is reused only while its endpoint is `HEALTHY` or- `UNKNOWN`. If it goes `DEGRADED` (or the call fails this request), the next turn- re-routes through normal selection and re-pins to whatever serves it. Locality yields- to health, so a session is never funneled into a failing endpoint.-- **Bounded memory.** Pins live in an in-process LRU map (the L1) capped at- `affinity_max_sessions` (default `10_000`); the least-recently-used pin is evicted past- the cap.-- **Cross-worker stickiness (optional shared store).** The L1 map is per process, so a- multi-worker/multi-pod deployment may pin the same conversation differently per worker,- and pins are lost on pod churn. Set `affinity_store: "redis"` (with `affinity_store_url`)- to add a shared, persistent L2: on an L1 miss the pin is read through from the store and- warmed back into L1, and every pin is written through to it. Pins then survive across- replicas and pod restarts (bounded by `affinity_store_ttl_seconds`, refreshed on each- write). The L2 is **best-effort** — an error or timeout falls back to L1 / unpinned and- never fails a request; fail-open operations are counted on- `switchyard_affinity_l2_errors_total`, and a circuit breaker (3 consecutive failures →- skip the store for 10 s, then probe; `switchyard_affinity_l2_breaker_open` gauge) keeps- a store outage from taxing every request with timeout waits. The store connection is released by the backend's- `shutdown()` (the server lifespan teardown hook awaits it). Requires the- `switchyard[affinity-redis]` extra. (Alternatively, front the workers with a- session-affinity load balancer.)--The `switchyard.route_decision` span tags each pick with `switchyard.affinity_hit` so you-can measure the warm-reuse rate. Affinity is **off by default**; existing per-turn latency-routing is unchanged unless opted in.--#### Deploying the shared store: the operational contract--What an operator needs to know to provision and run the Redis L2. Everything below is-verified against the implementation (`RedisPinStore`, `SessionAffinity`).--**Topology.** A **standalone Redis endpoint only** (the client is-`redis.asyncio.from_url`): a single instance, a primary endpoint of a replicated setup,-or a managed single-endpoint tier. **Redis Cluster and Sentinel URLs are not-supported** — cluster `MOVED` redirects surface as fail-open errors, meaning requests-keep succeeding while stickiness silently degrades. A sustained-`switchyard_affinity_l2_errors_total` rate from the first request is the-mis-provisioning signal; watch it when first enabling the store.--**Auth and TLS.** URL-driven: `rediss://` for TLS, credentials inside the URL. Route-YAML expands `${ENV}` references in every string value, so keep the URL in a secret and-configure `affinity_store_url: "${AFFINITY_REDIS_URL}"` rather than an inline literal.--**What is stored.** Key = `affinity_key_prefix` + a 16-hex-char hash of the-conversation's stable prefix (system prompt + first user message); value = the pinned-endpoint id string. **No prompt text, message content, or user identifier leaves the-process** — a pin is ~50–100 bytes.--**Traffic and sizing.** One `GET` per request whose conversation isn't in the local L1-(first turn on a pod), and one `SET` (which refreshes the TTL) per successful request.-Working-set memory ≈ active conversations × ~150 B, bounded by the sliding-`affinity_store_ttl_seconds` (default 1 h). **Persistence (RDB/AOF) and HA are optional**:-losing the store only resets stickiness — conversations re-route cold and re-pin — it-never affects correctness. `maxmemory-policy allkeys-lru` is the recommended eviction-policy; an evicted pin costs one cold re-route.--**Failure envelope.** Every store operation is bounded by a 0.1 s socket/connect-timeout (a colocated Redis answers in well under 1 ms, so this is ~100× headroom) and-fails open — requests never fail because of the store. A **circuit breaker** caps the-outage tax: after 3 consecutive failures, L2 operations are skipped without a network-attempt for a 10 s cooldown, then one operation probes; success closes the breaker,-failure re-arms it. Worst case per pod during an outage is therefore a brief window of-up to ~0.2 s added per request (one `GET` on the L1 miss + one `SET` after success)-before the breaker opens, then ~zero added latency with one 0.1 s probe per 10 s —-stickiness degrades to per-pod L1 throughout. Alert on-`switchyard_affinity_l2_breaker_open == 1` (the outage signal) and on-`rate(switchyard_affinity_l2_errors_total[5m])` (degradation without a full trip —-note the rate drops to ~0.1/s per pod while the breaker is open, since only probes-fail). Pod readiness (`/health`, `is_ready()`) never depends on Redis.--**Isolation.** Deployments or routes sharing one Redis should set distinct-`affinity_key_prefix` values (or separate logical databases). A pin is honored whenever-its stored endpoint id exists in the reading route's endpoint set, so shared prefixes-cross-pollinate stickiness between routes that reuse endpoint ids.--**Config changes, upgrades, rollback.** A persisted pin whose endpoint id is no longer-in the route's endpoint list is ignored gracefully (the next turn re-routes and-re-pins) — shrinking or renaming the endpoint set needs no store cleanup. The-conversation-key derivation is not a cross-version contract: a Switchyard upgrade may-invalidate existing pins (one-time cold re-route, self-healing). To roll back, set-`affinity_store: "memory"`; stale keys expire via TTL, and `FLUSHDB` is always safe-(stickiness reset only).--The deterministic LLM-classifier route also supports `affinity_warmup_turns` to delay-when a tier becomes sticky. Latency-service routing does **not** use that knob. Its-affinity objective is endpoint cache reuse, so it pins after the first successful-endpoint selection.--## Retry and failover--A single client request may try up to `1 + max_retries` endpoints (default `max_retries`-is 2, so 3 attempts). On each attempt:--1. Select an endpoint. If selection returns one already tried this request (and untried- endpoints remain), pick uniformly at random from the **untried** set. Dedup prevents- wasting an attempt re-hitting an endpoint that just failed.-2. Stamp the endpoint's `upstream_model` into `body["model"]` and call the upstream.-3. On a **transient** failure such as `429` (rate limit), `408` (request timeout), any- `5xx`, or a network / pre-status / SDK error, log it, record the error in stats, increment- the outcome metric, and continue to the next endpoint. These are exactly the faults a- health-aware router should absorb by trying a different endpoint.-4. On a **4xx client error** (`400`, `401`, `403`, `404`, `409`, `413`, `415`, `422`, …)- the loop **fails fast**: it does *not* retry. The request itself is malformed or- unauthorized, so every replica rejects the same payload identically; retrying only adds- latency before surfacing the same status. (Context-window overflow surfaces as a `400`- here and is also passed straight through. The `latency_service` route does not- participate in the chain-level evict-and-retry described in- [Context-Window Handling](../operations/context_window.md).)--If an attempt succeeds **after** at least one failure, the router records a-`retry_recovered` outcome. This is direct evidence the steering logic rescued a request that-would otherwise have failed.--Whenever the request ends on an upstream HTTP error, either a fail-fast 4xx or an-exhausted retryable error (e.g. a `503` after retries are exhausted), the backend records the-upstream status code and body on `ctx` so the endpoint passes the real upstream status-through to the client rather than masking it as a generic `500`.--## Configuration--`LatencyServiceBackendConfig` owns the router's policy, retry, and optional-session-affinity settings. In CLI YAML, configure the Python-`type: latency_service` router in a `routes:` bundle; there are no-policy-specific CLI flags. Run it with:--```bash-switchyard --routing-profiles my_routes.yaml -- serve --port 4100-```--### Config schema--`LatencyServiceBackendConfig`-(`switchyard/lib/config/latency_service_backend_config.py`):--| Field | Default | Purpose |-|---|---|---|-| `latency_service_url` | `""` | Base URL of the Latency Service. |-| `endpoints` | `[]` | The endpoints to route across (see below). At least one is required. |-| `poll_interval_s` | `10.0` | How often the background poller refreshes health. |-| `poll_timeout_s` | `5.0` | Timeout for each health API call. |-| `max_retries` | `2` | Extra endpoints to try on failure (total attempts = `1 + max_retries`). |-| `credential_policy` | `"configured_endpoint"` | Credential precedence for upstream calls. The default uses each endpoint's configured `api_key`; set `"caller_override"` only for BYO-key deployments where caller headers should replace endpoint keys. |-| `session_affinity` | `false` | Pin each conversation to one endpoint after the first turn (sticky routing: keeps the upstream cache warm). See [Endpoint selection §3](#3-optional-session-affinity-sticky-routing). |-| `affinity_max_sessions` | `10_000` | LRU cap on the in-process pin map (L1) when `session_affinity` is on. |-| `affinity_store` | `"memory"` | Shared L2 pin store behind the L1. `"memory"` = per-process only; `"redis"` shares pins across workers/pods and persists them across pod churn (requires `session_affinity: true`, `affinity_store_url`, and the `affinity-redis` extra). |-| `affinity_store_url` | `None` | Connection URL for the shared store (e.g. `redis://host:6379/0`). Required when `affinity_store` is `"redis"`. |-| `affinity_store_ttl_seconds` | `3_600` | Expiry for a shared pin; refreshed on each write, so an active conversation slides its TTL. |-| `affinity_key_prefix` | `"swyd:pin:"` | Namespace prefix for shared-store keys. |-| `enable_stats` | `true` | Wire the stats processors + `/metrics` exposition. |--Each `LatencyServiceEndpoint`:--| Field | Required | Purpose |-|---|---|---|-| `model` | yes | **Latency Service lookup key**: must be unique across the endpoint list. Also the value stamped into `body["model"]` unless `upstream_model` is set. |-| `upstream_model` | no | Override for `body["model"]` sent upstream. Defaults to `model`. |-| `api_key` | no | API key for the backing LLM API. This is the default upstream credential. |-| `base_url` | no | Base URL for the backing LLM API (include `/v1`). |-| `timeout` | no | Per-request timeout in seconds. |--### Why two `model` fields?--The Latency Service registers each endpoint under an ID *it* picked (e.g.-`openrouter-primary/openai/gpt-4o`). The upstream gateway may expect the OpenRouter-model name for the same destination (e.g. `openai/gpt-4o`). So:--- **`model`** is the Latency Service lookup key (and the metrics label).-- **`upstream_model`** is what gets stamped into `body["model"]` on the outbound call.--When they're equal, `upstream_model` can be omitted.--### Example--Create an OpenRouter key at <https://openrouter.ai/> and export it as-`OPENROUTER_API_KEY` before running this example.--```yaml-routes:- gpt-4o:- type: latency_service- latency_service_url: https://latency-service.example.com- poll_interval_s: 10.0- poll_timeout_s: 5.0- max_retries: 2- endpoints:- - model: openrouter-primary/openai/gpt-4o # Latency Service endpoint_id- upstream_model: openai/gpt-4o # OpenRouter model name to call- base_url: https://openrouter.ai/api/v1- api_key: ${OPENROUTER_API_KEY}- - model: openrouter-backup/openai/gpt-4o- upstream_model: openai/gpt-4o- base_url: https://openrouter.ai/api/v1- api_key: ${OPENROUTER_API_KEY}-```--A runnable version of this config, with extensive inline notes, lives at-[`experiments/latency_routing.yaml`](../../experiments/latency_routing.yaml).-A production-style Kubernetes manifest lives at-[`examples/k8s/latency-aware-deployment.yaml`](../../examples/k8s/latency-aware-deployment.yaml).--### BYO-key (multi-tenant) mode--By default, caller `Authorization: Bearer <key>` and `x-api-key` headers do not replace-the configured endpoint credentials. This keeps server-owned latency-service deployments-from accidentally forwarding a client placeholder or unrelated proxy credential upstream.--For a multi-tenant BYO-key deployment, set `credential_policy: caller_override` on the-latency-service route. In that mode, a non-empty caller key is forwarded as the upstream-credential for that request, and the endpoint `api_key` is used only when the caller omits-the header. Credential precedence is **`Authorization` > `x-api-key` > endpoint-`api_key`**.--## Response fidelity (Responses API)--For an endpoint configured `request_type: openai_responses`, the backend calls the-upstream `/v1/responses` surface and returns the upstream payload **verbatim** in both-modes:--- **Non-streaming** calls return the upstream's exact JSON body — the raw HTTP response- is used instead of round-tripping through the OpenAI SDK's typed model.-- **Streaming** calls forward the upstream's SSE frames as **verbatim strings**- (`RawSSEFrameStream` in `switchyard/lib/llm_client.py`): no typed-event parse happens,- so per-event provider extras, explicit nulls, event names, and comment keep-alives all- survive. Frames are byte-equivalent modulo CRLF → LF line normalization. Usage- accounting still works — the stats layers parse the frame's `data:` payload to find- `response.completed` usage without altering what flows to the client.--When the inbound request is also Responses-format, the terminal translation-short-circuits (same format in and out), so the client receives the upstream payload-as-is. **Cross-format** calls (e.g. a chat client served by a Responses endpoint, or-vice versa) are *re-synthesized* by the translation engine and are inherently lossy —-fields the upstream never produced in the target format cannot be invented. For-full-fidelity Responses routing, configure every endpoint on the route with-`request_type: openai_responses`.--### Field-fidelity matrix (Responses API)--How each field class fares per path. "Exact" means byte-equivalent to the upstream-(streaming: modulo CRLF normalization); "synthesized" means the translation engine-rebuilds the payload from its neutral representation and only mapped fields survive.--| Field class (examples) | Same-format, non-streaming | Same-format, streaming | Cross-format (either direction) |-|---|---|---|---|-| Standard scalars (`id`, `model`, `status`, `created_at`) | exact | exact | synthesized (`id`/`model` mapped; `created_at` → chat `created` defaults to `0`) |-| Sampling/config echoes (`temperature`, `top_p`, `store`, `text`) | exact | exact | dropped (no chat-format equivalent) |-| `reasoning` config + explicit-`null` fields | exact (nulls preserved) | exact (nulls preserved) | dropped |-| `usage` (incl. `*_tokens_details`) | exact | exact | token counts + reasoning detail mapped; cached-token detail drops |-| Output content (`output[]`, text deltas) | exact | exact | mapped (text/tool calls translate; unmappable block types drop) |-| Provider extras (unknown keys, e.g. Azure `content_filter_results`) | exact | exact | dropped |-| SSE event names / keep-alive comments | n/a | exact | re-framed to the target format's event contract |-| `metadata`, `previous_response_id` | exact | exact | dropped |--Wire-level proofs live in-`tests/test_upstream_error_passthrough.py`-(`test_responses_body_returned_exactly_on_the_wire`,-`test_responses_stream_forwarded_verbatim_on_the_wire`,-`test_azure_flavored_responses_stream_preserved_on_the_wire`) against-OpenAI-shaped and Azure-shaped samples; validation against live production-captures remains a deployment-side step.--## Failure-source annotation--When a request fails, the error response carries two headers so clients and-observability can tell **which layer** the failure came from without parsing bodies:--| Header | Values | Meaning |-|---|---|---|-| `x-switchyard-error-source` | `switchyard` \| `provider` | `provider`: an upstream LLM failure passed through (HTTP error, or the 500 rendered for a status-less network fault). `switchyard`: this proxy rejected or failed the request itself — credential-policy 401 (`missing_caller_api_key`), translation 400 (`invalid_value`), model-not-found 404, context-window 400, body validation 400, unexpected internal 500. |-| `x-switchyard-upstream-model` | model id | The model actually attempted upstream when the failure happened. Present only when a routing selection took place. |--The provider error **body** still passes through verbatim — the annotation is-header-only, deliberately, so the transparent-proxy contract on bodies is preserved.-The same vocabulary appears as `switchyard.error_source` / `switchyard.upstream_model`-tags on failed `switchyard.upstream_attempt` spans and as `error_source` /-`upstream_model` fields on the per-event error log (below).--Layering note: Switchyard can only distinguish *itself* from *its upstreams*. A proxy in-front of Switchyard (e.g. LiteLLM) should tag its own failures the same way and-propagate these headers from below. Mid-stream failures after HTTP 200 has been-committed cannot carry headers and are not annotated today.--## Route-selection spend attribution--For tokenomics reporting, a LiteLLM front proxy needs to tie its provider spend-log-rows back to the Switchyard logical route that selected them. The backend stamps-**every upstream attempt** with a spend-logs metadata request header (LiteLLM copies-its JSON value into the provider spend-log DB row):--```text-x-litellm-spend-logs-metadata: {"router_model": "<client-facing route id>",- "router_strategy": "latency",- "router_selected_endpoint": "<endpoint id>",- "router_selected_model": "<upstream model>",- "router_selected_provider": "<upstream model's leading path segment>",- "router_correlation_id": "<uuid4>"}-```--One correlation id is generated per client request and shared by every failover-attempt; the selection fields are re-stamped per attempt, so each provider row-records the endpoint it actually hit. `router_model` is the model the client asked-for, falling back to the configured `route_model` when the body carried no model.--**Successful** responses then return the selection that served the request, so the-front proxy can enrich its own (parent) spend-log row with the same correlation id-the provider row received:--| Header | Value |-|---|---|-| `x-switchyard-router-model` | `router_model` from the selection |-| `x-switchyard-selected-model` | `router_selected_model` |-| `x-switchyard-selected-provider` | `router_selected_provider` |-| `x-switchyard-router-correlation-id` | `router_correlation_id` |--Streaming responses carry the same headers (the upstream call completes before the-SSE response is committed). Error responses carry the failure-source headers above;-the selection headers join them only when an upstream call already **succeeded**-before the failure (e.g. a response-translation error after a billed 200), so the-provider row's correlation id stays joinable — a request that never reached a-successful upstream claims no selection. Because the client controls `router_model`,-header values are re-validated as header material: a value that is not-latin-1-encodable or contains control characters is omitted rather than breaking-(or splitting) the response; the outbound JSON header is immune (`json.dumps`-escapes it) and still records the raw value. The cross-component contract is-`CTX_ROUTE_SELECTION` in `switchyard/lib/proxy_context.py` (written by the backend,-read by the endpoint layer via `switchyard/lib/endpoints/route_selection.py`);-wire-level proofs live in `tests/test_route_selection_headers.py`.--## Observability--When `enable_stats` is `true` (the default), `LatencyServiceProfileConfig` wires a-`StatsRequestProcessor` + `StatsResponseProcessor` pair sharing one `StatsAccumulator`.-Because `LatencyServiceLLMBackend` is a Python-only backend, profile assembly cannot wrap-it with the Rust-native `StatsLlmBackend`; instead it records success / error /-call-latency in-place into the same accumulator the response processor records token usage-into. It also sets `ctx.selected_model` and `ctx.backend_call_latency_ms` so per-endpoint-attribution and routing-overhead figures land correctly on `/metrics`.--The backend publishes its in-memory health cache and poll-loop health to `/metrics`:--| Metric | Type | Meaning |-|---|---|---|-| `switchyard_endpoint_status{model,status}` | gauge | Current verdict per endpoint; exactly one status row per model is `1`. |-| `switchyard_endpoint_last_latency_ms{model}` | gauge | Last latency sample per endpoint. Absent when the upstream reported null. |-| `switchyard_latency_service_poll_ok` | gauge | `1` iff the most recent poll succeeded. |-| `switchyard_latency_service_poll_age_seconds` | gauge | Seconds since the last successful poll. Absent before the first success. |-| `switchyard_latency_service_polls_total` | counter | Total successful polls. |-| `switchyard_latency_service_poll_failures_total` | counter | Total failed polls; each resets the cache to `unknown`. |-| `switchyard_affinity_hits_total` | counter | Turns served by an existing session-affinity pin (warm reuse). **Emitted only when `session_affinity` is on.** |-| `switchyard_affinity_misses_total` | counter | First/unpinnable turns routed by the latency-aware picker while affinity was on. Reuse rate = `hits / (hits + misses)`. **Emitted only when `session_affinity` is on.** |-| `switchyard_affinity_l2_hits_total` | counter | Pins resolved from the shared (L2) store after an in-process miss — cross-worker/churn warm reuse. **Emitted only when a shared affinity store is configured.** |-| `switchyard_affinity_l2_errors_total` | counter | Shared-store get/put operations that failed open; routing fell back to in-process pins. Alert on a sustained rate — the store is degraded while requests keep succeeding. **Emitted only when a shared affinity store is configured.** |--See the [Metrics Reference](metrics_reference.md) for full label semantics, PromQL recipes-(traffic share per endpoint, retry-rescue rate), and a triage cheatsheet.--### Expected routing overhead--`switchyard_routing_overhead_ms{quantile}` is the source of truth for what Switchyard-itself adds to a request: per request it records `total_latency − backend_call_latency`,-which bundles format translation, endpoint selection, body mutation, response wrapping,-and retry wall time — everything except the upstream call.--Reference figures, measured on an idle dev box over loopback (stub upstream + stub-Latency Service, non-streaming `/v1/chat/completions`, sequential; commit `0289a415`):--| Figure | Value |-|---|---|-| `switchyard_routing_overhead_ms{quantile="0.5"}` | ~0.3 ms |-| `switchyard_routing_overhead_ms{quantile="0.99"}` | ~7 ms |-| Mean overhead (`_sum / _count`) | ~2 ms |-| Client-observed added wall time vs calling the upstream directly | ~2 ms mean |--Expected order of magnitude is therefore **single-digit milliseconds**. When an-end-to-end comparison ("direct provider call" vs "through the stack") shows a much-larger delta — e.g. ~100 ms — the difference lives **outside** this metric:--* the extra network hop and TLS handshake from the client to the Switchyard pod,-* any fronting layer (LiteLLM / inference-gateway) in front of Switchyard,-* per-pod queueing under concurrency — a single event loop serializes CPU work, so- client wall time grows with in-flight load while per-request overhead does not- (at loopback concurrency 10 the same box shows ~22 ms client p50 with an unchanged- overhead summary),-* upstream time-to-first-token variance between the two measurement runs.--Attribute before optimizing: read the deployment's own summary first.--```promql-# Median / p99 Switchyard-internal overhead-switchyard_routing_overhead_ms{quantile="0.5"}-switchyard_routing_overhead_ms{quantile="0.99"}--# Mean over a window-rate(switchyard_routing_overhead_ms_sum[5m]) / rate(switchyard_routing_overhead_ms_count[5m])-```--Two caveats. For streaming responses `switchyard_total_latency_ms` (and hence the-overhead summary) measures the **full turn**, not time-to-first-token — TTFT is what-interactive users feel, so don't read a long generation as router overhead. And the-summary is per-process: in a multi-pod deployment each pod reports its own.--The repeatable harness is the CI perf benchmark (`.github/workflows/perf.yml`): aiperf-against a no-op route measures the full server path with zero backend cost, non-streaming-and streaming, at configurable concurrency.--### Per-event error log (Loki)--`/metrics` answers *how many* of each error code, but Prometheus stores aggregates. It-carries no per-event timestamps. For audit, replay, or a "what failed and when" panel,-each failed upstream attempt also emits one structured JSON line on the-`switchyard.upstream_errors` logger-(`switchyard/lib/endpoints/upstream_error_log.py`):--```json-{"event":"upstream_attempt_failed","timestamp":"2026-05-29T12:00:00+00:00","model":"openai/gpt-5.5","upstream_model":"gpt-5.5","attempt":1,"status_code":429,"code":"429","outcome":"retryable_error","error_source":"provider","error_type":"APIStatusError","error":"..."}-```--`code` and `outcome` mirror the labels on `switchyard_upstream_attempts_total`, so the log-joins cleanly to the counter; `status_code` is `null` (and `code="none"`) for a non-HTTP-failure. Because the deployment uses plain-text logging, the message *is* the JSON object.-A Loki query parses it with `| json` and zero formatter config:--```logql-{app="switchyard"} | json | event="upstream_attempt_failed"-```--`is_ready()` returns `True` once the poller has completed at least one successful poll,-which makes it useful as a readiness gate during startup.--## Testing the router--Three tiers, cheapest first. The `switchyard-latency-router-e2e` skill carries the-runnable commands and helper scripts for each.--1. **Offline unit suite** (no keys): `tests/test_latency_service_*.py`,- `tests/test_session_key.py`, `tests/test_outcome_metrics.py`, and- `tests/test_upstream_error_passthrough.py`: full coverage of selection, tiering,- inverse-latency weighting, retry/fail-fast, session affinity, the health poller,- metrics, and spans.-2. **In-repo integration** (`tests/e2e/test_latency_service_llm_backend.py`, marked- `integration`): an in-process mock Latency Service feeds verdicts while the LLM- calls hit OpenRouter. Requires `OPENROUTER_API_KEY` (skips without it).-3. **Real-world**: point a real backend at a mock or live Latency Service and confirm- it routes. A mock LS pointed at `openrouter.ai` proves the- poll → tier → weighting path with no API key — a keyless dispatch returns `401`,- which confirms traffic reached the host.--## Source map--| File | Responsibility |-|---|---|-| `switchyard/lib/backends/latency_service_llm_backend.py` | Selection, session-affinity pins, retry, stats/metrics emission, per-attempt error log. |-| `switchyard/lib/session_key.py` | `session_key_from_body()`: stable per-conversation key for session affinity. |-| `switchyard/lib/session_affinity.py` | `SessionAffinity`: L1 (in-process LRU) + optional best-effort L2 pin store. |-| `switchyard/lib/affinity_pin_store.py` | `AffinityPinStore`: async protocol for a shared/persistent L2 store. |-| `switchyard/lib/redis_pin_store.py` | `RedisPinStore`: Redis-backed shared L2 (lazy `redis`, `affinity-redis` extra). |-| `switchyard/lib/endpoints/upstream_error_log.py` | Structured per-attempt failure log (`switchyard.upstream_errors`, Loki). |-| `switchyard/lib/backends/health_poller.py` | `EndpointHealthStatus`, `EndpointHealth`, `HealthPoller` daemon. |-| `switchyard/lib/config/latency_service_backend_config.py` | `LatencyServiceEndpoint`, `LatencyServiceBackendConfig`. |-| `switchyard/lib/profiles/latency_service.py` | `LatencyServiceProfileConfig`: profile assembly + stats wiring. |-| `tests/test_latency_service_llm_backend.py` | Selection, retry, tier-preference tests. |-| `tests/test_latency_service_health_metrics.py` | Poller and metrics tests. |diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.mdindex cdba2a318..e80974791 100644--- a/docs/internal/metrics_reference.md+++ b/docs/internal/metrics_reference.md@@ -12,7 +12,7 @@ a drop-in scrape config and starter alert rules.| Content-Type | `text/plain; version=0.0.4; charset=utf-8` || Format | Prometheus text format 0.0.4 || Auth | None. Designed as a public scrape endpoint |-| Default scrape interval | 15s (the Latency Service poll cycle is 10s by default, so finer scrape resolution captures no extra state) |+| Default scrape interval | 15s |`GET /metrics` is served by the Python route-bundle server started with`switchyard --routing-profiles PATH -- serve`.@@ -29,12 +29,11 @@ A JSON variant of the same underlying data lives at `GET /v1/stats`, with## Per-endpoint counters-The `model` label is the latency-service endpoint id (`openai/gpt-5.5`,+The `model` label is the configured endpoint id (`openai/gpt-5.5`,`azure_openai/gpt-5.5`, etc.).The `tier` label is optional: present only when a routing factory-supplied one. The `random_routing` factory emits `tier="strong"|"weak"`;-the `latency_service` factory does not.+supplied one. The `random_routing` factory emits `tier="strong"|"weak"`.| Metric | Type | Meaning ||---|---|---|@@ -64,44 +63,6 @@ Each summary emits `{quantile="0.5"}` and `{quantile="0.99"}` rows plusHealthy traffic typically sits at p50 ≈ 0.4 ms, p99 ≈ 0.6 ms.-## Latency Service state (gauges: latency-service chains only)--Published from the in-memory health cache the `LatencyServiceLLMBackend`-maintains, refreshed on each successful poll of the Latency Service.--| Metric | Type | Meaning |-|---|---|---|-| `switchyard_endpoint_status{model,status}` | gauge | Current Latency Service verdict per endpoint. `status` is one of `healthy`, `degraded`, `unknown`. Exactly one row per `model` is `1`; the rest are `0`, so `sum by (status)` gives a clean count of endpoints in each state. |-| `switchyard_endpoint_last_latency_ms{model}` | gauge | Last latency sample the Latency Service reported for this endpoint. **Absent** when the upstream reported `null`, and absence is meaningful. |--## Latency Service poll health (no labels: latency-service chains only)--| Metric | Type | Meaning |-|---|---|---|-| `switchyard_latency_service_poll_ok` | gauge | `1` iff the most recent poll succeeded. `0` means the next routing decisions are based on the cache-reset-to-unknown fallback state. |-| `switchyard_latency_service_poll_age_seconds` | gauge | Monotonic seconds since the last successful poll. **Absent** before the first success; combined with `poll_ok=0`, this distinguishes "never polled" from "polled but recently failed". |-| `switchyard_latency_service_polls_total` | counter | Total successful polls since process start. |-| `switchyard_latency_service_poll_failures_total` | counter | Total failed poll attempts. Each failure resets every endpoint in the cache to `unknown`. |--## Session affinity (no labels: latency-service chains with `session_affinity` on)--These counters are **absent unless `session_affinity: true`**, so the metric surface stays clean for the default per-turn-routing case.--| Metric | Type | Meaning |-|---|---|---|-| `switchyard_affinity_hits_total` | counter | Conversation turns served by an existing session-affinity pin, meaning the upstream prompt/KV cache was reused. Counted once per request (first attempt only), so failover retries don't inflate it. |-| `switchyard_affinity_misses_total` | counter | First turns of a conversation, or turns whose pin was broken by a health verdict, routed by the latency-aware picker. |-| `switchyard_affinity_l2_hits_total` | counter | Pins resolved from the shared (L2) affinity store after an in-process (L1) miss — cross-worker/churn warm reuse. **Also requires a shared store** (`affinity_store: "redis"`). |-| `switchyard_affinity_l2_errors_total` | counter | Shared-store get/put operations that failed open; routing fell back to in-process pins. A sustained rate means the store is degraded while requests keep succeeding — this is the alerting signal. While the breaker is open, only recovery probes fail, so the rate drops to ~1 per cooldown per pod. **Also requires a shared store.** |-| `switchyard_affinity_l2_breaker_open` | gauge | `1` while the shared-store circuit breaker is open (operations skipped without a network attempt after 3 consecutive failures; one probe per 10 s cooldown), `0` when closed. The unambiguous store-outage signal. **Also requires a shared store.** |--Warm-reuse rate (the fraction of turns that hit a warm endpoint):--```promql-rate(switchyard_affinity_hits_total[5m])- / (rate(switchyard_affinity_hits_total[5m]) + rate(switchyard_affinity_misses_total[5m]))-```-## Outcome counters for error-rate ratiosThe `outcome` label takes exactly three values:@@ -115,7 +76,6 @@ The `outcome` label takes exactly three values:| `switchyard_client_responses_total{outcome}` | counter | HTTP responses returned to clients on the LLM-serving routes (`/v1/chat/completions`, `/v1/messages`, `/v1/responses`). The denominator for the **router-served** error rate. || `switchyard_upstream_attempts_total{outcome,code}` | counter | Individual upstream call attempts. One client request can produce N attempts via retry. The denominator for the **direct-to-endpoint** baseline error rate. The `code` label carries the raw upstream HTTP status for plotting the error-code distribution (see below). || `switchyard_router_retry_recovered_total` | counter | Requests whose first upstream attempt failed but a subsequent attempt succeeded. This is direct evidence the routing logic rescued the request. |-| `switchyard_latency_upstream_attempts_total{requested_model,upstream_model,outcome,code}` | counter | Latency-service chains only: the per-model complement of `switchyard_upstream_attempts_total` (which stays model-free). `requested_model` is the client-supplied model bounded to a config-derived id — the route id (`route_model`, e.g. `nvidia/switchyard/gpt-5.4`) or a configured endpoint id — with `other` as the fallback sentinel. `upstream_model` is the selected endpoint's upstream name. Answers "for route X, which upstream failed or succeeded". |### The `code` label on `switchyard_upstream_attempts_total`@@ -154,7 +114,7 @@ error_rate_reduction = direct_error_rate − router_error_rate# Live Endpoint Routing rescue raterate(switchyard_router_retry_recovered_total[5m])-# Traffic share per endpoint (sanity-check inverse-latency weighting)+# Traffic share per endpointsum by (model) (rate(switchyard_requests_total[5m]))/ ignoring(model) group_left sum(rate(switchyard_requests_total[5m]))@@ -165,12 +125,6 @@ sum by (code) (rate(switchyard_upstream_attempts_total{code!="200"}[5m]))sum by (code) (rate(switchyard_upstream_attempts_total{code!="200"}[5m]))/ ignoring(code) group_leftsum (rate(switchyard_upstream_attempts_total{code!="200"}[5m]))--# Per-upstream outcome breakdown for one route — "for nvidia/switchyard/gpt-5.4,-# which upstream provider failed or succeeded?" (latency-service chains)-sum by (upstream_model, outcome, code) (- rate(switchyard_latency_upstream_attempts_total{requested_model="nvidia/switchyard/gpt-5.4"}[5m])-)```> **Note:** because `switchyard_upstream_attempts_total` now carries the@@ -189,18 +143,10 @@ into label space.| Label | Values | Where ||---|---|---|| `model` | One per configured endpoint, typically 2–6 per deployment. | All per-endpoint metrics. |-| `status` | Exactly 3: `healthy`, `degraded`, `unknown`. | `switchyard_endpoint_status` || `outcome` | Exactly 3: `success`, `retryable_error`, `other_error`. | Outcome counters || `code` | Bounded: the known-code allowlist (`200`, `400`, `401`, `403`, `404`, `408`, `409`, `422`, `429`, `500`, `502`, `503`, `504`), plus `none` and the per-class buckets `1xx`/`2xx`/`3xx`/`4xx`/`5xx`/`other`. About 20 values max. | `switchyard_upstream_attempts_total` |-| `requested_model` | Bounded to config-derived ids: the route id (`route_model`) plus configured endpoint ids, else the `other` sentinel. | `switchyard_latency_upstream_attempts_total` |-| `upstream_model` | One per configured endpoint (the endpoint's upstream name). | `switchyard_latency_upstream_attempts_total` || `quantile` | Exactly 2: `0.5`, `0.99`. | All summaries |-| `tier` | Small enumerated set, optional. Not emitted by latency-service chains. | Per-endpoint counters/summaries on routing chains that supply it |--For a latency-service deployment with `N` endpoints, expect roughly-`11N + 17` series at startup (five `code` series are seeded), growing by at-most a dozen more as additional upstream status codes appear. That is about 39-series for two endpoints. Well within single-Prometheus capacity.+| `tier` | Small enumerated set, optional. | Per-endpoint counters/summaries on routing chains that supply it |## Triage cheatsheet@@ -208,7 +154,5 @@ series for two endpoints. Well within single-Prometheus capacity.|---|---|| `model="<unknown>"` rows appear | The per-endpoint attribution wiring regressed. `ctx.selected_model` is not being set by the backend. || All counters at 0 after warm-up | Server just started with no traffic, or the scraper is hitting the wrong port. |-| `switchyard_latency_service_poll_ok` stuck at `0` | DNS / network unreachability to the Latency Service, **or** a schema mismatch on the LS response. Check server logs for `"Health poller: failed to reach Latency Service"`. |-| `switchyard_endpoint_last_latency_ms{...}` absent for an endpoint | The Latency Service reported `last_latency_ms: null`, or the endpoint was not returned in the most recent poll response. || `switchyard_routing_overhead_ms_count` stuck at `0` | The backend is not publishing `ctx.backend_call_latency_ms` (regression of the Python-backend routing-overhead wiring). |-| `switchyard_client_responses_total{outcome="retryable_error"}` rising | Either the upstream is genuinely flaky (cross-check `switchyard_endpoint_status`), or retries are exhausting (compare with `switchyard_router_retry_recovered_total` rate). |+| `switchyard_client_responses_total{outcome="retryable_error"}` rising | Either the upstream is genuinely flaky, or retries are exhausting (compare with `switchyard_router_retry_recovered_total` rate). |diff --git a/examples/latency_service_caller_required.yaml b/examples/latency_service_caller_required.yamldeleted file mode 100644index 3d6f78cfe..000000000--- a/examples/latency_service_caller_required.yaml+++ /dev/null@@ -1,70 +0,0 @@-# =============================================================================-# Latency-aware routing with per-caller key attribution (BYO key, required)-# =============================================================================-#-# Use this when every upstream call must be billed to the *caller's* own API key-# (e.g. per-user spend attribution in multi-tenant gateways) and the service-level key must-# NEVER authenticate caller inference.-#-# What `credential_policy: caller_required` does:-# * The caller key is read from the `x-switchyard-api-key` request header-# (preferred), then `Authorization: Bearer` / `x-api-key`.-# * Present -> forwarded upstream as `Authorization: Bearer <key>`; spend is-# attributed to that caller.-# * Missing -> the request is rejected with HTTP 401 BEFORE any upstream call;-# the configured endpoint key is never used.-# (Applies to every ingress path: /v1/chat/completions, /v1/responses, /v1/messages.)-#-# Why a dedicated header? A proxy in front of Switchyard (e.g. LiteLLM) consumes-# the `Authorization` header for its own auth and strips it before the upstream-# call. The custom `x-switchyard-api-key` header passes through untouched.-#-# How clients send the header:-# * Codex CLI: env_http_headers / http_headers-# * OpenCode: provider options.headers-# * OpenAI / Anthropic SDK: extra / default headers-# * Claude Code: limited (can't easily send two distinct auth values)-# -- may need a wrapper/gateway unless one key serves both.-#-# Rollout order (IMPORTANT -- avoid 401s):-# 1. Deploy this build first with the route still on its current policy-# (`configured_endpoint` default, or `caller_override`) -- no behavior change.-# 2. Update clients to send `x-switchyard-api-key` (harmless under those policies).-# 3. Only after clients are confirmed sending it, switch to `caller_required`.-#-# Run:-# uv run switchyard serve \-# --routing-profiles examples/latency_service_caller_required.yaml --port 4100-#-# # Caller key present -> billed to that key:-# curl -s -X POST http://127.0.0.1:4100/v1/chat/completions \-# -H 'Content-Type: application/json' \-# -H "x-switchyard-api-key: $USER_UPSTREAM_API_KEY" \-# -d '{"model":"my-model","messages":[{"role":"user","content":"hi"}]}'-#-# # Header omitted -> HTTP 401:-# # "missing caller API key; supply it via the x-switchyard-api-key header"-# # (the service key is never used for caller inference).-#-# Substitute your own Latency Service URL, gateway base URL, and model IDs below.-# =============================================================================--routes:- my-model:- type: latency_service- latency_service_url: https://your-latency-service.example.com- # Require a caller key; 401 when missing; never fall back to a service key.- credential_policy: caller_required- poll_interval_s: 10.0- max_retries: 2- endpoints:- # No `api_key:` on the endpoints: `caller_required` forwards the caller's- # key and never falls back, so there is no service credential to leak.- - model: primary/my-model # Latency Service endpoint_id- upstream_model: provider/my-model # model name the upstream expects- base_url: https://your-inference-gateway.example.com/v1- # request_type: openai_responses # set for Responses-only upstreams- # (e.g. gpt-5.1-codex-mini)- - model: backup/my-model- upstream_model: provider/my-model- base_url: https://your-inference-gateway.example.com/v1diff --git a/examples/prometheus/README.md b/examples/prometheus/README.mdindex 77311caba..ceea201f4 100644--- a/examples/prometheus/README.md+++ b/examples/prometheus/README.md@@ -25,12 +25,9 @@ Prometheus configuration (adjust the `targets:` list to point at yourSwitchyard pods or hosts). The endpoint is plain `GET /metrics` overHTTP, no auth.-If you're running Switchyard on Kubernetes via-[`examples/k8s/latency-aware-deployment.yaml`](../k8s/latency-aware-deployment.yaml),-the service-level DNS is-`switchyard-latency.llm-routing.svc.cluster.local`. The example scrape-job uses static targets; a `kubernetes_sd_configs` variant is a one-step-swap for cluster-native discovery.+If you're running Switchyard on Kubernetes, the example scrape job uses+static targets; a `kubernetes_sd_configs` variant is a one-step swap for+cluster-native discovery.### 2. Load the alert rules@@ -61,8 +58,7 @@ series:```You should see the families documented in `metrics_reference.md` —-top-line gauges, per-endpoint counters/summaries, Latency Service state,-poll health, and outcome counters.+top-line gauges, per-endpoint counters/summaries, and outcome counters.## Alert tuning notes@@ -71,8 +67,6 @@ conservative — tuned to avoid pages during routine scrape-config rollsor single-poll blips. Review them against your SLOs before enablingnotifications:-* `LatencyServiceUnreachable` fires after 60s of `poll_ok=0`. Bring this- down if your routing logic should react faster than one poll cycle.* `RouterOverheadHigh` is set at the success-criterion threshold (p99 >1 ms for 5 m). Retry-heavy workloads inflate this — cross-check`switchyard_upstream_attempts_total{outcome="retryable_error"}` ratediff --git a/examples/prometheus/prometheus.yml b/examples/prometheus/prometheus.ymlindex 229a0831b..cbecb6727 100644--- a/examples/prometheus/prometheus.yml+++ b/examples/prometheus/prometheus.yml@@ -16,16 +16,11 @@ scrape_configs:- job_name: switchyardmetrics_path: /metricsscheme: http- # Default scrape every 15s — the Latency Service poll cycle defaults- # to 10s, so finer scrape resolution captures no additional state.scrape_interval: 15sstatic_configs:- # Replace with your Switchyard host(s) and port(s). On Kubernetes- # with the manifest at examples/k8s/latency-aware-deployment.yaml- # the in-cluster DNS resolves to- # switchyard-latency.llm-routing.svc.cluster.local on port 80- # targeting container port 4100. Swap this static_configs block- # for kubernetes_sd_configs in a cluster-native deployment.+ # Replace with your Switchyard host(s) and port(s). Swap this+ # static_configs block for kubernetes_sd_configs in a+ # cluster-native deployment.- targets:- switchyard.example.internal:4100diff --git a/examples/prometheus/switchyard.rules.yaml b/examples/prometheus/switchyard.rules.yamlindex dfc8378f8..3b3400054 100644--- a/examples/prometheus/switchyard.rules.yaml+++ b/examples/prometheus/switchyard.rules.yaml@@ -9,60 +9,6 @@# examples/prometheus/README.md before wiring to a pager.groups:- - name: switchyard.latency_service.health- rules:- - alert: LatencyServiceUnreachable- expr: switchyard_latency_service_poll_ok == 0- for: 60s- labels:- severity: warning- component: switchyard- annotations:- summary: "Switchyard cannot reach the Latency Service"- description: |- switchyard_latency_service_poll_ok has been 0 for at least- 60s. Routing has fallen back to uniform random across all- endpoints (every verdict reset to unknown). Check the- Switchyard pod logs for "Health poller: failed to reach- Latency Service" and confirm DNS / network reachability to- the upstream LS.-- - alert: LatencyServicePollStale- # No `for:` — the age gauge only grows while polls keep failing, so- # the >60s threshold is already debounced; a `for:` would only delay- # the alert past the documented 60s staleness budget.- expr: switchyard_latency_service_poll_age_seconds > 60- labels:- severity: warning- component: switchyard- annotations:- summary: "Switchyard health verdicts are stale"- description: |- The most recent successful Latency Service poll was more- than 60 s ago. Routing decisions are still using cached- verdicts but they are increasingly out of date. Compare- switchyard_latency_service_poll_failures_total against- switchyard_latency_service_polls_total to see whether the- poller is failing intermittently or hanging entirely.-- - alert: LatencyServiceEndpointFlapping- expr: |- sum by (model) (- changes(switchyard_endpoint_status{status="healthy"}[10m])- ) > 5- for: 0s- labels:- severity: info- component: switchyard- annotations:- summary: "Endpoint {{ $labels.model }} is flapping"- description: |- switchyard_endpoint_status{model="{{ $labels.model }}"} has- transitioned in or out of "healthy" more than 5 times in- the last 10 minutes. Likely a real upstream instability —- cross-check switchyard_endpoint_last_latency_ms for that- model and the LS upstream's own dashboards.-- name: switchyard.routing.evidencerules:- alert: RouterNotRescuing@@ -84,7 +30,7 @@ groups:/ 504) but no request has been rescued by retry in the same5 m window. Likely cause: every configured endpoint isdegraded simultaneously, so the retry loop has nowhere to- steer. Check switchyard_endpoint_status by model.+ steer.- alert: RouterOverheadHighexpr: switchyard_routing_overhead_ms{quantile="0.99"} > 1.0@@ -127,6 +73,5 @@ groups:clients is at least as high as the per-attempt upstreamretryable_error rate. The router is not absorbing failuresvia retry — either retries are exhausting or every- endpoint is producing the same failure mode. Look at- switchyard_endpoint_status and the per-model breakdown of- switchyard_errors_total{model}.+ endpoint is producing the same failure mode. Look at the+ per-model breakdown of switchyard_errors_total{model}.diff --git a/switchyard/__init__.py b/switchyard/__init__.pyindex 34b5c35dd..5fc93f89e 100644--- a/switchyard/__init__.py+++ b/switchyard/__init__.py@@ -13,10 +13,6 @@ from typing import TYPE_CHECKING, Anyfrom switchyard.lib.backends import (AnthropicNativeBackend,- EndpointHealth,- EndpointHealthStatus,- HealthPoller,- LatencyServiceLLMBackend,OpenAiNativeBackend,)from switchyard.lib.backends.llm_target import (@@ -42,8 +38,6 @@ from switchyard.lib.chat_response import ()from switchyard.lib.config import (IntakeSinkConfig,- LatencyServiceBackendConfig,- LatencyServiceEndpoint,)from switchyard.lib.processors.intake_payload_builder import IntakePayloadBuilderfrom switchyard.lib.processors.intake_request_processor import IntakeRequestProcessor@@ -58,7 +52,6 @@ from switchyard.lib.profiles import (DeterministicRoutingProfileConfig,EscalationRouterConfig,EscalationRouterProfileConfig,- LatencyServiceProfileConfig,NoopProfile,NoopProfileConfig,PassthroughProfileConfig,@@ -156,7 +149,6 @@ __all__ = ["DeterministicRoutingPresets","EscalationRouterConfig","EscalationRouterProfileConfig",- "LatencyServiceProfileConfig","PassthroughProfileConfig","Profile","ProfileConfig",@@ -182,13 +174,6 @@ __all__ = ["build_switchyard_app",# Route dispatch table"RouteTable",- # Latency Service usage case- "EndpointHealth",- "EndpointHealthStatus",- "HealthPoller",- "LatencyServiceBackendConfig",- "LatencyServiceEndpoint",- "LatencyServiceLLMBackend","IntakePayloadBuilder","IntakeRequestProcessor","IntakeResponseProcessor",diff --git a/switchyard/cli/launch_command.py b/switchyard/cli/launch_command.pyindex b79a5164c..435e17383 100644--- a/switchyard/cli/launch_command.py+++ b/switchyard/cli/launch_command.py@@ -490,7 +490,7 @@ def cmd_launch_claude(args: argparse.Namespace) -> None:* ``--routing-profiles FILE`` (global flag) — YAML-driven multi-chain table.``--model`` is optional; falls back to the first YAML route.- Random / latency-aware routing live in the YAML schema.+ Random and other multi-chain routing live in the YAML schema."""if getattr(args, "startup_timing", False):startup_timing.enable()diff --git a/switchyard/cli/launchers/claude_code_launcher.py b/switchyard/cli/launchers/claude_code_launcher.pyindex 148764c13..c71eedc3a 100644--- a/switchyard/cli/launchers/claude_code_launcher.py+++ b/switchyard/cli/launchers/claude_code_launcher.py@@ -298,8 +298,8 @@ def _run_claude_with_switchyard(Takes a pre-built :class:`Switchyard` and a display-only modelname. The caller decides what's in the chain (single-model- passthrough, random routing across a preset, latency-service- pool, …) — this function owns only the uvicorn-in-a-thread ++ passthrough, random routing across a preset, …) — this function+ owns only the uvicorn-in-a-thread +``claude`` supervision + env-var injection boilerplate that isidentical across those modes.diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.pyindex 727c648b9..a465a3906 100644--- a/switchyard/cli/route_bundle.py+++ b/switchyard/cli/route_bundle.py@@ -34,13 +34,11 @@ from pydantic import ValidationErrorfrom switchyard.cli.model_catalog.model_discovery import fetch_model_idsfrom switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target-from switchyard.lib.config import LatencyServiceBackendConfig, LatencyServiceEndpointfrom switchyard.lib.processors.llm_classifier import DEFAULT_MAX_REQUEST_CHARSfrom switchyard.lib.processors.llm_classifier.presets import PROFILE_FACTORIESfrom switchyard.lib.profiles import (DeterministicRoutingProfileConfig,EscalationRouterProfileConfig,- LatencyServiceProfileConfig,ProfileSwitchyard,StageRouterProfileConfig,)@@ -193,32 +191,6 @@ _PASSTHROUGH_ROUTE_KEYS = (| frozenset({"defaults", "enable_stats"})| _PASSTHROUGH_SETTING_KEYS)-_LATENCY_ENDPOINT_DEFAULT_KEYS = frozenset({- "api_key",- "base_url",- "timeout",- "timeout_secs",-})-_LATENCY_ENDPOINT_KEYS = _LATENCY_ENDPOINT_DEFAULT_KEYS | frozenset(- {"model", "upstream_model", "request_type"}-)-_LATENCY_SERVICE_ROUTE_KEYS = _ROUTE_METADATA_KEYS | frozenset({- "defaults",- "endpoints",- "latency_service_url",- "latency_url",- "poll_interval_s",- "poll_timeout_s",- "max_retries",- "credential_policy",- "enable_stats",- "session_affinity",- "affinity_max_sessions",- "affinity_store",- "affinity_store_url",- "affinity_store_ttl_seconds",- "affinity_key_prefix",-}) | _LATENCY_ENDPOINT_DEFAULT_KEYS_NOOP_ROUTE_KEYS = _ROUTE_METADATA_KEYS_DETERMINISTIC_ROUTE_KEYS = (_ROUTE_METADATA_KEYS@@ -318,7 +290,6 @@ _CLASSIFIER_DEFAULT_KEYS = frozenset({_ROUTE_KEYS_BY_TYPE: Mapping[str, frozenset[str]] = {"model": _MODEL_ROUTE_KEYS,"random_routing": _RANDOM_ROUTING_ROUTE_KEYS,- "latency_service": _LATENCY_SERVICE_ROUTE_KEYS,"noop": _NOOP_ROUTE_KEYS,"passthrough": _PASSTHROUGH_ROUTE_KEYS,"deterministic": _DETERMINISTIC_ROUTE_KEYS,@@ -328,7 +299,6 @@ _ROUTE_KEYS_BY_TYPE: Mapping[str, frozenset[str]] = {_DEFAULT_KEYS_BY_TYPE: Mapping[str, frozenset[str]] = {"model": _TARGET_DEFAULT_KEYS,"random_routing": _TARGET_DEFAULT_KEYS,- "latency_service": _LATENCY_ENDPOINT_DEFAULT_KEYS,"passthrough": _PASSTHROUGH_SETTING_KEYS,"noop": frozenset(),"deterministic": _TARGET_DEFAULT_KEYS,@@ -869,16 +839,6 @@ def _build_route_chain(extra_response_processors=extra_response_processors,)- if route_type == "latency_service":- return _latency_service_switchyard(- model_id,- route,- target_defaults,- stats=stats,- extra_request_processors=pre_routing_request_processors,- extra_response_processors=extra_response_processors,- )-if route_type == "noop":from switchyard.lib.profiles.noop import NoopProfileConfig@@ -1305,62 +1265,6 @@ def _passthrough_target()-def _latency_service_switchyard(- model_id: str,- route: Mapping[str, object],- target_defaults: Mapping[str, object],- stats: StatsAccumulator,- extra_request_processors: Sequence[Any] = (),- extra_response_processors: Sequence[Any] = (),-) -> ChainRuntime:- endpoints_raw = _require_sequence(route.get("endpoints"), "latency_service.endpoints")- endpoints = [- _latency_endpoint(value, target_defaults, index=index)- for index, value in enumerate(endpoints_raw)- ]- enable_stats = _optional_bool(route.get("enable_stats"), default=True)- credential_policy = (- _optional_str(route["credential_policy"])- if "credential_policy" in route- else "configured_endpoint"- )- config = LatencyServiceBackendConfig.model_validate({- "latency_service_url": _required_str(- route.get("latency_service_url", route.get("latency_url")),- "latency_service.latency_service_url",- ),- "endpoints": endpoints,- # The YAML route key is what clients send as ``model``; hand it to the- # backend so the per-model metric can attribute route-key traffic.- "route_model": model_id,- "poll_interval_s": _optional_float(route.get("poll_interval_s"), default=10.0),- "poll_timeout_s": _optional_float(route.get("poll_timeout_s"), default=5.0),- "max_retries": _optional_int(route.get("max_retries"), default=2),- "credential_policy": credential_policy,- "enable_stats": enable_stats,- "session_affinity": _optional_bool(route.get("session_affinity"), default=False),- "affinity_max_sessions": _optional_int(- route.get("affinity_max_sessions"), default=10_000- ),- "affinity_store": _optional_str(route.get("affinity_store")) or "memory",- "affinity_store_url": _optional_str(route.get("affinity_store_url")),- "affinity_store_ttl_seconds": _optional_int(- route.get("affinity_store_ttl_seconds"), default=3_600- ),- "affinity_key_prefix": _optional_str(route.get("affinity_key_prefix")) or "swyd:pin:",- })- return ProfileSwitchyard(- LatencyServiceProfileConfig.from_config(config)- .build()- .with_runtime_components(- stats_accumulator=stats,- enable_stats=config.enable_stats,- pre_request_processors=extra_request_processors,- response_processors=extra_response_processors,- )- )--def _route_config(route: Mapping[str, object],target_defaults: Mapping[str, object],@@ -1438,33 +1342,6 @@ def _target_mapping(return target-def _latency_endpoint(- raw: object,- defaults: Mapping[str, object],- index: int,-) -> LatencyServiceEndpoint:- data = dict(defaults)- where = f"latency_service.endpoints[{index}]"- if isinstance(raw, str):- data["model"] = raw- elif isinstance(raw, Mapping):- endpoint_mapping = _require_mapping(raw, where)- _validate_allowed_keys(endpoint_mapping, _LATENCY_ENDPOINT_KEYS, where)- data.update(endpoint_mapping)- else:- raise RouteBundleConfigError(f"{where} must be a string or mapping")-- if "timeout" not in data and "timeout_secs" in data:- data["timeout"] = data["timeout_secs"]-- endpoint_data = {- key: data[key]- for key in ("model", "upstream_model", "api_key", "base_url", "timeout", "request_type")- if key in data- }- return LatencyServiceEndpoint.model_validate(endpoint_data)--def _target_defaults(bundle_defaults: Mapping[str, object],route: Mapping[str, object],@@ -1492,8 +1369,6 @@ def _route_type(model_id: str, route: Mapping[str, object]) -> str:if raw_type is None:if not route:return "noop"- if "latency_service_url" in route or "latency_url" in route:- return "latency_service"if "strong" in route and "weak" in route:return "random_routing"if "target" in route or "model" in route:@@ -1512,8 +1387,6 @@ def _route_type(model_id: str, route: Mapping[str, object]) -> str:"target": "model","random": "random_routing","random_routing": "random_routing",- "latency": "latency_service",- "latency_service": "latency_service","noop": "noop","no_op": "noop","passthrough": "passthrough",@@ -1615,12 +1488,6 @@ def _optional_mapping(value: object, where: str) -> dict[str, object]:return _require_mapping(value, where)-def _require_sequence(value: object, where: str) -> Sequence[object]:- if not isinstance(value, list):- raise RouteBundleConfigError(f"{where} must be a list")- return value--def _required_str(value: object, where: str) -> str:if not isinstance(value, str) or not value:raise RouteBundleConfigError(f"{where} must be a non-empty string")diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.pyindex 59c4de402..b0fdaf781 100644--- a/switchyard/cli/switchyard_cli.py+++ b/switchyard/cli/switchyard_cli.py@@ -44,7 +44,7 @@ Examples::switchyard --routing-profiles profiles.yaml -- launch claude -- --no-auto-approveRouting policies that used to be top-level CLI verbs (``passthrough``,-``random-routing``, ``latency-service``) and launcher flags+``random-routing``) and launcher flags(``--routing``, ``--weak-model``, ``--strong-probability``, ``--preset``)are expressed in routing-profile YAML files. ``serve`` andlaunchers still parse the YAML into profile-backed runtimes.diff --git a/switchyard/lib/affinity_pin_store.py b/switchyard/lib/affinity_pin_store.pyindex 1654bfeb1..39b766899 100644--- a/switchyard/lib/affinity_pin_store.py+++ b/switchyard/lib/affinity_pin_store.py@@ -20,8 +20,8 @@ from typing import Protocol, runtime_checkableclass AffinityPinStore(Protocol):"""Shared, out-of-process store for conversation→decision pins.- The pinned value is opaque to the store (an endpoint id for the latency- backend, a tier label for the classifier router). Implementations are+ The pinned value is opaque to the store (a tier label for the classifier+ or escalation router). Implementations arecalled on the request path, so they should be fast, and:class:`SessionAffinity` treats them as **best-effort**: an error or timeoutmust never break request routing (the caller falls back to L1 / unpinned).diff --git a/switchyard/lib/backends/__init__.py b/switchyard/lib/backends/__init__.pyindex 6251d02d1..53c20ae6f 100644--- a/switchyard/lib/backends/__init__.py+++ b/switchyard/lib/backends/__init__.py@@ -11,14 +11,6 @@ from switchyard.lib.backends.backend_format_resolver import (BackendFormatResolution,BackendFormatResolver,)-from switchyard.lib.backends.health_poller import (- EndpointHealth,- EndpointHealthStatus,- HealthPoller,-)-from switchyard.lib.backends.latency_service_llm_backend import (- LatencyServiceLLMBackend,-)from switchyard.lib.backends.stats_llm_backend import (StatsLlmBackend,)@@ -32,10 +24,6 @@ __all__ = ["AnthropicNativeBackend","BackendFormatResolution","BackendFormatResolver",- "EndpointHealth",- "EndpointHealthStatus",- "HealthPoller",- "LatencyServiceLLMBackend","OpenAiPassthroughBackend","OpenAiNativeBackend","StatsLlmBackend",diff --git a/switchyard/lib/backends/health_poller.py b/switchyard/lib/backends/health_poller.pydeleted file mode 100644index 5393e568f..000000000--- a/switchyard/lib/backends/health_poller.py+++ /dev/null@@ -1,181 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Health enum and background poller for the Latency Service usage case.--The Latency Service exposes a bulk health endpoint that returns one of-three states per endpoint. :class:`EndpointHealthStatus` is the shared-contract between Switchyard and the Latency Service — both sides must-agree on these string values.--:class:`HealthPoller` is a daemon thread that periodically pulls fresh-verdicts from the Latency Service and writes them into an in-memory-cache shared with :class:`LatencyServiceLLMBackend`. The backend's-request hot path only ever *reads* the cache (under a lock), so health-polling adds zero per-request latency.--Each cache entry is an :class:`EndpointHealth` snapshot carrying both-the discrete ``status`` and the most recent ``last_latency_ms`` sample-reported by the Latency Service. The status drives tier selection-(HEALTHY > UNKNOWN > DEGRADED); the latency drives inverse-latency-weighted selection *within* a tier.--Design properties:--- **Daemon thread, not asyncio task.** Simple lifecycle: starts when the- backend is constructed, dies with the process. The sync ``httpx.Client``- used inside the thread avoids event-loop contention for other work.-- **Fallback to UNKNOWN on poll failure.** If the Latency Service is- unreachable, all endpoints reset to ``UNKNOWN`` so the backend falls- back to random routing rather than acting on stale health data.-- **Graceful stop via ``stop()``.** Tests and ``shutdown()`` paths set- ``_stop_event`` to break the polling loop promptly.-"""--from __future__ import annotations--import enum-import logging-import threading-import time-from typing import NamedTuple--import httpx--logger = logging.getLogger(__name__)---class EndpointHealthStatus(enum.Enum):- """Shared contract between Switchyard and the Latency Service.-- Three states are sufficient for routing decisions — the Latency- Service may track finer-grained failure reasons internally for- observability but exposes only these values in the health API.- """-- HEALTHY = "healthy"- DEGRADED = "degraded"- UNKNOWN = "unknown"---class EndpointHealth(NamedTuple):- """Snapshot of one endpoint's health, written by :class:`HealthPoller`.-- ``status`` drives tier-based routing (HEALTHY > UNKNOWN > DEGRADED);- ``last_latency_ms`` — when present — drives inverse-latency weighted- selection within whichever tier is chosen. ``last_latency_ms`` is- ``None`` before the first successful poll and whenever the Latency- Service reports it as null.- """-- status: EndpointHealthStatus- last_latency_ms: float | None = None---class HealthPoller(threading.Thread):- """Background daemon thread that refreshes the backend's health cache.-- Polls ``{latency_service_url}/v1/endpoints/health`` every- ``poll_interval_s`` seconds and writes verdicts into- ``health_cache`` under ``cache_lock``. Holds references to the- same dict and lock the backend reads from — no message passing, no- extra copies.- """-- def __init__(- self,- latency_service_url: str,- model_ids: list[str],- health_cache: dict[str, EndpointHealth],- cache_lock: threading.Lock,- poll_interval_s: float,- poll_timeout_s: float,- ) -> None:- super().__init__(daemon=True, name="latency-health-poller")- self._url = latency_service_url.rstrip("/") + "/v1/endpoints/health"- self._model_ids = model_ids- self._health_cache = health_cache- self._cache_lock = cache_lock- self._poll_interval_s = poll_interval_s- self._http_client = httpx.Client(timeout=poll_timeout_s)- self._stop_event = threading.Event()- self._poll_count = 0- # Failure counter and last-success timestamp drive the /metrics- # poll-health gauges; tracked here because the poll loop is the- # single point where outcomes are observable.- self._poll_failures = 0- self._last_poll_ok = False- self._last_success_at: float | None = None-- def run(self) -> None:- while not self._stop_event.is_set():- try:- resp = self._http_client.get(- self._url,- params=[("endpoint_ids", mid) for mid in self._model_ids],- )- resp.raise_for_status()- data = resp.json()- with self._cache_lock:- for mid, info in data["endpoint_health"].items():- if mid in self._health_cache:- self._health_cache[mid] = EndpointHealth(- status=EndpointHealthStatus(info["status"]),- last_latency_ms=info.get("last_latency_ms"),- )- self._poll_count += 1- self._last_poll_ok = True- self._last_success_at = time.monotonic()- except Exception:- logger.warning(- "Health poller: failed to reach Latency Service, "- "resetting all endpoints to UNKNOWN (random routing)"- )- with self._cache_lock:- for mid in self._health_cache:- self._health_cache[mid] = EndpointHealth(- status=EndpointHealthStatus.UNKNOWN,- )- self._poll_failures += 1- self._last_poll_ok = False- self._stop_event.wait(timeout=self._poll_interval_s)-- @property- def has_polled(self) -> bool:- """True once at least one poll has successfully updated the cache."""- return self._poll_count > 0-- @property- def poll_successes(self) -> int:- """Total number of successful polls since the poller started."""- return self._poll_count-- @property- def poll_failures(self) -> int:- """Total number of poll attempts that failed (including timeouts).-- On every failure, all cached endpoint verdicts are reset to UNKNOWN- so routing falls back to random selection instead of acting on- stale health data.- """- return self._poll_failures-- @property- def last_poll_ok(self) -> bool:- """True iff the most recent poll attempt succeeded."""- return self._last_poll_ok-- @property- def seconds_since_last_success(self) -> float | None:- """Monotonic seconds since the last successful poll.-- ``None`` before the first success — useful as a "never polled"- signal on /metrics scrapes during startup or sustained outage.- """- if self._last_success_at is None:- return None- return time.monotonic() - self._last_success_at-- def stop(self) -> None:- """Signal the polling loop to exit at the next iteration boundary."""- self._stop_event.set()diff --git a/switchyard/lib/backends/latency_service_llm_backend.py b/switchyard/lib/backends/latency_service_llm_backend.pydeleted file mode 100644index 1942df790..000000000--- a/switchyard/lib/backends/latency_service_llm_backend.py+++ /dev/null@@ -1,1098 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""``LLMBackend`` that routes across many endpoints by Latency Service verdicts.--This is the usage case for Inference Hub deployments where a central-Latency Service owns heartbeat probing and statistical profiling. The-backend holds a pool of ``OpenAILLMClient`` instances keyed by model ID,-reads health verdicts from a locally-cached map maintained by a-:class:`HealthPoller` daemon thread, and picks a healthy endpoint on-every request.--Chain integration::-- request-side work → LatencyServiceLLMBackend → response-side work → TranslationEngine--Request-format translation is delegated to-:class:`switchyard_rust.translation.TranslationEngine` after an endpoint is-selected, so Chat endpoints keep receiving Chat Completions while-Responses-mode endpoints receive the OpenAI Responses API natively.-"""--import json-import logging-import random-import threading-import time-import uuid-from collections.abc import Mapping-from dataclasses import dataclass--from openai import APIStatusError, AsyncStream--from switchyard.lib.affinity_pin_store import AffinityPinStore-from switchyard.lib.backends.health_poller import (- EndpointHealth,- EndpointHealthStatus,- HealthPoller,-)-from switchyard.lib.chat_response.openai_chat import ResponseStream-from switchyard.lib.chat_response.openai_responses import ResponsesApiStream-from switchyard.lib.config.latency_service_backend_config import (- LatencyServiceBackendConfig,-)-from switchyard.lib.endpoints import outcome_metrics, prometheus_emitter-from switchyard.lib.endpoints.upstream_error_log import log_upstream_attempt_failure-from switchyard.lib.llm_client import OpenAILLMClient-from switchyard.lib.prometheus_exposition import format_number, render_labels-from switchyard.lib.proxy_context import (- CTX_CALLER_API_KEY,- CTX_ERROR_SOURCE,- CTX_ROUTE_SELECTION,- CTX_UPSTREAM_ATTEMPTS_RECORDED,- CTX_UPSTREAM_HTTP_BODY,- CTX_UPSTREAM_HTTP_STATUS,- CTX_UPSTREAM_MODEL,- ERROR_SOURCE_PROVIDER,- ERROR_SOURCE_SWITCHYARD,- ProxyContext,-)-from switchyard.lib.roles import LLMBackend-from switchyard.lib.session_affinity import SessionAffinity-from switchyard.lib.stats_accumulator import StatsAccumulator-from switchyard.lib.tracing import routing_span, set_tags-from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse, request_type_matches-from switchyard_rust.translation import TranslationEngine--log = logging.getLogger(__name__)--#: Request header read by a LiteLLM front proxy; its JSON value is copied into-#: the provider spend-log DB row, tying that row back to the Switchyard route-#: that selected it (see :data:`CTX_ROUTE_SELECTION` for the payload contract).-SPEND_LOGS_METADATA_HEADER = "x-litellm-spend-logs-metadata"--#: ``router_strategy`` value identifying this backend's selection algorithm.-ROUTER_STRATEGY_LATENCY = "latency"---@dataclass(frozen=True)-class _RouteDecision:- """One endpoint-selection outcome, carried to the ``route_decision`` span.-- ``candidates`` is the set the picker chose among (the winning health tier),- and ``was_fastest`` is whether ``selected`` had the lowest known latency in- that set. ``affinity`` is ``True`` when ``selected`` came from a session- pin (reused endpoint) rather than the latency-aware picker.- """-- selected: str- candidates: tuple[str, ...]- was_fastest: bool- affinity: bool = False---class LatencyServiceLLMBackend(LLMBackend):- """Routes to healthy endpoints based on Latency Service health verdicts.-- On construction, builds a pool of ``OpenAILLMClient`` instances- (one per configured endpoint, keyed by ``model``) and starts a- :class:`HealthPoller` daemon thread that refreshes the in-memory- health cache every ``poll_interval_s`` seconds.-- On each ``call()``:-- 1. Pick an endpoint from the health cache — ``HEALTHY`` preferred,- then ``UNKNOWN``, then ``DEGRADED``. Within the chosen tier,- selection is weighted by inverse ``last_latency_ms`` when every- candidate has a known sample; otherwise it falls back to uniform- random.- 2. Translate the request to that endpoint's configured OpenAI API- surface and override the body's ``model`` with the selected endpoint ID.- 3. Call ``OpenAILLMClient`` on the configured API surface. On a transient error- (429 / 408 / 5xx / network), retry with a different endpoint- (dedup prevents re-selecting the same one within a single- request). On a 4xx client error the request is rejected- identically by every replica, so the loop fails fast and passes- the upstream status through.- 4. Wrap the response into a Rust-owned ``ChatResponse``.- """-- def __init__(- self,- config: LatencyServiceBackendConfig,- *,- stats_accumulator: StatsAccumulator | None = None,- ) -> None:- if not config.endpoints:- raise ValueError("At least one endpoint must be configured")-- self._config = config- self._translation = TranslationEngine()- self._clients: dict[str, OpenAILLMClient] = {}- self._upstream_models: dict[str, str] = {}- self._request_types: dict[str, ChatRequestType] = {}- self._health_cache: dict[str, EndpointHealth] = {}- self._cache_lock = threading.Lock()- # When provided, the backend records success/error/latency directly- # into the accumulator on each attempt. This mirrors what the Rust- # ``StatsLlmBackend`` does for native backends; the Python-only- # latency-service backend can't be wrapped by it, so we record- # in-place to keep ``/metrics`` populated.- self._stats = stats_accumulator-- # Session affinity: pins a conversation to the endpoint that last served- # it so its upstream prompt/KV cache stays warm. Shared coordinator with- # the classifier router; the latency-specific reuse policy (health-gated,- # failover-aware) lives in ``_select_endpoint_decision``. An optional- # shared L2 (Redis) lets pins outlive this worker and be read by peers,- # so a conversation stays pinned across replicas and pod churn.- self._affinity = SessionAffinity(- enabled=config.session_affinity,- max_sessions=config.affinity_max_sessions,- l2=_build_affinity_l2(config),- )- # Cumulative warm-reuse counters, published on /metrics when affinity is- # enabled. A "hit" is a turn served by an existing pin; a "miss" is a- # first/unpinnable turn routed by the latency-aware picker. Counted once- # per request (first attempt only) so failover retries don't inflate- # them. Incremented and read only on the event loop, so no lock needed.- self._affinity_hits = 0- self._affinity_misses = 0-- # Per-model upstream-attempt counts for the latency-service-scoped- # /metrics breakdown, keyed by (requested model, selected upstream model,- # outcome, HTTP code). Kept separate from the layer-aggregate- # ``switchyard_upstream_attempts_total`` so that counter stays model-free.- # Cardinality is bounded: ``upstream_model`` comes from config, and- # ``requested_model`` is normalized to a config-derived id (route id or- # endpoint id) or the ``"other"`` sentinel in ``_record_model_attempt``- # so a caller-supplied model string can't grow the series set. Mutated- # and read only on the event loop (like the affinity counters), so no- # lock.- self._upstream_attempts_by_model: dict[tuple[str, str, str, str], int] = {}-- for ep_cfg in config.endpoints:- model_id = ep_cfg.model- if not model_id:- raise ValueError(- "Every endpoint must have a 'model' field — "- "this is the key used by the Latency Service"- )- if model_id in self._clients:- raise ValueError(f"Duplicate model ID: {model_id}")-- self._clients[model_id] = OpenAILLMClient(- api_key=ep_cfg.api_key,- base_url=ep_cfg.base_url,- timeout=ep_cfg.timeout,- # The ``call`` retry loop already retries on a *different*- # endpoint, which is what a health-aware router wants. Letting- # the SDK also retry (default 2) on the *same* endpoint stacks- # multiplicatively (up to (1+max_retries) x 3 attempts) and adds- # exponential-backoff sleeps that hold the request — and its- # buffered body — alive longer, amplifying connection-pool- # pressure during an upstream incident. Disable it here.- max_retries=0,- )- self._upstream_models[model_id] = ep_cfg.upstream_model or model_id- self._request_types[model_id] = _latency_endpoint_request_type(- ep_cfg.request_type- )- self._health_cache[model_id] = EndpointHealth(- status=EndpointHealthStatus.UNKNOWN,- )- log.info(- "LatencyServiceLLMBackend endpoint: model=%s upstream_model=%s "- "request_type=%s base_url=%s",- model_id,- self._upstream_models[model_id],- ep_cfg.request_type,- ep_cfg.base_url,- )-- # Bounded ``requested_model`` label set: the configured endpoint ids- # plus the client-facing route id when the deployment supplied one- # (IH clients request the route key, e.g. ``nvidia/switchyard/gpt-5.4``,- # never an endpoint id). Both sources are config-derived, so metric- # cardinality stays bounded.- self._requested_model_ids = frozenset(self._clients) | (- frozenset((config.route_model,)) if config.route_model else frozenset()- )-- self._poller = HealthPoller(- latency_service_url=config.latency_service_url,- model_ids=list(self._clients.keys()),- health_cache=self._health_cache,- cache_lock=self._cache_lock,- poll_interval_s=config.poll_interval_s,- poll_timeout_s=config.poll_timeout_s,- )- # Publish LS verdicts + poll health on /metrics. Registering here- # (and unregistering in ``shutdown``) ties emitter lifetime to the- # backend's lifetime, so a re-built chain doesn't leak a closure- # over a torn-down cache.- prometheus_emitter.register(self._render_prometheus_lines)- self._poller.start()-- @property- def supported_request_types(self) -> list[ChatRequestType]:- """OpenAI request types accepted by the configured endpoints."""- ordered = [ChatRequestType.OPENAI_CHAT, ChatRequestType.OPENAI_RESPONSES]- configured = set(self._request_types.values())- return [request_type for request_type in ordered if request_type in configured]-- # -- Endpoint selection (reads from cache, never blocks on network) ------- def _select_endpoint(self) -> str:- """Pick a model ID — see :meth:`_select_endpoint_decision`."""- return self._select_endpoint_decision().selected-- def _select_endpoint_decision(- self,- pinned_endpoint: str | None = None,- tried: frozenset[str] = frozenset(),- ) -> _RouteDecision:- """Pick a model ID and report the candidate set it chose among.-- When ``pinned_endpoint`` is set (this conversation's session-affinity- pin) and still usable — HEALTHY/UNKNOWN and not already failed this- request — it is reused directly, bypassing the latency-aware pick so the- conversation sticks to one endpoint and its upstream cache stays warm.-- Otherwise tier priority is HEALTHY > UNKNOWN > DEGRADED. Within the- chosen tier, selection is **inverse-latency weighted** when every- candidate has a known ``last_latency_ms`` — endpoints that the Latency- Service most recently saw as faster receive proportionally more- traffic. If any candidate's latency is unknown, the picker falls back- to uniform random for that tier; this keeps behaviour predictable while- the poller is warming up or when the upstream reports nulls.-- Returns the selected endpoint plus the candidates considered and- whether the pick was the lowest-latency one — the signals the- ``switchyard.route_decision`` span reports.- """- with self._cache_lock:- snapshot = dict(self._health_cache)-- # -- Affinity fast-path: reuse the pinned endpoint while it stays- # healthy and hasn't already failed this request. Skips latency- # weighting entirely — the whole point is to not chase a marginally- # faster endpoint at the cost of cache locality.- if (- pinned_endpoint is not None- and pinned_endpoint not in tried- and _affinity_usable(snapshot.get(pinned_endpoint))- ):- return _RouteDecision(- selected=pinned_endpoint,- candidates=(pinned_endpoint,),- was_fastest=False,- affinity=True,- )-- by_health: dict[EndpointHealthStatus, list[str]] = {- h: [] for h in EndpointHealthStatus- }- for mid, hp in snapshot.items():- by_health[hp.status].append(mid)-- for tier in (- EndpointHealthStatus.HEALTHY,- EndpointHealthStatus.UNKNOWN,- EndpointHealthStatus.DEGRADED,- ):- if by_health[tier]:- candidates = by_health[tier]- selected = self._pick_by_latency(candidates, snapshot)- return _RouteDecision(- selected=selected,- candidates=tuple(candidates),- was_fastest=self._is_fastest(selected, candidates, snapshot),- )-- candidates = list(self._clients.keys())- selected = random.choice(candidates)- return _RouteDecision(- selected=selected,- candidates=tuple(candidates),- was_fastest=self._is_fastest(selected, candidates, snapshot),- )-- @staticmethod- def _is_fastest(- selected: str,- candidates: list[str],- snapshot: dict[str, EndpointHealth],- ) -> bool:- """Whether *selected* had the lowest known ``last_latency_ms``.-- ``False`` when no candidate has a usable latency sample (selection was- uniform, so "fastest" is undefined). Over many requests the fraction of- ``True`` shows how often inverse-latency weighting picked the front-runner.- """- known: dict[str, float] = {}- for mid in candidates:- latency = snapshot[mid].last_latency_ms- if latency is not None and latency > 0:- known[mid] = latency- if not known or selected not in known:- return False- return known[selected] == min(known.values())-- def _poll_age_ms(self) -> float | None:- """Milliseconds since the last successful Latency Service poll, or ``None``."""- age_s = self._poller.seconds_since_last_success- return age_s * 1000.0 if age_s is not None else None-- @staticmethod- def _pick_by_latency(- candidates: list[str],- snapshot: dict[str, EndpointHealth],- ) -> str:- """Inverse-latency weighted random pick across ``candidates``.-- Returns a uniform random pick when only one candidate exists, when- any candidate's ``last_latency_ms`` is unknown, or when any sample- is non-positive (treated as bogus — the Latency Service should- never report 0 or negative, but be defensive).- """- if len(candidates) == 1:- return candidates[0]- latencies = [snapshot[c].last_latency_ms for c in candidates]- if any(lat is None or lat <= 0 for lat in latencies):- return random.choice(candidates)- weights = [1.0 / lat for lat in latencies] # type: ignore[operator]- return random.choices(candidates, weights=weights, k=1)[0]-- # -- Request processing (hot path — no Latency Service call) -------------- async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:- """Serve *request* on the best healthy endpoint, failing over on error.-- Candidates are ordered by health and observed latency (with optional- session affinity); every upstream attempt is stamped with the- spend-logs metadata header, and the attempt that succeeds records its- route selection on *ctx* (see :data:`CTX_ROUTE_SELECTION`). Exhausting- all candidates re-raises the last upstream error, annotated for the- client-facing error envelope.- """- # This backend records its own per-attempt ``outcome_metrics`` counters- # below (one per failover attempt). Claim attempt accounting for this- # request so the endpoint-layer fallback in ``dispatch`` /- # ``handle_chain_exception`` does not double-count the retry fan-out.- ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] = True- api_key_override = self._api_key_override_for_policy(- ctx.metadata.get(CTX_CALLER_API_KEY)- )- # ``caller_required`` never falls back to the configured endpoint key:- # reject before any upstream call so the service credential can't- # authenticate caller inference — and before the affinity lookup, so a- # doomed request never pays a pin resolution (possibly a shared-store- # round-trip). The rejection deliberately stays invisible to- # ``switchyard_upstream_attempts_total`` (the accounting flag above is- # already claimed): no upstream attempt happened, and counting a- # synthetic 401 would skew the direct-to-endpoint baseline error rate.- # ``api_key_override`` is ``None`` here only when no usable caller key- # was supplied.- if self._config.credential_policy == "caller_required" and api_key_override is None:- _reject_missing_caller_api_key(ctx)-- # Captured before the per-attempt ``body["model"]`` override so the span- # records the model the client asked for, not the selected endpoint.- incoming_model = request.model- # One correlation id per client request, shared by every failover- # attempt's spend-logs header and by the response headers — the join- # key between a front proxy's spend-log row and the provider rows.- correlation_id = str(uuid.uuid4())- # Resolve the session-affinity pin once (keyed on the stable conversation- # prefix). ``None`` when affinity is disabled or the conversation isn't- # pinned yet, so every attempt routes purely by health + latency.- pinned_endpoint = await self._affinity.pinned(ctx, request)-- last_exc: Exception | None = None- # Upstream model of the last failed attempt, surfaced on the error- # envelope/span/log so operators can tell *which* endpoint the- # provider-originated failure came from.- last_upstream_model: str | None = None- tried: set[str] = set()-- for attempt in range(1 + self._config.max_retries):- # -- Route decision span: which endpoint, out of which candidates --- with routing_span("switchyard.route_decision") as route_span:- decision = self._select_endpoint_decision(pinned_endpoint, frozenset(tried))- model_id = decision.selected- candidates = decision.candidates- was_fastest = decision.was_fastest- if model_id in tried and len(tried) < len(self._clients):- remaining = [m for m in self._clients if m not in tried]- model_id = random.choice(remaining)- # A forced dedup pick, not the latency-weighted choice.- candidates = tuple(remaining)- was_fastest = False- tried.add(model_id)- # Count the warm-reuse outcome once per request (the first- # decision); later attempts are failover, not an affinity signal.- if self._affinity.enabled and attempt == 0:- if decision.affinity:- self._affinity_hits += 1- else:- self._affinity_misses += 1- set_tags(route_span, {- "switchyard.model": incoming_model,- "switchyard.candidate_endpoints": ",".join(candidates),- "switchyard.selected_endpoint": model_id,- "switchyard.was_fastest_selected": was_fastest,- "switchyard.affinity_hit": decision.affinity,- "switchyard.latency_service_poll_age_ms": self._poll_age_ms(),- })-- upstream_model = self._upstream_models[model_id]- target_request_type = self._request_types[model_id]- body = self._body_for_endpoint_request_type(ctx, request, target_request_type)- body["model"] = upstream_model- # Per-attempt route-selection record: each upstream call is stamped- # with the endpoint it actually hits, so a failover retry's- # spend-log row doesn't inherit the failed attempt's selection.- route_selection = self._route_selection(- incoming_model=incoming_model,- model_id=model_id,- upstream_model=upstream_model,- correlation_id=correlation_id,- )- log.debug(- "LatencyServiceLLMBackend: attempt=%d model=%s upstream=%s "- "request_type=%s stream=%s",- attempt + 1,- model_id,- upstream_model,- target_request_type.value,- body.get("stream"),- )-- # -- Upstream attempt span: outcome of this one upstream call ------ with routing_span("switchyard.upstream_attempt") as attempt_span:- set_tags(attempt_span, {- "switchyard.model": incoming_model,- "switchyard.selected_endpoint": model_id,- "switchyard.retry_count": attempt,- })- started_at = time.monotonic()- try:- result = await self._call_endpoint(- model_id,- target_request_type,- api_key=api_key_override,- body=body,- extra_headers={- SPEND_LOGS_METADATA_HEADER: json.dumps(route_selection),- },- )- except APIStatusError as exc:- set_tags(attempt_span, {- "switchyard.outcome": outcome_metrics.classify(exc.status_code),- "switchyard.upstream_status_code": exc.status_code,- "switchyard.error_code": outcome_metrics.code_label(exc.status_code),- "switchyard.error_source": ERROR_SOURCE_PROVIDER,- "switchyard.upstream_model": upstream_model,- })- if self._stats is not None:- await self._stats.record_error(model_id)- outcome_metrics.record_upstream_attempt(exc.status_code)- self._record_model_attempt(- incoming_model,- upstream_model,- outcome_metrics.classify(exc.status_code),- outcome_metrics.code_label(exc.status_code),- )- # Per-event structured log (Loki) — the timestamped complement- # to the aggregate outcome counter recorded above.- log_upstream_attempt_failure(- model=model_id,- attempt=attempt + 1,- status_code=exc.status_code,- error=exc,- upstream_model=upstream_model,- )- last_exc = exc- last_upstream_model = upstream_model- # A client error (4xx other than 408/429) is deterministic —- # replicas reject the same payload identically — so fail fast- # instead of burning attempts; the post-loop passthrough- # surfaces the upstream status to the client.- if not _is_retryable_status(exc.status_code):- break- continue- except Exception as exc:- set_tags(attempt_span, {- "switchyard.outcome": "retryable_error",- "switchyard.error_code": outcome_metrics.NO_STATUS_CODE,- "switchyard.error_source": ERROR_SOURCE_PROVIDER,- "switchyard.upstream_model": upstream_model,- })- if self._stats is not None:- await self._stats.record_error(model_id)- # Non-HTTP failure (network, pre-status timeout, SDK error) —- # treated as a retryable_error in the outcome ratio: those- # are exactly the faults a health-aware router should absorb- # by selecting a different endpoint on the next attempt.- outcome_metrics.record_upstream_attempt(None)- self._record_model_attempt(- incoming_model,- upstream_model,- "retryable_error",- outcome_metrics.NO_STATUS_CODE,- )- # status_code=None → logged as code="none" (no HTTP status).- log_upstream_attempt_failure(- model=model_id,- attempt=attempt + 1,- status_code=None,- error=exc,- upstream_model=upstream_model,- )- last_exc = exc- last_upstream_model = upstream_model- continue-- backend_latency_ms = (time.monotonic() - started_at) * 1000.0- set_tags(attempt_span, {- "switchyard.outcome": "success",- "switchyard.upstream_status_code": 200,- })- if self._stats is not None:- await self._stats.record_success(model_id, backend_latency_ms)- outcome_metrics.record_upstream_attempt(200)- self._record_model_attempt(- incoming_model, upstream_model, "success", "200",- )- # A successful attempt after at least one failure is direct- # evidence the steering logic rescued this request. Counted- # once per client request, not per recovered retry.- if attempt > 0:- outcome_metrics.record_retry_recovered()-- # ``ctx.selected_model`` is the cross-language hook the Rust- # ``StatsResponseProcessor`` reads to attribute token usage and- # end-to-end latency per endpoint. Without it, the response- # processor sees no ``BackendSelection`` and buckets every call- # to ``model="<unknown>"`` on /metrics.- ctx.selected_model = model_id- # ``backend_call_latency_ms`` lets the Rust ``StatsResponseProcessor``- # compute ``routing_overhead_ms = total_latency - backend_latency``- # for this Python-only backend, which can't be wrapped by the- # Rust ``StatsLlmBackend`` that normally publishes this signal.- ctx.backend_call_latency_ms = backend_latency_ms-- # The selection that actually served the request, surfaced to- # the endpoint layer as ``x-switchyard-*`` response headers so- # a front proxy can enrich its own spend-log row with the same- # correlation id the provider row received.- ctx.metadata[CTX_ROUTE_SELECTION] = route_selection-- # Pin this conversation to the endpoint that served it so later- # turns reuse it (warm cache). Re-pinning on every success also- # follows a recovery: if the previous pin degraded and we- # re-routed, the endpoint that worked becomes the new pin.- # (No-op when affinity is disabled.)- await self._affinity.pin(ctx, request, model_id)-- return _chat_response_for_request_type(target_request_type, result)-- # All attempts failed. If the last failure was an upstream HTTP- # error (e.g. 401 from a bad API key), record the status code and- # body on ctx so the endpoint can passthrough the upstream status- # rather than masking it as a 500. The exception itself still- # propagates — the chain is errored — but the endpoint reads ctx- # to decide the response code.- if isinstance(last_exc, APIStatusError):- ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = last_exc.status_code- upstream_body = _extract_upstream_body(last_exc)- if upstream_body is not None:- ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = upstream_body-- # Failure-origin annotation for the client-facing error envelope:- # every exhausted attempt failed at a selected upstream, so what the- # endpoint surfaces — the HTTP passthrough above, or the 500 for- # status-less network failures — is provider-originated. (The stash- # helpers below mark Switchyard-originated rejections instead.)- ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_PROVIDER- if last_upstream_model is not None:- ctx.metadata[CTX_UPSTREAM_MODEL] = last_upstream_model-- raise last_exc # type: ignore[misc]-- def _body_for_endpoint_request_type(- self,- ctx: ProxyContext,- request: ChatRequest,- target_request_type: ChatRequestType,- ) -> dict[str, object]:- try:- normalized = self._translation.request_to_any_of(- request, [target_request_type],- )- except ValueError as exc:- # Transparent-router contract: a payload the upstream provider would- # reject (e.g. an unsupported message role like "api") must surface- # as a provider-compatible 400, not a generic 500. The Rust- # translation layer raises a ``ValueError`` whose message is- # prefixed with the stable error kind; an invalid-value rejection is- # recorded as an upstream 400 so the endpoint passes it through.- _stash_invalid_request_error(ctx, exc)- raise- if not request_type_matches(normalized, target_request_type):- raise TypeError(- "LatencyServiceLLMBackend expected request type "- f"{target_request_type.value} after translation"- )- return dict(normalized.body)-- def _route_selection(- self,- *,- incoming_model: str | None,- model_id: str,- upstream_model: str,- correlation_id: str,- ) -> dict[str, str | None]:- """Build the route-selection payload for one upstream attempt.-- This is the JSON value of the outbound ``x-litellm-spend-logs-metadata``- header and, for the attempt that succeeds, the- :data:`CTX_ROUTE_SELECTION` record behind the ``x-switchyard-*``- response headers. ``router_model`` falls back to the configured route id- when the client body carried no model; ``router_selected_provider`` is- the upstream model's leading path segment (IH/LiteLLM naming, e.g.- ``"openai/openai/gpt-5.4"`` → ``"openai"``).- """- return {- "router_model": incoming_model or self._config.route_model,- "router_strategy": ROUTER_STRATEGY_LATENCY,- "router_selected_endpoint": model_id,- "router_selected_model": upstream_model,- "router_selected_provider": upstream_model.split("/", 1)[0],- "router_correlation_id": correlation_id,- }-- async def _call_endpoint(- self,- model_id: str,- target_request_type: ChatRequestType,- *,- api_key: str | None,- body: dict[str, object],- extra_headers: dict[str, str],- ) -> object:- """Make one upstream SDK call with the protected *extra_headers* stamped on.-- ``extra_headers`` rides the client wrapper's ``**kwargs`` into the- OpenAI SDK's per-request header merge (over ``default_headers``). A- passthrough body may itself carry an SDK-style ``extra_headers`` field- (shape-preserving translation keeps unknown keys); it is still- forwarded — it used to ride ``**body`` into the same SDK parameter —- but the protected headers must win the merge so callers can't spoof- them, and header names are case-insensitive on the wire while dict- keys are not, so every case variant of a protected name is stripped- from the client mapping (a client-cased duplicate would otherwise- reach the wire alongside ours and win first-match parsing on the- receiving proxy). A non-mapping value is dropped: it could only ever- have broken the SDK call.- """- client_extra = body.pop("extra_headers", None)- protected = {name.lower() for name in extra_headers}- headers: dict[str, object] = (- {- name: value- for name, value in client_extra.items()- if not (isinstance(name, str) and name.lower() in protected)- }- if isinstance(client_extra, Mapping)- else {}- )- headers.update(extra_headers)- if target_request_type == ChatRequestType.OPENAI_RESPONSES:- return await self._clients[model_id].aresponses(- api_key=api_key,- extra_headers=headers,- **body,- )- return await self._clients[model_id].acompletion(- api_key=api_key,- extra_headers=headers,- **body,- )-- def _api_key_override_for_policy(self, caller_api_key: object) -> str | None:- """Per-request key to forward upstream, or ``None`` to use the endpoint key.-- ``configured_endpoint`` never forwards a caller key. ``caller_override``- and ``caller_required`` forward a usable caller key; they differ only in- the no-key case, which ``call`` handles before any upstream attempt- (``caller_override`` falls back to the endpoint key, ``caller_required``- returns 401).- """- if self._config.credential_policy == "configured_endpoint":- return None- if isinstance(caller_api_key, str) and caller_api_key.strip():- return caller_api_key- return None-- def _record_model_attempt(- self,- requested_model: str | None,- upstream_model: str,- outcome: str,- code: str,- ) -> None:- """Count one upstream attempt for the per-model /metrics breakdown.-- Event-loop only (no lock). ``requested_model`` is the client-supplied- model; it is bounded to a config-derived id — the route id- (``config.route_model``) or a configured endpoint id — with the- ``"other"`` sentinel as fallback before becoming a Prometheus label, so- caller-controlled input can't create unbounded metric-series- cardinality.- """- requested = (- requested_model if requested_model in self._requested_model_ids else "other"- )- key = (requested, upstream_model, outcome, code)- self._upstream_attempts_by_model[key] = (- self._upstream_attempts_by_model.get(key, 0) + 1- )-- # -- Lifecycle ------------------------------------------------------------ async def shutdown(self) -> None:- """Stop the background :class:`HealthPoller` daemon and release the- shared session-affinity store, when one is configured.-- Picked up automatically by the server's component teardown hook- (``_run_lifecycle_method``), which awaits awaitable ``shutdown``- results. Safe to call multiple times.- """- prometheus_emitter.unregister(self._render_prometheus_lines)- self._poller.stop()- await self._affinity.aclose()-- def is_ready(self) -> bool:- """True once the background poller has completed at least one successful poll."""- return self._poller.has_polled-- # -- Metrics emitter ------------------------------------------------------ def _render_prometheus_lines(self) -> list[str]:- """Emit per-endpoint health verdicts and poll-loop health gauges.-- Snapshotted under ``self._cache_lock`` so a poll concurrent with a- scrape can't produce a mixed view across endpoints. Output is- Prometheus exposition lines without trailing newline — composed- by :mod:`switchyard.lib.endpoints.prometheus_emitter`.- """- with self._cache_lock:- snapshot = dict(self._health_cache)-- lines: list[str] = []- lines.append(- "# HELP switchyard_endpoint_status "- "Latency-Service verdict per endpoint (1 = current status; "- "exactly one status row per model is non-zero)."- )- lines.append("# TYPE switchyard_endpoint_status gauge")- for model_id, health in sorted(snapshot.items()):- current = health.status.value- for status in EndpointHealthStatus:- value = 1 if status.value == current else 0- labels = render_labels({"model": model_id, "status": status.value})- lines.append(f"switchyard_endpoint_status{labels} {value}")-- lines.append(- "# HELP switchyard_endpoint_last_latency_ms "- "Last latency sample (ms) reported by the Latency Service per endpoint. "- "Absent until the first poll publishes a non-null sample."- )- lines.append("# TYPE switchyard_endpoint_last_latency_ms gauge")- for model_id, health in sorted(snapshot.items()):- if health.last_latency_ms is None:- continue- labels = render_labels({"model": model_id})- lines.append(- f"switchyard_endpoint_last_latency_ms{labels} "- f"{format_number(health.last_latency_ms)}"- )-- lines.append(- "# HELP switchyard_latency_service_poll_ok "- "1 when the last poll succeeded, 0 when it failed or has not yet run."- )- lines.append("# TYPE switchyard_latency_service_poll_ok gauge")- last_age = self._poller.seconds_since_last_success- # ``poll_ok`` reflects the latest poll attempt. Combined with- # ``poll_age_seconds``, scrapers can tell "never polled" (no age line)- # from "polled, but the latest attempt failed" (age present, ok=0).- poll_ok = 1 if self._poller.last_poll_ok else 0- lines.append(f"switchyard_latency_service_poll_ok {poll_ok}")-- lines.append(- "# HELP switchyard_latency_service_poll_age_seconds "- "Monotonic seconds since the last successful poll. Absent before the "- "first success."- )- lines.append("# TYPE switchyard_latency_service_poll_age_seconds gauge")- if last_age is not None:- lines.append(- "switchyard_latency_service_poll_age_seconds "- f"{format_number(last_age)}"- )-- lines.append(- "# HELP switchyard_latency_service_polls_total "- "Total successful health polls since the poller started."- )- lines.append("# TYPE switchyard_latency_service_polls_total counter")- lines.append(- f"switchyard_latency_service_polls_total {self._poller.poll_successes}"- )-- lines.append(- "# HELP switchyard_latency_service_poll_failures_total "- "Total failed health polls; each failure resets every endpoint to "- "UNKNOWN."- )- lines.append("# TYPE switchyard_latency_service_poll_failures_total counter")- lines.append(- "switchyard_latency_service_poll_failures_total "- f"{self._poller.poll_failures}"- )-- # Warm-reuse counters — only meaningful (and only emitted) when session- # affinity is enabled, so the metric surface stays clean for the common- # per-turn-routing case. Reuse rate = hits / (hits + misses).- if self._config.session_affinity:- lines.append(- "# HELP switchyard_affinity_hits_total "- "Conversation turns served by an existing session-affinity pin "- "(warm endpoint reuse)."- )- lines.append("# TYPE switchyard_affinity_hits_total counter")- lines.append(f"switchyard_affinity_hits_total {self._affinity_hits}")-- lines.append(- "# HELP switchyard_affinity_misses_total "- "First or unpinnable turns routed by the latency-aware picker "- "while session affinity was enabled."- )- lines.append("# TYPE switchyard_affinity_misses_total counter")- lines.append(f"switchyard_affinity_misses_total {self._affinity_misses}")-- # Shared-store (L2) counters — emitted only when a shared pin store- # is configured. Hits measure cross-worker/churn pin reuse; errors- # count fail-open store operations (the alerting signal for a store- # that is degraded while requests keep succeeding).- if self._affinity.l2_enabled:- lines.append(- "# HELP switchyard_affinity_l2_hits_total "- "Pins resolved from the shared (L2) affinity store after an "- "in-process (L1) miss — cross-worker warm reuse."- )- lines.append("# TYPE switchyard_affinity_l2_hits_total counter")- lines.append(- f"switchyard_affinity_l2_hits_total {self._affinity.l2_hits}"- )-- lines.append(- "# HELP switchyard_affinity_l2_errors_total "- "Shared (L2) affinity-store operations that failed open "- "(get or put); routing fell back to in-process pins only."- )- lines.append("# TYPE switchyard_affinity_l2_errors_total counter")- lines.append(- f"switchyard_affinity_l2_errors_total {self._affinity.l2_errors}"- )-- lines.append(- "# HELP switchyard_affinity_l2_breaker_open "- "1 while the shared-store circuit breaker is open "- "(operations skipped without a network attempt); 0 when closed."- )- lines.append("# TYPE switchyard_affinity_l2_breaker_open gauge")- lines.append(- "switchyard_affinity_l2_breaker_open "- f"{int(self._affinity.l2_breaker_open)}"- )-- # Per-model upstream-attempt breakdown — the latency-service-scoped- # complement to the layer-aggregate ``switchyard_upstream_attempts_total``.- # Series are created lazily per (requested_model, upstream_model, outcome,- # code), so the surface stays empty until traffic flows. Snapshotted to a- # plain dict for a stable view across the emission loop.- attempts_by_model = dict(self._upstream_attempts_by_model)- if attempts_by_model:- lines.append(- "# HELP switchyard_latency_upstream_attempts_total "- "Upstream call attempts from the latency-service backend, labelled "- "by requested (route) model, selected upstream model, outcome, and "- "HTTP status code (code=\"none\" for non-HTTP failures). Per-model "- "complement to the layer-aggregate switchyard_upstream_attempts_total; "- "cardinality is bounded by the configured endpoint set."- )- lines.append("# TYPE switchyard_latency_upstream_attempts_total counter")- # Sorted for deterministic exposition order across scrapes.- for (requested_model, upstream_model, outcome, code), count in sorted(- attempts_by_model.items()- ):- labels = render_labels({- "requested_model": requested_model,- "upstream_model": upstream_model,- "outcome": outcome,- "code": code,- })- lines.append(- f"switchyard_latency_upstream_attempts_total{labels} {count}"- )- return lines---def _latency_endpoint_request_type(value: str) -> ChatRequestType:- if value == "openai_responses":- return ChatRequestType.OPENAI_RESPONSES- return ChatRequestType.OPENAI_CHAT---def _chat_response_for_request_type(- request_type: ChatRequestType,- result: object,-) -> ChatResponse:- if request_type == ChatRequestType.OPENAI_RESPONSES:- # Streaming yields raw SSE frame strings (``RawSSEFrameStream``) so the- # upstream events pass through verbatim; anything async-iterable is a- # stream, a plain mapping is the exact non-streaming JSON body.- if isinstance(result, AsyncStream) or hasattr(result, "__aiter__"):- return ChatResponse.openai_responses_stream(ResponsesApiStream(result))- return ChatResponse.openai_responses_completion(result)- if isinstance(result, AsyncStream):- return ChatResponse.openai_stream(ResponseStream(result))- return ChatResponse.openai_completion(result)---def _is_retryable_status(status_code: int) -> bool:- """Whether an upstream status warrants failing over to a different endpoint.-- Retries 5xx + 408 + 429 (transient/capacity/server-side); other 4xx are the- client's payload, which every replica rejects identically, so fail fast.- Deliberately broader than ``outcome_metrics.RETRYABLE_STATUSES`` (metrics- bucketing) — e.g. 502/503 must fail over but aren't metrics-retryable.- """- return status_code >= 500 or status_code in (408, 429)---def _affinity_usable(health: EndpointHealth | None) -> bool:- """Whether a pinned endpoint may still serve its session.-- The pin holds while the endpoint is HEALTHY or UNKNOWN — the same tiers the- latency-aware picker trusts. A DEGRADED verdict (or an endpoint that- vanished from the health cache) breaks the pin so the next turn re-routes:- locality must yield to health, or sticky routing would funnel a session- into a failing endpoint.- """- return health is not None and health.status in (- EndpointHealthStatus.HEALTHY,- EndpointHealthStatus.UNKNOWN,- )---def _build_affinity_l2(config: LatencyServiceBackendConfig) -> AffinityPinStore | None:- """Build the optional shared L2 pin store from config (``None`` = L1-only).-- ``affinity_store="memory"`` (the default) keeps pins per process. ``"redis"``- imports :class:`RedisPinStore` lazily so the ``redis`` dependency stays- optional. Config validation guarantees a URL when the store is Redis.- """- if config.affinity_store != "redis":- return None- from switchyard.lib.redis_pin_store import RedisPinStore-- assert config.affinity_store_url is not None # enforced by config validator- return RedisPinStore(- config.affinity_store_url,- ttl_seconds=config.affinity_store_ttl_seconds,- key_prefix=config.affinity_key_prefix,- )---# Prefix of the stringified translation error for a client-invalid payload.-# ``TranslationError::kind()`` is the stable FFI signal for the error variant-# (``crates/switchyard-translation/src/error.rs``); ``py_translation_error``-# formats translation errors as ``"{kind}: {message}"``, so an invalid-value-# rejection (e.g. an unsupported message role) is recognizable by this prefix.-_TRANSLATION_INVALID_VALUE_PREFIX = "InvalidValue:"---def _stash_invalid_request_error(ctx: ProxyContext, error: Exception) -> None:- """Record a translation invalid-value error as an upstream HTTP 400 on ``ctx``.-- Mirrors the upstream-status passthrough used for provider HTTP errors: the- Compatibility execution flattens the raised exception to a generic error- string, but ``ctx.metadata`` survives the error path, so the endpoint's- ``upstream_response_from_ctx`` can return a provider-compatible 400. This is- a no-op for any other error (it keeps propagating and FastAPI maps it to a- 500 as before).- """- message = str(error)- if not message.startswith(_TRANSLATION_INVALID_VALUE_PREFIX):- return- detail = message[len(_TRANSLATION_INVALID_VALUE_PREFIX) :].strip()- ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 400- ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = {- "error": {- "message": detail or "invalid request payload",- "type": "invalid_request_error",- "code": "invalid_value",- }- }- # This 400 rides the upstream-status channel but is Switchyard's own- # translation rejection — label it so the error-source header is truthful.- ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_SWITCHYARD---def _reject_missing_caller_api_key(ctx: ProxyContext) -> None:- """Reject a ``caller_required`` request that carries no caller API key.-- Stashes an HTTP 401 and a provider-compatible error envelope on ``ctx`` the- same way upstream-status passthrough does, then raises so the chain errors- out before any upstream call. No upstream is contacted or counted;- ``upstream_response_from_ctx`` reads the stashed status to return a 401. This- is what keeps the configured endpoint key from ever authenticating caller- inference under the ``caller_required`` policy.- """- ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 401- ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = {- "error": {- "message": "missing caller API key; supply it via the x-switchyard-api-key header",- "type": "invalid_request_error",- "code": "missing_caller_api_key",- }- }- # Rides the upstream-status channel but no upstream was contacted — this- # is Switchyard's own credential-policy rejection.- ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_SWITCHYARD- raise PermissionError("caller_required policy: missing caller API key")---def _extract_upstream_body(error: APIStatusError) -> object | None:- """Best-effort extraction of the upstream response body for passthrough.-- Prefers the structured ``body`` attribute (set by the OpenAI SDK for- JSON responses). Falls back to ``response.text`` for responses the- SDK couldn't decode. Returns ``None`` when nothing usable is present- — the endpoint then synthesizes a generic error envelope.- """- body: object | None = getattr(error, "body", None)- if body is not None:- return body- response = getattr(error, "response", None)- if response is None:- return None- text: object | None = getattr(response, "text", None)- return text if text else Nonediff --git a/switchyard/lib/config/__init__.py b/switchyard/lib/config/__init__.pyindex f21a9bf7d..1c74cba12 100644--- a/switchyard/lib/config/__init__.py+++ b/switchyard/lib/config/__init__.py@@ -7,14 +7,8 @@ from switchyard.lib.config.intake_sink_config import (IntakeQueueFullPolicy,IntakeSinkConfig,)-from switchyard.lib.config.latency_service_backend_config import (- LatencyServiceBackendConfig,- LatencyServiceEndpoint,-)__all__ = ["IntakeQueueFullPolicy","IntakeSinkConfig",- "LatencyServiceBackendConfig",- "LatencyServiceEndpoint",]diff --git a/switchyard/lib/config/latency_service_backend_config.py b/switchyard/lib/config/latency_service_backend_config.pydeleted file mode 100644index f52da2c39..000000000--- a/switchyard/lib/config/latency_service_backend_config.py+++ /dev/null@@ -1,168 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Configuration models for the Latency Service usage case.--``LatencyServiceEndpoint`` describes one LLM backend monitored by the-Latency Service. ``LatencyServiceBackendConfig`` bundles the full-backend configuration — URL of the Latency Service, the endpoint list,-and polling/retry parameters.--The ``model`` field on each endpoint doubles as the endpoint ID used by-the Latency Service's health API — mirroring the routing-by-model-name-convention the rest of the library already follows.-"""--from typing import Literal, Self--from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator--LatencyServiceRequestType = Literal["openai_chat", "openai_responses"]-LatencyServiceCredentialPolicy = Literal[- "configured_endpoint", "caller_override", "caller_required"-]---class LatencyServiceEndpoint(BaseModel):- """One LLM backend registered with the Latency Service.-- The ``model`` field is the endpoint ID used by the Latency Service —- it must be unique across the endpoint list and it is the value the- Latency Service returns health verdicts under. By default it is also- the value written into ``body["model"]`` when calling the upstream;- set ``upstream_model`` when the upstream expects a different name- (e.g. routing the latency-service key ``"openai/gpt-5.5"`` through an- IH gateway that expects ``"openai/openai/gpt-5.5"``).-- Attributes:- model: Latency-service lookup key. Must be unique across the- endpoint list. Also used as ``body["model"]`` unless- ``upstream_model`` is set.- upstream_model: Optional override for ``body["model"]`` sent to- the upstream LLM. Defaults to ``model`` when ``None``.- api_key: API key for the backing LLM API.- base_url: Base URL for the backing LLM API (include ``/v1``).- timeout: Request timeout in seconds, forwarded to the underlying- ``OpenAILLMClient``. ``None`` uses the client default.- request_type: Upstream OpenAI API surface used for this endpoint.- ``"openai_chat"`` sends ``/v1/chat/completions``; ``"openai_responses"``- sends ``/v1/responses`` for Responses-only upstream models.- """-- model_config = ConfigDict(frozen=True)-- model: str- upstream_model: str | None = None- api_key: str | None = None- base_url: str | None = None- timeout: float | None = None- request_type: LatencyServiceRequestType = "openai_chat"-- @field_validator("request_type", mode="before")- @classmethod- def _normalize_request_type(cls, value: object) -> object:- if value == "chat":- return "openai_chat"- if value == "responses":- return "openai_responses"- return value---class LatencyServiceBackendConfig(BaseModel):- """Configuration for :class:`LatencyServiceLLMBackend`.-- Attributes:- latency_service_url: Base URL of the Latency Service- (e.g. ``"http://latency-service.inference-hub.svc:8080"``).- endpoints: LLM backends to route across. Each must have a- unique ``model`` — this is the routing + health-lookup key.- poll_interval_s: How often the background poller refreshes- health from the Latency Service. Health is cached between- polls; the request hot path never blocks on a network call.- poll_timeout_s: Timeout for the health API call.- max_retries: On error, retry on a different endpoint up to this- many times. Dedup prevents re-selecting an endpoint that- already failed for the same request.- route_model: Client-facing route id this backend serves (the- route-table / YAML route key, e.g.- ``"nvidia/switchyard/gpt-5.4"``). Metrics-only: it joins the- bounded ``requested_model`` label set on- ``switchyard_latency_upstream_attempts_total`` so route-key- traffic is attributed instead of collapsing to ``"other"``.- Has no effect on routing.- credential_policy: Which credential authenticates the upstream call.- The caller key is read from the ``x-switchyard-api-key`` header- (preferred — it survives proxies such as LiteLLM that strip- ``Authorization``), then ``Authorization: Bearer`` / ``x-api-key``, on- every ingress path (``/chat/completions``, ``/responses``,- ``/messages``). ``"configured_endpoint"`` always uses each endpoint's- configured ``api_key`` and ignores any caller key. ``"caller_override"``- opts into BYO-key forwarding: a caller key is used when present, else- the call falls back to the configured ``api_key``. ``"caller_required"``- forwards the caller key but never falls back — a request with no caller- key is rejected with HTTP 401 and the configured ``api_key`` is never- used for upstream inference (use this for per-user spend attribution,- e.g. multi-tenant gateway routes).- session_affinity: When ``True``, pin each conversation to the endpoint- that first served it (cache stays warm); a pin is broken only when- its endpoint degrades or the call fails. Per process. Default off.- affinity_max_sessions: Bounded-LRU cap on pinned conversations; ignored- when ``session_affinity`` is off.- affinity_store: Shared L2 pin store behind the in-process LRU. ``"memory"``- (default) keeps pins per process; ``"redis"`` shares them across- workers/pods and persists them across pod churn. The store is- best-effort — an L2 error never fails a request.- affinity_store_url: Connection URL for the shared store (e.g.- ``"redis://host:6379/0"``). Required when ``affinity_store`` is- ``"redis"``.- affinity_store_ttl_seconds: Expiry for a shared pin. The backend re-pins- on every successful turn, so an active conversation slides its TTL.- affinity_key_prefix: Namespace prefix for shared-store keys.- enable_stats: When ``True`` (default), the factory wires a- :class:`StatsRequestProcessor` + :class:`StatsResponseProcessor`- pair sharing one :class:`StatsAccumulator` and wraps the- backend in :class:`StatsLlmBackend`, so the chain contributes- ``GET /metrics``, ``GET /v1/stats``, and the legacy- ``GET /v1/routing/stats`` aliases via the standard- ``get_endpoint()`` mechanism in :func:`build_switchyard_app`.- """-- model_config = ConfigDict(frozen=True, extra="forbid")-- latency_service_url: str = ""- endpoints: list[LatencyServiceEndpoint] = Field(default_factory=list)- route_model: str | None = None- poll_interval_s: float = 10.0- poll_timeout_s: float = 5.0- max_retries: int = 2- credential_policy: LatencyServiceCredentialPolicy = "configured_endpoint"- session_affinity: bool = False- affinity_max_sessions: int = Field(default=10_000, ge=0)- affinity_store: Literal["memory", "redis"] = "memory"- affinity_store_url: str | None = None- affinity_store_ttl_seconds: int = Field(default=3_600, gt=0)- affinity_key_prefix: str = "swyd:pin:"- enable_stats: bool = True-- @model_validator(mode="after")- def _affinity_capacity_nonzero_when_enabled(self) -> Self:- # A zero-capacity affinity store retains nothing — silently non-sticky.- if self.session_affinity and self.affinity_max_sessions == 0:- raise ValueError(- "affinity_max_sessions must be > 0 when session_affinity is enabled"- )- return self-- @model_validator(mode="after")- def _redis_store_requires_url_and_affinity(self) -> Self:- # A shared store is dead config unless affinity is on and reachable.- if self.affinity_store == "redis":- if not self.session_affinity:- raise ValueError(- 'affinity_store="redis" requires session_affinity to be enabled'- )- if not self.affinity_store_url:- raise ValueError(- 'affinity_store="redis" requires affinity_store_url to be set'- )- return selfdiff --git a/switchyard/lib/endpoints/prometheus_emitter.py b/switchyard/lib/endpoints/prometheus_emitter.pyindex e49299285..aafe0274c 100644--- a/switchyard/lib/endpoints/prometheus_emitter.py+++ b/switchyard/lib/endpoints/prometheus_emitter.py@@ -6,9 +6,8 @@The accumulator-derived exposition rendered by:func:`switchyard.lib.prometheus_exposition.render_prometheus` coversrequest flow (counts, tokens, latency). Components that own state which-is not request-derived — Latency Service verdicts, poll-loop health,-endpoint-level live samples — register an emitter here so their lines-appear on the same ``/metrics`` scrape rather than a sidecar URL.+is not request-derived register an emitter here so their lines appear on+the same ``/metrics`` scrape rather than a sidecar URL.Single-process table by design: a Switchyard server is one process,emitters are owned by component lifetimes, and the table isdiff --git a/switchyard/lib/endpoints/stats_endpoint.py b/switchyard/lib/endpoints/stats_endpoint.pyindex 31d96f94b..3ffdc2674 100644--- a/switchyard/lib/endpoints/stats_endpoint.py+++ b/switchyard/lib/endpoints/stats_endpoint.py@@ -68,10 +68,9 @@ class StatsEndpoint(NemoSwitchyardEndpoint):async def get_metrics() -> Response:"""Prometheus text-format exposition of the shared stats snapshot.- Components that own non-request-derived state (Latency Service- verdicts, poll-loop health) contribute extra lines via- :mod:`switchyard.lib.endpoints.prometheus_emitter` so a single- ``/metrics`` scrape carries both surfaces.+ Components that own non-request-derived state contribute extra+ lines via :mod:`switchyard.lib.endpoints.prometheus_emitter` so a+ single ``/metrics`` scrape carries both surfaces."""snapshot = await stats.snapshot()outcome_block = "\n".join(outcome_metrics.render_lines()) + "\n"diff --git a/switchyard/lib/endpoints/upstream_error.py b/switchyard/lib/endpoints/upstream_error.pyindex 285eda79b..66de361d2 100644--- a/switchyard/lib/endpoints/upstream_error.py+++ b/switchyard/lib/endpoints/upstream_error.py@@ -3,9 +3,9 @@"""Endpoint-side helper: convert chain exceptions into upstream-status responses.-Python LLM backends (e.g. :class:`LatencyServiceLLMBackend`) stash the-upstream HTTP status / body into ``ctx.metadata`` before raising the-upstream provider's exception. Rust backends attach typed+Python LLM backends stash the upstream HTTP status / body into+``ctx.metadata`` before raising the upstream provider's exception. Rust+backends attach typed``status_code`` and ``body`` attributes to ``SwitchyardUpstreamError`` soendpoints can preserve provider failures without parsing exception text."""@@ -53,9 +53,8 @@ def record_upstream_attempt_success(ctx: ProxyContext) -> None:to one successful attempt observable here.No-op when a backend already counted its own attempts- (:data:`CTX_UPSTREAM_ATTEMPTS_RECORDED` set) — e.g.- :class:`LatencyServiceLLMBackend`, whose retry fan-out must not be- double-counted here.+ (:data:`CTX_UPSTREAM_ATTEMPTS_RECORDED` set) — its retry fan-out must not+ be double-counted here."""if ctx.metadata.get(CTX_UPSTREAM_ATTEMPTS_RECORDED):returndiff --git a/switchyard/lib/profiles/__init__.py b/switchyard/lib/profiles/__init__.pyindex b7d3de6dc..1b06d983f 100644--- a/switchyard/lib/profiles/__init__.py+++ b/switchyard/lib/profiles/__init__.py@@ -21,7 +21,6 @@ from switchyard.lib.profiles.header_routing import (HeaderRoutingDecision,HeaderRoutingProfile,)-from switchyard.lib.profiles.latency_service import LatencyServiceProfileConfigfrom switchyard.lib.profiles.noop import NoopProfile, NoopProfileConfigfrom switchyard.lib.profiles.passthrough import PassthroughProfileConfigfrom switchyard.lib.profiles.protocols import (@@ -61,7 +60,6 @@ __all__ = ["HeaderRoutingConfig","HeaderRoutingDecision","HeaderRoutingProfile",- "LatencyServiceProfileConfig","NoopProfile","NoopProfileConfig","PassthroughProfileConfig",diff --git a/switchyard/lib/profiles/escalation_router_profile_config.py b/switchyard/lib/profiles/escalation_router_profile_config.pyindex 96c59db29..f6f5b4ce8 100644--- a/switchyard/lib/profiles/escalation_router_profile_config.py+++ b/switchyard/lib/profiles/escalation_router_profile_config.py@@ -138,9 +138,9 @@ class EscalationRouterProfileConfig:def _build_affinity_l2(config: EscalationRouterConfig) -> AffinityPinStore | None:"""Build the optional shared L2 latch store (``None`` = L1-only).- Mirrors the latency route's ``affinity_store`` semantics; ``"redis"``- imports :class:`RedisPinStore` lazily so the ``redis`` dependency stays- optional. Config validation guarantees a URL when the store is Redis.+ Uses the shared ``affinity_store`` semantics; ``"redis"`` imports+ :class:`RedisPinStore` lazily so the ``redis`` dependency stays optional.+ Config validation guarantees a URL when the store is Redis."""if config.affinity_store != "redis":return Nonediff --git a/switchyard/lib/profiles/latency_service.py b/switchyard/lib/profiles/latency_service.pydeleted file mode 100644index abdb144df..000000000--- a/switchyard/lib/profiles/latency_service.py+++ /dev/null@@ -1,36 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Profile-owned latency-service routing construction."""--from __future__ import annotations--from typing import Self--from switchyard.lib.config import LatencyServiceBackendConfig-from switchyard.lib.profiles.chain import ComponentChainProfile-from switchyard.lib.profiles.table import profile_config---@profile_config("latency_service")-class LatencyServiceProfileConfig:- """Profile config wrapper for health-aware latency-service profiles."""-- config: LatencyServiceBackendConfig-- @classmethod- def from_config(cls, config: LatencyServiceBackendConfig) -> Self:- """Create a profile config from the validated parsing model."""- return cls(config=config)-- def build(self) -> ComponentChainProfile:- """Build the latency-service profile runtime."""- from switchyard.lib.backends import LatencyServiceLLMBackend-- backend = LatencyServiceLLMBackend(self.config)- return ComponentChainProfile(- backend=backend,- )---__all__ = ["LatencyServiceProfileConfig"]diff --git a/switchyard/lib/proxy_context.py b/switchyard/lib/proxy_context.pyindex 864ddfb61..01c9b78a4 100644--- a/switchyard/lib/proxy_context.py+++ b/switchyard/lib/proxy_context.py@@ -58,8 +58,8 @@ CTX_UPSTREAM_HTTP_BODY = "_upstream_http_body"#: Set truthy by a backend that records its own per-attempt#: ``switchyard.lib.endpoints.outcome_metrics`` upstream-attempt counters-#: (e.g. :class:`LatencyServiceLLMBackend`, which retries across endpoints and-#: must count each attempt). When present, the endpoint layer skips its+#: (e.g. a backend that retries across endpoints and must count each+#: attempt). When present, the endpoint layer skips its#: single-attempt fallback recording for this request so retry fan-out is not#: double-counted. Absent for the Rust native / passthrough / multi backends,#: which issue exactly one upstream attempt per call and have no Python retrydiff --git a/switchyard/lib/redis_pin_store.py b/switchyard/lib/redis_pin_store.pyindex 0cc193540..0b1b24910 100644--- a/switchyard/lib/redis_pin_store.py+++ b/switchyard/lib/redis_pin_store.py@@ -6,8 +6,8 @@Implements :class:`~switchyard.lib.affinity_pin_store.AffinityPinStore` against astandalone Redis so pins are visible to every Switchyard worker/pod and survivepod churn. Keys are namespaced ``{key_prefix}{session_key}`` and expire after-``ttl_seconds``; the latency backend re-pins on every successful turn, so an-active conversation slides its own TTL.+``ttl_seconds``; callers re-pin on every successful turn, so an active+conversation slides its own TTL.The ``redis`` dependency is optional (``switchyard[affinity-redis]``) and isimported lazily, so the default install never pulls it in. Socket timeouts arediff --git a/switchyard/lib/session_affinity.py b/switchyard/lib/session_affinity.pyindex a4ef6e571..1d5f48d74 100644--- a/switchyard/lib/session_affinity.py+++ b/switchyard/lib/session_affinity.py@@ -71,8 +71,8 @@ class SessionAffinity:stickiness: derive a stable conversation key (system prompt + first usermessage, memoized on the request context) and store/look up a pinned valuein a bounded LRU. The pinned value is opaque — a tier label for the- classifier router, an endpoint id for the latency backend. Each caller keeps- its own *policy* (when to write a pin, whether to honor one) on top.+ classifier or escalation router. Each caller keeps its own *policy*+ (when to write a pin, whether to honor one) on top.The in-process ``SessionCache`` is the L1. An optional ``l2`` implementing:class:`~switchyard.lib.affinity_pin_store.AffinityPinStore` is a shared,diff --git a/switchyard/lib/tracing.py b/switchyard/lib/tracing.pyindex ff2954873..5f80a7f15 100644--- a/switchyard/lib/tracing.py+++ b/switchyard/lib/tracing.py@@ -64,8 +64,8 @@ def routing_span(name: str) -> Iterator[Span]:def set_tags(span: Span, tags: Mapping[str, Any]) -> None:"""Set each non-``None`` tag on *span*.- ``None`` values are skipped so an unavailable signal (e.g. no latency-service- poll has completed yet) leaves no empty/misleading tag on the span.+ ``None`` values are skipped so an unavailable signal (e.g. a signal that+ has not been recorded yet) leaves no empty/misleading tag on the span."""for key, value in tags.items():if value is not None:diff --git a/tests/e2e/test_latency_service_llm_backend.py b/tests/e2e/test_latency_service_llm_backend.pydeleted file mode 100644index fbd93dc50..000000000--- a/tests/e2e/test_latency_service_llm_backend.py+++ /dev/null@@ -1,427 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Integration tests for :class:`LatencyServiceLLMBackend`.--These tests make real LLM calls through the latency-service backend-to validate end-to-end routing against an external health source.--A lightweight mock Latency Service runs in-process to supply health-verdicts. The LLM calls go to a real backend (OpenRouter by default,-with NVIDIA Inference Hub as a fallback).--Prerequisites:- - OPENROUTER_API_KEY or NVIDIA_API_KEY environment variable--Run with:- OPENROUTER_API_KEY=sk-or-... pytest tests/e2e/test_latency_service_llm_backend.py -v-"""--import json-import os-import threading-import time-from http.server import BaseHTTPRequestHandler, HTTPServer-from urllib.parse import parse_qs, urlparse--import pytest--from switchyard.lib.backends.health_poller import (- EndpointHealthStatus,-)-from switchyard.lib.backends.latency_service_llm_backend import (- LatencyServiceLLMBackend,-)-from switchyard.lib.config.latency_service_backend_config import (- LatencyServiceBackendConfig,- LatencyServiceEndpoint,-)-from switchyard.lib.proxy_context import ProxyContext-from switchyard_rust.core import (- ChatRequest,- ChatResponseType,- response_type_matches,-)--from .conftest import find_free_port, get_nvidia_config--pytestmark = pytest.mark.integration--_nvidia = get_nvidia_config()-_skip_reason = "OPENROUTER_API_KEY or NVIDIA_API_KEY not set"--BACKEND_BASE_URL = _nvidia["base_url"]--if _nvidia["provider"] == "openrouter":- MODEL_A = _nvidia["model"]- MODEL_B = os.environ.get("OPENROUTER_MODEL_B") or "anthropic/claude-opus-4.7"-else:- MODEL_A = "openai/openai/gpt-5.2"- MODEL_B = "azure/openai/gpt-5.2"--SIMPLE_MESSAGES = [- {"role": "system", "content": "You are a helpful assistant."},- {"role": "user", "content": "Hello! Can you help me?"},-]---# ----------------------------------------------------------------------------# Mock Latency Service-# ------------------------------------------------------------------------------class _HealthState:- """Thread-safe mutable health state for the mock Latency Service."""-- def __init__(self) -> None:- self._lock = threading.Lock()- self._health: dict[str, str] = {}- self.request_count = 0-- def set_health(self, model_id: str, status: str) -> None:- with self._lock:- self._health[model_id] = status-- def get_response(self, requested_ids: list[str]) -> dict:- with self._lock:- self.request_count += 1- return {- "endpoint_health": {- mid: {- "status": self._health.get(mid, "unknown"),- "last_latency_ms": None,- }- for mid in requested_ids- }- }---def _make_handler(state: _HealthState):- """Create an HTTP request handler class bound to the given health state."""-- class _Handler(BaseHTTPRequestHandler):- def do_GET(self) -> None:- parsed = urlparse(self.path)- if parsed.path == "/v1/endpoints/health":- qs = parse_qs(parsed.query)- endpoint_ids = qs.get("endpoint_ids", [])- body = json.dumps(state.get_response(endpoint_ids))- self.send_response(200)- self.send_header("Content-Type", "application/json")- self.end_headers()- self.wfile.write(body.encode())- else:- self.send_response(404)- self.end_headers()-- def log_message(self, format, *args) -> None: # noqa: A002- pass # Suppress request logging in test output-- return _Handler---class MockLatencyService:- """Manages a mock Latency Service HTTP server for testing."""-- def __init__(self) -> None:- self.state = _HealthState()- self.port = find_free_port()- handler = _make_handler(self.state)- self._server = HTTPServer(("127.0.0.1", self.port), handler)- self._thread = threading.Thread(- target=self._server.serve_forever,- daemon=True,- name="mock-latency-service",- )- self._thread.start()-- @property- def url(self) -> str:- return f"http://127.0.0.1:{self.port}"-- def set_health(self, model_id: str, status: str) -> None:- self.state.set_health(model_id, status)-- def shutdown(self) -> None:- self._server.shutdown()- self._thread.join(timeout=5.0)---def mock_latency_service():- """Provide a mock Latency Service for the duration of a test."""- svc = MockLatencyService()- yield svc- svc.shutdown()---# ----------------------------------------------------------------------------# Helpers-# ------------------------------------------------------------------------------def _endpoint(model: str) -> LatencyServiceEndpoint:- return LatencyServiceEndpoint(- model=model,- api_key=_nvidia["api_key"],- base_url=BACKEND_BASE_URL,- )---def _make_backend(- latency_service_url: str,- models: list[str],- poll_interval_s: float = 1.0,- **kwargs,-) -> LatencyServiceLLMBackend:- config = LatencyServiceBackendConfig(- latency_service_url=latency_service_url,- endpoints=[_endpoint(m) for m in models],- poll_interval_s=poll_interval_s,- **kwargs,- )- return LatencyServiceLLMBackend(config)---def _chat_request(stream: bool = False) -> ChatRequest:- # ``max_tokens`` is deliberately generous so reasoning-capable- # backends (e.g. the GPT-5 / Qwen 3.5 families) have room to spend- # the first several hundred tokens on internal reasoning before- # emitting user-visible ``content``. Matches the value used by- # the sibling passthrough e2e suites.- body: dict = {- "messages": SIMPLE_MESSAGES,- "max_tokens": 2048,- "stream": stream,- }- return ChatRequest.openai_chat(body) # type: ignore[arg-type]---def _wait_for_first_poll(- backend: LatencyServiceLLMBackend,- timeout: float = 10.0,-) -> None:- """Block until the background poller completes at least one poll."""- deadline = time.monotonic() + timeout- while not backend.is_ready():- if time.monotonic() > deadline:- raise TimeoutError("Poller did not complete first poll within timeout")- time.sleep(0.05)---# ================================================================== #-# Non-streaming tests-# ================================================================== #--[email protected](not _nvidia["api_key"], reason=_skip_reason)-class TestLatencyServiceBackendNonStreaming:- """Non-streaming requests through the latency-service backend."""-- async def test_healthy_endpoint_serves_request(self, mock_latency_service):- """A HEALTHY endpoint should successfully serve a non-streaming request."""- mock_latency_service.set_health(MODEL_A, "healthy")- backend = _make_backend(mock_latency_service.url, [MODEL_A])- try:- _wait_for_first_poll(backend)-- ctx = ProxyContext()- response = await backend.call(ctx, _chat_request())-- assert response_type_matches(response, ChatResponseType.OPENAI_COMPLETION)- assert response.body["choices"][0]["message"]["content"]- assert response.body["choices"][0]["message"]["role"] == "assistant"- assert ctx.selected_model == MODEL_A- finally:- await backend.shutdown()-- async def test_response_has_usage(self, mock_latency_service):- """Response should include token usage statistics."""- mock_latency_service.set_health(MODEL_A, "healthy")- backend = _make_backend(mock_latency_service.url, [MODEL_A])- try:- _wait_for_first_poll(backend)-- ctx = ProxyContext()- response = await backend.call(ctx, _chat_request())-- assert response_type_matches(response, ChatResponseType.OPENAI_COMPLETION)- assert response.body["usage"] is not None- assert response.body["usage"]["prompt_tokens"] > 0- assert response.body["usage"]["completion_tokens"] > 0- finally:- await backend.shutdown()---# ================================================================== #-# Streaming tests-# ================================================================== #--[email protected](not _nvidia["api_key"], reason=_skip_reason)-class TestLatencyServiceBackendStreaming:- """Streaming requests through the latency-service backend."""-- async def test_streaming_yields_chunks(self, mock_latency_service):- """Streaming request should yield content chunks."""- mock_latency_service.set_health(MODEL_A, "healthy")- backend = _make_backend(mock_latency_service.url, [MODEL_A])- try:- _wait_for_first_poll(backend)-- ctx = ProxyContext()- response = await backend.call(ctx, _chat_request(stream=True))- assert response_type_matches(response, ChatResponseType.OPENAI_STREAM)-- chunks = [chunk async for chunk in response.stream]- assert len(chunks) >= 2, "Expected at least one content chunk + stop chunk"-- text_parts = [- c.choices[0].delta.content- for c in chunks- if c.choices and c.choices[0].delta and c.choices[0].delta.content- ]- full_text = "".join(text_parts)- assert len(full_text) > 0- finally:- await backend.shutdown()---# ================================================================== #-# Health-based routing-# ================================================================== #--[email protected](not _nvidia["api_key"], reason=_skip_reason)-class TestLatencyServiceBackendHealth:- """Verify health-based endpoint selection with real LLM calls."""-- async def test_healthy_preferred_over_degraded(self, mock_latency_service):- """When one endpoint is HEALTHY and another DEGRADED, all traffic- should go to the HEALTHY one."""- mock_latency_service.set_health(MODEL_A, "healthy")- mock_latency_service.set_health(MODEL_B, "degraded")- backend = _make_backend(mock_latency_service.url, [MODEL_A, MODEL_B])- try:- _wait_for_first_poll(backend)-- selected_models = set()- for _ in range(3):- ctx = ProxyContext()- await backend.call(ctx, _chat_request())- selected_models.add(ctx.selected_model)-- assert selected_models == {MODEL_A}, (- f"Expected all traffic to HEALTHY model, got: {selected_models}"- )- finally:- await backend.shutdown()-- async def test_unknown_falls_back_to_random(self, mock_latency_service):- """When all endpoints are UNKNOWN, the backend should distribute- traffic randomly across all endpoints."""- mock_latency_service.set_health(MODEL_A, "unknown")- mock_latency_service.set_health(MODEL_B, "unknown")- backend = _make_backend(mock_latency_service.url, [MODEL_A, MODEL_B])- try:- _wait_for_first_poll(backend)-- selected_models = set()- for _ in range(6):- ctx = ProxyContext()- await backend.call(ctx, _chat_request())- selected_models.add(ctx.selected_model)-- assert len(selected_models) > 1, (- f"Expected random distribution across models, got: {selected_models}"- )- finally:- await backend.shutdown()-- async def test_poller_receives_health_updates(self, mock_latency_service):- """The background poller should pick up health changes from the- Latency Service and reflect them in the health cache."""- mock_latency_service.set_health(MODEL_A, "healthy")- backend = _make_backend(- mock_latency_service.url, [MODEL_A], poll_interval_s=0.5,- )- try:- _wait_for_first_poll(backend)- assert backend._health_cache[MODEL_A].status == EndpointHealthStatus.HEALTHY-- mock_latency_service.set_health(MODEL_A, "degraded")- time.sleep(1.5) # Wait for at least one poll cycle-- assert backend._health_cache[MODEL_A].status == EndpointHealthStatus.DEGRADED- finally:- await backend.shutdown()---# ================================================================== #-# Readiness-# ================================================================== #--[email protected](not _nvidia["api_key"], reason=_skip_reason)-class TestLatencyServiceBackendReadiness:- """Verify is_ready() behavior with a real poller."""-- async def test_ready_after_first_poll(self, mock_latency_service):- """is_ready() should return True once the poller has fetched health."""- mock_latency_service.set_health(MODEL_A, "healthy")- backend = _make_backend(mock_latency_service.url, [MODEL_A])- try:- assert not backend.is_ready()- _wait_for_first_poll(backend)- assert backend.is_ready()- finally:- await backend.shutdown()---# ================================================================== #-# Session affinity (sticky routing)-# ================================================================== #--[email protected](not _nvidia["api_key"], reason=_skip_reason)-class TestLatencyServiceBackendStickiness:- """Verify session affinity pins a multi-turn conversation to one endpoint."""-- async def test_conversation_sticks_to_one_endpoint(self, mock_latency_service):- """With ``session_affinity`` on, every turn of one conversation routes- to the same endpoint — even though both endpoints are HEALTHY with no- latency sample and would otherwise be picked at random (the foil is- ``test_unknown_falls_back_to_random``, which spreads without affinity)."""- mock_latency_service.set_health(MODEL_A, "healthy")- mock_latency_service.set_health(MODEL_B, "healthy")- backend = _make_backend(- mock_latency_service.url, [MODEL_A, MODEL_B], session_affinity=True,- )- try:- _wait_for_first_poll(backend)-- # One conversation that grows each turn but keeps a stable prefix- # (same system + first user message → same session key).- messages = list(SIMPLE_MESSAGES)- selected: list[str] = []- for turn in range(6):- ctx = ProxyContext()- request = ChatRequest.openai_chat(- {"messages": messages, "max_tokens": 128}, # type: ignore[arg-type]- )- response = await backend.call(ctx, request)- assert response_type_matches(- response, ChatResponseType.OPENAI_COMPLETION- )- selected.append(ctx.selected_model)- messages = messages + [- {"role": "assistant", "content": "Sure."},- {"role": "user", "content": f"Follow-up {turn + 1}?"},- ]-- assert len(set(selected)) == 1, (- f"affinity should pin the conversation to one endpoint, got {selected}"- )- assert selected[0] in {MODEL_A, MODEL_B}- finally:- await backend.shutdown()diff --git a/tests/readme/test_readme.py b/tests/readme/test_readme.pyindex 6af37699c..b1686e62e 100644--- a/tests/readme/test_readme.py+++ b/tests/readme/test_readme.py@@ -71,10 +71,9 @@ def test_python_snippet_tripwire(readme_text: str) -> None:def _validate_route_blocks(text: str, source: Path) -> int:# Schema/key validation, not a full chain build: building documented routes# is NOT hermetic — `passthrough` with `discover: true` does a live catalog- # fetch and `latency_service` polls. The- # schema layer (route type + per-type key allowlist) is what we can check- # offline, and it catches the likeliest drift: a renamed `type:` or a key- # that no longer exists on that type.+ # fetch. The schema layer (route type + per-type key allowlist) is what we+ # can check offline, and it catches the likeliest drift: a renamed `type:`+ # or a key that no longer exists on that type.blocks = _code_blocks(text, "yaml")validated_routes = 0for idx, block in enumerate(blocks):diff --git a/tests/test_error_source_annotation.py b/tests/test_error_source_annotation.pyindex 6613fd256..57710da9f 100644--- a/tests/test_error_source_annotation.py+++ b/tests/test_error_source_annotation.py@@ -5,9 +5,9 @@Every client-facing error carries ``x-switchyard-error-source`` naming thelayer that originated it (``switchyard`` | ``provider``), plus-``x-switchyard-upstream-model`` when a routing selection had happened. The-backend-side ctx stamps are covered in ``test_latency_service_llm_backend.py``;-these tests cover the endpoint layer that renders them.+``x-switchyard-upstream-model`` when a routing selection had happened. These+tests cover the endpoint layer that renders those annotations from the ctx+stamps a backend sets."""from __future__ import annotationsdiff --git a/tests/test_inference_e2e.py b/tests/test_inference_e2e.pyindex 2164ede1c..f8420b46d 100644--- a/tests/test_inference_e2e.py+++ b/tests/test_inference_e2e.py@@ -13,10 +13,12 @@ from __future__ import annotationsimport jsonfrom collections.abc import AsyncIterator+from unittest.mock import AsyncMock, patchimport httpximport pytestfrom fastapi import FastAPI+from fastapi.testclient import TestClientfrom openai.types.chat import ChatCompletion, ChatCompletionChunkfrom openai.types.chat.chat_completion import Choicefrom openai.types.chat.chat_completion_chunk import Choice as ChunkChoice@@ -93,6 +95,23 @@ class _RaisingMockLLMBackend(LLMBackend):raise self._exc+class _TranslatingMockLLMBackend(LLMBackend):+ """Chat-only backend that translates the inbound request to Chat at the top of+ ``call`` — mirroring any real chat-completions backend (native / passthrough).++ An unsupported inbound field (e.g. a bad message role) is rejected by the+ translation engine here, before any upstream call, exactly as it would be for+ a production chat backend.+ """++ def supported_request_types(self) -> list[ChatRequestType]:+ return [ChatRequestType.OPENAI_CHAT]++ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:+ request = TranslationEngine().request_to_any_of(request, [ChatRequestType.OPENAI_CHAT])+ return ChatResponse.openai_completion(_make_completion())++# ---------------------------------------------------------------------------# Helpers# ---------------------------------------------------------------------------@@ -257,6 +276,17 @@ async def raising_client(raising_app: FastAPI) -> AsyncIterator[httpx.AsyncClienyield c+def translating_app() -> FastAPI:+ """App whose chat-only backend translates the inbound request to Chat, so an+ invalid inbound role is rejected during translation before any upstream call."""+ switchyard = Switchyard(+ backend=_TranslatingMockLLMBackend(),+ translator=TranslationEngine(),+ )+ return build_switchyard_app(switchyard)++# ---------------------------------------------------------------------------# Tests# ---------------------------------------------------------------------------@@ -502,3 +532,68 @@ class TestToolCallRoundTrip:block = tool_use_blocks[0]assert block["name"] == "get_weather"assert block["input"] == {"city": "Paris"}+++class TestEndpointErrorContract:+ """Endpoint error-mapping contracts previously covered only through the+ removed latency-service backend, re-vehicled onto in-process backends."""++ def test_post_dispatch_exception_returns_json_500(self, app: FastAPI) -> None:+ """An exception raised AFTER dispatch (e.g. during result serialization)+ must surface as a JSON 500 envelope, not FastAPI's plain-text 500."""++ class _Unserializable:+ def model_dump(self) -> None:+ raise RuntimeError("serialization exploded")++ with patch(+ "switchyard.lib.endpoints.openai_chat_endpoint.dispatch_chat_request",+ new_callable=AsyncMock,+ return_value=_Unserializable(),+ ):+ with TestClient(app, raise_server_exceptions=False) as client:+ response = client.post(+ "/v1/chat/completions",+ json={"model": "any-model", "messages": [{"role": "user", "content": "hi"}]},+ )++ assert response.status_code == 500+ assert response.headers["content-type"].startswith("application/json")+ body = response.json()+ assert body["error"]["type"] == "internal_error"+ assert body["error"]["code"] == "internal_chain_error"+ assert "serialization exploded" in body["error"]["message"]++ @pytest.mark.parametrize(+ ("path", "payload"),+ [+ (+ "/v1/responses",+ {"model": "m", "input": [{"type": "message", "role": "api", "content": "ping"}]},+ ),+ (+ "/v1/messages",+ {"model": "m", "max_tokens": 16, "messages": [{"role": "api", "content": "ping"}]},+ ),+ ],+ )+ def test_invalid_inbound_role_surfaces_as_internal_error(+ self, translating_app: FastAPI, path: str, payload: dict[str, object]+ ) -> None:+ """An unsupported inbound message role is rejected by the translation engine+ when a chat-only backend translates the request, before any upstream call.++ NOTE: this currently surfaces as a 500 ``internal_error``. The removed+ latency-service backend special-cased translation ``invalid_value`` errors+ into a provider-compatible 400; no surviving backend reproduces that, so+ the standalone-server contract is now a 500. Update this test (and the+ endpoint/translation layer) if a 400 is restored.+ """+ with TestClient(translating_app, raise_server_exceptions=False) as client:+ response = client.post(path, json=payload)++ assert response.status_code == 500+ body = response.json()+ assert body["error"]["type"] == "internal_error"+ assert '"api"' in body["error"]["message"]+ assert "role" in body["error"]["message"]diff --git a/tests/test_latency_service_health_metrics.py b/tests/test_latency_service_health_metrics.pydeleted file mode 100644index 9329137e4..000000000--- a/tests/test_latency_service_health_metrics.py+++ /dev/null@@ -1,311 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""End-to-end gates for latency-service routing-state metrics.--The Prometheus exposition rendered from the accumulator covers request-flow but says nothing about *why* a request landed on a given endpoint.-The latency-service backend now contributes per-endpoint verdict gauges-and poll-loop health gauges so dashboards can see the routing inputs,-not just outputs.-"""--from __future__ import annotations--from unittest.mock import MagicMock, patch--import pytest-from fastapi.testclient import TestClient--from switchyard.lib.backends.health_poller import (- EndpointHealth,- EndpointHealthStatus,- HealthPoller,-)-from switchyard.lib.backends.latency_service_llm_backend import (- LatencyServiceLLMBackend,-)-from switchyard.lib.config.latency_service_backend_config import (- LatencyServiceBackendConfig,- LatencyServiceEndpoint,-)-from switchyard.lib.endpoints import prometheus_emitter-from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard-from switchyard.server.switchyard_app import build_switchyard_app--[email protected](autouse=True)-def _clean_table():- prometheus_emitter._clear_for_tests()- yield- prometheus_emitter._clear_for_tests()---def _config(*models: str) -> LatencyServiceBackendConfig:- return LatencyServiceBackendConfig(- latency_service_url="http://latency.test:8080",- endpoints=[- LatencyServiceEndpoint(- model=model,- base_url=f"http://llm-{model}.test/v1",- api_key="test-key",- )- for model in models- ],- )---def _latency_service_switchyard(- config: LatencyServiceBackendConfig,-) -> ProfileSwitchyard:- """Build the latency-service profile-backed serving adapter."""- return ProfileSwitchyard(- LatencyServiceProfileConfig.from_config(config)- .build()- .with_runtime_components(enable_stats=config.enable_stats)- )---def _make_backend(config: LatencyServiceBackendConfig) -> LatencyServiceLLMBackend:- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"):- return LatencyServiceLLMBackend(config)---def _emit(backend: LatencyServiceLLMBackend) -> str:- """Render this backend's emitter output as a single string."""- return "\n".join(backend._render_prometheus_lines())---class TestEndpointStatusGauge:- def test_emits_one_row_per_status_per_endpoint(self) -> None:- """Three-state encoding lets `sum by (status)` give a clean histogram."""- backend = _make_backend(_config("model-A", "model-B"))- with backend._cache_lock:- backend._health_cache["model-A"] = EndpointHealth(- EndpointHealthStatus.HEALTHY, 100.0,- )- backend._health_cache["model-B"] = EndpointHealth(- EndpointHealthStatus.DEGRADED, 800.0,- )-- out = _emit(backend)- assert 'switchyard_endpoint_status{model="model-A",status="healthy"} 1' in out- assert 'switchyard_endpoint_status{model="model-A",status="degraded"} 0' in out- assert 'switchyard_endpoint_status{model="model-A",status="unknown"} 0' in out- assert 'switchyard_endpoint_status{model="model-B",status="degraded"} 1' in out- assert 'switchyard_endpoint_status{model="model-B",status="healthy"} 0' in out-- def test_unknown_default_before_first_poll(self) -> None:- """New backends start with every endpoint UNKNOWN."""- backend = _make_backend(_config("model-A"))- out = _emit(backend)- assert 'switchyard_endpoint_status{model="model-A",status="unknown"} 1' in out- assert 'switchyard_endpoint_status{model="model-A",status="healthy"} 0' in out---class TestEndpointLatencyGauge:- def test_emits_only_for_endpoints_with_sample(self) -> None:- """Absence is meaningful: don't emit a zero where no sample exists."""- backend = _make_backend(_config("with-sample", "no-sample"))- with backend._cache_lock:- backend._health_cache["with-sample"] = EndpointHealth(- EndpointHealthStatus.HEALTHY, 250.5,- )- backend._health_cache["no-sample"] = EndpointHealth(- EndpointHealthStatus.HEALTHY, None,- )-- out = _emit(backend)- assert (- 'switchyard_endpoint_last_latency_ms{model="with-sample"} 250.5' in out- )- assert 'switchyard_endpoint_last_latency_ms{model="no-sample"}' not in out---class TestPollHealthGauges:- def test_before_first_poll_signals_never_polled(self) -> None:- """poll_ok=0 + no poll_age line lets a scraper detect "never polled"."""- backend = _make_backend(_config("model-A"))- out = _emit(backend)- assert "switchyard_latency_service_poll_ok 0" in out- assert "switchyard_latency_service_poll_age_seconds" not in out.replace(- "# HELP switchyard_latency_service_poll_age_seconds", ""- ).replace(- "# TYPE switchyard_latency_service_poll_age_seconds", ""- )- assert "switchyard_latency_service_polls_total 0" in out- assert "switchyard_latency_service_poll_failures_total 0" in out-- def test_after_successful_poll_emits_age_and_ok(self) -> None:- backend = _make_backend(_config("model-A"))- backend._poller._poll_count = 3- backend._poller._last_poll_ok = True- backend._poller._last_success_at = __import__("time").monotonic() - 1.0-- out = _emit(backend)- assert "switchyard_latency_service_poll_ok 1" in out- assert "switchyard_latency_service_polls_total 3" in out- # Age value is positive — exact value depends on test wall time.- age_lines = [- line for line in out.splitlines()- if line.startswith("switchyard_latency_service_poll_age_seconds ")- ]- assert len(age_lines) == 1- age_value = float(age_lines[0].split()[-1])- assert age_value > 0-- def test_poll_failure_flips_ok_to_zero(self) -> None:- """Even with prior successes, a recorded failure flips poll_ok off- so dashboards alarm on the *latest* poll result, not history."""- backend = _make_backend(_config("model-A"))- backend._poller._poll_count = 5- backend._poller._last_success_at = __import__("time").monotonic() - 1.0- backend._poller._poll_failures = 1- backend._poller._last_poll_ok = False-- out = _emit(backend)- assert "switchyard_latency_service_poll_ok 0" in out- assert "switchyard_latency_service_poll_failures_total 1" in out-- def test_success_after_prior_failure_emits_ok(self) -> None:- """Prior failures are historical counters; poll_ok tracks latest outcome."""- backend = _make_backend(_config("model-A"))- backend._poller._poll_count = 5- backend._poller._poll_failures = 1- backend._poller._last_poll_ok = True- backend._poller._last_success_at = __import__("time").monotonic() - 1.0-- out = _emit(backend)- assert "switchyard_latency_service_poll_ok 1" in out- assert "switchyard_latency_service_poll_failures_total 1" in out---class TestEmitterLifecycle:- async def test_construction_registers_emitter(self) -> None:- assert prometheus_emitter._EMITTERS == []- backend = _make_backend(_config("model-A"))- assert len(prometheus_emitter._EMITTERS) == 1- # Confirm table output includes this backend's lines.- assert "switchyard_endpoint_status" in prometheus_emitter.render()- await backend.shutdown()- assert prometheus_emitter._EMITTERS == []-- async def test_shutdown_idempotent(self) -> None:- """Lifespan tear-down may call shutdown more than once."""- backend = _make_backend(_config("model-A"))- await backend.shutdown()- await backend.shutdown()- assert prometheus_emitter._EMITTERS == []---class TestRoutingOverhead:- async def test_metrics_record_routing_overhead_for_python_backend(self) -> None:- """End-to-end: a request through the latency-service chain must- publish ``switchyard_routing_overhead_ms`` with non-zero count.-- Before ``ctx.backend_call_latency_ms`` existed, the Rust response- processor had no backend-latency reading to subtract from total,- and the summary stayed at ``count=0`` even under heavy load.- """- from unittest.mock import AsyncMock-- from openai.types.chat import ChatCompletion- from openai.types.chat.chat_completion import Choice- from openai.types.chat.chat_completion_message import ChatCompletionMessage-- from switchyard_rust.core import ChatRequest-- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- switchyard = _latency_service_switchyard(- LatencyServiceBackendConfig(- latency_service_url="http://latency.test:8080",- endpoints=[- LatencyServiceEndpoint(- model="model-A",- api_key="test-key",- base_url="http://llm.test/v1",- ),- ],- )- )-- backend = next(- component- for component in switchyard.iter_components()- if hasattr(component, "_clients")- )- completion = ChatCompletion(- id="cmpl-test",- object="chat.completion",- created=1700000000,- model="model-A",- choices=[- Choice(- index=0,- message=ChatCompletionMessage(role="assistant", content="ok"),- finish_reason="stop",- )- ],- )- backend._clients["model-A"].acompletion = AsyncMock(return_value=completion)-- await switchyard.call(ChatRequest.openai_chat({- "model": "model-A",- "messages": [{"role": "user", "content": "hi"}],- }))-- app = build_switchyard_app(switchyard)- with TestClient(app, raise_server_exceptions=False) as client:- metrics = client.get("/metrics")-- assert metrics.status_code == 200- body = metrics.text- count_line = next(- line for line in body.splitlines()- if line.startswith("switchyard_routing_overhead_ms_count")- )- count = int(count_line.split()[-1])- assert count >= 1, f"expected non-zero routing_overhead samples, body=\n{body}"---class TestEndToEnd:- def test_metrics_endpoint_includes_health_lines(self) -> None:- """Wire the full chain via the recipe and verify /metrics carries- the new lines on top of the standard accumulator exposition."""- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- switchyard = _latency_service_switchyard(- LatencyServiceBackendConfig(- latency_service_url="http://latency.test:8080",- endpoints=[- LatencyServiceEndpoint(- model="model-A",- api_key="test-key",- base_url="http://llm.test/v1",- ),- ],- )- )- app = build_switchyard_app(switchyard)-- with TestClient(app, raise_server_exceptions=False) as client:- metrics = client.get("/metrics")-- assert metrics.status_code == 200- body = metrics.text- # Both surfaces coexist on the same scrape.- assert "switchyard_requests_total" in body- assert "switchyard_endpoint_status" in body- assert "switchyard_latency_service_poll_ok" in body- assert "switchyard_latency_service_polls_total" in bodydiff --git a/tests/test_latency_service_llm_backend.py b/tests/test_latency_service_llm_backend.pydeleted file mode 100644index 00e9addaf..000000000--- a/tests/test_latency_service_llm_backend.py+++ /dev/null@@ -1,1976 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Unit tests for :class:`LatencyServiceLLMBackend` (usage case)."""--import json-import random-import threading-import time-import uuid-from unittest.mock import AsyncMock, MagicMock, patch--import httpx-import openai-import pytest-from openai.types.chat import ChatCompletion-from openai.types.chat.chat_completion import Choice-from openai.types.chat.chat_completion_message import ChatCompletionMessage-from pydantic import ValidationError--from switchyard.lib.backends.health_poller import (- EndpointHealth,- EndpointHealthStatus,- HealthPoller,-)-from switchyard.lib.backends.latency_service_llm_backend import (- SPEND_LOGS_METADATA_HEADER,- LatencyServiceLLMBackend,-)-from switchyard.lib.config.latency_service_backend_config import (- LatencyServiceBackendConfig,- LatencyServiceEndpoint,-)-from switchyard.lib.proxy_context import (- CTX_ERROR_SOURCE,- CTX_ROUTE_SELECTION,- CTX_UPSTREAM_HTTP_STATUS,- CTX_UPSTREAM_MODEL,- ProxyContext,-)-from switchyard.lib.switchyard import Switchyard-from switchyard.server.switchyard_app import build_switchyard_app-from switchyard_rust.core import (- ChatRequest,- ChatRequestType,- ChatResponseType,- response_type_matches,-)-from switchyard_rust.translation import TranslationEngine--LATENCY_SERVICE_URL = "http://latency-service.test:8080"---# ----------------------------------------------------------------------------# Helpers-# ------------------------------------------------------------------------------def _ep(- model: str,- base_url: str | None = None,- request_type: str = "openai_chat",-) -> LatencyServiceEndpoint:- return LatencyServiceEndpoint(- model=model,- base_url=base_url or f"http://llm-{model}.test",- api_key="test-key",- request_type=request_type, # type: ignore[arg-type]- )---def _config(*models: str, **kwargs) -> LatencyServiceBackendConfig:- request_type = str(kwargs.pop("request_type", "openai_chat"))- return LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[_ep(m, request_type=request_type) for m in models],- **kwargs,- )---def _make_backend(config: LatencyServiceBackendConfig) -> LatencyServiceLLMBackend:- """Construct a backend with a mocked OpenAILLMClient and stopped poller."""- with patch(- "switchyard.lib.backends"- ".latency_service_llm_backend.OpenAILLMClient"- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(- name=f"client-{kw.get('base_url')}"- )- with patch.object(HealthPoller, "start"):- backend = LatencyServiceLLMBackend(config)- return backend---def _set_health(- backend: LatencyServiceLLMBackend,- health_map: dict[str, EndpointHealthStatus | EndpointHealth],-) -> None:- """Write directly to the backend's health cache for deterministic tests.-- Accepts either a bare ``EndpointHealthStatus`` (auto-wrapped with no- latency sample) or a full ``EndpointHealth`` snapshot.- """- with backend._cache_lock:- for model_id, value in health_map.items():- if isinstance(value, EndpointHealthStatus):- value = EndpointHealth(status=value)- backend._health_cache[model_id] = value---def _make_completion(- *, model: str = "test-model", content: str = "hello",-) -> ChatCompletion:- return ChatCompletion(- id="chatcmpl-test",- object="chat.completion",- created=1700000000,- model=model,- choices=[- Choice(- index=0,- message=ChatCompletionMessage(role="assistant", content=content),- finish_reason="stop",- )- ],- )---def _openai_request(**overrides) -> ChatRequest:- body: dict = {- "model": "incoming-model",- "messages": [{"role": "user", "content": "hi"}],- }- body.update(overrides)- return ChatRequest.openai_chat(body) # type: ignore[arg-type]---def _api_status_error(status_code: int) -> openai.APIStatusError:- """Synthetic APIStatusError carrying the given upstream status code."""- response = httpx.Response(- status_code,- request=httpx.Request("POST", "http://llm.test/v1/chat/completions"),- json={"error": {"message": "synthetic"}},- )- return openai.APIStatusError(- "synthetic", response=response, body={"error": "synthetic"}- )---# ----------------------------------------------------------------------------# Health poller helpers-# ------------------------------------------------------------------------------def _make_poller(- model_ids: list[str],- health_cache: dict[str, EndpointHealth],- poll_interval_s: float = 100.0,-) -> HealthPoller:- cache_lock = threading.Lock()- return HealthPoller(- latency_service_url=LATENCY_SERVICE_URL,- model_ids=model_ids,- health_cache=health_cache,- cache_lock=cache_lock,- poll_interval_s=poll_interval_s,- poll_timeout_s=5.0,- )---def _health_response(status_code: int, **kwargs) -> httpx.Response:- return httpx.Response(- status_code,- request=httpx.Request("GET", LATENCY_SERVICE_URL + "/v1/endpoints/health"),- **kwargs,- )---def _mock_health_response(poller: HealthPoller, response: httpx.Response) -> None:- mock_client = MagicMock()- mock_client.get.return_value = response- poller._http_client = mock_client---def _run_one_poll(poller: HealthPoller) -> None:- """Execute exactly one poll iteration and then exit the run loop."""- original_wait = poller._stop_event.wait- call_count = 0-- def _wait_then_stop(timeout=None):- nonlocal call_count- call_count += 1- if call_count >= 1:- poller._stop_event.set()- return original_wait(timeout=0)-- poller._stop_event.wait = _wait_then_stop # type: ignore[method-assign]- poller.run()---# ----------------------------------------------------------------------------# Config validation-# ------------------------------------------------------------------------------class TestConfigValidation:- def test_supported_request_types(self):- backend = _make_backend(_config("model-A"))- assert backend.supported_request_types == [ChatRequestType.OPENAI_CHAT]-- def test_responses_endpoint_supported_request_types(self):- backend = _make_backend(_config("model-A", request_type="openai_responses"))- assert backend.supported_request_types == [ChatRequestType.OPENAI_RESPONSES]-- def test_mixed_endpoint_supported_request_types_are_stable(self):- config = LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[- _ep("chat-model"),- _ep("responses-model", request_type="openai_responses"),- ],- )- backend = _make_backend(config)- assert backend.supported_request_types == [- ChatRequestType.OPENAI_CHAT,- ChatRequestType.OPENAI_RESPONSES,- ]-- def test_request_type_aliases_normalize(self):- assert LatencyServiceEndpoint(model="m", request_type="chat").request_type == "openai_chat"- assert (- LatencyServiceEndpoint(model="m", request_type="responses").request_type- == "openai_responses"- )-- def test_unknown_request_type_raises(self):- with pytest.raises(ValidationError):- LatencyServiceEndpoint(model="m", request_type="anthropic") # type: ignore[arg-type]-- def test_unknown_backend_config_field_raises(self):- with pytest.raises(ValidationError):- LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[_ep("model-A")],- credential_polciy="caller_override",- )-- def test_no_endpoints_raises(self):- with pytest.raises(ValueError, match="At least one endpoint"):- _make_backend(_config())-- def test_single_endpoint_ok(self):- backend = _make_backend(_config("model-A"))- assert "model-A" in backend._clients-- def test_missing_model_raises(self):- # dataclass enforces ``model`` as required, but an empty string- # should still be rejected by the backend as a routing key.- config = LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[LatencyServiceEndpoint(model="")],- )- with pytest.raises(ValueError, match="must have a 'model' field"):- _make_backend(config)-- def test_duplicate_model_raises(self):- config = LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[_ep("model-A"), _ep("model-A", base_url="http://other")],- )- with pytest.raises(ValueError, match="Duplicate model ID"):- _make_backend(config)---# ----------------------------------------------------------------------------# Client construction-# ------------------------------------------------------------------------------class TestClientConstruction:- def test_disables_sdk_retries(self):- """The SDK retry layer is disabled so the health-aware ``call`` loop is- the single source of retries.-- The loop already retries on a *different* endpoint; letting the SDK also- retry the *same* endpoint stacks multiplicatively and adds backoff- sleeps that hold each request (and its buffered body) alive longer,- amplifying connection-pool pressure during an upstream incident.- """- with patch(- "switchyard.lib.backends"- ".latency_service_llm_backend.OpenAILLMClient"- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock()- with patch.object(HealthPoller, "start"):- LatencyServiceLLMBackend(_config("model-A", "model-B"))-- assert mock_cls.call_args_list, "expected one OpenAILLMClient per endpoint"- for call in mock_cls.call_args_list:- assert call.kwargs.get("max_retries") == 0---# ----------------------------------------------------------------------------# Tiered endpoint selection-# ------------------------------------------------------------------------------class TestEndpointSelection:- def test_healthy_preferred_over_unknown(self):- backend = _make_backend(_config("model-A", "model-B"))- _set_health(backend, {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.UNKNOWN,- })- picks = {backend._select_endpoint() for _ in range(50)}- assert picks == {"model-A"}-- def test_unknown_preferred_over_degraded(self):- backend = _make_backend(_config("model-A", "model-B"))- _set_health(backend, {- "model-A": EndpointHealthStatus.UNKNOWN,- "model-B": EndpointHealthStatus.DEGRADED,- })- picks = {backend._select_endpoint() for _ in range(50)}- assert picks == {"model-A"}-- def test_all_healthy_random_distribution(self):- backend = _make_backend(_config("model-A", "model-B", "model-C"))- _set_health(backend, {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.HEALTHY,- "model-C": EndpointHealthStatus.HEALTHY,- })- picks = {backend._select_endpoint() for _ in range(200)}- assert len(picks) > 1-- def test_all_degraded_still_picks(self):- backend = _make_backend(_config("model-A", "model-B"))- _set_health(backend, {- "model-A": EndpointHealthStatus.DEGRADED,- "model-B": EndpointHealthStatus.DEGRADED,- })- pick = backend._select_endpoint()- assert pick in {"model-A", "model-B"}-- def test_initial_state_is_unknown(self):- backend = _make_backend(_config("model-A", "model-B"))- assert backend._health_cache["model-A"].status == EndpointHealthStatus.UNKNOWN- assert backend._health_cache["model-B"].status == EndpointHealthStatus.UNKNOWN- assert backend._health_cache["model-A"].last_latency_ms is None---# ----------------------------------------------------------------------------# Inverse-latency weighted selection within a tier-# ------------------------------------------------------------------------------class TestLatencyWeightedSelection:- def test_picks_skew_toward_lower_latency(self):- """Two HEALTHY endpoints; the faster one should attract most traffic."""- backend = _make_backend(_config("fast", "slow"))- _set_health(backend, {- "fast": EndpointHealth(EndpointHealthStatus.HEALTHY, 50.0),- "slow": EndpointHealth(EndpointHealthStatus.HEALTHY, 500.0),- })- random.seed(0)- picks = [backend._select_endpoint() for _ in range(2000)]- fast = picks.count("fast")- slow = picks.count("slow")- # weights are 1/50 vs 1/500 → expected 10:1; bound loosely to avoid flake.- assert fast > slow * 5, f"fast={fast} slow={slow}"-- def test_unknown_latency_falls_back_to_uniform(self):- """If any candidate's last_latency_ms is None, the tier picks uniformly."""- backend = _make_backend(_config("model-A", "model-B"))- _set_health(backend, {- "model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 100.0),- "model-B": EndpointHealth(EndpointHealthStatus.HEALTHY, None),- })- random.seed(0)- picks = [backend._select_endpoint() for _ in range(2000)]- a = picks.count("model-A")- b = picks.count("model-B")- # Uniform within ±15% of 50/50 over 2000 draws is comfortably non-flaky.- assert 850 < a < 1150, f"model-A picked {a}/2000"- assert 850 < b < 1150, f"model-B picked {b}/2000"-- def test_zero_latency_falls_back_to_uniform(self):- """Non-positive samples are treated as bogus and trigger uniform fallback."""- backend = _make_backend(_config("model-A", "model-B"))- _set_health(backend, {- "model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 0.0),- "model-B": EndpointHealth(EndpointHealthStatus.HEALTHY, 100.0),- })- # Must not raise (no 1/0); just exercise selection a few times.- picks = {backend._select_endpoint() for _ in range(50)}- assert picks <= {"model-A", "model-B"}-- def test_weighting_only_within_winning_tier(self):- """A faster DEGRADED endpoint must not beat a slower HEALTHY one."""- backend = _make_backend(_config("slow-healthy", "fast-degraded"))- _set_health(backend, {- "slow-healthy": EndpointHealth(EndpointHealthStatus.HEALTHY, 500.0),- "fast-degraded": EndpointHealth(EndpointHealthStatus.DEGRADED, 10.0),- })- picks = {backend._select_endpoint() for _ in range(50)}- assert picks == {"slow-healthy"}---# ----------------------------------------------------------------------------# Retry with dedup-# ------------------------------------------------------------------------------class TestRetryDedup:- async def test_retry_avoids_same_endpoint(self):- backend = _make_backend(_config("model-A", "model-B", max_retries=1))- _set_health(backend, {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.HEALTHY,- })-- fail_mock = AsyncMock(side_effect=RuntimeError("down"))- success_mock = AsyncMock(return_value=_make_completion())- backend._clients["model-A"].acompletion = fail_mock- backend._clients["model-B"].acompletion = success_mock-- ctx = ProxyContext()- result = await backend.call(ctx, _openai_request())-- assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION)- assert success_mock.called-- async def test_all_retries_exhausted_raises(self):- backend = _make_backend(_config("model-A", "model-B", max_retries=1))-- for mid in backend._clients:- backend._clients[mid].acompletion = AsyncMock(- side_effect=RuntimeError("down")- )-- ctx = ProxyContext()- with pytest.raises(RuntimeError, match="down"):- await backend.call(ctx, _openai_request())---# ----------------------------------------------------------------------------# Retry policy: transient errors retry, 4xx client errors fail fast-# ------------------------------------------------------------------------------class TestRetryPolicy:- @pytest.mark.parametrize("status", [400, 401, 403, 404, 409, 413, 415, 422])- def test_client_errors_are_not_retryable(self, status):- from switchyard.lib.backends.latency_service_llm_backend import (- _is_retryable_status,- )-- assert _is_retryable_status(status) is False-- @pytest.mark.parametrize("status", [408, 429, 500, 502, 503, 504])- def test_transient_errors_are_retryable(self, status):- from switchyard.lib.backends.latency_service_llm_backend import (- _is_retryable_status,- )-- assert _is_retryable_status(status) is True-- async def test_400_fails_fast_without_failover(self):- """A 400 on the first endpoint must not retry the second."""- from switchyard.lib.proxy_context import CTX_UPSTREAM_HTTP_STATUS-- backend = _make_backend(_config("model-A", "model-B", max_retries=2))- # model-A is the sole HEALTHY endpoint → tried first deterministically.- _set_health(backend, {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.UNKNOWN,- })- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(400)- )- backend._clients["model-B"].acompletion = AsyncMock(- return_value=_make_completion()- )-- ctx = ProxyContext()- with pytest.raises(openai.APIStatusError):- await backend.call(ctx, _openai_request())-- # Failed once on model-A, never failed over to model-B.- assert backend._clients["model-A"].acompletion.call_count == 1- assert backend._clients["model-B"].acompletion.call_count == 0- # Upstream status is stashed so the endpoint passes the 400 through.- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 400-- async def test_5xx_retries_to_another_endpoint(self):- """A 503 on the first endpoint fails over and recovers on the second."""- backend = _make_backend(_config("model-A", "model-B", max_retries=2))- _set_health(backend, {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.UNKNOWN,- })- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(503)- )- backend._clients["model-B"].acompletion = AsyncMock(- return_value=_make_completion()- )-- ctx = ProxyContext()- result = await backend.call(ctx, _openai_request())-- assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION)- assert backend._clients["model-B"].acompletion.call_count == 1- assert ctx.selected_model == "model-B"---# ----------------------------------------------------------------------------# Session affinity (pin a conversation to one endpoint)-# ------------------------------------------------------------------------------def _conv_request(text: str) -> ChatRequest:- """A request whose first user message anchors a distinct session."""- return _openai_request(messages=[{"role": "user", "content": text}])---class TestSessionAffinity:- async def test_disabled_by_default_pins_nothing(self):- """Without session_affinity, the affinity map stays empty."""- backend = _make_backend(_config("model-A", "model-B"))- _set_health(backend, {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.HEALTHY,- })- for mid in backend._clients:- backend._clients[mid].acompletion = AsyncMock(- return_value=_make_completion()- )-- await backend.call(ProxyContext(), _conv_request("task"))- assert len(backend._affinity) == 0-- async def test_first_turn_latency_aware_then_sticks(self):- """Turn 1 uses the latency-aware pick; later turns reuse that endpoint- even after latencies flip to favour the other one."""- backend = _make_backend(_config("fast", "slow", session_affinity=True))- for mid in backend._clients:- backend._clients[mid].acompletion = AsyncMock(- return_value=_make_completion()- )- # Turn 1: only "fast" is selectable (slow is DEGRADED) → deterministic pin.- _set_health(backend, {- "fast": EndpointHealthStatus.HEALTHY,- "slow": EndpointHealthStatus.DEGRADED,- })- req = _conv_request("the task")- ctx1 = ProxyContext()- await backend.call(ctx1, req)- assert ctx1.selected_model == "fast"-- # Flip: "slow" is now healthy AND the latency winner. Affinity must- # still keep the conversation on "fast".- _set_health(backend, {- "fast": EndpointHealth(EndpointHealthStatus.HEALTHY, 1000.0),- "slow": EndpointHealth(EndpointHealthStatus.HEALTHY, 1.0),- })- for _ in range(20):- ctx = ProxyContext()- await backend.call(ctx, req)- assert ctx.selected_model == "fast"-- async def test_distinct_conversations_pin_independently(self):- """Two conversations resolve to independent pins."""- backend = _make_backend(_config("model-A", "model-B", session_affinity=True))- for mid in backend._clients:- backend._clients[mid].acompletion = AsyncMock(- return_value=_make_completion()- )-- # Conversation 1 pins to model-A (only A selectable).- _set_health(backend, {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.DEGRADED,- })- req_a = _conv_request("conversation one")- ctx_a = ProxyContext()- await backend.call(ctx_a, req_a)- assert ctx_a.selected_model == "model-A"-- # Conversation 2 pins to model-B (only B selectable).- _set_health(backend, {- "model-A": EndpointHealthStatus.DEGRADED,- "model-B": EndpointHealthStatus.HEALTHY,- })- req_b = _conv_request("conversation two")- ctx_b = ProxyContext()- await backend.call(ctx_b, req_b)- assert ctx_b.selected_model == "model-B"-- # Independent pins: each conversation resolves to its own endpoint.- assert len(backend._affinity) == 2- assert await backend._affinity.pinned(ProxyContext(), req_a) == "model-A"- assert await backend._affinity.pinned(ProxyContext(), req_b) == "model-B"-- async def test_degraded_pin_reroutes_and_repins(self):- """When the pinned endpoint degrades, the next turn re-routes to a- healthy endpoint and the pin follows."""- backend = _make_backend(_config("model-A", "model-B", session_affinity=True))- for mid in backend._clients:- backend._clients[mid].acompletion = AsyncMock(- return_value=_make_completion()- )- _set_health(backend, {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.DEGRADED,- })- req = _conv_request("task")- await backend.call(ProxyContext(), req)- assert await backend._affinity.pinned(ProxyContext(), req) == "model-A"-- # model-A degrades; model-B becomes the only healthy endpoint.- _set_health(backend, {- "model-A": EndpointHealthStatus.DEGRADED,- "model-B": EndpointHealthStatus.HEALTHY,- })- ctx2 = ProxyContext()- await backend.call(ctx2, req)- assert ctx2.selected_model == "model-B"- assert await backend._affinity.pinned(ProxyContext(), req) == "model-B"-- async def test_pin_follows_recovery_after_call_failure(self):- """If the first-turn endpoint fails the call, the pin records the- endpoint that actually served the request."""- backend = _make_backend(- _config("model-A", "model-B", max_retries=1, session_affinity=True)- )- _set_health(backend, {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.HEALTHY,- })- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=RuntimeError("down")- )- backend._clients["model-B"].acompletion = AsyncMock(- return_value=_make_completion()- )-- req = _conv_request("task")- ctx = ProxyContext()- await backend.call(ctx, req)- # Regardless of which endpoint was tried first, only model-B succeeds.- assert ctx.selected_model == "model-B"- assert await backend._affinity.pinned(ProxyContext(), req) == "model-B"-- async def test_lru_eviction_bounds_map(self):- """The affinity map never exceeds affinity_max_sessions."""- backend = _make_backend(- _config("model-A", session_affinity=True, affinity_max_sessions=2)- )- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})-- for i in range(3):- await backend.call(ProxyContext(), _conv_request(f"conversation {i}"))-- assert len(backend._affinity) == 2-- def test_affinity_config_reaches_backend(self):- """A config dict's affinity settings reach the backend's pin map."""- config = LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[{"model": "model-A", "base_url": "http://a.test", "api_key": "k"}],- session_affinity=True,- affinity_max_sessions=5,- )- assert config.session_affinity is True- assert config.affinity_max_sessions == 5-- backend = _make_backend(config)- assert backend._affinity.max_sessions == 5-- async def test_affinity_counters_rendered_on_metrics(self):- """hits/misses counters appear on /metrics when affinity is enabled."""- backend = _make_backend(_config("model-A", session_affinity=True))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})-- req = _conv_request("the task")- await backend.call(ProxyContext(), req) # first turn → miss (no pin yet)- await backend.call(ProxyContext(), req) # reuse → hit- await backend.call(ProxyContext(), req) # reuse → hit-- out = "\n".join(backend._render_prometheus_lines())- assert "switchyard_affinity_hits_total 2" in out- assert "switchyard_affinity_misses_total 1" in out-- async def test_affinity_counters_absent_when_disabled(self):- """The warm-reuse counters stay off the metric surface by default."""- backend = _make_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})-- await backend.call(ProxyContext(), _conv_request("task"))-- out = "\n".join(backend._render_prometheus_lines())- assert "switchyard_affinity_hits_total" not in out- assert "switchyard_affinity_misses_total" not in out-- def test_affinity_l2_counters_rendered_when_shared_store_configured(self):- """L2 hit/error counters appear on /metrics when a shared store is on.-- Constructing the Redis store never connects (the client is lazy), so- this stays hermetic.- """- backend = _make_backend(_config(- "model-A",- session_affinity=True,- affinity_store="redis",- affinity_store_url="redis://cache.test:6379/0",- ))-- out = "\n".join(backend._render_prometheus_lines())- assert "switchyard_affinity_l2_hits_total 0" in out- assert "switchyard_affinity_l2_errors_total 0" in out- assert "switchyard_affinity_l2_breaker_open 0" in out-- def test_affinity_l2_breaker_gauge_reads_1_while_open(self):- """The breaker gauge flips to 1 once the failure streak opens it."""- backend = _make_backend(_config(- "model-A",- session_affinity=True,- affinity_store="redis",- affinity_store_url="redis://cache.test:6379/0",- ))- affinity = backend._affinity- for _ in range(affinity._l2_breaker_threshold):- affinity._record_l2_failure("test-induced failure")-- out = "\n".join(backend._render_prometheus_lines())- assert "switchyard_affinity_l2_breaker_open 1" in out-- def test_affinity_l2_counters_absent_for_memory_store(self):- """Without a shared store the L2 counters stay off the metric surface."""- backend = _make_backend(_config("model-A", session_affinity=True))-- out = "\n".join(backend._render_prometheus_lines())- assert "switchyard_affinity_l2_hits_total" not in out- assert "switchyard_affinity_l2_errors_total" not in out- assert "switchyard_affinity_l2_breaker_open" not in out-- def test_negative_affinity_max_rejected_at_construction(self):- """A negative cap is rejected when the config is built — otherwise the- LRU eviction loop in SessionCache.put would pop past an empty map."""- from pydantic import ValidationError-- with pytest.raises(ValidationError):- LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[_ep("model-A")],- session_affinity=True,- affinity_max_sessions=-1,- )-- def test_zero_affinity_max_rejected_when_enabled(self):- """A zero cap with affinity on is rejected — it would retain nothing."""- from pydantic import ValidationError-- with pytest.raises(ValidationError):- LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[_ep("model-A")],- session_affinity=True,- affinity_max_sessions=0,- )---# ----------------------------------------------------------------------------# Request processing-# ------------------------------------------------------------------------------class TestCall:- async def test_non_streaming_returns_completion_chat_response(self):- backend = _make_backend(_config("model-A"))- completion = _make_completion(content="world")- backend._clients["model-A"].acompletion = AsyncMock(return_value=completion)-- ctx = ProxyContext()- result = await backend.call(ctx, _openai_request())-- assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION)- assert result.body["choices"][0]["message"]["content"] == "world"- # ``selected_model`` is the cross-language ctx field the Rust- # ``StatsResponseProcessor`` reads to attribute tokens and- # end-to-end latency per endpoint on /metrics.- assert ctx.selected_model == "model-A"- # ``backend_call_latency_ms`` is the second cross-language hook —- # the response processor uses it to compute ``routing_overhead_ms``.- # Mocked upstream returns instantly so latency is small but non-None.- assert ctx.backend_call_latency_ms is not None- assert ctx.backend_call_latency_ms >= 0.0-- async def test_responses_endpoint_dispatches_responses_natively(self):- backend = _make_backend(_config("model-A", request_type="openai_responses"))- backend._clients["model-A"].aresponses = AsyncMock(- return_value={- "id": "resp-test",- "object": "response",- "model": "model-A",- "output": [],- }- )- backend._clients["model-A"].acompletion = AsyncMock()-- ctx = ProxyContext()- result = await backend.call(- ctx,- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),- )-- assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_COMPLETION)- backend._clients["model-A"].aresponses.assert_awaited_once()- backend._clients["model-A"].acompletion.assert_not_called()- call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs- assert call_kwargs["model"] == "model-A"- assert call_kwargs["input"] == "hi"- assert "messages" not in call_kwargs- assert ctx.selected_model == "model-A"-- async def test_responses_endpoint_preserves_exact_upstream_body(self):- """Same-format Responses passthrough returns the upstream JSON untouched.-- Provider-specific extras and explicit-null fields must survive the- wrap into ``ChatResponse`` — nothing may be normalized,- re-synthesized, or dropped on this path.- """- upstream_body = {- "id": "resp-raw",- "object": "response",- "created_at": 1719890000,- "model": "model-A",- "status": "completed",- "output": [],- "store": False,- "temperature": 1.0,- "top_p": 0.9,- "previous_response_id": None,- "provider_meta": {"region": "eastus"},- }- backend = _make_backend(_config("model-A", request_type="openai_responses"))- backend._clients["model-A"].aresponses = AsyncMock(return_value=dict(upstream_body))-- result = await backend.call(- ProxyContext(),- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),- )-- assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_COMPLETION)- assert result.body == upstream_body-- async def test_chat_endpoint_translates_responses_to_chat_fallback(self):- backend = _make_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )- backend._clients["model-A"].aresponses = AsyncMock()-- result = await backend.call(- ProxyContext(),- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),- )-- assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION)- backend._clients["model-A"].acompletion.assert_awaited_once()- backend._clients["model-A"].aresponses.assert_not_called()- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs- assert call_kwargs["model"] == "model-A"- assert call_kwargs["messages"][0]["content"] == "hi"-- async def test_streaming_wraps_into_streaming_chat_response(self):- backend = _make_backend(_config("model-A"))- stream_mock = MagicMock(spec=openai.AsyncStream)- backend._clients["model-A"].acompletion = AsyncMock(return_value=stream_mock)-- result = await backend.call(ProxyContext(), _openai_request(stream=True))- assert response_type_matches(result, ChatResponseType.OPENAI_STREAM)-- async def test_responses_streaming_wraps_into_responses_stream(self):- backend = _make_backend(_config("model-A", request_type="openai_responses"))- stream_mock = MagicMock(spec=openai.AsyncStream)- backend._clients["model-A"].aresponses = AsyncMock(return_value=stream_mock)-- result = await backend.call(- ProxyContext(),- ChatRequest.openai_responses({- "model": "incoming-model",- "input": "hi",- "stream": True,- }),- )-- assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_STREAM)-- async def test_model_id_overrides_incoming_model(self):- """The selected endpoint model ID should replace the request body's model."""- backend = _make_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )-- await backend.call(ProxyContext(), _openai_request(model="incoming-model"))-- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs- assert call_kwargs["model"] == "model-A"-- async def test_accepts_anthropic_request_via_translation(self):- """Non-OpenAI inbound formats go through TranslationEngine."""- backend = _make_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )-- anthropic_req = ChatRequest.anthropic({- "model": "claude-sonnet-4-20250514",- "messages": [{"role": "user", "content": "Hi"}],- "max_tokens": 1024,- })- result = await backend.call(ProxyContext(), anthropic_req)-- assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION)- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs- assert call_kwargs["model"] == "model-A"- assert "messages" in call_kwargs-- async def test_upstream_model_override_used_in_body(self):- """When upstream_model is set, it's the value sent in body['model']."""- config = LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[- LatencyServiceEndpoint(- model="openai/gpt-5.5",- upstream_model="openai/openai/gpt-5.5",- base_url="https://inference-api.nvidia.com/v1",- api_key="nvapi-test",- ),- ],- )- backend = _make_backend(config)- backend._clients["openai/gpt-5.5"].acompletion = AsyncMock(- return_value=_make_completion()- )-- ctx = ProxyContext()- await backend.call(ctx, _openai_request(model="incoming-model"))-- call_kwargs = backend._clients["openai/gpt-5.5"].acompletion.call_args.kwargs- assert call_kwargs["model"] == "openai/openai/gpt-5.5"- assert ctx.selected_model == "openai/gpt-5.5"-- async def test_upstream_model_default_falls_back_to_model(self):- """Without upstream_model, body['model'] should be the lookup key."""- config = LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[- LatencyServiceEndpoint(- model="openai/gpt-5.5",- base_url="http://llm.test",- api_key="k",- ),- ],- )- backend = _make_backend(config)- backend._clients["openai/gpt-5.5"].acompletion = AsyncMock(- return_value=_make_completion()- )-- await backend.call(ProxyContext(), _openai_request())-- call_kwargs = backend._clients["openai/gpt-5.5"].acompletion.call_args.kwargs- assert call_kwargs["model"] == "openai/gpt-5.5"---# ----------------------------------------------------------------------------# Route-selection spend-logs metadata (tokenomics attribution)-# ------------------------------------------------------------------------------def _spend_logs_payload(mock: AsyncMock, call_index: int = -1) -> dict[str, object]:- """Parse the spend-logs metadata header stamped on one recorded SDK call."""- call_kwargs = mock.call_args_list[call_index].kwargs- header_value = call_kwargs["extra_headers"][SPEND_LOGS_METADATA_HEADER]- payload = json.loads(header_value)- assert isinstance(payload, dict)- return payload---class TestRouteSelectionSpendLogs:- """Every upstream attempt is stamped with route-selection metadata.-- The ``x-litellm-spend-logs-metadata`` request header ties a LiteLLM- provider spend-log row back to the Switchyard route that selected it, and- the successful attempt's selection is recorded on- ``ctx.metadata[CTX_ROUTE_SELECTION]`` so the endpoint layer can return the- matching ``x-switchyard-*`` response headers.- """-- async def test_outbound_header_carries_route_selection(self):- """The outbound header records the full selection for the chosen endpoint."""- config = LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[- LatencyServiceEndpoint(- model="openai/gpt-5.5",- upstream_model="openai/openai/gpt-5.5",- base_url="https://inference-api.test/v1",- api_key="k",- ),- ],- )- backend = _make_backend(config)- backend._clients["openai/gpt-5.5"].acompletion = AsyncMock(- return_value=_make_completion()- )-- await backend.call(ProxyContext(), _openai_request(model="nvidia/switchyard/gpt-5.5"))-- payload = _spend_logs_payload(backend._clients["openai/gpt-5.5"].acompletion)- assert payload["router_model"] == "nvidia/switchyard/gpt-5.5"- assert payload["router_strategy"] == "latency"- assert payload["router_selected_endpoint"] == "openai/gpt-5.5"- assert payload["router_selected_model"] == "openai/openai/gpt-5.5"- assert payload["router_selected_provider"] == "openai"- # Must parse as a UUID — the join key between spend-log rows.- uuid.UUID(str(payload["router_correlation_id"]))-- async def test_ctx_records_selection_matching_outbound_header(self):- """ctx records exactly the payload the wire header carried."""- backend = _make_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )-- ctx = ProxyContext()- await backend.call(ctx, _openai_request())-- payload = _spend_logs_payload(backend._clients["model-A"].acompletion)- assert ctx.metadata[CTX_ROUTE_SELECTION] == payload- # Endpoint id without a provider prefix: the provider segment- # degenerates to the whole id (split on the first "/").- assert payload["router_selected_provider"] == "model-A"-- async def test_failover_restamps_selection_and_keeps_correlation_id(self):- """Each attempt is stamped with its own endpoint under one correlation id."""- backend = _make_backend(_config("model-A", "model-B"))- _set_health(- backend,- {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.DEGRADED,- },- )- failing = AsyncMock(side_effect=_api_status_error(500))- succeeding = AsyncMock(return_value=_make_completion())- backend._clients["model-A"].acompletion = failing- backend._clients["model-B"].acompletion = succeeding-- ctx = ProxyContext()- await backend.call(ctx, _openai_request())-- first = _spend_logs_payload(failing)- second = _spend_logs_payload(succeeding)- # Each attempt records the endpoint it actually hit…- assert first["router_selected_endpoint"] == "model-A"- assert second["router_selected_endpoint"] == "model-B"- # …while sharing one correlation id per client request.- assert first["router_correlation_id"] == second["router_correlation_id"]- # ctx carries the selection that served the request, not the failure.- assert ctx.metadata[CTX_ROUTE_SELECTION] == second-- async def test_correlation_id_is_fresh_per_request(self):- """Two client requests never share a correlation id."""- backend = _make_backend(_config("model-A"))- mock = AsyncMock(return_value=_make_completion())- backend._clients["model-A"].acompletion = mock-- await backend.call(ProxyContext(), _openai_request())- await backend.call(ProxyContext(), _openai_request())-- first = _spend_logs_payload(mock, call_index=0)- second = _spend_logs_payload(mock, call_index=1)- assert first["router_correlation_id"] != second["router_correlation_id"]-- async def test_router_model_falls_back_to_configured_route_model(self):- """A model-less client body attributes to the configured route id."""- backend = _make_backend(- _config("model-A", route_model="nvidia/switchyard/gpt-5.5")- )- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )-- body: dict = {"messages": [{"role": "user", "content": "hi"}]}- await backend.call(ProxyContext(), ChatRequest.openai_chat(body))-- payload = _spend_logs_payload(backend._clients["model-A"].acompletion)- assert payload["router_model"] == "nvidia/switchyard/gpt-5.5"-- async def test_no_selection_recorded_when_all_attempts_fail(self):- """No billed success → no selection recorded on ctx."""- backend = _make_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(401),- )-- ctx = ProxyContext()- with pytest.raises(openai.APIStatusError):- await backend.call(ctx, _openai_request())-- assert CTX_ROUTE_SELECTION not in ctx.metadata-- async def test_responses_surface_is_stamped_too(self):- """The Responses-API surface is stamped like the Chat surface."""- backend = _make_backend(_config("model-A", request_type="openai_responses"))- backend._clients["model-A"].aresponses = AsyncMock(- return_value={"id": "resp-test", "object": "response", "output": []}- )-- await backend.call(- ProxyContext(),- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),- )-- payload = _spend_logs_payload(backend._clients["model-A"].aresponses)- assert payload["router_selected_endpoint"] == "model-A"- assert payload["router_strategy"] == "latency"-- async def test_cased_spoof_of_spend_logs_header_is_stripped(self):- """A differently-cased spoof key cannot ride client extra_headers to the wire.-- Header names are case-insensitive on the wire while dict merges are- not: exactly one instance of the protected header — ours — may reach- the SDK, while benign client headers still pass through.- """- backend = _make_backend(_config("model-A"))- mock = AsyncMock(return_value=_make_completion())- backend._clients["model-A"].acompletion = mock-- await backend.call(- ProxyContext(),- _openai_request(- extra_headers={- "X-LiteLLM-Spend-Logs-Metadata": "spoofed",- "x-client-tag": "42",- },- ),- )-- headers = mock.call_args.kwargs["extra_headers"]- spoof_keys = [k for k in headers if k.lower() == SPEND_LOGS_METADATA_HEADER]- assert spoof_keys == [SPEND_LOGS_METADATA_HEADER]- payload = _spend_logs_payload(mock)- assert payload["router_selected_endpoint"] == "model-A"- assert headers["x-client-tag"] == "42"---# ----------------------------------------------------------------------------# Credential policy-# ------------------------------------------------------------------------------class TestCallerApiKey:- """Per-request caller-key forwarding is opt-in for multi-tenant deployments.-- When the HTTP endpoint extracts an ``Authorization: Bearer ...`` header- and writes the value into ``ctx.metadata[CTX_CALLER_API_KEY]``, the- backend ignores that key by default. ``credential_policy="caller_override"``- makes the caller key a per-call SDK override.- """-- async def test_default_policy_ignores_caller_key_for_chat_sdk(self):- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY-- backend = _make_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )-- ctx = ProxyContext()- ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied"- await backend.call(ctx, _openai_request())-- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs- assert call_kwargs["api_key"] is None-- async def test_caller_override_policy_passes_caller_key_to_chat_sdk(self):- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY-- backend = _make_backend(_config("model-A", credential_policy="caller_override"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )-- ctx = ProxyContext()- ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied"- await backend.call(ctx, _openai_request())-- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs- assert call_kwargs["api_key"] == "nvapi-caller-supplied"-- async def test_default_policy_ignores_caller_key_for_responses_sdk(self):- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY-- backend = _make_backend(_config("model-A", request_type="openai_responses"))- backend._clients["model-A"].aresponses = AsyncMock(- return_value={"id": "resp-test", "output": []}- )-- ctx = ProxyContext()- ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied"- await backend.call(- ctx,- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),- )-- call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs- assert call_kwargs["api_key"] is None-- async def test_caller_override_policy_passes_caller_key_to_responses_sdk(self):- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY-- backend = _make_backend(_config(- "model-A",- request_type="openai_responses",- credential_policy="caller_override",- ))- backend._clients["model-A"].aresponses = AsyncMock(- return_value={"id": "resp-test", "output": []}- )-- ctx = ProxyContext()- ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied"- await backend.call(- ctx,- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),- )-- call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs- assert call_kwargs["api_key"] == "nvapi-caller-supplied"-- async def test_missing_caller_key_falls_back_to_config(self):- """ctx without a caller key sends api_key=None to OpenAILLMClient.-- ``OpenAILLMClient.acompletion`` then uses the construction-time- config key (no ``with_options`` override). The test only asserts- the contract at the seam — that we pass ``api_key=None`` so the- client falls back, rather than passing a stale or empty string.- """- backend = _make_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )-- await backend.call(ProxyContext(), _openai_request())-- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs- assert call_kwargs["api_key"] is None-- async def test_missing_caller_key_falls_back_to_config_for_responses(self):- backend = _make_backend(_config("model-A", request_type="openai_responses"))- backend._clients["model-A"].aresponses = AsyncMock(- return_value={"id": "resp-test", "output": []}- )-- await backend.call(- ProxyContext(),- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),- )-- call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs- assert call_kwargs["api_key"] is None-- async def test_caller_required_policy_passes_caller_key_to_chat_sdk(self):- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY-- backend = _make_backend(_config("model-A", credential_policy="caller_required"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )-- ctx = ProxyContext()- ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied"- await backend.call(ctx, _openai_request())-- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs- assert call_kwargs["api_key"] == "nvapi-caller-supplied"-- async def test_caller_required_policy_rejects_missing_caller_key(self):- """No caller key under caller_required → 401 stashed, no upstream call."""- from switchyard.lib.proxy_context import (- CTX_UPSTREAM_ATTEMPTS_RECORDED,- CTX_UPSTREAM_HTTP_STATUS,- )-- backend = _make_backend(_config("model-A", credential_policy="caller_required"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )-- ctx = ProxyContext()- with pytest.raises(PermissionError):- await backend.call(ctx, _openai_request())-- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401- backend._clients["model-A"].acompletion.assert_not_called()- # Attempt accounting stays claimed: no upstream attempt happened, so- # the endpoint fallback must not count this 401 in- # ``switchyard_upstream_attempts_total``.- assert ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] is True- assert "switchyard_latency_upstream_attempts_total" not in "\n".join(- backend._render_prometheus_lines()- )-- async def test_caller_required_policy_rejects_blank_caller_key(self):- """A blank caller key is unusable → 401, never the configured service key."""- from switchyard.lib.proxy_context import (- CTX_CALLER_API_KEY,- CTX_UPSTREAM_ATTEMPTS_RECORDED,- CTX_UPSTREAM_HTTP_STATUS,- )-- backend = _make_backend(_config("model-A", credential_policy="caller_required"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )-- ctx = ProxyContext()- ctx.metadata[CTX_CALLER_API_KEY] = " "- with pytest.raises(PermissionError):- await backend.call(ctx, _openai_request())-- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401- backend._clients["model-A"].acompletion.assert_not_called()- assert ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] is True-- async def test_caller_required_rejection_skips_affinity_lookup(self):- """The credential check runs before pin resolution, so a doomed- request never touches the session-affinity store."""- backend = _make_backend(_config(- "model-A", credential_policy="caller_required", session_affinity=True,- ))- backend._affinity.pinned = AsyncMock()-- with pytest.raises(PermissionError):- await backend.call(ProxyContext(), _openai_request())-- backend._affinity.pinned.assert_not_awaited()--- ("path", "body"),- [- ("/v1/messages", {"model": "incoming-model", "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]}),- ("/v1/responses", {"model": "incoming-model", "input": "hi"}),- ],-)-async def test_non_chat_http_endpoints_default_to_endpoint_credentials(- path: str,- body: dict[str, object],-) -> None:- """HTTP ingress may attach caller keys, but the default backend policy ignores them."""- backend = _make_backend(_config("model-A"))- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion(model="model-A")- )- app = build_switchyard_app(Switchyard(backend=backend, translator=TranslationEngine()))-- async with httpx.AsyncClient(- transport=httpx.ASGITransport(app=app, raise_app_exceptions=False),- base_url="http://test",- ) as client:- response = await client.post(- path,- json=body,- headers={"Authorization": "Bearer CALLER-BYO-KEY"},- )-- assert response.status_code == 200, response.text- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs- assert call_kwargs["api_key"] is None--- ("path", "body"),- [- ("/v1/messages", {"model": "incoming-model", "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]}),- ("/v1/responses", {"model": "incoming-model", "input": "hi"}),- ],-)-async def test_non_chat_http_endpoints_forward_caller_key_in_byo_mode(- path: str,- body: dict[str, object],-) -> None:- """Opt-in BYO mode preserves caller keys across Messages/Responses ingress."""- backend = _make_backend(_config("model-A", credential_policy="caller_override"))- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion(model="model-A")- )- app = build_switchyard_app(Switchyard(backend=backend, translator=TranslationEngine()))-- async with httpx.AsyncClient(- transport=httpx.ASGITransport(app=app, raise_app_exceptions=False),- base_url="http://test",- ) as client:- response = await client.post(- path,- json=body,- headers={"Authorization": "Bearer CALLER-BYO-KEY"},- )-- assert response.status_code == 200, response.text- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs- assert call_kwargs["api_key"] == "CALLER-BYO-KEY"---async def test_responses_http_endpoint_can_use_native_latency_responses_mode() -> None:- backend = _make_backend(_config("model-A", request_type="openai_responses"))- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})- backend._clients["model-A"].aresponses = AsyncMock(- return_value={- "id": "resp-test",- "object": "response",- "model": "model-A",- "output": [],- }- )- backend._clients["model-A"].acompletion = AsyncMock()- app = build_switchyard_app(Switchyard(backend=backend, translator=TranslationEngine()))-- async with httpx.AsyncClient(- transport=httpx.ASGITransport(app=app, raise_app_exceptions=False),- base_url="http://test",- ) as client:- response = await client.post(- "/v1/responses",- json={"model": "incoming-model", "input": "hi"},- headers={"Authorization": "Bearer CALLER-BYO-KEY"},- )-- assert response.status_code == 200, response.text- backend._clients["model-A"].aresponses.assert_awaited_once()- backend._clients["model-A"].acompletion.assert_not_called()- call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs- assert call_kwargs["api_key"] is None- assert call_kwargs["input"] == "hi"---class TestEndpointApiKeyReachesUpstream:- """End-to-end auth: which credential the upstream HTTP call carries.-- The seam test above only asserts the ``api_key`` value handed to- ``acompletion``; it cannot catch a regression where a blank caller key- clobbers the configured endpoint key and the upstream call goes out- unauthenticated. These tests run the real- ``OpenAILLMClient`` against a mocked HTTP transport and inspect the- ``Authorization`` header that actually reaches the wire.- """-- @staticmethod- def _backend_with_captured_auth(- captured: dict[str, str | None],- credential_policy: str = "configured_endpoint",- ) -> LatencyServiceLLMBackend:- """Real-client backend whose upstream auth header is captured."""-- def handler(request: httpx.Request) -> httpx.Response:- captured["authorization"] = request.headers.get("authorization")- return httpx.Response(- 200,- json={- "id": "chatcmpl-test",- "object": "chat.completion",- "created": 1700000000,- "model": "model-A",- "choices": [{- "index": 0,- "message": {"role": "assistant", "content": "ok"},- "finish_reason": "stop",- }],- },- )-- config = LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[LatencyServiceEndpoint(- model="model-A",- base_url="http://llm.test/v1",- api_key="ENDPOINT-CONFIGURED-KEY",- )],- credential_policy=credential_policy,- )- with patch.object(HealthPoller, "start"):- backend = LatencyServiceLLMBackend(config)- backend._clients["model-A"].async_client._client = httpx.AsyncClient(- transport=httpx.MockTransport(handler)- )- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})- return backend-- async def test_no_caller_key_uses_configured_endpoint_key(self):- """No caller key → the endpoint's configured key authenticates upstream."""- captured: dict[str, str | None] = {}- backend = self._backend_with_captured_auth(captured)-- await backend.call(ProxyContext(), _openai_request())-- assert captured["authorization"] == "Bearer ENDPOINT-CONFIGURED-KEY"-- async def test_default_policy_ignores_caller_key(self):- """Default policy keeps using the endpoint key even when a caller key exists."""- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY-- captured: dict[str, str | None] = {}- backend = self._backend_with_captured_auth(captured)-- ctx = ProxyContext()- ctx.metadata[CTX_CALLER_API_KEY] = "CALLER-BYO-KEY"- await backend.call(ctx, _openai_request())-- assert captured["authorization"] == "Bearer ENDPOINT-CONFIGURED-KEY"-- async def test_caller_override_policy_uses_caller_key(self):- """BYO mode lets a caller-supplied key override the endpoint key."""- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY-- captured: dict[str, str | None] = {}- backend = self._backend_with_captured_auth(- captured,- credential_policy="caller_override",- )-- ctx = ProxyContext()- ctx.metadata[CTX_CALLER_API_KEY] = "CALLER-BYO-KEY"- await backend.call(ctx, _openai_request())-- assert captured["authorization"] == "Bearer CALLER-BYO-KEY"-- async def test_blank_caller_key_does_not_clobber_endpoint_key(self):- """A blank caller key in BYO mode still falls back to the configured key."""- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY-- captured: dict[str, str | None] = {}- backend = self._backend_with_captured_auth(- captured,- credential_policy="caller_override",- )-- ctx = ProxyContext()- ctx.metadata[CTX_CALLER_API_KEY] = " "- await backend.call(ctx, _openai_request())-- assert captured["authorization"] == "Bearer ENDPOINT-CONFIGURED-KEY"-- async def test_caller_required_policy_uses_caller_key(self):- """caller_required forwards the caller key to the upstream wire."""- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY-- captured: dict[str, str | None] = {}- backend = self._backend_with_captured_auth(- captured,- credential_policy="caller_required",- )-- ctx = ProxyContext()- ctx.metadata[CTX_CALLER_API_KEY] = "CALLER-BYO-KEY"- await backend.call(ctx, _openai_request())-- assert captured["authorization"] == "Bearer CALLER-BYO-KEY"-- async def test_caller_required_rejects_missing_key_without_upstream_call(self):- """caller_required with no caller key 401s before reaching the upstream.-- The configured ``ENDPOINT-CONFIGURED-KEY`` must never authenticate the- call — the mock transport handler should not run at all.- """- from switchyard.lib.proxy_context import CTX_UPSTREAM_HTTP_STATUS-- captured: dict[str, str | None] = {}- backend = self._backend_with_captured_auth(- captured,- credential_policy="caller_required",- )-- ctx = ProxyContext()- with pytest.raises(PermissionError):- await backend.call(ctx, _openai_request())-- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401- assert captured == {}---# ----------------------------------------------------------------------------# Per-model upstream-attempt metrics-# ------------------------------------------------------------------------------class TestPerModelUpstreamMetrics:- """``switchyard_latency_upstream_attempts_total`` breaks attempts out by- requested route model, selected upstream model, outcome, and HTTP code —- the per-model complement to the layer-aggregate counter."""-- async def test_success_attempt_recorded_per_model(self):- backend = _make_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})-- # Request a configured endpoint id so ``requested_model`` is preserved.- await backend.call(ProxyContext(), _openai_request(model="model-A"))-- out = "\n".join(backend._render_prometheus_lines())- assert (- 'switchyard_latency_upstream_attempts_total{'- 'requested_model="model-A",upstream_model="model-A",'- 'outcome="success",code="200"} 1'- ) in out-- async def test_error_attempt_recorded_per_model(self):- backend = _make_backend(_config("model-A", max_retries=0))- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(500)- )- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})-- with pytest.raises(openai.APIStatusError):- await backend.call(ProxyContext(), _openai_request(model="model-A"))-- out = "\n".join(backend._render_prometheus_lines())- assert (- 'switchyard_latency_upstream_attempts_total{'- 'requested_model="model-A",upstream_model="model-A",'- 'outcome="retryable_error",code="500"} 1'- ) in out-- async def test_route_model_preserved_as_requested_model_label(self):- # Clients of a deployed route send the route key (e.g.- # ``nvidia/switchyard/gpt-5.4``), never an endpoint id. The route id is- # config-derived, so it joins the bounded label set instead of- # collapsing to the ``"other"`` sentinel.- config = LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[_ep("azure/openai/gpt-5.4")],- route_model="nvidia/switchyard/gpt-5.4",- )- backend = _make_backend(config)- backend._clients["azure/openai/gpt-5.4"].acompletion = AsyncMock(- return_value=_make_completion()- )- _set_health(backend, {"azure/openai/gpt-5.4": EndpointHealthStatus.HEALTHY})-- await backend.call(- ProxyContext(), _openai_request(model="nvidia/switchyard/gpt-5.4")- )-- out = "\n".join(backend._render_prometheus_lines())- assert (- 'switchyard_latency_upstream_attempts_total{'- 'requested_model="nvidia/switchyard/gpt-5.4",'- 'upstream_model="azure/openai/gpt-5.4",'- 'outcome="success",code="200"} 1'- ) in out- assert 'requested_model="other"' not in out-- async def test_unknown_requested_model_normalized_to_sentinel(self):- # A client-supplied model that is not a configured endpoint id must not- # become a raw Prometheus label (unbounded cardinality); it collapses to- # the ``"other"`` sentinel.- backend = _make_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})-- await backend.call(ProxyContext(), _openai_request()) # model="incoming-model"-- out = "\n".join(backend._render_prometheus_lines())- assert 'requested_model="other"' in out- assert 'requested_model="incoming-model"' not in out-- async def test_upstream_model_override_labels_the_upstream_name(self):- config = LatencyServiceBackendConfig(- latency_service_url=LATENCY_SERVICE_URL,- endpoints=[LatencyServiceEndpoint(- model="model-A",- base_url="http://a.test",- api_key="k",- upstream_model="azure/openai/gpt-5.1-codex-mini",- )],- )- backend = _make_backend(config)- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})-- await backend.call(ProxyContext(), _openai_request())-- out = "\n".join(backend._render_prometheus_lines())- assert 'upstream_model="azure/openai/gpt-5.1-codex-mini"' in out-- async def test_metric_absent_until_traffic(self):- backend = _make_backend(_config("model-A"))-- out = "\n".join(backend._render_prometheus_lines())-- assert "switchyard_latency_upstream_attempts_total" not in out---# ----------------------------------------------------------------------------# Failure-source annotation-# ------------------------------------------------------------------------------class TestErrorSourceAnnotation:- """The backend stamps the failure origin + upstream model on ctx."""-- async def test_http_failure_stamps_provider_source_and_upstream_model(self):- backend = _make_backend(_config("model-A"))- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(401)- )-- ctx = ProxyContext()- with pytest.raises(openai.APIStatusError):- await backend.call(ctx, _openai_request())-- assert ctx.metadata[CTX_ERROR_SOURCE] == "provider"- assert ctx.metadata[CTX_UPSTREAM_MODEL] == "model-A"- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401-- async def test_network_failure_stamps_provider_without_status(self):- """A status-less upstream fault is still provider-originated, so the- endpoint's 500 envelope can say so instead of implying an internal bug."""- backend = _make_backend(_config("model-A"))- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=RuntimeError("connection reset")- )-- ctx = ProxyContext()- with pytest.raises(RuntimeError):- await backend.call(ctx, _openai_request())-- assert ctx.metadata[CTX_ERROR_SOURCE] == "provider"- assert ctx.metadata[CTX_UPSTREAM_MODEL] == "model-A"- assert CTX_UPSTREAM_HTTP_STATUS not in ctx.metadata-- async def test_caller_required_rejection_is_switchyard_source(self):- """The caller_required 401 rides the upstream-status channel but is- Switchyard's own rejection — the source label must say so."""- backend = _make_backend(- _config("model-A", credential_policy="caller_required")- )-- ctx = ProxyContext()- with pytest.raises(PermissionError):- await backend.call(ctx, _openai_request())-- assert ctx.metadata[CTX_ERROR_SOURCE] == "switchyard"- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401- assert CTX_UPSTREAM_MODEL not in ctx.metadata-- def test_translation_rejection_is_switchyard_source(self):- """The translation invalid-value 400 is likewise Switchyard-originated."""- from switchyard.lib.backends.latency_service_llm_backend import (- _stash_invalid_request_error,- )-- ctx = ProxyContext()- _stash_invalid_request_error(- ctx, ValueError("InvalidValue: unsupported message role 'api'")- )-- assert ctx.metadata[CTX_ERROR_SOURCE] == "switchyard"- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 400---# ----------------------------------------------------------------------------# Readiness and shutdown-# ------------------------------------------------------------------------------class TestReadinessAndShutdown:- def test_not_ready_before_first_poll(self):- backend = _make_backend(_config("model-A"))- assert backend.is_ready() is False-- def test_ready_after_first_poll(self):- backend = _make_backend(_config("model-A"))- backend._poller._poll_count = 1- assert backend.is_ready() is True-- async def test_shutdown_stops_poller(self):- backend = _make_backend(_config("model-A"))- await backend.shutdown()- assert backend._poller._stop_event.is_set()-- async def test_shutdown_closes_shared_affinity_store(self):- """Backend teardown releases the shared (L2) pin store's connections."""-- class _ClosableL2:- def __init__(self) -> None:- self.closed = False-- async def get(self, key: str) -> str | None:- return None-- async def put(self, key: str, value: str) -> None:- return None-- async def aclose(self) -> None:- self.closed = True-- backend = _make_backend(_config("model-A", session_affinity=True))- l2 = _ClosableL2()- backend._affinity._l2 = l2-- await backend.shutdown()- assert l2.closed is True---# ----------------------------------------------------------------------------# Health poller-# ------------------------------------------------------------------------------class TestHealthPoller:- def test_poll_updates_cache(self):- health_cache: dict[str, EndpointHealth] = {- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),- "model-B": EndpointHealth(EndpointHealthStatus.UNKNOWN),- }- poller = _make_poller(["model-A", "model-B"], health_cache)-- response_data = {- "endpoint_health": {- "model-A": {"status": "healthy", "last_latency_ms": 100.0},- "model-B": {"status": "degraded", "last_latency_ms": 1500.0},- }- }- _mock_health_response(poller, _health_response(200, json=response_data))- _run_one_poll(poller)-- assert health_cache["model-A"].status == EndpointHealthStatus.HEALTHY- assert health_cache["model-A"].last_latency_ms == 100.0- assert health_cache["model-B"].status == EndpointHealthStatus.DEGRADED- assert health_cache["model-B"].last_latency_ms == 1500.0- assert poller.has_polled-- def test_poll_captures_null_latency(self):- """``last_latency_ms`` is nullable in the spec; absence becomes None."""- health_cache: dict[str, EndpointHealth] = {- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),- }- poller = _make_poller(["model-A"], health_cache)-- response_data = {- "endpoint_health": {- "model-A": {"status": "healthy", "last_latency_ms": None},- }- }- _mock_health_response(poller, _health_response(200, json=response_data))- _run_one_poll(poller)-- assert health_cache["model-A"].status == EndpointHealthStatus.HEALTHY- assert health_cache["model-A"].last_latency_ms is None-- def test_poll_failure_resets_to_unknown(self):- """If the Latency Service is unreachable, all endpoints reset to UNKNOWN- so the backend falls back to random routing rather than stale data."""- health_cache: dict[str, EndpointHealth] = {- "model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 100.0),- "model-B": EndpointHealth(EndpointHealthStatus.DEGRADED, 800.0),- }- poller = _make_poller(["model-A", "model-B"], health_cache)- _mock_health_response(poller, _health_response(500))- _run_one_poll(poller)-- assert health_cache["model-A"].status == EndpointHealthStatus.UNKNOWN- assert health_cache["model-A"].last_latency_ms is None- assert health_cache["model-B"].status == EndpointHealthStatus.UNKNOWN- assert health_cache["model-B"].last_latency_ms is None- assert not poller.has_polled-- def test_poll_ignores_unknown_model_ids(self):- health_cache: dict[str, EndpointHealth] = {- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),- }- poller = _make_poller(["model-A"], health_cache)-- response_data = {- "endpoint_health": {- "model-A": {"status": "healthy", "last_latency_ms": 100.0},- "model-UNKNOWN": {"status": "degraded", "last_latency_ms": 999.0},- }- }- _mock_health_response(poller, _health_response(200, json=response_data))- _run_one_poll(poller)-- assert health_cache["model-A"].status == EndpointHealthStatus.HEALTHY- assert "model-UNKNOWN" not in health_cache-- def test_success_increments_polls_and_records_timestamp(self):- """``/metrics`` reads these fields to expose poll-loop health."""- health_cache: dict[str, EndpointHealth] = {- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),- }- poller = _make_poller(["model-A"], health_cache)- _mock_health_response(- poller,- _health_response(200, json={"endpoint_health": {- "model-A": {"status": "healthy", "last_latency_ms": 50.0},- }}),- )- _run_one_poll(poller)-- assert poller.poll_successes == 1- assert poller.poll_failures == 0- assert poller.last_poll_ok is True- assert poller.seconds_since_last_success is not None- assert poller.seconds_since_last_success >= 0-- def test_failure_increments_failures_and_leaves_age_unset(self):- """Pre-success failure leaves ``seconds_since_last_success`` None so- scrapers can distinguish "never polled" from "polled but stale"."""- health_cache: dict[str, EndpointHealth] = {- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),- }- poller = _make_poller(["model-A"], health_cache)- _mock_health_response(poller, _health_response(500))- _run_one_poll(poller)-- assert poller.poll_successes == 0- assert poller.poll_failures == 1- assert poller.last_poll_ok is False- assert poller.seconds_since_last_success is None-- def test_stop_event_terminates_poller(self):- health_cache: dict[str, EndpointHealth] = {- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),- }- poller = _make_poller(["model-A"], health_cache, poll_interval_s=0.05)- _mock_health_response(- poller,- _health_response(200, json={"endpoint_health": {- "model-A": {"status": "healthy", "last_latency_ms": 50.0},- }}),- )-- poller.start()- time.sleep(0.15)- poller.stop()- poller.join(timeout=2.0)-- assert not poller.is_alive()- assert poller.has_polleddiff --git a/tests/test_latency_service_spans.py b/tests/test_latency_service_spans.pydeleted file mode 100644index 09d7be99b..000000000--- a/tests/test_latency_service_spans.py+++ /dev/null@@ -1,273 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""Datadog routing spans/tags emitted by ``LatencyServiceLLMBackend``.--The backend emits a ``switchyard.route_decision`` span around endpoint selection-and a ``switchyard.upstream_attempt`` span around each upstream call. These tests-install a fake tracer (so no ddtrace install is needed) and assert the tag set.-"""--from __future__ import annotations--import time-from typing import Any-from unittest.mock import AsyncMock, MagicMock, patch--import httpx-import openai-import pytest-from openai.types.chat import ChatCompletion-from openai.types.chat.chat_completion import Choice-from openai.types.chat.chat_completion_message import ChatCompletionMessage--from switchyard.lib import tracing-from switchyard.lib.backends.health_poller import (- EndpointHealth,- EndpointHealthStatus,- HealthPoller,-)-from switchyard.lib.backends.latency_service_llm_backend import (- LatencyServiceLLMBackend,-)-from switchyard.lib.config.latency_service_backend_config import (- LatencyServiceBackendConfig,- LatencyServiceEndpoint,-)-from switchyard.lib.proxy_context import ProxyContext-from switchyard_rust.core import ChatRequest--# ----------------------------------------------------------------------------# Fake tracer (stands in for ddtrace so no install is required)-# ------------------------------------------------------------------------------class _FakeSpan:- def __init__(self, name: str) -> None:- self.name = name- self.tags: dict[str, Any] = {}-- def set_tag(self, key: str, value: Any) -> None:- self.tags[key] = value-- def __enter__(self) -> _FakeSpan:- return self-- def __exit__(self, *_args: object) -> bool:- return False---class _FakeTracer:- def __init__(self) -> None:- self.spans: list[_FakeSpan] = []-- def trace(self, name: str) -> _FakeSpan:- span = _FakeSpan(name)- self.spans.append(span)- return span-- def named(self, name: str) -> list[_FakeSpan]:- return [span for span in self.spans if span.name == name]---def tracer(monkeypatch: pytest.MonkeyPatch) -> _FakeTracer:- fake = _FakeTracer()- monkeypatch.setattr(tracing, "_dd_tracer", fake)- return fake---# ----------------------------------------------------------------------------# Helpers-# ------------------------------------------------------------------------------def _config(*models: str, **kwargs: Any) -> LatencyServiceBackendConfig:- return LatencyServiceBackendConfig(- latency_service_url="http://ls.test:8080",- endpoints=[- LatencyServiceEndpoint(model=m, base_url=f"http://{m}.test", api_key="k")- for m in models- ],- **kwargs,- )---def _make_backend(config: LatencyServiceBackendConfig) -> LatencyServiceLLMBackend:- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient"- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock()- with patch.object(HealthPoller, "start"):- return LatencyServiceLLMBackend(config)---def _set_health(- backend: LatencyServiceLLMBackend,- mapping: dict[str, EndpointHealthStatus | EndpointHealth],-) -> None:- with backend._cache_lock:- for mid, value in mapping.items():- if isinstance(value, EndpointHealthStatus):- value = EndpointHealth(status=value)- backend._health_cache[mid] = value---def _completion(content: str = "ok") -> ChatCompletion:- return ChatCompletion(- id="chatcmpl-test",- object="chat.completion",- created=1700000000,- model="m",- choices=[- Choice(- index=0,- message=ChatCompletionMessage(role="assistant", content=content),- finish_reason="stop",- )- ],- )---def _request(**overrides: Any) -> ChatRequest:- body: dict[str, Any] = {"model": "incoming", "messages": [{"role": "user", "content": "hi"}]}- body.update(overrides)- return ChatRequest.openai_chat(body) # type: ignore[arg-type]---def _api_status_error(status_code: int) -> openai.APIStatusError:- response = httpx.Response(- status_code, request=httpx.Request("POST", "http://upstream.test/v1/chat/completions")- )- return openai.APIStatusError("upstream error", response=response, body=None)---# ----------------------------------------------------------------------------# Tests-# ------------------------------------------------------------------------------async def test_success_emits_route_and_attempt_spans(tracer: _FakeTracer) -> None:- backend = _make_backend(_config("model-A"))- _set_health(backend, {"model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 50.0)})- backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion())-- await backend.call(ProxyContext(), _request(model="incoming"))-- route = tracer.named("switchyard.route_decision")- attempt = tracer.named("switchyard.upstream_attempt")- assert len(route) == 1- assert len(attempt) == 1-- # poll-age is absent (poller never recorded a success) — None tags are skipped.- assert route[0].tags == {- "switchyard.model": "incoming",- "switchyard.candidate_endpoints": "model-A",- "switchyard.selected_endpoint": "model-A",- "switchyard.was_fastest_selected": True,- "switchyard.affinity_hit": False,- }-- assert attempt[0].tags == {- "switchyard.model": "incoming",- "switchyard.selected_endpoint": "model-A",- "switchyard.retry_count": 0,- "switchyard.outcome": "success",- "switchyard.upstream_status_code": 200,- }---async def test_poll_age_tag_present_after_a_poll(tracer: _FakeTracer) -> None:- backend = _make_backend(_config("model-A"))- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})- backend._poller._last_success_at = time.monotonic()- backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion())-- await backend.call(ProxyContext(), _request())-- route = tracer.named("switchyard.route_decision")[0]- assert route.tags["switchyard.latency_service_poll_age_ms"] >= 0.0---async def test_api_status_error_tags(tracer: _FakeTracer) -> None:- backend = _make_backend(_config("model-A"))- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(429)- )-- with pytest.raises(openai.APIStatusError):- await backend.call(ProxyContext(), _request())-- attempt = tracer.named("switchyard.upstream_attempt")[0]- assert attempt.tags["switchyard.upstream_status_code"] == 429- assert attempt.tags["switchyard.outcome"] == "retryable_error"- assert attempt.tags["switchyard.error_code"] == "429"- # Failure-source annotation: the attempt failed upstream.- assert attempt.tags["switchyard.error_source"] == "provider"- assert attempt.tags["switchyard.upstream_model"] == "model-A"---async def test_generic_error_tags(tracer: _FakeTracer) -> None:- backend = _make_backend(_config("model-A"))- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})- backend._clients["model-A"].acompletion = AsyncMock(side_effect=RuntimeError("down"))-- with pytest.raises(RuntimeError, match="down"):- await backend.call(ProxyContext(), _request())-- attempt = tracer.named("switchyard.upstream_attempt")[0]- assert attempt.tags["switchyard.outcome"] == "retryable_error"- assert attempt.tags["switchyard.error_code"] == "none"- # A non-HTTP failure has no status line.- assert "switchyard.upstream_status_code" not in attempt.tags- # A network-level fault is still an upstream-side failure.- assert attempt.tags["switchyard.error_source"] == "provider"- assert attempt.tags["switchyard.upstream_model"] == "model-A"---async def test_retry_count_is_sequential_and_ends_success(tracer: _FakeTracer) -> None:- backend = _make_backend(_config("model-A", "model-B", max_retries=1))- _set_health(backend, {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.HEALTHY,- })- backend._clients["model-A"].acompletion = AsyncMock(side_effect=RuntimeError("down"))- backend._clients["model-B"].acompletion = AsyncMock(return_value=_completion())-- await backend.call(ProxyContext(), _request())-- attempts = tracer.named("switchyard.upstream_attempt")- # retry_count is the 0-based attempt index, monotonically increasing.- assert [a.tags["switchyard.retry_count"] for a in attempts] == list(range(len(attempts)))- assert attempts[-1].tags["switchyard.outcome"] == "success"---async def test_was_fastest_false_when_latency_unknown(tracer: _FakeTracer) -> None:- backend = _make_backend(_config("model-A", "model-B"))- # Both HEALTHY but no latency samples -> uniform pick, "fastest" undefined.- _set_health(backend, {- "model-A": EndpointHealthStatus.HEALTHY,- "model-B": EndpointHealthStatus.HEALTHY,- })- selected = {"model-A", "model-B"}- backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion())- backend._clients["model-B"].acompletion = AsyncMock(return_value=_completion())-- await backend.call(ProxyContext(), _request())-- route = tracer.named("switchyard.route_decision")[0]- assert route.tags["switchyard.was_fastest_selected"] is False- assert route.tags["switchyard.selected_endpoint"] in selected---async def test_call_works_without_tracer() -> None:- # No ``tracer`` fixture: _dd_tracer is None (ddtrace absent in dev), so the- # spans are no-ops and call() must proceed normally.- backend = _make_backend(_config("model-A"))- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})- backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion())-- ctx = ProxyContext()- await backend.call(ctx, _request())- assert ctx.selected_model == "model-A"diff --git a/tests/test_latency_service_stats_metrics.py b/tests/test_latency_service_stats_metrics.pydeleted file mode 100644index ec011c917..000000000--- a/tests/test_latency_service_stats_metrics.py+++ /dev/null@@ -1,273 +0,0 @@-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-# SPDX-License-Identifier: Apache-2.0--"""End-to-end gate that a latency-service-routed chain exposes ``/metrics``.--Pins the fix for the production 404: the profile-backed latency service chain-must contribute a stats response processor so Datadog / OTel scrapers do not-see 404 on every scrape.-"""--from __future__ import annotations--from unittest.mock import MagicMock, patch--from fastapi.testclient import TestClient--from switchyard.lib.backends.health_poller import HealthPoller-from switchyard.lib.backends.latency_service_llm_backend import (- LatencyServiceLLMBackend,-)-from switchyard.lib.config.latency_service_backend_config import (- LatencyServiceBackendConfig,- LatencyServiceEndpoint,-)-from switchyard.lib.processors.stats_request_processor import (- StatsRequestProcessor,-)-from switchyard.lib.processors.stats_response_processor_accumulator import (- StatsResponseProcessor,-)-from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard-from switchyard.lib.stats_accumulator import StatsAccumulator-from switchyard.server.switchyard_app import build_switchyard_app---def _config(*, enable_stats: bool = True) -> LatencyServiceBackendConfig:- return LatencyServiceBackendConfig(- latency_service_url="http://latency-service.test:8080",- endpoints=[- LatencyServiceEndpoint(- model="model-A",- api_key="test-key",- base_url="http://llm.test/v1",- ),- ],- enable_stats=enable_stats,- )---def _build_switchyard(- config: LatencyServiceBackendConfig,- *,- stats_accumulator: StatsAccumulator | None = None,-) -> ProfileSwitchyard:- """Build a latency-service profile without starting real poller threads."""- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- return ProfileSwitchyard(- LatencyServiceProfileConfig.from_config(config)- .build()- .with_runtime_components(- stats_accumulator=stats_accumulator,- enable_stats=config.enable_stats,- )- )---# ----------------------------------------------------------------------------# Profile component assembly-# ------------------------------------------------------------------------------def test_profile_wires_stats_processors_by_default() -> None:- switchyard = _build_switchyard(_config())-- components = list(switchyard.iter_components())-- assert any(isinstance(p, StatsRequestProcessor) for p in components), (- "expected StatsRequestProcessor in the latency-service profile"- )- assert any(isinstance(p, StatsResponseProcessor) for p in components), (- "expected StatsResponseProcessor in the latency-service profile"- )---def test_profile_skips_stats_when_disabled() -> None:- switchyard = _build_switchyard(_config(enable_stats=False))- components = list(switchyard.iter_components())-- assert not any(isinstance(p, StatsRequestProcessor) for p in components)- assert not any(isinstance(p, StatsResponseProcessor) for p in components)---async def test_profile_shares_one_accumulator_across_components() -> None:- """Response processor and backend must record into the same accumulator.-- The shared bucket is how ``/metrics`` ends up with both per-call counts- (recorded by the backend) and token usage (recorded by the response- processor) under the same model label. The Rust binding's- ``accumulator`` getter returns a fresh Python wrapper each time, so- we verify shared identity by writing through one surface and reading- through the other.- """- switchyard = _build_switchyard(_config())- components = list(switchyard.iter_components())-- stats_processor = next(- p for p in components- if isinstance(p, StatsResponseProcessor)- )- backend = next(- p for p in components- if isinstance(p, LatencyServiceLLMBackend)- )-- # Write through the backend's accumulator handle, then read through the- # response processor's accumulator handle. If construction wired- # two separate accumulators, the read would not see the write.- assert backend._stats is not None- await backend._stats.record_success("model-A", 12.5)-- snapshot = await stats_processor.accumulator.snapshot()- assert snapshot["total_requests"] == 1- assert snapshot["models"]["model-A"]["calls"] == 1-- assert stats_processor.get_endpoint() is not None---def test_profile_honors_externally_provided_stats_accumulator() -> None:- shared = StatsAccumulator()- switchyard = _build_switchyard(_config(), stats_accumulator=shared)-- stats_processor = next(- p for p in switchyard.iter_components()- if isinstance(p, StatsResponseProcessor)- )- # The processor's stats source must be the shared accumulator so an- # externally-mounted StatsEndpoint surfaces the same data.- assert stats_processor.get_endpoint() is not None---# ----------------------------------------------------------------------------# End-to-end: build_switchyard_app exposes /metrics + /v1/stats-# ------------------------------------------------------------------------------def _build_app(*, enable_stats: bool):- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- switchyard = ProfileSwitchyard(- LatencyServiceProfileConfig.from_config(_config(enable_stats=enable_stats))- .build()- .with_runtime_components(enable_stats=enable_stats)- )- return build_switchyard_app(switchyard)---def test_recipe_app_exposes_metrics_endpoint() -> None:- """Regression: latency-service deployments must serve ``/metrics``.-- Before this fix, Datadog/OTel scrapers got 404 on every poll because the- latency-service chain contributed no ``StatsEndpoint`` via- :func:`build_switchyard_app`'s component iteration.- """- app = _build_app(enable_stats=True)-- with TestClient(app, raise_server_exceptions=False) as client:- metrics = client.get("/metrics")- stats = client.get("/v1/stats")- routing_stats = client.get("/v1/routing/stats")-- assert metrics.status_code == 200, (- f"/metrics should be 200 on a latency-service chain, got {metrics.status_code}"- )- assert metrics.headers["content-type"].startswith("text/plain")- # Prometheus exposition includes the canonical counter family even at zero.- assert "switchyard_requests_total" in metrics.text-- assert stats.status_code == 200- assert routing_stats.status_code == 200---def test_recipe_app_omits_metrics_when_stats_disabled() -> None:- """``enable_stats=False`` keeps ``/metrics`` unmounted — opt-out is intentional."""- app = _build_app(enable_stats=False)-- with TestClient(app, raise_server_exceptions=False) as client:- metrics = client.get("/metrics")- stats = client.get("/v1/stats")-- assert metrics.status_code == 404- assert stats.status_code == 404---# ----------------------------------------------------------------------------# YAML route-bundle path-# ------------------------------------------------------------------------------def test_route_bundle_latency_service_exposes_metrics() -> None:- """Deployment path (YAML bundle) must also surface ``/metrics``."""- from switchyard.cli.route_bundle import build_route_bundle_table-- bundle = {- "routes": {- "ls-route": {- "type": "latency_service",- "latency_service_url": "http://latency-service.test:8080",- "endpoints": [- {- "model": "model-A",- "api_key": "test-key",- "base_url": "http://llm.test/v1",- },- ],- },- },- }-- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- table = build_route_bundle_table(bundle)- app = build_switchyard_app(table)-- with TestClient(app, raise_server_exceptions=False) as client:- metrics = client.get("/metrics")-- assert metrics.status_code == 200- assert "switchyard_requests_total" in metrics.text---def test_route_bundle_latency_service_honors_enable_stats_false() -> None:- """YAML ``enable_stats: false`` opts out of the metrics surface."""- from switchyard.cli.route_bundle import build_route_bundle_table-- bundle = {- "routes": {- "ls-route": {- "type": "latency_service",- "latency_service_url": "http://latency-service.test:8080",- "enable_stats": False,- "endpoints": [- {- "model": "model-A",- "api_key": "test-key",- "base_url": "http://llm.test/v1",- },- ],- },- },- }-- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- table = build_route_bundle_table(bundle)- app = build_switchyard_app(table)-- with TestClient(app, raise_server_exceptions=False) as client:- metrics = client.get("/metrics")-- assert metrics.status_code == 404diff --git a/tests/test_outcome_metrics.py b/tests/test_outcome_metrics.pyindex 9cb27d23b..3d8397785 100644--- a/tests/test_outcome_metrics.py+++ b/tests/test_outcome_metrics.py@@ -5,39 +5,33 @@from __future__ import annotations-import json-import logging-from unittest.mock import AsyncMock, MagicMock, patch--import openaiimport pytestfrom fastapi.testclient import TestClientfrom openai.types.chat import ChatCompletionfrom openai.types.chat.chat_completion import Choicefrom openai.types.chat.chat_completion_message import ChatCompletionMessage+from openai.types.completion_usage import CompletionUsage-from switchyard.lib.backends.health_poller import (- EndpointHealth,- EndpointHealthStatus,- HealthPoller,-)-from switchyard.lib.config.latency_service_backend_config import (- LatencyServiceBackendConfig,- LatencyServiceEndpoint,-)from switchyard.lib.endpoints import outcome_metricsfrom switchyard.lib.endpoints.upstream_error import (record_upstream_attempt_failure,record_upstream_attempt_success,)-from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyardfrom switchyard.lib.proxy_context import (CTX_UPSTREAM_ATTEMPTS_RECORDED,CTX_UPSTREAM_HTTP_STATUS,ProxyContext,)+from switchyard.lib.roles import LLMBackend+from switchyard.lib.switchyard import Switchyardfrom switchyard.server.switchyard_app import build_switchyard_app-from switchyard_rust.core import ChatRequest, SwitchyardUpstreamError+from switchyard_rust.core import (+ ChatRequest,+ ChatRequestType,+ ChatResponse,+ SwitchyardUpstreamError,+)+from switchyard_rust.translation import TranslationEngine@pytest.fixture(autouse=True)@@ -47,17 +41,6 @@ def _reset_counters():outcome_metrics._reset_for_tests()-def _latency_service_switchyard(- config: LatencyServiceBackendConfig,-) -> ProfileSwitchyard:- """Build the latency-service profile-backed serving adapter."""- return ProfileSwitchyard(- LatencyServiceProfileConfig.from_config(config)- .build()- .with_runtime_components(enable_stats=config.enable_stats)- )--# ---------------------------------------------------------------------------# Classification# ---------------------------------------------------------------------------@@ -194,295 +177,6 @@ class TestRender:)-# ----------------------------------------------------------------------------# Latency-service backend wiring-# ------------------------------------------------------------------------------def _config(*models: str) -> LatencyServiceBackendConfig:- return LatencyServiceBackendConfig(- latency_service_url="http://latency.test:8080",- endpoints=[- LatencyServiceEndpoint(- model=model,- base_url=f"http://llm-{model}.test/v1",- api_key="test-key",- )- for model in models- ],- max_retries=2,- )---def _build_backend(config: LatencyServiceBackendConfig):- from switchyard.lib.backends.latency_service_llm_backend import (- LatencyServiceLLMBackend,- )-- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"):- return LatencyServiceLLMBackend(config)---def _make_completion() -> ChatCompletion:- return ChatCompletion(- id="cmpl-test",- object="chat.completion",- created=1700000000,- model="any",- choices=[- Choice(- index=0,- message=ChatCompletionMessage(role="assistant", content="ok"),- finish_reason="stop",- )- ],- )---def _api_status_error(status_code: int) -> openai.APIStatusError:- """Build a synthetic APIStatusError carrying the given status code.-- Mirrors what ``OpenAILLMClient.acompletion`` raises when the upstream- HTTP call returns a non-2xx response.- """- import httpx-- response = httpx.Response(- status_code,- request=httpx.Request("POST", "http://llm.test/v1/chat/completions"),- json={"error": {"message": "synthetic"}},- )- return openai.APIStatusError(- "synthetic", response=response, body={"error": "synthetic"}- )---class TestBackendCounters:- async def test_success_records_one_attempt_success(self) -> None:- backend = _build_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion()- )-- await backend.call(ProxyContext(), ChatRequest.openai_chat({- "model": "model-A",- "messages": [{"role": "user", "content": "hi"}],- }))-- out = "\n".join(outcome_metrics.render_lines())- assert 'switchyard_upstream_attempts_total{outcome="success",code="200"} 1' in out- assert (- 'switchyard_upstream_attempts_total{outcome="retryable_error",code="429"} 0'- in out- )- assert "switchyard_router_retry_recovered_total 0" in out-- async def test_429_then_success_increments_recovered(self) -> None:- """First attempt 429, retry succeeds — the steering signal we care about."""- backend = _build_backend(_config("model-A", "model-B"))- # Pin model-A as the sole HEALTHY endpoint so it is tried first- # deterministically; otherwise selection is a random coin between two- # UNKNOWN endpoints and the 429-then-recover path only fires by chance.- with backend._cache_lock:- backend._health_cache["model-A"] = EndpointHealth(- status=EndpointHealthStatus.HEALTHY,- )- backend._health_cache["model-B"] = EndpointHealth(- status=EndpointHealthStatus.UNKNOWN,- )- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(429),- )- backend._clients["model-B"].acompletion = AsyncMock(- return_value=_make_completion(),- )- with backend._cache_lock:- backend._health_cache["model-A"] = EndpointHealth(- EndpointHealthStatus.HEALTHY,- 10.0,- )- backend._health_cache["model-B"] = EndpointHealth(- EndpointHealthStatus.DEGRADED,- 100.0,- )-- await backend.call(ProxyContext(), ChatRequest.openai_chat({- "model": "x",- "messages": [{"role": "user", "content": "hi"}],- }))-- out = "\n".join(outcome_metrics.render_lines())- assert (- 'switchyard_upstream_attempts_total{outcome="retryable_error",code="429"} 1'- in out- )- assert 'switchyard_upstream_attempts_total{outcome="success",code="200"} 1' in out- assert "switchyard_router_retry_recovered_total 1" in out-- async def test_401_does_not_count_as_retryable(self) -> None:- """A 401 (bad key) is ``other_error`` and is not retried — fail fast."""- backend = _build_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(401),- )-- import pytest as _pytest- with _pytest.raises(openai.APIStatusError):- await backend.call(ProxyContext(), ChatRequest.openai_chat({- "model": "x",- "messages": [{"role": "user", "content": "hi"}],- }))-- # A 4xx client error is deterministic, so the loop fails fast: exactly- # one attempt, no failover retries.- assert backend._clients["model-A"].acompletion.call_count == 1- out = "\n".join(outcome_metrics.render_lines())- assert (- 'switchyard_upstream_attempts_total{outcome="other_error",code="401"} 1'- in out- )- assert (- 'switchyard_upstream_attempts_total{outcome="retryable_error",code="429"} 0'- in out- )- assert "switchyard_router_retry_recovered_total 0" in out-- async def test_network_error_counts_as_retryable(self) -> None:- """Non-HTTP exceptions (network, pre-status timeout) map to retryable_error."""- backend = _build_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=RuntimeError("connection refused"),- )-- import pytest as _pytest- with _pytest.raises(RuntimeError):- await backend.call(ProxyContext(), ChatRequest.openai_chat({- "model": "x",- "messages": [{"role": "user", "content": "hi"}],- }))-- out = "\n".join(outcome_metrics.render_lines())- assert (- 'switchyard_upstream_attempts_total{outcome="retryable_error",code="none"} 3'- in out- )-- async def test_failure_emits_structured_error_log(- self, caplog: pytest.LogCaptureFixture- ) -> None:- """Every failed attempt emits one per-event structured log for Loki.-- Single endpoint that always 429s — deterministic, unlike a- multi-endpoint setup whose first pick is a random choice.- """- backend = _build_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(429),- )-- with caplog.at_level(logging.WARNING, logger="switchyard.upstream_errors"):- with pytest.raises(openai.APIStatusError):- await backend.call(ProxyContext(), ChatRequest.openai_chat({- "model": "x",- "messages": [{"role": "user", "content": "hi"}],- }))-- records = [- json.loads(r.getMessage())- for r in caplog.records- if r.name == "switchyard.upstream_errors"- ]- # 3 attempts (1 + max_retries=2), all 429 → 3 structured records,- # attempt numbers 1-based and increasing.- assert len(records) == 3- assert all(r["event"] == "upstream_attempt_failed" for r in records)- assert all(r["status_code"] == 429 and r["code"] == "429" for r in records)- assert all(r["model"] == "model-A" for r in records)- assert [r["attempt"] for r in records] == [1, 2, 3]---# ----------------------------------------------------------------------------# FastAPI middleware — client-side counter-# ------------------------------------------------------------------------------class TestClientResponseMiddleware:- def test_successful_chat_completion_counts_as_success(self) -> None:- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- switchyard = _latency_service_switchyard(- LatencyServiceBackendConfig(- latency_service_url="http://latency.test:8080",- endpoints=[- LatencyServiceEndpoint(- model="model-A",- api_key="test-key",- base_url="http://llm.test/v1",- ),- ],- )- )- backend = next(- component for component in switchyard.iter_components()- if hasattr(component, "_clients")- )- backend._clients["model-A"].acompletion = AsyncMock(- return_value=_make_completion(),- )-- app = build_switchyard_app(switchyard)- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/chat/completions",- json={- "model": "model-A",- "messages": [{"role": "user", "content": "hi"}],- },- )- assert response.status_code == 200- metrics = client.get("/metrics").text-- assert 'switchyard_client_responses_total{outcome="success"} 1' in metrics- assert 'switchyard_client_responses_total{outcome="retryable_error"} 0' in metrics-- def test_metrics_route_is_not_counted_as_client_response(self) -> None:- """Only the LLM routes feed the client outcome counter. Otherwise a- scraper polling /metrics every 10s would inflate the success count."""- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- switchyard = _latency_service_switchyard(- LatencyServiceBackendConfig(- latency_service_url="http://latency.test:8080",- endpoints=[- LatencyServiceEndpoint(- model="model-A",- api_key="test-key",- base_url="http://llm.test/v1",- ),- ],- )- )- app = build_switchyard_app(switchyard)- with TestClient(app, raise_server_exceptions=False) as client:- for _ in range(5):- client.get("/metrics")- client.get("/health")- client.get("/v1/models")- metrics = client.get("/metrics").text-- assert 'switchyard_client_responses_total{outcome="success"} 0' in metrics--# ---------------------------------------------------------------------------# Endpoint-layer fallback — wires the upstream-attempt counter for backends# (Rust native / passthrough / multi) that issue one attempt and can't reach@@ -540,17 +234,71 @@ class TestEndpointUpstreamAttemptFallback:assert f"{_upstream_count(out, 'success', '200')} 0" in outassert f"{_upstream_count(out, 'retryable_error', 'none')} 0" in out- async def test_latency_service_backend_sets_dedup_flag(self) -> None:- """The latency-service backend claims attempt accounting on its ctx."""- backend = _build_backend(_config("model-A"))- backend._clients["model-A"].acompletion = AsyncMock(return_value=_make_completion())- ctx = ProxyContext()- await backend.call(ctx, ChatRequest.openai_chat({- "model": "model-A",- "messages": [{"role": "user", "content": "hi"}],- }))- assert ctx.metadata.get(CTX_UPSTREAM_ATTEMPTS_RECORDED) is True- # It recorded exactly one attempt itself — and the flag would stop the- # endpoint fallback from adding a second.- out = "\n".join(outcome_metrics.render_lines())- assert f"{_upstream_count(out, 'success', '200')} 1" in out++# ---------------------------------------------------------------------------+# Client-response middleware — only the /v1/* LLM routes feed the client+# outcome counter; the operational routes (/metrics, /health, /v1/models) do+# not, so a scraper polling them cannot inflate the success count.+# ---------------------------------------------------------------------------+++class _CannedBackend(LLMBackend):+ """Passthrough-style backend: returns one canned OpenAI completion per call."""++ def supported_request_types(self) -> list[ChatRequestType]:+ return [+ ChatRequestType.OPENAI_CHAT,+ ChatRequestType.OPENAI_RESPONSES,+ ChatRequestType.ANTHROPIC,+ ]++ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:+ return ChatResponse.openai_completion(+ ChatCompletion(+ id="chatcmpl-test",+ object="chat.completion",+ created=1700000000,+ model="mock-model",+ choices=[+ Choice(+ index=0,+ message=ChatCompletionMessage(role="assistant", content="hi"),+ finish_reason="stop",+ )+ ],+ usage=CompletionUsage(prompt_tokens=5, completion_tokens=3, total_tokens=8),+ )+ )+++def _canned_app() -> object:+ return build_switchyard_app(+ Switchyard(backend=_CannedBackend(), translator=TranslationEngine())+ )+++class TestClientResponseMiddleware:+ def test_successful_chat_completion_counts_as_success(self) -> None:+ """A served /v1/chat/completions success increments the client outcome counter."""+ with TestClient(_canned_app(), raise_server_exceptions=False) as client:+ response = client.post(+ "/v1/chat/completions",+ json={"model": "mock-model", "messages": [{"role": "user", "content": "hi"}]},+ )+ assert response.status_code == 200++ metrics = "\n".join(outcome_metrics.render_lines())+ assert 'switchyard_client_responses_total{outcome="success"} 1' in metrics+ assert 'switchyard_client_responses_total{outcome="retryable_error"} 0' in metrics++ def test_operational_routes_are_not_counted_as_client_responses(self) -> None:+ """Polling /metrics, /health, /v1/models must not feed the client counter —+ otherwise a scraper on a fixed cadence would inflate the success count."""+ with TestClient(_canned_app(), raise_server_exceptions=False) as client:+ for _ in range(5):+ client.get("/metrics")+ client.get("/health")+ client.get("/v1/models")++ metrics = "\n".join(outcome_metrics.render_lines())+ assert 'switchyard_client_responses_total{outcome="success"} 0' in metricsdiff --git a/tests/test_passthrough_profile_e2e.py b/tests/test_passthrough_profile_e2e.pyindex 1e96a2878..3a42ee280 100644--- a/tests/test_passthrough_profile_e2e.py+++ b/tests/test_passthrough_profile_e2e.py@@ -399,7 +399,7 @@ class TestPassthroughProfileBackendErrors:class TestPassthroughProfileUpstreamAttemptCounters:- """`switchyard_upstream_attempts_total` must populate for non-latency chains.+ """`switchyard_upstream_attempts_total` must populate for passthrough chains.The endpoint-layer fallback records one upstream attempt per request forbackends (here the Rust ``OpenAiPassthroughBackend``) that issue exactlydiff --git a/tests/test_redis_pin_store.py b/tests/test_redis_pin_store.pyindex 49a7269d0..1ccf5fafa 100644--- a/tests/test_redis_pin_store.py+++ b/tests/test_redis_pin_store.py@@ -14,11 +14,8 @@ from __future__ import annotationsimport pytestfrom pydantic import ValidationError-from switchyard.lib.backends.latency_service_llm_backend import _build_affinity_l2-from switchyard.lib.config.latency_service_backend_config import (- LatencyServiceBackendConfig,- LatencyServiceEndpoint,-)+from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig+from switchyard.lib.profiles.escalation_router_profile_config import _build_affinity_l2from switchyard.lib.redis_pin_store import RedisPinStore@@ -86,15 +83,17 @@ async def test_aclose_before_first_use_is_noop() -> None:# --- config validation + backend wiring --------------------------------------def _redis_config(**overrides: object) -> LatencyServiceBackendConfig:+def _redis_config(**overrides: object) -> EscalationRouterConfig:base: dict[str, object] = {- "endpoints": [LatencyServiceEndpoint(model="m")],- "session_affinity": True,+ "strong": {"model": "strong-model"},+ "weak": {"model": "weak-model"},+ "judge": {"model": "judge-model"},+ "fallback_target_on_evict": "weak","affinity_store": "redis","affinity_store_url": "redis://cache:6379/0",}base.update(overrides)- return LatencyServiceBackendConfig(**base) # type: ignore[arg-type]+ return EscalationRouterConfig(**base) # type: ignore[arg-type]def test_redis_store_requires_url() -> None:@@ -102,13 +101,13 @@ def test_redis_store_requires_url() -> None:_redis_config(affinity_store_url=None)-def test_redis_store_requires_session_affinity() -> None:- with pytest.raises(ValidationError):- _redis_config(session_affinity=False)--def test_memory_is_the_default_store() -> None:- cfg = LatencyServiceBackendConfig(endpoints=[LatencyServiceEndpoint(model="m")])+ cfg = EscalationRouterConfig(+ strong={"model": "strong-model"},+ weak={"model": "weak-model"},+ judge={"model": "judge-model"},+ fallback_target_on_evict="weak",+ )assert cfg.affinity_store == "memory"assert _build_affinity_l2(cfg) is Nonediff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.pyindex b1a7e68a1..2f7365544 100644--- a/tests/test_route_bundle.py+++ b/tests/test_route_bundle.py@@ -50,18 +50,6 @@ def _random_processor(chain: object) -> Any:)-def _latency_backend(chain: object) -> Any:- from switchyard.lib.backends.latency_service_llm_backend import (- LatencyServiceLLMBackend,- )-- return next(- component- for component in chain.iter_components()- if isinstance(component, LatencyServiceLLMBackend)- )--class _NoopRequestProcessor:async def process(self, _ctx: ProxyContext, request: ChatRequest) -> ChatRequest:return request@@ -1486,115 +1474,15 @@ _AFFINITY_DET_ROUTE = {"strong": {"model": "s", "api_key": "k", "base_url": "https://ls.test/v1"},"weak": {"model": "w", "api_key": "k", "base_url": "https://ls.test/v1"},}-_AFFINITY_LAT_ROUTE = {- "type": "latency_service",- "latency_service_url": "http://ls.test:8080",- "session_affinity": True,- "affinity_max_sessions": 50,- "endpoints": [- {"model": "s", "api_key": "k", "base_url": "https://ls.test/v1"},- {"model": "w", "api_key": "k", "base_url": "https://ls.test/v1"},- ],-}[email protected]("route", [_AFFINITY_DET_ROUTE, _AFFINITY_LAT_ROUTE])[email protected]("route", [_AFFINITY_DET_ROUTE])def test_session_affinity_keys_accepted_by_route_bundle(route: dict[str, Any]) -> None:- """session_affinity / affinity_max_sessions are valid keys on both route types."""+ """session_affinity / affinity_max_sessions are valid keys on the deterministic route."""table = build_route_bundle_table({"routes": {"r": route}})assert isinstance(table, RouteTable)-def test_latency_service_credential_policy_defaults_to_endpoint_keys() -> None:- """Omitting credential_policy keeps the server-configured endpoint keys authoritative."""- table = build_route_bundle_table({"routes": {"r": _AFFINITY_LAT_ROUTE}})-- backend = _latency_backend(table.lookup_switchyard("r"))-- assert backend._config.credential_policy == "configured_endpoint"---def test_latency_service_credential_policy_reaches_backend_config() -> None:- """YAML credential_policy opts into BYO-key caller overrides."""- table = build_route_bundle_table({- "routes": {- "r": {- **_AFFINITY_LAT_ROUTE,- "credential_policy": "caller_override",- },- },- })-- backend = _latency_backend(table.lookup_switchyard("r"))-- assert backend._config.credential_policy == "caller_override"---def test_latency_service_invalid_credential_policy_rejected_via_bundle() -> None:- """Invalid latency-service credential policy values fail closed."""- from pydantic import ValidationError-- with pytest.raises(ValidationError):- build_route_bundle_table({- "routes": {- "r": {- **_AFFINITY_LAT_ROUTE,- "credential_policy": "caller-overrides",- },- },- })---def test_latency_endpoint_request_type_reaches_backend_config() -> None:- """Endpoint-level request_type in YAML selects the upstream API surface."""- table = build_route_bundle_table({- "routes": {- "r": {- "type": "latency_service",- "latency_service_url": "http://ls.test:8080",- "endpoints": [- {- "model": "codex-mini",- "api_key": "k",- "base_url": "https://ls.test/v1",- "request_type": "openai_responses",- },- {"model": "w", "api_key": "k", "base_url": "https://ls.test/v1"},- ],- },- },- })-- backend = _latency_backend(table.lookup_switchyard("r"))-- by_model = {endpoint.model: endpoint for endpoint in backend._config.endpoints}- assert by_model["codex-mini"].request_type == "openai_responses"- assert by_model["w"].request_type == "openai_chat"---def test_latency_route_key_reaches_backend_as_route_model() -> None:- """The YAML route key becomes the backend's metrics route_model id."""- table = build_route_bundle_table({- "routes": {- "nvidia/switchyard/gpt-5.4": {- "type": "latency_service",- "latency_service_url": "http://ls.test:8080",- "endpoints": [- {- "model": "azure/openai/gpt-5.4",- "api_key": "k",- "base_url": "https://ls.test/v1",- },- ],- },- },- })-- backend = _latency_backend(table.lookup_switchyard("nvidia/switchyard/gpt-5.4"))-- assert backend._config.route_model == "nvidia/switchyard/gpt-5.4"--def test_deterministic_affinity_warmup_turns_accepted_by_route_bundle() -> None:"""affinity_warmup_turns is a deterministic-route knob."""table = build_route_bundle_table({@@ -1603,7 +1491,7 @@ def test_deterministic_affinity_warmup_turns_accepted_by_route_bundle() -> None:assert isinstance(table, RouteTable)[email protected]("route", [_AFFINITY_DET_ROUTE, _AFFINITY_LAT_ROUTE])[email protected]("route", [_AFFINITY_DET_ROUTE])def test_zero_capacity_affinity_rejected_via_bundle(route: dict[str, Any]) -> None:"""A zero cap with affinity on is rejected — proving the keys reach the config."""from pydantic import ValidationError@@ -1612,37 +1500,6 @@ def test_zero_capacity_affinity_rejected_via_bundle(route: dict[str, Any]) -> Nobuild_route_bundle_table({"routes": {"r": {**route, "affinity_max_sessions": 0}}})-def test_latency_affinity_redis_reaches_backend_via_bundle() -> None:- """Redis L2 keys parse and construct a RedisPinStore on the latency backend."""- from switchyard.lib.redis_pin_store import RedisPinStore-- table = build_route_bundle_table({- "routes": {- "r": {- **_AFFINITY_LAT_ROUTE,- "affinity_store": "redis",- "affinity_store_url": "redis://cache:6379/0",- "affinity_store_ttl_seconds": 120,- "affinity_key_prefix": "k:",- },- },- })- backend = _latency_backend(table.lookup_switchyard("r"))- assert backend._config.affinity_store == "redis"- assert backend._config.affinity_store_url == "redis://cache:6379/0"- assert isinstance(backend._affinity._l2, RedisPinStore)---def test_latency_affinity_redis_requires_url_via_bundle() -> None:- """affinity_store=redis without a URL fails closed at config load."""- from pydantic import ValidationError-- with pytest.raises(ValidationError):- build_route_bundle_table({- "routes": {"r": {**_AFFINITY_LAT_ROUTE, "affinity_store": "redis"}},- })--def test_negative_affinity_warmup_turns_rejected_via_bundle() -> None:"""The deterministic config validates affinity_warmup_turns as non-negative."""from pydantic import ValidationErrordiff --git a/tests/test_route_selection_headers.py b/tests/test_route_selection_headers.pyindex 350498f46..4b1dabacd 100644--- a/tests/test_route_selection_headers.py+++ b/tests/test_route_selection_headers.py@@ -1,43 +1,22 @@# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.# SPDX-License-Identifier: Apache-2.0-"""End-to-end gate for route-selection spend-attribution headers.+"""Route-selection spend-attribution response headers.For tokenomics reporting, a front proxy (e.g. LiteLLM) must be able to tieits provider spend-log rows back to the Switchyard logical route that-selected them:--* outbound — every upstream attempt carries an ``x-litellm-spend-logs-metadata``- request header whose JSON payload records the route selection plus a- per-request correlation id;-* inbound — the Switchyard response returns ``x-switchyard-*`` headers with- the successful attempt's selection and the same correlation id, so the- parent spend-log row can be enriched to match the provider row.--The app-level tests run the real ``OpenAILLMClient`` against a loopback HTTP-stub, so the outbound header is asserted on the wire — not on a mock.+selected them: the Switchyard response returns ``x-switchyard-*`` headers with+the successful attempt's selection plus a per-request correlation id, so the+parent spend-log row can be enriched to match the provider row."""from __future__ import annotationsimport json-from collections.abc import Iteratorfrom typing import Any-from fastapi import FastAPIfrom fastapi.responses import JSONResponse-from fastapi.testclient import TestClient-from pytest import fixture-from pytest_mock import MockerFixture-from switchyard.lib.backends.health_poller import HealthPoller-from switchyard.lib.backends.latency_service_llm_backend import (- SPEND_LOGS_METADATA_HEADER,-)-from switchyard.lib.config.latency_service_backend_config import (- LatencyServiceBackendConfig,- LatencyServiceEndpoint,-)from switchyard.lib.endpoints.dispatch import serialize_chain_resultfrom switchyard.lib.endpoints.route_selection import (ROUTER_CORRELATION_ID_HEADER,@@ -47,10 +26,7 @@ from switchyard.lib.endpoints.route_selection import (route_selection_headers,)from switchyard.lib.endpoints.upstream_error import handle_chain_exception-from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyardfrom switchyard.lib.proxy_context import CTX_ROUTE_SELECTION, ProxyContext-from switchyard.server.switchyard_app import build_switchyard_app-from tests._chain_test_helpers import _backend_payload, _OpenAICompatStub, _sse_body, _stream_chunkROUTE_MODEL = "nvidia/switchyard/test-route"ENDPOINT_ID = "openai/test-model"@@ -58,10 +34,9 @@ UPSTREAM_MODEL = "openai/openai/test-model"def _selection(**overrides: object) -> dict[str, object]:- """Selection record as the latency backend writes it, with *overrides* applied."""+ """Recorded route selection, with *overrides* applied."""payload: dict[str, object] = {"router_model": ROUTE_MODEL,- "router_strategy": "latency","router_selected_endpoint": ENDPOINT_ID,"router_selected_model": UPSTREAM_MODEL,"router_selected_provider": "openai",@@ -256,240 +231,3 @@ class TestErrorPathSelectionHeaders:assert response.status_code == 500assert ROUTER_CORRELATION_ID_HEADER not in response.headers---# ----------------------------------------------------------------------------# End-to-end: HTTP in → wire header out → response headers back-# ------------------------------------------------------------------------------@fixture-def latency_app(mocker: MockerFixture) -> Iterator[tuple[FastAPI, _OpenAICompatStub]]:- """Latency-service app whose single endpoint targets a loopback stub.-- The backend uses the real ``OpenAILLMClient``/OpenAI SDK, so the stub- records the actual HTTP headers the upstream would receive. Only the- health poller is stubbed out (no Latency Service in tests).- """- with _OpenAICompatStub() as stub:- config = LatencyServiceBackendConfig(- latency_service_url="http://latency-service.test:8080",- route_model=ROUTE_MODEL,- endpoints=[- LatencyServiceEndpoint(- model=ENDPOINT_ID,- upstream_model=UPSTREAM_MODEL,- base_url=stub.base_url,- api_key="test-key",- ),- ],- )- mocker.patch.object(HealthPoller, "start")- mocker.patch.object(HealthPoller, "stop")- switchyard = ProfileSwitchyard(- LatencyServiceProfileConfig.from_config(config)- .build()- .with_runtime_components(enable_stats=config.enable_stats)- )- yield build_switchyard_app(switchyard), stub---def _wire_spend_logs_payload(stub: _OpenAICompatStub) -> dict[str, object]:- """Parse the spend-logs metadata header the stub received on the wire."""- headers = stub.requests[-1]["headers"]- assert isinstance(headers, dict)- payload = json.loads(str(headers[SPEND_LOGS_METADATA_HEADER]))- assert isinstance(payload, dict)- return payload---def test_chat_completion_roundtrips_selection_and_correlation_id(- latency_app: tuple[FastAPI, _OpenAICompatStub],-) -> None:- """The wire spend-logs header and the response headers share one selection."""- app, stub = latency_app- stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL))-- with TestClient(app) as client:- response = client.post(- "/v1/chat/completions",- json={- "model": ROUTE_MODEL,- "messages": [{"role": "user", "content": "hi"}],- },- )-- assert response.status_code == 200- payload = _wire_spend_logs_payload(stub)- assert payload["router_model"] == ROUTE_MODEL- assert payload["router_strategy"] == "latency"- assert payload["router_selected_endpoint"] == ENDPOINT_ID- assert payload["router_selected_model"] == UPSTREAM_MODEL- assert payload["router_selected_provider"] == "openai"- assert response.headers[ROUTER_MODEL_HEADER] == ROUTE_MODEL- assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL- assert response.headers[SELECTED_PROVIDER_HEADER] == "openai"- # The join key: the response header must equal the id the provider row got.- assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"]---def test_streaming_chat_completion_carries_selection_headers(- latency_app: tuple[FastAPI, _OpenAICompatStub],-) -> None:- """A streaming completion returns the same selection headers before the body."""- app, stub = latency_app- stub.respond_sse(_sse_body([_stream_chunk(content="hel"), _stream_chunk(finish="stop")]))-- with TestClient(app) as client:- response = client.post(- "/v1/chat/completions",- json={- "model": ROUTE_MODEL,- "messages": [{"role": "user", "content": "hi"}],- "stream": True,- },- )-- assert response.status_code == 200- assert response.headers["content-type"].startswith("text/event-stream")- assert "data:" in response.text- payload = _wire_spend_logs_payload(stub)- assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"]- assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL---def test_anthropic_messages_endpoint_carries_selection_headers(- latency_app: tuple[FastAPI, _OpenAICompatStub],-) -> None:- """Anthropic inbound rides the same dispatch path and gets the same headers."""- app, stub = latency_app- stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL))-- with TestClient(app) as client:- response = client.post(- "/v1/messages",- json={- "model": ROUTE_MODEL,- "max_tokens": 64,- "messages": [{"role": "user", "content": "hi"}],- },- )-- assert response.status_code == 200- payload = _wire_spend_logs_payload(stub)- assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"]- assert response.headers[SELECTED_PROVIDER_HEADER] == "openai"---def test_responses_endpoint_carries_selection_headers(- latency_app: tuple[FastAPI, _OpenAICompatStub],-) -> None:- """OpenAI Responses inbound rides the same dispatch path and gets the same headers."""- app, stub = latency_app- stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL))-- with TestClient(app) as client:- response = client.post(- "/v1/responses",- json={"model": ROUTE_MODEL, "input": "hi"},- )-- assert response.status_code == 200- payload = _wire_spend_logs_payload(stub)- assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"]- assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL---def test_upstream_error_response_has_no_selection_headers(- latency_app: tuple[FastAPI, _OpenAICompatStub],-) -> None:- """No upstream call succeeded — the error response must not claim a selection.-- The failed attempt itself is still stamped on the wire (the provider row- for the failure keeps its attribution); the client-facing headers are- reserved for a recorded selection, i.e. a billed upstream success — which- never happened here. (A failure *after* a billed success does carry them;- see TestErrorPathSelectionHeaders.)- """- app, stub = latency_app- stub.respond_json({"error": {"message": "bad key"}}, status=401)-- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/chat/completions",- json={- "model": ROUTE_MODEL,- "messages": [{"role": "user", "content": "hi"}],- },- )-- assert response.status_code == 401- assert ROUTER_CORRELATION_ID_HEADER not in response.headers- assert SPEND_LOGS_METADATA_HEADER in stub.requests[-1]["headers"] # type: ignore[operator]---def test_hostile_model_string_never_breaks_the_response(- latency_app: tuple[FastAPI, _OpenAICompatStub],-) -> None:- """A model string that is not legal header material must not 500 or inject.-- Single-chain profile serving forwards any client model string, so- router_model is client-controlled; the billed 200 must survive, with the- unsafe header skipped and the config-derived headers intact.- """- app, stub = latency_app-- with TestClient(app) as client:- for hostile_model in ("gpt-4中文", "gpt\r\nx-evil: 1"):- stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL))- response = client.post(- "/v1/chat/completions",- json={- "model": hostile_model,- "messages": [{"role": "user", "content": "hi"}],- },- )-- assert response.status_code == 200- assert ROUTER_MODEL_HEADER not in response.headers- assert "x-evil" not in response.headers- assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL- # The outbound header is JSON (ensure_ascii escapes control and- # non-ASCII chars), so the wire payload still records the model.- assert _wire_spend_logs_payload(stub)["router_model"] == hostile_model---def test_client_body_extra_headers_forwarded_not_collided(- latency_app: tuple[FastAPI, _OpenAICompatStub],-) -> None:- """A passthrough body carrying an SDK-style extra_headers field keeps working.-- Pre-spend-logs it rode ``**body`` into the SDK's header kwarg; it must- still reach the wire — and must not be able to spoof the spend-logs- header under any key casing, since header names are case-insensitive on- the wire.- """- app, stub = latency_app- stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL))-- with TestClient(app) as client:- response = client.post(- "/v1/chat/completions",- json={- "model": ROUTE_MODEL,- "messages": [{"role": "user", "content": "hi"}],- "extra_headers": {- "x-client-tag": "42",- SPEND_LOGS_METADATA_HEADER: "spoofed",- "X-LiteLLM-Spend-Logs-Metadata": "spoofed-cased",- },- },- )-- assert response.status_code == 200- headers = stub.requests[-1]["headers"]- assert isinstance(headers, dict)- assert headers["x-client-tag"] == "42"- payload = _wire_spend_logs_payload(stub)- assert payload["router_selected_endpoint"] == ENDPOINT_ID # not "spoofed"- assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"]diff --git a/tests/test_session_affinity.py b/tests/test_session_affinity.pyindex 26cd88975..d06a8ac57 100644--- a/tests/test_session_affinity.py+++ b/tests/test_session_affinity.py@@ -3,7 +3,7 @@"""Unit tests for SessionAffinity's L1 (in-process) + optional L2 (shared) tiering.-The latency backend and classifier router integration tests cover affinity+The classifier and escalation router integration tests cover affinity*policy*; these tests cover the store mechanics in isolation: write-through,read-through with L1 warming, and best-effort (fail-open) degradation when theL2 errors.diff --git a/tests/test_stream_close_chain.py b/tests/test_stream_close_chain.pyindex 134844998..81b6e085e 100644--- a/tests/test_stream_close_chain.py+++ b/tests/test_stream_close_chain.py@@ -3,7 +3,7 @@"""Close ownership survives the Python -> Rust-core -> Python round trip.-The latency-router production path is: a Python backend returns+The production path is: a Python backend returns``ChatResponse.openai_stream(ResponseStream(sdk_stream))``; the backend adapterconverts it to a Rust-core stream via ``take_core``; response processors run incore; the terminal translator rebuilds a Python ``ChatResponseStream`` viadiff --git a/tests/test_stream_leak_repro.py b/tests/test_stream_leak_repro.pyindex ec00b6f98..22c1f571c 100644--- a/tests/test_stream_leak_repro.py+++ b/tests/test_stream_leak_repro.py@@ -195,7 +195,7 @@ async def test_our_close_releases_pool_connection() -> None:class _RealSdkStreamingBackend(LLMBackend):"""Backend that issues a real streaming SDK call and wraps the AsyncStream.- Mirrors the latency-service production shape: ``call`` returns+ Mirrors the production shape: ``call`` returns``ChatResponse.openai_stream(ResponseStream(sdk_stream))`` where``sdk_stream`` is a genuine OpenAI ``AsyncStream`` holding a pooled httpxconnection. Running this through ``Switchyard`` exercises the realdiff --git a/tests/test_upstream_error_passthrough.py b/tests/test_upstream_error_passthrough.pyindex e2be77f59..5952906da 100644--- a/tests/test_upstream_error_passthrough.py+++ b/tests/test_upstream_error_passthrough.py@@ -19,99 +19,23 @@ Switchyard error envelope instead of FastAPI's default plain-text 500.from __future__ import annotationsimport json-from collections.abc import AsyncIterator-from unittest.mock import AsyncMock, MagicMock, patch-import httpx-import openai-import pytest-from fastapi import FastAPIfrom fastapi.testclient import TestClientfrom switchyard.cli.route_bundle import build_route_bundle_table-from switchyard.lib.backends.health_poller import HealthPoller-from switchyard.lib.backends.latency_service_llm_backend import (- LatencyServiceLLMBackend,-)-from switchyard.lib.config.latency_service_backend_config import (- LatencyServiceBackendConfig,- LatencyServiceEndpoint,-)-from switchyard.lib.endpoints.error_envelope import (- ERROR_SOURCE_HEADER,- UPSTREAM_MODEL_HEADER,-)+from switchyard.lib.endpoints.error_envelope import ERROR_SOURCE_HEADERfrom switchyard.lib.endpoints.upstream_error import (internal_chain_error_response,upstream_response_from_ctx,)-from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyardfrom switchyard.lib.proxy_context import (CTX_UPSTREAM_HTTP_BODY,CTX_UPSTREAM_HTTP_STATUS,ProxyContext,)from switchyard.server.switchyard_app import build_switchyard_app-from switchyard_rust.core import ChatRequestfrom tests._chain_test_helpers import _OpenAICompatStub-# ----------------------------------------------------------------------------# Helper-# ------------------------------------------------------------------------------def _api_status_error(status: int, body: object = None) -> openai.APIStatusError:- """Build an ``openai.APIStatusError`` with a realistic response object."""- request = httpx.Request("POST", "http://upstream.test/v1/chat/completions")- response = httpx.Response(- status_code=status,- json=body if body is not None else {"error": {"message": f"upstream {status}"}},- request=request,- )- return openai.APIStatusError(- message=f"upstream returned {status}",- response=response,- body=body,- )---def _latency_service_switchyard(- config: LatencyServiceBackendConfig,-) -> ProfileSwitchyard:- """Build the latency-service profile-backed serving adapter."""- return ProfileSwitchyard(- LatencyServiceProfileConfig.from_config(config)- .build()- .with_runtime_components(enable_stats=config.enable_stats)- )---def _make_backend() -> LatencyServiceLLMBackend:- config = LatencyServiceBackendConfig(- latency_service_url="http://latency-service.test:8080",- endpoints=[- LatencyServiceEndpoint(- model="model-A",- api_key="bad-key",- base_url="http://llm.test/v1",- ),- ],- )- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"):- return LatencyServiceLLMBackend(config)---def _openai_request() -> ChatRequest:- return ChatRequest.openai_chat({- "model": "model-A",- "messages": [{"role": "user", "content": "hi"}],- })--# ---------------------------------------------------------------------------# Helper unit tests# ---------------------------------------------------------------------------@@ -228,266 +152,11 @@ class TestUpstreamResponseFromCtx:assert len(body["error"]["message"]) <= 200-# ----------------------------------------------------------------------------# Backend records status on ctx before raising-# ------------------------------------------------------------------------------class TestLatencyServiceBackendStashesStatus:- async def test_401_recorded_on_ctx_before_raise(self) -> None:- backend = _make_backend()- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(401, {"error": {"message": "bad key"}}),- )-- ctx = ProxyContext()- with pytest.raises(openai.APIStatusError):- await backend.call(ctx, _openai_request())-- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401- body = ctx.metadata[CTX_UPSTREAM_HTTP_BODY]- assert isinstance(body, dict)- assert body["error"]["message"] == "bad key"-- async def test_429_recorded_on_ctx_before_raise(self) -> None:- backend = _make_backend()- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(429, {"error": {"message": "slow down"}}),- )-- ctx = ProxyContext()- with pytest.raises(openai.APIStatusError):- await backend.call(ctx, _openai_request())-- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 429-- async def test_non_http_error_leaves_ctx_clean(self) -> None:- """Network errors etc shouldn't trip the upstream-status passthrough."""- backend = _make_backend()- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=RuntimeError("connection refused"),- )-- ctx = ProxyContext()- with pytest.raises(RuntimeError, match="connection refused"):- await backend.call(ctx, _openai_request())-- assert CTX_UPSTREAM_HTTP_STATUS not in ctx.metadata- assert CTX_UPSTREAM_HTTP_BODY not in ctx.metadata---# ----------------------------------------------------------------------------# End-to-end: HTTP request through the recipe → upstream 401 → client 401-# ------------------------------------------------------------------------------async def latency_service_app() -> AsyncIterator[tuple[FastAPI, LatencyServiceLLMBackend]]:- """Build a latency-service-backed FastAPI app sharing one backend instance.-- Returns ``(app, backend)`` so each test can patch ``backend._clients[...].acompletion``- to simulate a specific upstream response.- """- config = LatencyServiceBackendConfig(- latency_service_url="http://latency-service.test:8080",- endpoints=[- LatencyServiceEndpoint(- model="model-A",- api_key="bad-key",- base_url="http://llm.test/v1",- ),- ],- )- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- switchyard = _latency_service_switchyard(config)- app = build_switchyard_app(switchyard)- # Reach into the chain to find the LatencyServiceLLMBackend- # instance so tests can stub the upstream call.- backend = _find_latency_backend(switchyard)- yield app, backend---def _find_latency_backend(switchyard: object) -> LatencyServiceLLMBackend:- iter_components = getattr(switchyard, "iter_components", None)- assert iter_components is not None- for component in iter_components():- # The latency-service backend may be wrapped (it isn't today, but- # be tolerant) — match by isinstance, with attribute-walk fallback.- if isinstance(component, LatencyServiceLLMBackend):- return component- inner = getattr(component, "_inner", None) or getattr(component, "inner", None)- if isinstance(inner, LatencyServiceLLMBackend):- return inner- raise AssertionError("LatencyServiceLLMBackend not found in chain")---def test_upstream_401_passes_through_as_401(- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],-) -> None:- """A 401 from upstream must become a 401 to the client, not a 500."""- app, backend = latency_service_app- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(- 401, {"error": {"message": "bad key", "type": "invalid_api_key"}},- ),- )-- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/chat/completions",- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},- )-- assert response.status_code == 401, (- f"expected upstream 401 to pass through; got {response.status_code}, "- f"body={response.text!r}"- )- body = response.json()- assert body["error"]["message"] == "bad key"- assert body["error"]["type"] == "invalid_api_key"- assert body["error"]["code"] == "invalid_api_key"---def test_upstream_429_passes_through_as_429(- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],-) -> None:- app, backend = latency_service_app- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(- 429, {"error": {"message": "rate limit", "type": "rate_limit_exceeded"}},- ),- )-- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/chat/completions",- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},- )-- assert response.status_code == 429- assert response.json() == {- "error": {- "message": "rate limit",- "type": "rate_limit_exceeded",- "code": "rate_limit_exceeded",- }- }---def test_non_http_error_returns_wrapped_500(- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],-) -> None:- """Errors without an upstream HTTP status still return a JSON error envelope."""- app, backend = latency_service_app- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=RuntimeError("connection refused"),- )-- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/chat/completions",- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},- )-- assert response.status_code == 500- assert response.headers["content-type"].startswith("application/json")- body = response.json()- assert body["error"]["type"] == "internal_error"- assert body["error"]["code"] == "internal_chain_error"- assert "connection refused" in body["error"]["message"]---def test_post_dispatch_exception_returns_json_500(- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],-) -> None:- """Exception raised after dispatch (e.g. during result serialization) must not- fall through to FastAPI's plain-text 500 handler."""- app, _ = latency_service_app-- class _UnserializableResult:- def model_dump(self) -> None:- raise RuntimeError("serialization exploded")-- with patch(- "switchyard.lib.endpoints.openai_chat_endpoint.dispatch_chat_request",- new_callable=AsyncMock,- return_value=_UnserializableResult(),- ):- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/chat/completions",- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},- )-- assert response.status_code == 500- assert response.headers["content-type"].startswith("application/json")- body = response.json()- assert body["error"]["type"] == "internal_error"- assert body["error"]["code"] == "internal_chain_error"- assert "serialization exploded" in body["error"]["message"]--# ---------------------------------------------------------------------------# Anthropic and Responses endpoints share the same helper# ----------------------------------------------------------------------------def test_anthropic_endpoint_passes_through_upstream_401(- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],-) -> None:- app, backend = latency_service_app- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(401, {"error": {"message": "bad key"}}),- )-- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/messages",- json={- "model": "model-A",- "max_tokens": 16,- "messages": [{"role": "user", "content": "ping"}],- },- )-- assert response.status_code == 401- assert response.json() == {- "error": {- "message": "bad key",- "type": "upstream_error",- "code": "upstream_error",- }- }---def test_responses_endpoint_passes_through_upstream_401(- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],-) -> None:- app, backend = latency_service_app- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(401, {"error": {"message": "bad key"}}),- )-- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/responses",- json={"model": "model-A", "input": "ping"},- )-- assert response.status_code == 401- assert response.json() == {- "error": {- "message": "bad key",- "type": "upstream_error",- "code": "upstream_error",- }- }--def test_rust_openai_route_upstream_401_returns_structured_openai_error() -> None:"""Rust OpenAI-native backend errors must not fall through to FastAPI's 500."""with _OpenAICompatStub() as upstream:@@ -609,111 +278,6 @@ def test_rust_openai_route_upstream_401_returns_same_error_shape_for_responses_i}-# ----------------------------------------------------------------------------# Invalid request payloads-# ----------------------------------------------------------------------------# A transparent router must reject the same payloads the upstream provider-# would. An unsupported message role (e.g. "api") was previously coerced to-# "user" and returned 200; it now surfaces as a provider-compatible 400 via the-# same ctx-stash → endpoint-passthrough mechanism used for upstream HTTP errors.---class TestInvalidRoleRejection:- async def test_invalid_role_stashes_400_without_upstream_call(self) -> None:- """An unsupported role is rejected at translation, before any upstream call."""- backend = _make_backend()- backend._clients["model-A"].acompletion = AsyncMock()-- ctx = ProxyContext()- request = ChatRequest.openai_responses({- "model": "model-A",- "input": [{"type": "message", "role": "api", "content": "hi"}],- })- with pytest.raises(ValueError):- await backend.call(ctx, request)-- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 400- body = ctx.metadata[CTX_UPSTREAM_HTTP_BODY]- assert isinstance(body, dict)- assert body["error"]["code"] == "invalid_value"- # The upstream must never be called for an invalid payload.- backend._clients["model-A"].acompletion.assert_not_awaited()-- async def test_valid_role_passes_translation_and_reaches_upstream(self) -> None:- """A valid role must not be over-rejected — translation succeeds and the- request proceeds to the upstream call (no invalid-value 400 stashed)."""- backend = _make_backend()- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=RuntimeError("reached upstream"),- )-- ctx = ProxyContext()- request = ChatRequest.openai_responses({- "model": "model-A",- "input": [{"type": "message", "role": "user", "content": "hi"}],- })- with pytest.raises(RuntimeError, match="reached upstream"):- await backend.call(ctx, request)-- assert CTX_UPSTREAM_HTTP_STATUS not in ctx.metadata---def test_responses_invalid_role_returns_400(- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],-) -> None:- """role:"api" on the Responses endpoint must return 400, not a coerced 200/500."""- app, backend = latency_service_app- # No upstream stub: the request must be rejected before any upstream call.- backend._clients["model-A"].acompletion = AsyncMock()-- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/responses",- json={- "model": "model-A",- "input": [{"type": "message", "role": "api", "content": "ping"}],- },- )-- assert response.status_code == 400, (- f"expected invalid role to be rejected with 400; got {response.status_code}, "- f"body={response.text!r}"- )- body = response.json()- assert body["error"]["code"] == "invalid_value"- backend._clients["model-A"].acompletion.assert_not_awaited()---def test_anthropic_invalid_role_returns_400(- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],-) -> None:- """role:"api" on the Anthropic endpoint is rejected with a 400.-- The Anthropic inbound format must be translated to OpenAI Chat for the- latency backend, so the request is decoded and the unsupported role is- rejected before any upstream call. (A native OpenAI-Chat request is a- passthrough for this backend, so its contract is enforced upstream.)- """- app, backend = latency_service_app- backend._clients["model-A"].acompletion = AsyncMock()-- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/messages",- json={- "model": "model-A",- "max_tokens": 16,- "messages": [{"role": "api", "content": "ping"}],- },- )-- assert response.status_code == 400, (- f"expected invalid role to be rejected with 400; got {response.status_code}, "- f"body={response.text!r}"- )- backend._clients["model-A"].acompletion.assert_not_awaited()--# ---------------------------------------------------------------------------# Failure-source headers on the wire# ---------------------------------------------------------------------------@@ -722,71 +286,6 @@ def test_anthropic_invalid_role_returns_400(# the HTTP response a client receives.-def test_upstream_401_header_labels_provider_on_the_wire(- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],-) -> None:- app, backend = latency_service_app- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=_api_status_error(401, {"error": {"message": "bad key"}}),- )-- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/chat/completions",- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},- )-- assert response.status_code == 401- assert response.headers[ERROR_SOURCE_HEADER] == "provider"- assert response.headers[UPSTREAM_MODEL_HEADER] == "model-A"- # The provider body still passes through unmodified — annotation is header-only.- assert response.json()["error"]["message"] == "bad key"---def test_network_failure_500_header_labels_provider_on_the_wire(- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],-) -> None:- """A status-less upstream fault renders as the internal 500 envelope but- the header says provider — clients can tell it wasn't a Switchyard bug."""- app, backend = latency_service_app- backend._clients["model-A"].acompletion = AsyncMock(- side_effect=RuntimeError("connection refused"),- )-- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/chat/completions",- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},- )-- assert response.status_code == 500- assert response.json()["error"]["code"] == "internal_chain_error"- assert response.headers[ERROR_SOURCE_HEADER] == "provider"- assert response.headers[UPSTREAM_MODEL_HEADER] == "model-A"---def test_invalid_role_400_header_labels_switchyard_on_the_wire(- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],-) -> None:- """The translation rejection rides the upstream-status channel but must- surface as switchyard-originated on the wire."""- app, backend = latency_service_app- backend._clients["model-A"].acompletion = AsyncMock()-- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/responses",- json={- "model": "model-A",- "input": [{"type": "message", "role": "api", "content": "ping"}],- },- )-- assert response.status_code == 400- assert response.json()["error"]["code"] == "invalid_value"- assert response.headers[ERROR_SOURCE_HEADER] == "switchyard"--def test_model_not_found_404_header_labels_switchyard_on_the_wire() -> None:"""RouteTable dispatch 404s carry the switchyard source header."""with _OpenAICompatStub() as upstream:@@ -816,232 +315,3 @@ def test_model_not_found_404_header_labels_switchyard_on_the_wire() -> None:assert response.status_code == 404assert response.json()["error"]["code"] == "model_not_found"assert response.headers[ERROR_SOURCE_HEADER] == "switchyard"---def test_caller_required_401_header_labels_switchyard_on_the_wire() -> None:- """The caller_required rejection is Switchyard's own 401: same status as a- provider auth failure, distinguishable by the source header on the wire."""- config = LatencyServiceBackendConfig(- latency_service_url="http://latency-service.test:8080",- endpoints=[- LatencyServiceEndpoint(- model="model-A",- api_key="svc-key", # pragma: allowlist secret- base_url="http://llm.test/v1",- ),- ],- credential_policy="caller_required",- )- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- app = build_switchyard_app(_latency_service_switchyard(config))- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/chat/completions",- json={- "model": "model-A",- "messages": [{"role": "user", "content": "ping"}],- },- )-- assert response.status_code == 401- assert response.json()["error"]["code"] == "missing_caller_api_key"- assert response.headers[ERROR_SOURCE_HEADER] == "switchyard"- assert UPSTREAM_MODEL_HEADER not in response.headers---# ----------------------------------------------------------------------------# Responses API fidelity on the wire-# ------------------------------------------------------------------------------def test_responses_body_returned_exactly_on_the_wire() -> None:- """A Responses request through the full app returns the upstream JSON- byte-for-byte: provider extras and explicit-null fields included.-- This is the composed fidelity proof — backend raw wrap, the- terminal translation short-circuit, and endpoint serialization together.- """- upstream_body = {- "id": "resp-e2e",- "object": "response",- "created_at": 1719890000,- "model": "model-A",- "status": "completed",- "output": [- {- "type": "message",- "id": "msg-1",- "role": "assistant",- "status": "completed",- "content": [{"type": "output_text", "text": "hello", "annotations": []}],- }- ],- "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5},- "store": False,- "temperature": 1.0,- "top_p": 0.9,- "text": {"format": {"type": "text"}},- "reasoning": {"effort": None, "summary": None},- "previous_response_id": None,- "provider_meta": {"azure_region": "eastus"},- }- config = LatencyServiceBackendConfig(- latency_service_url="http://latency-service.test:8080",- endpoints=[- LatencyServiceEndpoint(- model="model-A",- api_key="k",- base_url="http://llm.test/v1",- request_type="openai_responses",- ),- ],- )- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- switchyard = _latency_service_switchyard(config)- backend = _find_latency_backend(switchyard)- backend._clients["model-A"].aresponses = AsyncMock(- return_value=dict(upstream_body)- )- app = build_switchyard_app(switchyard)- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/responses",- json={"model": "model-A", "input": "ping"},- )-- assert response.status_code == 200, response.text- assert response.json() == upstream_body---def _latency_responses_app(frames: list[str]) -> FastAPI:- """Full app over one ``openai_responses`` endpoint streaming *frames*."""- config = LatencyServiceBackendConfig(- latency_service_url="http://latency-service.test:8080",- endpoints=[- LatencyServiceEndpoint(- model="model-A",- api_key="k",- base_url="http://llm.test/v1",- request_type="openai_responses",- ),- ],- )- with patch(- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",- ) as mock_cls:- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):- switchyard = _latency_service_switchyard(config)- backend = _find_latency_backend(switchyard)-- async def _frame_iter() -> AsyncIterator[str]:- for frame in frames:- yield frame-- backend._clients["model-A"].aresponses = AsyncMock(- return_value=_frame_iter()- )- return build_switchyard_app(switchyard)---def test_responses_stream_forwarded_verbatim_on_the_wire() -> None:- """A same-format streaming Responses request returns the upstream SSE- frames byte-for-byte: unknown provider fields, explicit nulls, event- names, and comment keep-alives all survive (streaming fidelity leg)."""- frames = [- (- 'event: response.created\n'- 'data: {"type":"response.created","response":{"id":"resp_s1","store":false,'- '"temperature":1.0,"reasoning":{"effort":null,"summary":null}}}\n\n'- ),- ": keep-alive\n\n",- (- 'event: response.output_text.delta\n'- 'data: {"type":"response.output_text.delta","delta":"hello","vendor_extra":1}\n\n'- ),- (- 'event: response.completed\n'- 'data: {"type":"response.completed","response":{"id":"resp_s1","usage":'- '{"input_tokens":3,"output_tokens":2}}}\n\n'- ),- ]- app = _latency_responses_app(frames)- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/responses",- json={"model": "model-A", "input": "ping", "stream": True},- )-- assert response.status_code == 200, response.text- assert response.text == "".join(frames)---def test_azure_flavored_responses_stream_preserved_on_the_wire() -> None:- """Azure-shaped Responses events (content-filter annotations and other- provider extras) pass through the streaming path unmodified."""- frames = [- (- 'event: response.output_text.delta\n'- 'data: {"type":"response.output_text.delta","delta":"hi",'- '"content_filter_results":{"hate":{"filtered":false,"severity":"safe"}}}\n\n'- ),- (- 'event: response.completed\n'- 'data: {"type":"response.completed","response":{"id":"resp_az",'- '"prompt_filter_results":[{"prompt_index":0}],"usage":'- '{"input_tokens":1,"output_tokens":1},"previous_response_id":null}}\n\n'- ),- ]- app = _latency_responses_app(frames)- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/responses",- json={"model": "model-A", "input": "ping", "stream": True},- )-- assert response.status_code == 200, response.text- assert response.text == "".join(frames)---def test_responses_raw_frames_translate_for_chat_client_on_the_wire() -> None:- """Cross-format regression: a chat client served by a Responses endpoint- still gets translated chat chunks — raw frame strings are parsed back into- events for the translator instead of passing through."""- frames = [- (- 'event: response.created\n'- 'data: {"type":"response.created","response":{"id":"resp_x","model":"m"}}\n\n'- ),- (- 'event: response.output_text.delta\n'- 'data: {"type":"response.output_text.delta","delta":"hello"}\n\n'- ),- (- 'event: response.completed\n'- 'data: {"type":"response.completed","response":{"id":"resp_x","usage":'- '{"input_tokens":3,"output_tokens":2}}}\n\n'- ),- ]- app = _latency_responses_app(frames)- with TestClient(app, raise_server_exceptions=False) as client:- response = client.post(- "/v1/chat/completions",- json={- "model": "model-A",- "messages": [{"role": "user", "content": "ping"}],- "stream": True,- },- )-- assert response.status_code == 200, response.text- assert "chat.completion.chunk" in response.text- assert "hello" in response.text- assert "response.output_text.delta" not in response.text