Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(routing)!: remove latency-aware router (#162)

eric-liu-nvidiac6a850f2f19e
diff --git a/AGENTS.md b/AGENTS.md
index 0a3117ad4..e214dc9b9 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -182,12 +182,10 @@ switchyard/
│ │ ├── openai_llm_backend.py # OpenAiPassthroughBackend
│ │ ├── openai_native_backend.py # OpenAiNativeBackend
│ │ ├── anthropic_native_llm_backend.py # AnthropicNativeBackend
-│ │ ├── latency_service_llm_backend.py # LatencyServiceLLMBackend
│ │ ├── llm_target.py # LlmTarget, BackendFormat
│ │ ├── multi_llm_backend.py # MultiLlmBackend helpers
│ │ ├── stats_llm_backend.py # StatsLlmBackend
-│ │ ├── backend_format_resolver.py # BackendFormatResolver
-│ │ └── health_poller.py # HealthPoller, EndpointHealthStatus
+│ │ └── backend_format_resolver.py # BackendFormatResolver
│ ├── processors/ # Request-side / response-side component implementations
│ │ ├── format_translate.py
│ │ ├── random_routing_request_processor.py
@@ -205,8 +203,7 @@ switchyard/
│ │ ├── sse_helpers.py
│ │ └── base.py
│ └── config/
-│ ├── intake_sink_config.py
-│ └── latency_service_backend_config.py
+│ └── intake_sink_config.py
├── cli/ # CLI (requires `nemo-switchyard[cli]`)
│ ├── switchyard_cli.py # `switchyard` entry point
│ ├── launch_command.py # `switchyard launch claude/codex`
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a711afd12..bd9d04b45 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,20 @@ All notable changes to Switchyard are documented here. The format is based on
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [Unreleased]
+
+### Removed
+
+- **Latency-aware router** — the `latency_service` route type and its
+ `LatencyServiceLLMBackend`, `LatencyServiceBackendConfig`,
+ `LatencyServiceEndpoint`, and `LatencyServiceProfileConfig` public API are
+ removed. It depended on NVIDIA Inference Hub's latency endpoint and schema.
+ Deployments that need multi-endpoint, load- or latency-aware routing should
+ migrate to [Dynamo](https://github.com/ai-dynamo/dynamo) (backend-load /
+ KV-cache-aware routing with request failover) or an external load balancer
+ such as [Traefik](https://doc.traefik.io/traefik/reference/routing-configuration/http/load-balancing/service/)
+ or HAProxy.
+
## [0.1.0] — Initial release
First public release of Switchyard — a typed, composable control plane for LLM
diff --git a/crates/switchyard-components/tests/stats_accumulator.rs b/crates/switchyard-components/tests/stats_accumulator.rs
index d6cd37401..1f2e5134d 100644
--- a/crates/switchyard-components/tests/stats_accumulator.rs
+++ b/crates/switchyard-components/tests/stats_accumulator.rs
@@ -751,7 +751,7 @@ fn routing_decision_counts_are_grouped_by_profile_type() -> Result<()> {
accumulator.record_routing_decision("stage_router", "dimensions")?;
accumulator.record_routing_decision("stage_router", "dimensions")?;
accumulator.record_routing_decision("stage_router", "llm-classifier")?;
- accumulator.record_routing_decision("latency-service", "health")?;
+ accumulator.record_routing_decision("escalation_router", "judge")?;
let snapshot = accumulator.snapshot()?;
assert_eq!(
@@ -771,8 +771,8 @@ fn routing_decision_counts_are_grouped_by_profile_type() -> Result<()> {
assert_eq!(
snapshot
.routing_decisions
- .get("latency-service")
- .and_then(|sources| sources.get("health")),
+ .get("escalation_router")
+ .and_then(|sources| sources.get("judge")),
Some(&1)
);
diff --git a/crates/switchyard-py/src/core_bindings/context.rs b/crates/switchyard-py/src/core_bindings/context.rs
index 57c8fd4b5..1b361f819 100644
--- a/crates/switchyard-py/src/core_bindings/context.rs
+++ b/crates/switchyard-py/src/core_bindings/context.rs
@@ -614,11 +614,10 @@ impl PyProxyContext {
/// Records the measured backend-call latency in ms.
///
/// The Rust ``StatsLlmBackend`` wraps native backends and writes this
- /// slot automatically. Python-only backends (e.g. ``LatencyServiceLLMBackend``)
- /// that can't be wrapped record their measurement here so the
- /// downstream ``StatsResponseProcessor`` can compute
- /// ``routing_overhead_ms = total_latency - backend_latency`` and emit
- /// it on ``/metrics``. Setting ``None`` clears the slot.
+ /// slot automatically. Python-only backends that can't be wrapped record
+ /// their measurement here so the downstream ``StatsResponseProcessor`` can
+ /// compute ``routing_overhead_ms = total_latency - backend_latency`` and
+ /// emit it on ``/metrics``. Setting ``None`` clears the slot.
#[setter]
fn set_backend_call_latency_ms(&self, value: Option<f64>) -> PyResult<()> {
let mut inner = self.lock()?;
diff --git a/crates/switchyard-py/src/core_bindings/response.rs b/crates/switchyard-py/src/core_bindings/response.rs
index 85fcb4442..e9a02eded 100644
--- a/crates/switchyard-py/src/core_bindings/response.rs
+++ b/crates/switchyard-py/src/core_bindings/response.rs
@@ -492,9 +492,9 @@ impl PyResponseStream {
/// ``Switchyard.call`` round trip (``source`` is ``None``): the source was
/// moved onto the core stream by ``take_core_stream`` as a
/// ``SourceClosingStream``; dropping the inner adapter here drops that
- /// wrapper, which closes the source on drop. This is the latency-router
- /// production path, where response processors re-wrap the core stream and
- /// ``source`` cannot be recovered on this object.
+ /// wrapper, which closes the source on drop. This is the production path
+ /// where response processors re-wrap the core stream and ``source``
+ /// cannot be recovered on this object.
fn aclose<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
self.consumed.store(true, Ordering::Release);
let stream = Arc::clone(&self.stream);
diff --git a/docs/internal/latency_service_routing.md b/docs/internal/latency_service_routing.md
deleted file mode 100644
index d57b731b3..000000000
--- a/docs/internal/latency_service_routing.md
+++ /dev/null
@@ -1,609 +0,0 @@
-# Latency Service Routing
-
-The **latency-aware router** (`type: latency_service`) routes each request to one of
-several equivalent endpoints, biasing traffic toward whichever endpoint a central
-**Latency Service** most recently observed as healthy and fast. It is the production
-routing policy for OpenRouter-backed deployments, where a separate Latency Service
-owns heartbeat probing and statistical latency profiling, and Switchyard consumes its
-verdicts to make per-request steering decisions.
-
-This document explains how the router works, the design choices behind it, and how to
-configure and operate it.
-
-## When to use it
-
-Use `latency_service` when you have **multiple interchangeable endpoints** serving the
-same logical model (e.g. the same model behind different providers or regions) and you
-want Switchyard to:
-
-- bias traffic toward the lowest-latency healthy endpoint,
-- automatically steer away from degraded endpoints, and
-- retry on a different endpoint when one fails, without the client ever seeing it.
-
-If you only have a single backend, use `passthrough`. If you want a fixed weighted
-strong/weak split for benchmarking, use
-[Random Routing](../routing_algorithms/random_routing.md).
-
-## Architecture
-
-The router is implemented as a single custom `LLMBackend`,
-`LatencyServiceLLMBackend` (`switchyard/lib/backends/latency_service_llm_backend.py`),
-rather than a monolithic routing strategy. It slots into the standard four-role chain:
-
-```
-[RequestProcessor*] → LatencyServiceLLMBackend → [ResponseProcessor*] → TranslationEngine
-```
-
-The backend owns four things:
-
-1. **A pool of `OpenAILLMClient` instances**, one per configured endpoint, keyed by the
- endpoint's `model` ID.
-2. **A thread-safe in-memory health cache** (`dict[str, EndpointHealth]` guarded by a
- `threading.Lock`).
-3. **A background `HealthPoller` daemon thread**
- (`switchyard/lib/backends/health_poller.py`) that refreshes the cache from the
- Latency Service.
-4. **Endpoint-selection + retry logic** on the request hot path.
-
-Inbound format translation (Anthropic Messages / OpenAI Responses → OpenAI Chat) is
-delegated to the Rust `TranslationEngine`, so the backend transparently accepts any
-inbound wire format and only ever speaks OpenAI Chat Completions upstream.
-
-### Two-clock design: poll loop vs. hot path
-
-The defining property of this router is that **health polling and request serving run on
-separate clocks**:
-
-- The **`HealthPoller` daemon thread** is the only thing that ever talks to the Latency
- Service. It polls on a fixed interval and *writes* verdicts into the cache.
-- The **request hot path** only ever *reads* the cache (under the lock). It never makes a
- network call to the Latency Service.
-
-Health awareness adds **zero per-request latency**. Selection is a dictionary
-read and a weighted random pick. The poller is a plain daemon thread (not an asyncio
-task), so its lifecycle is trivial. It starts when the backend is constructed and dies
-with the process. Its synchronous `httpx.Client` never contends with the event loop
-serving requests.
-
-## The health poller
-
-`HealthPoller` polls `{latency_service_url}/v1/endpoints/health` every `poll_interval_s`
-seconds (default 10s), passing the configured endpoint IDs as `endpoint_ids` query
-params. The Latency Service responds with a verdict and a latency sample per endpoint:
-
-```json
-{
- "endpoint_health": {
- "openai/gpt-5.5": {"status": "healthy", "last_latency_ms": 420.0},
- "azure_openai/gpt-5.5": {"status": "degraded", "last_latency_ms": 980.0}
- }
-}
-```
-
-Each cache entry is an `EndpointHealth` snapshot carrying the discrete `status` and the
-most recent `last_latency_ms`. The status enum is the **shared contract** between
-Switchyard and the Latency Service: both sides must agree on these three string values:
-
-| Status | Meaning |
-|---|---|
-| `healthy` | Endpoint is up and within latency expectations. |
-| `degraded` | Endpoint is reachable but slow or partially failing. |
-| `unknown` | No current verdict (warming up, or fallback after a poll failure). |
-
-### Failure handling: fall back to UNKNOWN, not stale data
-
-If a poll fails for any reason (DNS, network, timeout, non-2xx, or a schema mismatch on
-the response), the poller **resets every endpoint in the cache to `UNKNOWN`** rather than
-leaving stale verdicts in place. With every endpoint at `UNKNOWN`, the router degrades to
-uniform random routing across the full pool. This is the safe default when health
-information is unavailable. Acting on stale "healthy" data after the Latency Service has
-gone dark would be strictly worse.
-
-`last_latency_ms` is `None` before the first successful poll and whenever the Latency
-Service reports it as null.
-
-## Endpoint selection
-
-On each request, `_select_endpoint()` picks a `model` ID from the cache in two steps:
-
-### 1. Tier preference: `HEALTHY > UNKNOWN > DEGRADED`
-
-Endpoints are bucketed by status, and the router picks from the **best non-empty tier**.
-`UNKNOWN` is deliberately preferred over `DEGRADED`: an endpoint we have no verdict for is
-a better bet than one the Latency Service explicitly flagged as degraded.
-
-### 2. Within a tier: inverse-latency weighted random
-
-Within the chosen tier, `_pick_by_latency()` selects an endpoint with probability
-proportional to `1 / last_latency_ms`, so an endpoint the Latency Service recently saw
-at 200 ms receives roughly twice the traffic of one at 400 ms. This is **weighted random,
-not always-pick-the-fastest**, which spreads load and avoids stampeding a single endpoint.
-
-The picker falls back to **uniform random** within the tier when:
-
-- there is only one candidate, or
-- any candidate's `last_latency_ms` is unknown (`None`), or
-- any sample is non-positive (defensive: the Latency Service should never report ≤ 0).
-
-This keeps behavior predictable while the poller is warming up or when the upstream
-reports nulls.
-
-### 3. Optional: session affinity (sticky routing)
-
-By default the router picks per **turn**. Every request runs the tier + latency steps
-above independently. For a multi-turn conversation this interleaves endpoints, which lets
-each endpoint's upstream prompt/KV cache lapse and forces expensive cache *re-writes* on
-the next turn that lands there.
-
-Set `session_affinity: true` to route per **conversation** instead. The latency-aware
-picker decides only the **first** turn, then pins that conversation to the endpoint
-that served it. Every later turn reuses that endpoint (the affinity fast-path bypasses
-tier + latency selection entirely), so the upstream cache stays warm for the life of
-the conversation.
-
-- **Conversation identity.** A streaming proxy holds no session ID. The client re-sends
- the full history each turn. `session_key_from_body()`
- (`switchyard/lib/session_key.py`) derives a stable key by hashing the stable harness
- prefix made from the system prompt + the first user message. Later turns only append, so
- every turn of one conversation hashes the same; distinct conversations differ.
-- **Health overrides the pin.** A pin is reused only while its endpoint is `HEALTHY` or
- `UNKNOWN`. If it goes `DEGRADED` (or the call fails this request), the next turn
- re-routes through normal selection and re-pins to whatever serves it. Locality yields
- to health, so a session is never funneled into a failing endpoint.
-- **Bounded memory.** Pins live in an in-process LRU map (the L1) capped at
- `affinity_max_sessions` (default `10_000`); the least-recently-used pin is evicted past
- the cap.
-- **Cross-worker stickiness (optional shared store).** The L1 map is per process, so a
- multi-worker/multi-pod deployment may pin the same conversation differently per worker,
- and pins are lost on pod churn. Set `affinity_store: "redis"` (with `affinity_store_url`)
- to add a shared, persistent L2: on an L1 miss the pin is read through from the store and
- warmed back into L1, and every pin is written through to it. Pins then survive across
- replicas and pod restarts (bounded by `affinity_store_ttl_seconds`, refreshed on each
- write). The L2 is **best-effort** — an error or timeout falls back to L1 / unpinned and
- never fails a request; fail-open operations are counted on
- `switchyard_affinity_l2_errors_total`, and a circuit breaker (3 consecutive failures →
- skip the store for 10 s, then probe; `switchyard_affinity_l2_breaker_open` gauge) keeps
- a store outage from taxing every request with timeout waits. The store connection is released by the backend's
- `shutdown()` (the server lifespan teardown hook awaits it). Requires the
- `switchyard[affinity-redis]` extra. (Alternatively, front the workers with a
- session-affinity load balancer.)
-
-The `switchyard.route_decision` span tags each pick with `switchyard.affinity_hit` so you
-can measure the warm-reuse rate. Affinity is **off by default**; existing per-turn latency
-routing is unchanged unless opted in.
-
-#### Deploying the shared store: the operational contract
-
-What an operator needs to know to provision and run the Redis L2. Everything below is
-verified against the implementation (`RedisPinStore`, `SessionAffinity`).
-
-**Topology.** A **standalone Redis endpoint only** (the client is
-`redis.asyncio.from_url`): a single instance, a primary endpoint of a replicated setup,
-or a managed single-endpoint tier. **Redis Cluster and Sentinel URLs are not
-supported** — cluster `MOVED` redirects surface as fail-open errors, meaning requests
-keep succeeding while stickiness silently degrades. A sustained
-`switchyard_affinity_l2_errors_total` rate from the first request is the
-mis-provisioning signal; watch it when first enabling the store.
-
-**Auth and TLS.** URL-driven: `rediss://` for TLS, credentials inside the URL. Route
-YAML expands `${ENV}` references in every string value, so keep the URL in a secret and
-configure `affinity_store_url: "${AFFINITY_REDIS_URL}"` rather than an inline literal.
-
-**What is stored.** Key = `affinity_key_prefix` + a 16-hex-char hash of the
-conversation's stable prefix (system prompt + first user message); value = the pinned
-endpoint id string. **No prompt text, message content, or user identifier leaves the
-process** — a pin is ~50–100 bytes.
-
-**Traffic and sizing.** One `GET` per request whose conversation isn't in the local L1
-(first turn on a pod), and one `SET` (which refreshes the TTL) per successful request.
-Working-set memory ≈ active conversations × ~150 B, bounded by the sliding
-`affinity_store_ttl_seconds` (default 1 h). **Persistence (RDB/AOF) and HA are optional**:
-losing the store only resets stickiness — conversations re-route cold and re-pin — it
-never affects correctness. `maxmemory-policy allkeys-lru` is the recommended eviction
-policy; an evicted pin costs one cold re-route.
-
-**Failure envelope.** Every store operation is bounded by a 0.1 s socket/connect
-timeout (a colocated Redis answers in well under 1 ms, so this is ~100× headroom) and
-fails open — requests never fail because of the store. A **circuit breaker** caps the
-outage tax: after 3 consecutive failures, L2 operations are skipped without a network
-attempt for a 10 s cooldown, then one operation probes; success closes the breaker,
-failure re-arms it. Worst case per pod during an outage is therefore a brief window of
-up to ~0.2 s added per request (one `GET` on the L1 miss + one `SET` after success)
-before the breaker opens, then ~zero added latency with one 0.1 s probe per 10 s —
-stickiness degrades to per-pod L1 throughout. Alert on
-`switchyard_affinity_l2_breaker_open == 1` (the outage signal) and on
-`rate(switchyard_affinity_l2_errors_total[5m])` (degradation without a full trip —
-note the rate drops to ~0.1/s per pod while the breaker is open, since only probes
-fail). Pod readiness (`/health`, `is_ready()`) never depends on Redis.
-
-**Isolation.** Deployments or routes sharing one Redis should set distinct
-`affinity_key_prefix` values (or separate logical databases). A pin is honored whenever
-its stored endpoint id exists in the reading route's endpoint set, so shared prefixes
-cross-pollinate stickiness between routes that reuse endpoint ids.
-
-**Config changes, upgrades, rollback.** A persisted pin whose endpoint id is no longer
-in the route's endpoint list is ignored gracefully (the next turn re-routes and
-re-pins) — shrinking or renaming the endpoint set needs no store cleanup. The
-conversation-key derivation is not a cross-version contract: a Switchyard upgrade may
-invalidate existing pins (one-time cold re-route, self-healing). To roll back, set
-`affinity_store: "memory"`; stale keys expire via TTL, and `FLUSHDB` is always safe
-(stickiness reset only).
-
-The deterministic LLM-classifier route also supports `affinity_warmup_turns` to delay
-when a tier becomes sticky. Latency-service routing does **not** use that knob. Its
-affinity objective is endpoint cache reuse, so it pins after the first successful
-endpoint selection.
-
-## Retry and failover
-
-A single client request may try up to `1 + max_retries` endpoints (default `max_retries`
-is 2, so 3 attempts). On each attempt:
-
-1. Select an endpoint. If selection returns one already tried this request (and untried
- endpoints remain), pick uniformly at random from the **untried** set. Dedup prevents
- wasting an attempt re-hitting an endpoint that just failed.
-2. Stamp the endpoint's `upstream_model` into `body["model"]` and call the upstream.
-3. On a **transient** failure such as `429` (rate limit), `408` (request timeout), any
- `5xx`, or a network / pre-status / SDK error, log it, record the error in stats, increment
- the outcome metric, and continue to the next endpoint. These are exactly the faults a
- health-aware router should absorb by trying a different endpoint.
-4. On a **4xx client error** (`400`, `401`, `403`, `404`, `409`, `413`, `415`, `422`, …)
- the loop **fails fast**: it does *not* retry. The request itself is malformed or
- unauthorized, so every replica rejects the same payload identically; retrying only adds
- latency before surfacing the same status. (Context-window overflow surfaces as a `400`
- here and is also passed straight through. The `latency_service` route does not
- participate in the chain-level evict-and-retry described in
- [Context-Window Handling](../operations/context_window.md).)
-
-If an attempt succeeds **after** at least one failure, the router records a
-`retry_recovered` outcome. This is direct evidence the steering logic rescued a request that
-would otherwise have failed.
-
-Whenever the request ends on an upstream HTTP error, either a fail-fast 4xx or an
-exhausted retryable error (e.g. a `503` after retries are exhausted), the backend records the
-upstream status code and body on `ctx` so the endpoint passes the real upstream status
-through to the client rather than masking it as a generic `500`.
-
-## Configuration
-
-`LatencyServiceBackendConfig` owns the router's policy, retry, and optional
-session-affinity settings. In CLI YAML, configure the Python
-`type: latency_service` router in a `routes:` bundle; there are no
-policy-specific CLI flags. Run it with:
-
-```bash
-switchyard --routing-profiles my_routes.yaml -- serve --port 4100
-```
-
-### Config schema
-
-`LatencyServiceBackendConfig`
-(`switchyard/lib/config/latency_service_backend_config.py`):
-
-| Field | Default | Purpose |
-|---|---|---|
-| `latency_service_url` | `""` | Base URL of the Latency Service. |
-| `endpoints` | `[]` | The endpoints to route across (see below). At least one is required. |
-| `poll_interval_s` | `10.0` | How often the background poller refreshes health. |
-| `poll_timeout_s` | `5.0` | Timeout for each health API call. |
-| `max_retries` | `2` | Extra endpoints to try on failure (total attempts = `1 + max_retries`). |
-| `credential_policy` | `"configured_endpoint"` | Credential precedence for upstream calls. The default uses each endpoint's configured `api_key`; set `"caller_override"` only for BYO-key deployments where caller headers should replace endpoint keys. |
-| `session_affinity` | `false` | Pin each conversation to one endpoint after the first turn (sticky routing: keeps the upstream cache warm). See [Endpoint selection §3](#3-optional-session-affinity-sticky-routing). |
-| `affinity_max_sessions` | `10_000` | LRU cap on the in-process pin map (L1) when `session_affinity` is on. |
-| `affinity_store` | `"memory"` | Shared L2 pin store behind the L1. `"memory"` = per-process only; `"redis"` shares pins across workers/pods and persists them across pod churn (requires `session_affinity: true`, `affinity_store_url`, and the `affinity-redis` extra). |
-| `affinity_store_url` | `None` | Connection URL for the shared store (e.g. `redis://host:6379/0`). Required when `affinity_store` is `"redis"`. |
-| `affinity_store_ttl_seconds` | `3_600` | Expiry for a shared pin; refreshed on each write, so an active conversation slides its TTL. |
-| `affinity_key_prefix` | `"swyd:pin:"` | Namespace prefix for shared-store keys. |
-| `enable_stats` | `true` | Wire the stats processors + `/metrics` exposition. |
-
-Each `LatencyServiceEndpoint`:
-
-| Field | Required | Purpose |
-|---|---|---|
-| `model` | yes | **Latency Service lookup key**: must be unique across the endpoint list. Also the value stamped into `body["model"]` unless `upstream_model` is set. |
-| `upstream_model` | no | Override for `body["model"]` sent upstream. Defaults to `model`. |
-| `api_key` | no | API key for the backing LLM API. This is the default upstream credential. |
-| `base_url` | no | Base URL for the backing LLM API (include `/v1`). |
-| `timeout` | no | Per-request timeout in seconds. |
-
-### Why two `model` fields?
-
-The Latency Service registers each endpoint under an ID *it* picked (e.g.
-`openrouter-primary/openai/gpt-4o`). The upstream gateway may expect the OpenRouter
-model name for the same destination (e.g. `openai/gpt-4o`). So:
-
-- **`model`** is the Latency Service lookup key (and the metrics label).
-- **`upstream_model`** is what gets stamped into `body["model"]` on the outbound call.
-
-When they're equal, `upstream_model` can be omitted.
-
-### Example
-
-Create an OpenRouter key at <https://openrouter.ai/> and export it as
-`OPENROUTER_API_KEY` before running this example.
-
-```yaml
-routes:
- gpt-4o:
- type: latency_service
- latency_service_url: https://latency-service.example.com
- poll_interval_s: 10.0
- poll_timeout_s: 5.0
- max_retries: 2
- endpoints:
- - model: openrouter-primary/openai/gpt-4o # Latency Service endpoint_id
- upstream_model: openai/gpt-4o # OpenRouter model name to call
- base_url: https://openrouter.ai/api/v1
- api_key: ${OPENROUTER_API_KEY}
- - model: openrouter-backup/openai/gpt-4o
- upstream_model: openai/gpt-4o
- base_url: https://openrouter.ai/api/v1
- api_key: ${OPENROUTER_API_KEY}
-```
-
-A runnable version of this config, with extensive inline notes, lives at
-[`experiments/latency_routing.yaml`](../../experiments/latency_routing.yaml).
-A production-style Kubernetes manifest lives at
-[`examples/k8s/latency-aware-deployment.yaml`](../../examples/k8s/latency-aware-deployment.yaml).
-
-### BYO-key (multi-tenant) mode
-
-By default, caller `Authorization: Bearer <key>` and `x-api-key` headers do not replace
-the configured endpoint credentials. This keeps server-owned latency-service deployments
-from accidentally forwarding a client placeholder or unrelated proxy credential upstream.
-
-For a multi-tenant BYO-key deployment, set `credential_policy: caller_override` on the
-latency-service route. In that mode, a non-empty caller key is forwarded as the upstream
-credential for that request, and the endpoint `api_key` is used only when the caller omits
-the header. Credential precedence is **`Authorization` > `x-api-key` > endpoint
-`api_key`**.
-
-## Response fidelity (Responses API)
-
-For an endpoint configured `request_type: openai_responses`, the backend calls the
-upstream `/v1/responses` surface and returns the upstream payload **verbatim** in both
-modes:
-
-- **Non-streaming** calls return the upstream's exact JSON body — the raw HTTP response
- is used instead of round-tripping through the OpenAI SDK's typed model.
-- **Streaming** calls forward the upstream's SSE frames as **verbatim strings**
- (`RawSSEFrameStream` in `switchyard/lib/llm_client.py`): no typed-event parse happens,
- so per-event provider extras, explicit nulls, event names, and comment keep-alives all
- survive. Frames are byte-equivalent modulo CRLF → LF line normalization. Usage
- accounting still works — the stats layers parse the frame's `data:` payload to find
- `response.completed` usage without altering what flows to the client.
-
-When the inbound request is also Responses-format, the terminal translation
-short-circuits (same format in and out), so the client receives the upstream payload
-as-is. **Cross-format** calls (e.g. a chat client served by a Responses endpoint, or
-vice versa) are *re-synthesized* by the translation engine and are inherently lossy —
-fields the upstream never produced in the target format cannot be invented. For
-full-fidelity Responses routing, configure every endpoint on the route with
-`request_type: openai_responses`.
-
-### Field-fidelity matrix (Responses API)
-
-How each field class fares per path. "Exact" means byte-equivalent to the upstream
-(streaming: modulo CRLF normalization); "synthesized" means the translation engine
-rebuilds the payload from its neutral representation and only mapped fields survive.
-
-| Field class (examples) | Same-format, non-streaming | Same-format, streaming | Cross-format (either direction) |
-|---|---|---|---|
-| Standard scalars (`id`, `model`, `status`, `created_at`) | exact | exact | synthesized (`id`/`model` mapped; `created_at` → chat `created` defaults to `0`) |
-| Sampling/config echoes (`temperature`, `top_p`, `store`, `text`) | exact | exact | dropped (no chat-format equivalent) |
-| `reasoning` config + explicit-`null` fields | exact (nulls preserved) | exact (nulls preserved) | dropped |
-| `usage` (incl. `*_tokens_details`) | exact | exact | token counts + reasoning detail mapped; cached-token detail drops |
-| Output content (`output[]`, text deltas) | exact | exact | mapped (text/tool calls translate; unmappable block types drop) |
-| Provider extras (unknown keys, e.g. Azure `content_filter_results`) | exact | exact | dropped |
-| SSE event names / keep-alive comments | n/a | exact | re-framed to the target format's event contract |
-| `metadata`, `previous_response_id` | exact | exact | dropped |
-
-Wire-level proofs live in
-`tests/test_upstream_error_passthrough.py`
-(`test_responses_body_returned_exactly_on_the_wire`,
-`test_responses_stream_forwarded_verbatim_on_the_wire`,
-`test_azure_flavored_responses_stream_preserved_on_the_wire`) against
-OpenAI-shaped and Azure-shaped samples; validation against live production
-captures remains a deployment-side step.
-
-## Failure-source annotation
-
-When a request fails, the error response carries two headers so clients and
-observability can tell **which layer** the failure came from without parsing bodies:
-
-| Header | Values | Meaning |
-|---|---|---|
-| `x-switchyard-error-source` | `switchyard` \| `provider` | `provider`: an upstream LLM failure passed through (HTTP error, or the 500 rendered for a status-less network fault). `switchyard`: this proxy rejected or failed the request itself — credential-policy 401 (`missing_caller_api_key`), translation 400 (`invalid_value`), model-not-found 404, context-window 400, body validation 400, unexpected internal 500. |
-| `x-switchyard-upstream-model` | model id | The model actually attempted upstream when the failure happened. Present only when a routing selection took place. |
-
-The provider error **body** still passes through verbatim — the annotation is
-header-only, deliberately, so the transparent-proxy contract on bodies is preserved.
-The same vocabulary appears as `switchyard.error_source` / `switchyard.upstream_model`
-tags on failed `switchyard.upstream_attempt` spans and as `error_source` /
-`upstream_model` fields on the per-event error log (below).
-
-Layering note: Switchyard can only distinguish *itself* from *its upstreams*. A proxy in
-front of Switchyard (e.g. LiteLLM) should tag its own failures the same way and
-propagate these headers from below. Mid-stream failures after HTTP 200 has been
-committed cannot carry headers and are not annotated today.
-
-## Route-selection spend attribution
-
-For tokenomics reporting, a LiteLLM front proxy needs to tie its provider spend-log
-rows back to the Switchyard logical route that selected them. The backend stamps
-**every upstream attempt** with a spend-logs metadata request header (LiteLLM copies
-its JSON value into the provider spend-log DB row):
-
-```text
-x-litellm-spend-logs-metadata: {"router_model": "<client-facing route id>",
- "router_strategy": "latency",
- "router_selected_endpoint": "<endpoint id>",
- "router_selected_model": "<upstream model>",
- "router_selected_provider": "<upstream model's leading path segment>",
- "router_correlation_id": "<uuid4>"}
-```
-
-One correlation id is generated per client request and shared by every failover
-attempt; the selection fields are re-stamped per attempt, so each provider row
-records the endpoint it actually hit. `router_model` is the model the client asked
-for, falling back to the configured `route_model` when the body carried no model.
-
-**Successful** responses then return the selection that served the request, so the
-front proxy can enrich its own (parent) spend-log row with the same correlation id
-the provider row received:
-
-| Header | Value |
-|---|---|
-| `x-switchyard-router-model` | `router_model` from the selection |
-| `x-switchyard-selected-model` | `router_selected_model` |
-| `x-switchyard-selected-provider` | `router_selected_provider` |
-| `x-switchyard-router-correlation-id` | `router_correlation_id` |
-
-Streaming responses carry the same headers (the upstream call completes before the
-SSE response is committed). Error responses carry the failure-source headers above;
-the selection headers join them only when an upstream call already **succeeded**
-before the failure (e.g. a response-translation error after a billed 200), so the
-provider row's correlation id stays joinable — a request that never reached a
-successful upstream claims no selection. Because the client controls `router_model`,
-header values are re-validated as header material: a value that is not
-latin-1-encodable or contains control characters is omitted rather than breaking
-(or splitting) the response; the outbound JSON header is immune (`json.dumps`
-escapes it) and still records the raw value. The cross-component contract is
-`CTX_ROUTE_SELECTION` in `switchyard/lib/proxy_context.py` (written by the backend,
-read by the endpoint layer via `switchyard/lib/endpoints/route_selection.py`);
-wire-level proofs live in `tests/test_route_selection_headers.py`.
-
-## Observability
-
-When `enable_stats` is `true` (the default), `LatencyServiceProfileConfig` wires a
-`StatsRequestProcessor` + `StatsResponseProcessor` pair sharing one `StatsAccumulator`.
-Because `LatencyServiceLLMBackend` is a Python-only backend, profile assembly cannot wrap
-it with the Rust-native `StatsLlmBackend`; instead it records success / error /
-call-latency in-place into the same accumulator the response processor records token usage
-into. It also sets `ctx.selected_model` and `ctx.backend_call_latency_ms` so per-endpoint
-attribution and routing-overhead figures land correctly on `/metrics`.
-
-The backend publishes its in-memory health cache and poll-loop health to `/metrics`:
-
-| Metric | Type | Meaning |
-|---|---|---|
-| `switchyard_endpoint_status{model,status}` | gauge | Current verdict per endpoint; exactly one status row per model is `1`. |
-| `switchyard_endpoint_last_latency_ms{model}` | gauge | Last latency sample per endpoint. Absent when the upstream reported null. |
-| `switchyard_latency_service_poll_ok` | gauge | `1` iff the most recent poll succeeded. |
-| `switchyard_latency_service_poll_age_seconds` | gauge | Seconds since the last successful poll. Absent before the first success. |
-| `switchyard_latency_service_polls_total` | counter | Total successful polls. |
-| `switchyard_latency_service_poll_failures_total` | counter | Total failed polls; each resets the cache to `unknown`. |
-| `switchyard_affinity_hits_total` | counter | Turns served by an existing session-affinity pin (warm reuse). **Emitted only when `session_affinity` is on.** |
-| `switchyard_affinity_misses_total` | counter | First/unpinnable turns routed by the latency-aware picker while affinity was on. Reuse rate = `hits / (hits + misses)`. **Emitted only when `session_affinity` is on.** |
-| `switchyard_affinity_l2_hits_total` | counter | Pins resolved from the shared (L2) store after an in-process miss — cross-worker/churn warm reuse. **Emitted only when a shared affinity store is configured.** |
-| `switchyard_affinity_l2_errors_total` | counter | Shared-store get/put operations that failed open; routing fell back to in-process pins. Alert on a sustained rate — the store is degraded while requests keep succeeding. **Emitted only when a shared affinity store is configured.** |
-
-See the [Metrics Reference](metrics_reference.md) for full label semantics, PromQL recipes
-(traffic share per endpoint, retry-rescue rate), and a triage cheatsheet.
-
-### Expected routing overhead
-
-`switchyard_routing_overhead_ms{quantile}` is the source of truth for what Switchyard
-itself adds to a request: per request it records `total_latency − backend_call_latency`,
-which bundles format translation, endpoint selection, body mutation, response wrapping,
-and retry wall time — everything except the upstream call.
-
-Reference figures, measured on an idle dev box over loopback (stub upstream + stub
-Latency Service, non-streaming `/v1/chat/completions`, sequential; commit `0289a415`):
-
-| Figure | Value |
-|---|---|
-| `switchyard_routing_overhead_ms{quantile="0.5"}` | ~0.3 ms |
-| `switchyard_routing_overhead_ms{quantile="0.99"}` | ~7 ms |
-| Mean overhead (`_sum / _count`) | ~2 ms |
-| Client-observed added wall time vs calling the upstream directly | ~2 ms mean |
-
-Expected order of magnitude is therefore **single-digit milliseconds**. When an
-end-to-end comparison ("direct provider call" vs "through the stack") shows a much
-larger delta — e.g. ~100 ms — the difference lives **outside** this metric:
-
-* the extra network hop and TLS handshake from the client to the Switchyard pod,
-* any fronting layer (LiteLLM / inference-gateway) in front of Switchyard,
-* per-pod queueing under concurrency — a single event loop serializes CPU work, so
- client wall time grows with in-flight load while per-request overhead does not
- (at loopback concurrency 10 the same box shows ~22 ms client p50 with an unchanged
- overhead summary),
-* upstream time-to-first-token variance between the two measurement runs.
-
-Attribute before optimizing: read the deployment's own summary first.
-
-```promql
-# Median / p99 Switchyard-internal overhead
-switchyard_routing_overhead_ms{quantile="0.5"}
-switchyard_routing_overhead_ms{quantile="0.99"}
-
-# Mean over a window
-rate(switchyard_routing_overhead_ms_sum[5m]) / rate(switchyard_routing_overhead_ms_count[5m])
-```
-
-Two caveats. For streaming responses `switchyard_total_latency_ms` (and hence the
-overhead summary) measures the **full turn**, not time-to-first-token — TTFT is what
-interactive users feel, so don't read a long generation as router overhead. And the
-summary is per-process: in a multi-pod deployment each pod reports its own.
-
-The repeatable harness is the CI perf benchmark (`.github/workflows/perf.yml`): aiperf
-against a no-op route measures the full server path with zero backend cost, non-streaming
-and streaming, at configurable concurrency.
-
-### Per-event error log (Loki)
-
-`/metrics` answers *how many* of each error code, but Prometheus stores aggregates. It
-carries no per-event timestamps. For audit, replay, or a "what failed and when" panel,
-each failed upstream attempt also emits one structured JSON line on the
-`switchyard.upstream_errors` logger
-(`switchyard/lib/endpoints/upstream_error_log.py`):
-
-```json
-{"event":"upstream_attempt_failed","timestamp":"2026-05-29T12:00:00+00:00","model":"openai/gpt-5.5","upstream_model":"gpt-5.5","attempt":1,"status_code":429,"code":"429","outcome":"retryable_error","error_source":"provider","error_type":"APIStatusError","error":"..."}
-```
-
-`code` and `outcome` mirror the labels on `switchyard_upstream_attempts_total`, so the log
-joins cleanly to the counter; `status_code` is `null` (and `code="none"`) for a non-HTTP
-failure. Because the deployment uses plain-text logging, the message *is* the JSON object.
-A Loki query parses it with `| json` and zero formatter config:
-
-```logql
-{app="switchyard"} | json | event="upstream_attempt_failed"
-```
-
-`is_ready()` returns `True` once the poller has completed at least one successful poll,
-which makes it useful as a readiness gate during startup.
-
-## Testing the router
-
-Three tiers, cheapest first. The `switchyard-latency-router-e2e` skill carries the
-runnable commands and helper scripts for each.
-
-1. **Offline unit suite** (no keys): `tests/test_latency_service_*.py`,
- `tests/test_session_key.py`, `tests/test_outcome_metrics.py`, and
- `tests/test_upstream_error_passthrough.py`: full coverage of selection, tiering,
- inverse-latency weighting, retry/fail-fast, session affinity, the health poller,
- metrics, and spans.
-2. **In-repo integration** (`tests/e2e/test_latency_service_llm_backend.py`, marked
- `integration`): an in-process mock Latency Service feeds verdicts while the LLM
- calls hit OpenRouter. Requires `OPENROUTER_API_KEY` (skips without it).
-3. **Real-world**: point a real backend at a mock or live Latency Service and confirm
- it routes. A mock LS pointed at `openrouter.ai` proves the
- poll → tier → weighting path with no API key — a keyless dispatch returns `401`,
- which confirms traffic reached the host.
-
-## Source map
-
-| File | Responsibility |
-|---|---|
-| `switchyard/lib/backends/latency_service_llm_backend.py` | Selection, session-affinity pins, retry, stats/metrics emission, per-attempt error log. |
-| `switchyard/lib/session_key.py` | `session_key_from_body()`: stable per-conversation key for session affinity. |
-| `switchyard/lib/session_affinity.py` | `SessionAffinity`: L1 (in-process LRU) + optional best-effort L2 pin store. |
-| `switchyard/lib/affinity_pin_store.py` | `AffinityPinStore`: async protocol for a shared/persistent L2 store. |
-| `switchyard/lib/redis_pin_store.py` | `RedisPinStore`: Redis-backed shared L2 (lazy `redis`, `affinity-redis` extra). |
-| `switchyard/lib/endpoints/upstream_error_log.py` | Structured per-attempt failure log (`switchyard.upstream_errors`, Loki). |
-| `switchyard/lib/backends/health_poller.py` | `EndpointHealthStatus`, `EndpointHealth`, `HealthPoller` daemon. |
-| `switchyard/lib/config/latency_service_backend_config.py` | `LatencyServiceEndpoint`, `LatencyServiceBackendConfig`. |
-| `switchyard/lib/profiles/latency_service.py` | `LatencyServiceProfileConfig`: profile assembly + stats wiring. |
-| `tests/test_latency_service_llm_backend.py` | Selection, retry, tier-preference tests. |
-| `tests/test_latency_service_health_metrics.py` | Poller and metrics tests. |
diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md
index cdba2a318..e80974791 100644
--- a/docs/internal/metrics_reference.md
+++ b/docs/internal/metrics_reference.md
@@ -12,7 +12,7 @@ a drop-in scrape config and starter alert rules.
| Content-Type | `text/plain; version=0.0.4; charset=utf-8` |
| Format | Prometheus text format 0.0.4 |
| Auth | None. Designed as a public scrape endpoint |
-| Default scrape interval | 15s (the Latency Service poll cycle is 10s by default, so finer scrape resolution captures no extra state) |
+| Default scrape interval | 15s |
`GET /metrics` is served by the Python route-bundle server started with
`switchyard --routing-profiles PATH -- serve`.
@@ -29,12 +29,11 @@ A JSON variant of the same underlying data lives at `GET /v1/stats`, with
## Per-endpoint counters
-The `model` label is the latency-service endpoint id (`openai/gpt-5.5`,
+The `model` label is the configured endpoint id (`openai/gpt-5.5`,
`azure_openai/gpt-5.5`, etc.).
The `tier` label is optional: present only when a routing factory
-supplied one. The `random_routing` factory emits `tier="strong"|"weak"`;
-the `latency_service` factory does not.
+supplied one. The `random_routing` factory emits `tier="strong"|"weak"`.
| Metric | Type | Meaning |
|---|---|---|
@@ -64,44 +63,6 @@ Each summary emits `{quantile="0.5"}` and `{quantile="0.99"}` rows plus
Healthy traffic typically sits at p50 ≈ 0.4 ms, p99 ≈ 0.6 ms.
-## Latency Service state (gauges: latency-service chains only)
-
-Published from the in-memory health cache the `LatencyServiceLLMBackend`
-maintains, refreshed on each successful poll of the Latency Service.
-
-| Metric | Type | Meaning |
-|---|---|---|
-| `switchyard_endpoint_status{model,status}` | gauge | Current Latency Service verdict per endpoint. `status` is one of `healthy`, `degraded`, `unknown`. Exactly one row per `model` is `1`; the rest are `0`, so `sum by (status)` gives a clean count of endpoints in each state. |
-| `switchyard_endpoint_last_latency_ms{model}` | gauge | Last latency sample the Latency Service reported for this endpoint. **Absent** when the upstream reported `null`, and absence is meaningful. |
-
-## Latency Service poll health (no labels: latency-service chains only)
-
-| Metric | Type | Meaning |
-|---|---|---|
-| `switchyard_latency_service_poll_ok` | gauge | `1` iff the most recent poll succeeded. `0` means the next routing decisions are based on the cache-reset-to-unknown fallback state. |
-| `switchyard_latency_service_poll_age_seconds` | gauge | Monotonic seconds since the last successful poll. **Absent** before the first success; combined with `poll_ok=0`, this distinguishes "never polled" from "polled but recently failed". |
-| `switchyard_latency_service_polls_total` | counter | Total successful polls since process start. |
-| `switchyard_latency_service_poll_failures_total` | counter | Total failed poll attempts. Each failure resets every endpoint in the cache to `unknown`. |
-
-## Session affinity (no labels: latency-service chains with `session_affinity` on)
-
-These counters are **absent unless `session_affinity: true`**, so the metric surface stays clean for the default per-turn-routing case.
-
-| Metric | Type | Meaning |
-|---|---|---|
-| `switchyard_affinity_hits_total` | counter | Conversation turns served by an existing session-affinity pin, meaning the upstream prompt/KV cache was reused. Counted once per request (first attempt only), so failover retries don't inflate it. |
-| `switchyard_affinity_misses_total` | counter | First turns of a conversation, or turns whose pin was broken by a health verdict, routed by the latency-aware picker. |
-| `switchyard_affinity_l2_hits_total` | counter | Pins resolved from the shared (L2) affinity store after an in-process (L1) miss — cross-worker/churn warm reuse. **Also requires a shared store** (`affinity_store: "redis"`). |
-| `switchyard_affinity_l2_errors_total` | counter | Shared-store get/put operations that failed open; routing fell back to in-process pins. A sustained rate means the store is degraded while requests keep succeeding — this is the alerting signal. While the breaker is open, only recovery probes fail, so the rate drops to ~1 per cooldown per pod. **Also requires a shared store.** |
-| `switchyard_affinity_l2_breaker_open` | gauge | `1` while the shared-store circuit breaker is open (operations skipped without a network attempt after 3 consecutive failures; one probe per 10 s cooldown), `0` when closed. The unambiguous store-outage signal. **Also requires a shared store.** |
-
-Warm-reuse rate (the fraction of turns that hit a warm endpoint):
-
-```promql
-rate(switchyard_affinity_hits_total[5m])
- / (rate(switchyard_affinity_hits_total[5m]) + rate(switchyard_affinity_misses_total[5m]))
-```
-
## Outcome counters for error-rate ratios
The `outcome` label takes exactly three values:
@@ -115,7 +76,6 @@ The `outcome` label takes exactly three values:
| `switchyard_client_responses_total{outcome}` | counter | HTTP responses returned to clients on the LLM-serving routes (`/v1/chat/completions`, `/v1/messages`, `/v1/responses`). The denominator for the **router-served** error rate. |
| `switchyard_upstream_attempts_total{outcome,code}` | counter | Individual upstream call attempts. One client request can produce N attempts via retry. The denominator for the **direct-to-endpoint** baseline error rate. The `code` label carries the raw upstream HTTP status for plotting the error-code distribution (see below). |
| `switchyard_router_retry_recovered_total` | counter | Requests whose first upstream attempt failed but a subsequent attempt succeeded. This is direct evidence the routing logic rescued the request. |
-| `switchyard_latency_upstream_attempts_total{requested_model,upstream_model,outcome,code}` | counter | Latency-service chains only: the per-model complement of `switchyard_upstream_attempts_total` (which stays model-free). `requested_model` is the client-supplied model bounded to a config-derived id — the route id (`route_model`, e.g. `nvidia/switchyard/gpt-5.4`) or a configured endpoint id — with `other` as the fallback sentinel. `upstream_model` is the selected endpoint's upstream name. Answers "for route X, which upstream failed or succeeded". |
### The `code` label on `switchyard_upstream_attempts_total`
@@ -154,7 +114,7 @@ error_rate_reduction = direct_error_rate − router_error_rate
# Live Endpoint Routing rescue rate
rate(switchyard_router_retry_recovered_total[5m])
-# Traffic share per endpoint (sanity-check inverse-latency weighting)
+# Traffic share per endpoint
sum by (model) (rate(switchyard_requests_total[5m]))
/ ignoring(model) group_left sum(rate(switchyard_requests_total[5m]))
@@ -165,12 +125,6 @@ sum by (code) (rate(switchyard_upstream_attempts_total{code!="200"}[5m]))
sum by (code) (rate(switchyard_upstream_attempts_total{code!="200"}[5m]))
/ ignoring(code) group_left
sum (rate(switchyard_upstream_attempts_total{code!="200"}[5m]))
-
-# Per-upstream outcome breakdown for one route — "for nvidia/switchyard/gpt-5.4,
-# which upstream provider failed or succeeded?" (latency-service chains)
-sum by (upstream_model, outcome, code) (
- rate(switchyard_latency_upstream_attempts_total{requested_model="nvidia/switchyard/gpt-5.4"}[5m])
-)
```
> **Note:** because `switchyard_upstream_attempts_total` now carries the
@@ -189,18 +143,10 @@ into label space.
| Label | Values | Where |
|---|---|---|
| `model` | One per configured endpoint, typically 2–6 per deployment. | All per-endpoint metrics. |
-| `status` | Exactly 3: `healthy`, `degraded`, `unknown`. | `switchyard_endpoint_status` |
| `outcome` | Exactly 3: `success`, `retryable_error`, `other_error`. | Outcome counters |
| `code` | Bounded: the known-code allowlist (`200`, `400`, `401`, `403`, `404`, `408`, `409`, `422`, `429`, `500`, `502`, `503`, `504`), plus `none` and the per-class buckets `1xx`/`2xx`/`3xx`/`4xx`/`5xx`/`other`. About 20 values max. | `switchyard_upstream_attempts_total` |
-| `requested_model` | Bounded to config-derived ids: the route id (`route_model`) plus configured endpoint ids, else the `other` sentinel. | `switchyard_latency_upstream_attempts_total` |
-| `upstream_model` | One per configured endpoint (the endpoint's upstream name). | `switchyard_latency_upstream_attempts_total` |
| `quantile` | Exactly 2: `0.5`, `0.99`. | All summaries |
-| `tier` | Small enumerated set, optional. Not emitted by latency-service chains. | Per-endpoint counters/summaries on routing chains that supply it |
-
-For a latency-service deployment with `N` endpoints, expect roughly
-`11N + 17` series at startup (five `code` series are seeded), growing by at
-most a dozen more as additional upstream status codes appear. That is about 39
-series for two endpoints. Well within single-Prometheus capacity.
+| `tier` | Small enumerated set, optional. | Per-endpoint counters/summaries on routing chains that supply it |
## Triage cheatsheet
@@ -208,7 +154,5 @@ series for two endpoints. Well within single-Prometheus capacity.
|---|---|
| `model="<unknown>"` rows appear | The per-endpoint attribution wiring regressed. `ctx.selected_model` is not being set by the backend. |
| All counters at 0 after warm-up | Server just started with no traffic, or the scraper is hitting the wrong port. |
-| `switchyard_latency_service_poll_ok` stuck at `0` | DNS / network unreachability to the Latency Service, **or** a schema mismatch on the LS response. Check server logs for `"Health poller: failed to reach Latency Service"`. |
-| `switchyard_endpoint_last_latency_ms{...}` absent for an endpoint | The Latency Service reported `last_latency_ms: null`, or the endpoint was not returned in the most recent poll response. |
| `switchyard_routing_overhead_ms_count` stuck at `0` | The backend is not publishing `ctx.backend_call_latency_ms` (regression of the Python-backend routing-overhead wiring). |
-| `switchyard_client_responses_total{outcome="retryable_error"}` rising | Either the upstream is genuinely flaky (cross-check `switchyard_endpoint_status`), or retries are exhausting (compare with `switchyard_router_retry_recovered_total` rate). |
+| `switchyard_client_responses_total{outcome="retryable_error"}` rising | Either the upstream is genuinely flaky, or retries are exhausting (compare with `switchyard_router_retry_recovered_total` rate). |
diff --git a/examples/latency_service_caller_required.yaml b/examples/latency_service_caller_required.yaml
deleted file mode 100644
index 3d6f78cfe..000000000
--- a/examples/latency_service_caller_required.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
-# =============================================================================
-# Latency-aware routing with per-caller key attribution (BYO key, required)
-# =============================================================================
-#
-# Use this when every upstream call must be billed to the *caller's* own API key
-# (e.g. per-user spend attribution in multi-tenant gateways) and the service-level key must
-# NEVER authenticate caller inference.
-#
-# What `credential_policy: caller_required` does:
-# * The caller key is read from the `x-switchyard-api-key` request header
-# (preferred), then `Authorization: Bearer` / `x-api-key`.
-# * Present -> forwarded upstream as `Authorization: Bearer <key>`; spend is
-# attributed to that caller.
-# * Missing -> the request is rejected with HTTP 401 BEFORE any upstream call;
-# the configured endpoint key is never used.
-# (Applies to every ingress path: /v1/chat/completions, /v1/responses, /v1/messages.)
-#
-# Why a dedicated header? A proxy in front of Switchyard (e.g. LiteLLM) consumes
-# the `Authorization` header for its own auth and strips it before the upstream
-# call. The custom `x-switchyard-api-key` header passes through untouched.
-#
-# How clients send the header:
-# * Codex CLI: env_http_headers / http_headers
-# * OpenCode: provider options.headers
-# * OpenAI / Anthropic SDK: extra / default headers
-# * Claude Code: limited (can't easily send two distinct auth values)
-# -- may need a wrapper/gateway unless one key serves both.
-#
-# Rollout order (IMPORTANT -- avoid 401s):
-# 1. Deploy this build first with the route still on its current policy
-# (`configured_endpoint` default, or `caller_override`) -- no behavior change.
-# 2. Update clients to send `x-switchyard-api-key` (harmless under those policies).
-# 3. Only after clients are confirmed sending it, switch to `caller_required`.
-#
-# Run:
-# uv run switchyard serve \
-# --routing-profiles examples/latency_service_caller_required.yaml --port 4100
-#
-# # Caller key present -> billed to that key:
-# curl -s -X POST http://127.0.0.1:4100/v1/chat/completions \
-# -H 'Content-Type: application/json' \
-# -H "x-switchyard-api-key: $USER_UPSTREAM_API_KEY" \
-# -d '{"model":"my-model","messages":[{"role":"user","content":"hi"}]}'
-#
-# # Header omitted -> HTTP 401:
-# # "missing caller API key; supply it via the x-switchyard-api-key header"
-# # (the service key is never used for caller inference).
-#
-# Substitute your own Latency Service URL, gateway base URL, and model IDs below.
-# =============================================================================
-
-routes:
- my-model:
- type: latency_service
- latency_service_url: https://your-latency-service.example.com
- # Require a caller key; 401 when missing; never fall back to a service key.
- credential_policy: caller_required
- poll_interval_s: 10.0
- max_retries: 2
- endpoints:
- # No `api_key:` on the endpoints: `caller_required` forwards the caller's
- # key and never falls back, so there is no service credential to leak.
- - model: primary/my-model # Latency Service endpoint_id
- upstream_model: provider/my-model # model name the upstream expects
- base_url: https://your-inference-gateway.example.com/v1
- # request_type: openai_responses # set for Responses-only upstreams
- # (e.g. gpt-5.1-codex-mini)
- - model: backup/my-model
- upstream_model: provider/my-model
- base_url: https://your-inference-gateway.example.com/v1
diff --git a/examples/prometheus/README.md b/examples/prometheus/README.md
index 77311caba..ceea201f4 100644
--- a/examples/prometheus/README.md
+++ b/examples/prometheus/README.md
@@ -25,12 +25,9 @@ Prometheus configuration (adjust the `targets:` list to point at your
Switchyard pods or hosts). The endpoint is plain `GET /metrics` over
HTTP, no auth.
-If you're running Switchyard on Kubernetes via
-[`examples/k8s/latency-aware-deployment.yaml`](../k8s/latency-aware-deployment.yaml),
-the service-level DNS is
-`switchyard-latency.llm-routing.svc.cluster.local`. The example scrape
-job uses static targets; a `kubernetes_sd_configs` variant is a one-step
-swap for cluster-native discovery.
+If you're running Switchyard on Kubernetes, the example scrape job uses
+static targets; a `kubernetes_sd_configs` variant is a one-step swap for
+cluster-native discovery.
### 2. Load the alert rules
@@ -61,8 +58,7 @@ series:
```
You should see the families documented in `metrics_reference.md` —
-top-line gauges, per-endpoint counters/summaries, Latency Service state,
-poll health, and outcome counters.
+top-line gauges, per-endpoint counters/summaries, and outcome counters.
## Alert tuning notes
@@ -71,8 +67,6 @@ conservative — tuned to avoid pages during routine scrape-config rolls
or single-poll blips. Review them against your SLOs before enabling
notifications:
-* `LatencyServiceUnreachable` fires after 60s of `poll_ok=0`. Bring this
- down if your routing logic should react faster than one poll cycle.
* `RouterOverheadHigh` is set at the success-criterion threshold (p99 >
1 ms for 5 m). Retry-heavy workloads inflate this — cross-check
`switchyard_upstream_attempts_total{outcome="retryable_error"}` rate
diff --git a/examples/prometheus/prometheus.yml b/examples/prometheus/prometheus.yml
index 229a0831b..cbecb6727 100644
--- a/examples/prometheus/prometheus.yml
+++ b/examples/prometheus/prometheus.yml
@@ -16,16 +16,11 @@ scrape_configs:
- job_name: switchyard
metrics_path: /metrics
scheme: http
- # Default scrape every 15s — the Latency Service poll cycle defaults
- # to 10s, so finer scrape resolution captures no additional state.
scrape_interval: 15s
static_configs:
- # Replace with your Switchyard host(s) and port(s). On Kubernetes
- # with the manifest at examples/k8s/latency-aware-deployment.yaml
- # the in-cluster DNS resolves to
- # switchyard-latency.llm-routing.svc.cluster.local on port 80
- # targeting container port 4100. Swap this static_configs block
- # for kubernetes_sd_configs in a cluster-native deployment.
+ # Replace with your Switchyard host(s) and port(s). Swap this
+ # static_configs block for kubernetes_sd_configs in a
+ # cluster-native deployment.
- targets:
- switchyard.example.internal:4100
diff --git a/examples/prometheus/switchyard.rules.yaml b/examples/prometheus/switchyard.rules.yaml
index dfc8378f8..3b3400054 100644
--- a/examples/prometheus/switchyard.rules.yaml
+++ b/examples/prometheus/switchyard.rules.yaml
@@ -9,60 +9,6 @@
# examples/prometheus/README.md before wiring to a pager.
groups:
- - name: switchyard.latency_service.health
- rules:
- - alert: LatencyServiceUnreachable
- expr: switchyard_latency_service_poll_ok == 0
- for: 60s
- labels:
- severity: warning
- component: switchyard
- annotations:
- summary: "Switchyard cannot reach the Latency Service"
- description: |
- switchyard_latency_service_poll_ok has been 0 for at least
- 60s. Routing has fallen back to uniform random across all
- endpoints (every verdict reset to unknown). Check the
- Switchyard pod logs for "Health poller: failed to reach
- Latency Service" and confirm DNS / network reachability to
- the upstream LS.
-
- - alert: LatencyServicePollStale
- # No `for:` — the age gauge only grows while polls keep failing, so
- # the >60s threshold is already debounced; a `for:` would only delay
- # the alert past the documented 60s staleness budget.
- expr: switchyard_latency_service_poll_age_seconds > 60
- labels:
- severity: warning
- component: switchyard
- annotations:
- summary: "Switchyard health verdicts are stale"
- description: |
- The most recent successful Latency Service poll was more
- than 60 s ago. Routing decisions are still using cached
- verdicts but they are increasingly out of date. Compare
- switchyard_latency_service_poll_failures_total against
- switchyard_latency_service_polls_total to see whether the
- poller is failing intermittently or hanging entirely.
-
- - alert: LatencyServiceEndpointFlapping
- expr: |
- sum by (model) (
- changes(switchyard_endpoint_status{status="healthy"}[10m])
- ) > 5
- for: 0s
- labels:
- severity: info
- component: switchyard
- annotations:
- summary: "Endpoint {{ $labels.model }} is flapping"
- description: |
- switchyard_endpoint_status{model="{{ $labels.model }}"} has
- transitioned in or out of "healthy" more than 5 times in
- the last 10 minutes. Likely a real upstream instability —
- cross-check switchyard_endpoint_last_latency_ms for that
- model and the LS upstream's own dashboards.
-
- name: switchyard.routing.evidence
rules:
- alert: RouterNotRescuing
@@ -84,7 +30,7 @@ groups:
/ 504) but no request has been rescued by retry in the same
5 m window. Likely cause: every configured endpoint is
degraded simultaneously, so the retry loop has nowhere to
- steer. Check switchyard_endpoint_status by model.
+ steer.
- alert: RouterOverheadHigh
expr: switchyard_routing_overhead_ms{quantile="0.99"} > 1.0
@@ -127,6 +73,5 @@ groups:
clients is at least as high as the per-attempt upstream
retryable_error rate. The router is not absorbing failures
via retry — either retries are exhausting or every
- endpoint is producing the same failure mode. Look at
- switchyard_endpoint_status and the per-model breakdown of
- switchyard_errors_total{model}.
+ endpoint is producing the same failure mode. Look at the
+ per-model breakdown of switchyard_errors_total{model}.
diff --git a/switchyard/__init__.py b/switchyard/__init__.py
index 34b5c35dd..5fc93f89e 100644
--- a/switchyard/__init__.py
+++ b/switchyard/__init__.py
@@ -13,10 +13,6 @@ from typing import TYPE_CHECKING, Any
from switchyard.lib.backends import (
AnthropicNativeBackend,
- EndpointHealth,
- EndpointHealthStatus,
- HealthPoller,
- LatencyServiceLLMBackend,
OpenAiNativeBackend,
)
from switchyard.lib.backends.llm_target import (
@@ -42,8 +38,6 @@ from switchyard.lib.chat_response import (
)
from switchyard.lib.config import (
IntakeSinkConfig,
- LatencyServiceBackendConfig,
- LatencyServiceEndpoint,
)
from switchyard.lib.processors.intake_payload_builder import IntakePayloadBuilder
from switchyard.lib.processors.intake_request_processor import IntakeRequestProcessor
@@ -58,7 +52,6 @@ from switchyard.lib.profiles import (
DeterministicRoutingProfileConfig,
EscalationRouterConfig,
EscalationRouterProfileConfig,
- LatencyServiceProfileConfig,
NoopProfile,
NoopProfileConfig,
PassthroughProfileConfig,
@@ -156,7 +149,6 @@ __all__ = [
"DeterministicRoutingPresets",
"EscalationRouterConfig",
"EscalationRouterProfileConfig",
- "LatencyServiceProfileConfig",
"PassthroughProfileConfig",
"Profile",
"ProfileConfig",
@@ -182,13 +174,6 @@ __all__ = [
"build_switchyard_app",
# Route dispatch table
"RouteTable",
- # Latency Service usage case
- "EndpointHealth",
- "EndpointHealthStatus",
- "HealthPoller",
- "LatencyServiceBackendConfig",
- "LatencyServiceEndpoint",
- "LatencyServiceLLMBackend",
"IntakePayloadBuilder",
"IntakeRequestProcessor",
"IntakeResponseProcessor",
diff --git a/switchyard/cli/launch_command.py b/switchyard/cli/launch_command.py
index b79a5164c..435e17383 100644
--- a/switchyard/cli/launch_command.py
+++ b/switchyard/cli/launch_command.py
@@ -490,7 +490,7 @@ def cmd_launch_claude(args: argparse.Namespace) -> None:
* ``--routing-profiles FILE`` (global flag) — YAML-driven multi-chain table.
``--model`` is optional; falls back to the first YAML route.
- Random / latency-aware routing live in the YAML schema.
+ Random and other multi-chain routing live in the YAML schema.
"""
if getattr(args, "startup_timing", False):
startup_timing.enable()
diff --git a/switchyard/cli/launchers/claude_code_launcher.py b/switchyard/cli/launchers/claude_code_launcher.py
index 148764c13..c71eedc3a 100644
--- a/switchyard/cli/launchers/claude_code_launcher.py
+++ b/switchyard/cli/launchers/claude_code_launcher.py
@@ -298,8 +298,8 @@ def _run_claude_with_switchyard(
Takes a pre-built :class:`Switchyard` and a display-only model
name. The caller decides what's in the chain (single-model
- passthrough, random routing across a preset, latency-service
- pool, …) — this function owns only the uvicorn-in-a-thread +
+ passthrough, random routing across a preset, …) — this function
+ owns only the uvicorn-in-a-thread +
``claude`` supervision + env-var injection boilerplate that is
identical across those modes.
diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py
index 727c648b9..a465a3906 100644
--- a/switchyard/cli/route_bundle.py
+++ b/switchyard/cli/route_bundle.py
@@ -34,13 +34,11 @@ from pydantic import ValidationError
from switchyard.cli.model_catalog.model_discovery import fetch_model_ids
from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target
-from switchyard.lib.config import LatencyServiceBackendConfig, LatencyServiceEndpoint
from switchyard.lib.processors.llm_classifier import DEFAULT_MAX_REQUEST_CHARS
from switchyard.lib.processors.llm_classifier.presets import PROFILE_FACTORIES
from switchyard.lib.profiles import (
DeterministicRoutingProfileConfig,
EscalationRouterProfileConfig,
- LatencyServiceProfileConfig,
ProfileSwitchyard,
StageRouterProfileConfig,
)
@@ -193,32 +191,6 @@ _PASSTHROUGH_ROUTE_KEYS = (
| frozenset({"defaults", "enable_stats"})
| _PASSTHROUGH_SETTING_KEYS
)
-_LATENCY_ENDPOINT_DEFAULT_KEYS = frozenset({
- "api_key",
- "base_url",
- "timeout",
- "timeout_secs",
-})
-_LATENCY_ENDPOINT_KEYS = _LATENCY_ENDPOINT_DEFAULT_KEYS | frozenset(
- {"model", "upstream_model", "request_type"}
-)
-_LATENCY_SERVICE_ROUTE_KEYS = _ROUTE_METADATA_KEYS | frozenset({
- "defaults",
- "endpoints",
- "latency_service_url",
- "latency_url",
- "poll_interval_s",
- "poll_timeout_s",
- "max_retries",
- "credential_policy",
- "enable_stats",
- "session_affinity",
- "affinity_max_sessions",
- "affinity_store",
- "affinity_store_url",
- "affinity_store_ttl_seconds",
- "affinity_key_prefix",
-}) | _LATENCY_ENDPOINT_DEFAULT_KEYS
_NOOP_ROUTE_KEYS = _ROUTE_METADATA_KEYS
_DETERMINISTIC_ROUTE_KEYS = (
_ROUTE_METADATA_KEYS
@@ -318,7 +290,6 @@ _CLASSIFIER_DEFAULT_KEYS = frozenset({
_ROUTE_KEYS_BY_TYPE: Mapping[str, frozenset[str]] = {
"model": _MODEL_ROUTE_KEYS,
"random_routing": _RANDOM_ROUTING_ROUTE_KEYS,
- "latency_service": _LATENCY_SERVICE_ROUTE_KEYS,
"noop": _NOOP_ROUTE_KEYS,
"passthrough": _PASSTHROUGH_ROUTE_KEYS,
"deterministic": _DETERMINISTIC_ROUTE_KEYS,
@@ -328,7 +299,6 @@ _ROUTE_KEYS_BY_TYPE: Mapping[str, frozenset[str]] = {
_DEFAULT_KEYS_BY_TYPE: Mapping[str, frozenset[str]] = {
"model": _TARGET_DEFAULT_KEYS,
"random_routing": _TARGET_DEFAULT_KEYS,
- "latency_service": _LATENCY_ENDPOINT_DEFAULT_KEYS,
"passthrough": _PASSTHROUGH_SETTING_KEYS,
"noop": frozenset(),
"deterministic": _TARGET_DEFAULT_KEYS,
@@ -869,16 +839,6 @@ def _build_route_chain(
extra_response_processors=extra_response_processors,
)
- if route_type == "latency_service":
- return _latency_service_switchyard(
- model_id,
- route,
- target_defaults,
- stats=stats,
- extra_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
-
if route_type == "noop":
from switchyard.lib.profiles.noop import NoopProfileConfig
@@ -1305,62 +1265,6 @@ def _passthrough_target(
)
-def _latency_service_switchyard(
- model_id: str,
- route: Mapping[str, object],
- target_defaults: Mapping[str, object],
- stats: StatsAccumulator,
- extra_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
-) -> ChainRuntime:
- endpoints_raw = _require_sequence(route.get("endpoints"), "latency_service.endpoints")
- endpoints = [
- _latency_endpoint(value, target_defaults, index=index)
- for index, value in enumerate(endpoints_raw)
- ]
- enable_stats = _optional_bool(route.get("enable_stats"), default=True)
- credential_policy = (
- _optional_str(route["credential_policy"])
- if "credential_policy" in route
- else "configured_endpoint"
- )
- config = LatencyServiceBackendConfig.model_validate({
- "latency_service_url": _required_str(
- route.get("latency_service_url", route.get("latency_url")),
- "latency_service.latency_service_url",
- ),
- "endpoints": endpoints,
- # The YAML route key is what clients send as ``model``; hand it to the
- # backend so the per-model metric can attribute route-key traffic.
- "route_model": model_id,
- "poll_interval_s": _optional_float(route.get("poll_interval_s"), default=10.0),
- "poll_timeout_s": _optional_float(route.get("poll_timeout_s"), default=5.0),
- "max_retries": _optional_int(route.get("max_retries"), default=2),
- "credential_policy": credential_policy,
- "enable_stats": enable_stats,
- "session_affinity": _optional_bool(route.get("session_affinity"), default=False),
- "affinity_max_sessions": _optional_int(
- route.get("affinity_max_sessions"), default=10_000
- ),
- "affinity_store": _optional_str(route.get("affinity_store")) or "memory",
- "affinity_store_url": _optional_str(route.get("affinity_store_url")),
- "affinity_store_ttl_seconds": _optional_int(
- route.get("affinity_store_ttl_seconds"), default=3_600
- ),
- "affinity_key_prefix": _optional_str(route.get("affinity_key_prefix")) or "swyd:pin:",
- })
- return ProfileSwitchyard(
- LatencyServiceProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats,
- enable_stats=config.enable_stats,
- pre_request_processors=extra_request_processors,
- response_processors=extra_response_processors,
- )
- )
-
-
def _route_config(
route: Mapping[str, object],
target_defaults: Mapping[str, object],
@@ -1438,33 +1342,6 @@ def _target_mapping(
return target
-def _latency_endpoint(
- raw: object,
- defaults: Mapping[str, object],
- index: int,
-) -> LatencyServiceEndpoint:
- data = dict(defaults)
- where = f"latency_service.endpoints[{index}]"
- if isinstance(raw, str):
- data["model"] = raw
- elif isinstance(raw, Mapping):
- endpoint_mapping = _require_mapping(raw, where)
- _validate_allowed_keys(endpoint_mapping, _LATENCY_ENDPOINT_KEYS, where)
- data.update(endpoint_mapping)
- else:
- raise RouteBundleConfigError(f"{where} must be a string or mapping")
-
- if "timeout" not in data and "timeout_secs" in data:
- data["timeout"] = data["timeout_secs"]
-
- endpoint_data = {
- key: data[key]
- for key in ("model", "upstream_model", "api_key", "base_url", "timeout", "request_type")
- if key in data
- }
- return LatencyServiceEndpoint.model_validate(endpoint_data)
-
-
def _target_defaults(
bundle_defaults: Mapping[str, object],
route: Mapping[str, object],
@@ -1492,8 +1369,6 @@ def _route_type(model_id: str, route: Mapping[str, object]) -> str:
if raw_type is None:
if not route:
return "noop"
- if "latency_service_url" in route or "latency_url" in route:
- return "latency_service"
if "strong" in route and "weak" in route:
return "random_routing"
if "target" in route or "model" in route:
@@ -1512,8 +1387,6 @@ def _route_type(model_id: str, route: Mapping[str, object]) -> str:
"target": "model",
"random": "random_routing",
"random_routing": "random_routing",
- "latency": "latency_service",
- "latency_service": "latency_service",
"noop": "noop",
"no_op": "noop",
"passthrough": "passthrough",
@@ -1615,12 +1488,6 @@ def _optional_mapping(value: object, where: str) -> dict[str, object]:
return _require_mapping(value, where)
-def _require_sequence(value: object, where: str) -> Sequence[object]:
- if not isinstance(value, list):
- raise RouteBundleConfigError(f"{where} must be a list")
- return value
-
-
def _required_str(value: object, where: str) -> str:
if not isinstance(value, str) or not value:
raise RouteBundleConfigError(f"{where} must be a non-empty string")
diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.py
index 59c4de402..b0fdaf781 100644
--- a/switchyard/cli/switchyard_cli.py
+++ b/switchyard/cli/switchyard_cli.py
@@ -44,7 +44,7 @@ Examples::
switchyard --routing-profiles profiles.yaml -- launch claude -- --no-auto-approve
Routing policies that used to be top-level CLI verbs (``passthrough``,
-``random-routing``, ``latency-service``) and launcher flags
+``random-routing``) and launcher flags
(``--routing``, ``--weak-model``, ``--strong-probability``, ``--preset``)
are expressed in routing-profile YAML files. ``serve`` and
launchers still parse the YAML into profile-backed runtimes.
diff --git a/switchyard/lib/affinity_pin_store.py b/switchyard/lib/affinity_pin_store.py
index 1654bfeb1..39b766899 100644
--- a/switchyard/lib/affinity_pin_store.py
+++ b/switchyard/lib/affinity_pin_store.py
@@ -20,8 +20,8 @@ from typing import Protocol, runtime_checkable
class AffinityPinStore(Protocol):
"""Shared, out-of-process store for conversation→decision pins.
- The pinned value is opaque to the store (an endpoint id for the latency
- backend, a tier label for the classifier router). Implementations are
+ The pinned value is opaque to the store (a tier label for the classifier
+ or escalation router). Implementations are
called on the request path, so they should be fast, and
:class:`SessionAffinity` treats them as **best-effort**: an error or timeout
must never break request routing (the caller falls back to L1 / unpinned).
diff --git a/switchyard/lib/backends/__init__.py b/switchyard/lib/backends/__init__.py
index 6251d02d1..53c20ae6f 100644
--- a/switchyard/lib/backends/__init__.py
+++ b/switchyard/lib/backends/__init__.py
@@ -11,14 +11,6 @@ from switchyard.lib.backends.backend_format_resolver import (
BackendFormatResolution,
BackendFormatResolver,
)
-from switchyard.lib.backends.health_poller import (
- EndpointHealth,
- EndpointHealthStatus,
- HealthPoller,
-)
-from switchyard.lib.backends.latency_service_llm_backend import (
- LatencyServiceLLMBackend,
-)
from switchyard.lib.backends.stats_llm_backend import (
StatsLlmBackend,
)
@@ -32,10 +24,6 @@ __all__ = [
"AnthropicNativeBackend",
"BackendFormatResolution",
"BackendFormatResolver",
- "EndpointHealth",
- "EndpointHealthStatus",
- "HealthPoller",
- "LatencyServiceLLMBackend",
"OpenAiPassthroughBackend",
"OpenAiNativeBackend",
"StatsLlmBackend",
diff --git a/switchyard/lib/backends/health_poller.py b/switchyard/lib/backends/health_poller.py
deleted file mode 100644
index 5393e568f..000000000
--- a/switchyard/lib/backends/health_poller.py
+++ /dev/null
@@ -1,181 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Health enum and background poller for the Latency Service usage case.
-
-The Latency Service exposes a bulk health endpoint that returns one of
-three states per endpoint. :class:`EndpointHealthStatus` is the shared
-contract between Switchyard and the Latency Service — both sides must
-agree on these string values.
-
-:class:`HealthPoller` is a daemon thread that periodically pulls fresh
-verdicts from the Latency Service and writes them into an in-memory
-cache shared with :class:`LatencyServiceLLMBackend`. The backend's
-request hot path only ever *reads* the cache (under a lock), so health
-polling adds zero per-request latency.
-
-Each cache entry is an :class:`EndpointHealth` snapshot carrying both
-the discrete ``status`` and the most recent ``last_latency_ms`` sample
-reported by the Latency Service. The status drives tier selection
-(HEALTHY > UNKNOWN > DEGRADED); the latency drives inverse-latency
-weighted selection *within* a tier.
-
-Design properties:
-
-- **Daemon thread, not asyncio task.** Simple lifecycle: starts when the
- backend is constructed, dies with the process. The sync ``httpx.Client``
- used inside the thread avoids event-loop contention for other work.
-- **Fallback to UNKNOWN on poll failure.** If the Latency Service is
- unreachable, all endpoints reset to ``UNKNOWN`` so the backend falls
- back to random routing rather than acting on stale health data.
-- **Graceful stop via ``stop()``.** Tests and ``shutdown()`` paths set
- ``_stop_event`` to break the polling loop promptly.
-"""
-
-from __future__ import annotations
-
-import enum
-import logging
-import threading
-import time
-from typing import NamedTuple
-
-import httpx
-
-logger = logging.getLogger(__name__)
-
-
-class EndpointHealthStatus(enum.Enum):
- """Shared contract between Switchyard and the Latency Service.
-
- Three states are sufficient for routing decisions — the Latency
- Service may track finer-grained failure reasons internally for
- observability but exposes only these values in the health API.
- """
-
- HEALTHY = "healthy"
- DEGRADED = "degraded"
- UNKNOWN = "unknown"
-
-
-class EndpointHealth(NamedTuple):
- """Snapshot of one endpoint's health, written by :class:`HealthPoller`.
-
- ``status`` drives tier-based routing (HEALTHY > UNKNOWN > DEGRADED);
- ``last_latency_ms`` — when present — drives inverse-latency weighted
- selection within whichever tier is chosen. ``last_latency_ms`` is
- ``None`` before the first successful poll and whenever the Latency
- Service reports it as null.
- """
-
- status: EndpointHealthStatus
- last_latency_ms: float | None = None
-
-
-class HealthPoller(threading.Thread):
- """Background daemon thread that refreshes the backend's health cache.
-
- Polls ``{latency_service_url}/v1/endpoints/health`` every
- ``poll_interval_s`` seconds and writes verdicts into
- ``health_cache`` under ``cache_lock``. Holds references to the
- same dict and lock the backend reads from — no message passing, no
- extra copies.
- """
-
- def __init__(
- self,
- latency_service_url: str,
- model_ids: list[str],
- health_cache: dict[str, EndpointHealth],
- cache_lock: threading.Lock,
- poll_interval_s: float,
- poll_timeout_s: float,
- ) -> None:
- super().__init__(daemon=True, name="latency-health-poller")
- self._url = latency_service_url.rstrip("/") + "/v1/endpoints/health"
- self._model_ids = model_ids
- self._health_cache = health_cache
- self._cache_lock = cache_lock
- self._poll_interval_s = poll_interval_s
- self._http_client = httpx.Client(timeout=poll_timeout_s)
- self._stop_event = threading.Event()
- self._poll_count = 0
- # Failure counter and last-success timestamp drive the /metrics
- # poll-health gauges; tracked here because the poll loop is the
- # single point where outcomes are observable.
- self._poll_failures = 0
- self._last_poll_ok = False
- self._last_success_at: float | None = None
-
- def run(self) -> None:
- while not self._stop_event.is_set():
- try:
- resp = self._http_client.get(
- self._url,
- params=[("endpoint_ids", mid) for mid in self._model_ids],
- )
- resp.raise_for_status()
- data = resp.json()
- with self._cache_lock:
- for mid, info in data["endpoint_health"].items():
- if mid in self._health_cache:
- self._health_cache[mid] = EndpointHealth(
- status=EndpointHealthStatus(info["status"]),
- last_latency_ms=info.get("last_latency_ms"),
- )
- self._poll_count += 1
- self._last_poll_ok = True
- self._last_success_at = time.monotonic()
- except Exception:
- logger.warning(
- "Health poller: failed to reach Latency Service, "
- "resetting all endpoints to UNKNOWN (random routing)"
- )
- with self._cache_lock:
- for mid in self._health_cache:
- self._health_cache[mid] = EndpointHealth(
- status=EndpointHealthStatus.UNKNOWN,
- )
- self._poll_failures += 1
- self._last_poll_ok = False
- self._stop_event.wait(timeout=self._poll_interval_s)
-
- @property
- def has_polled(self) -> bool:
- """True once at least one poll has successfully updated the cache."""
- return self._poll_count > 0
-
- @property
- def poll_successes(self) -> int:
- """Total number of successful polls since the poller started."""
- return self._poll_count
-
- @property
- def poll_failures(self) -> int:
- """Total number of poll attempts that failed (including timeouts).
-
- On every failure, all cached endpoint verdicts are reset to UNKNOWN
- so routing falls back to random selection instead of acting on
- stale health data.
- """
- return self._poll_failures
-
- @property
- def last_poll_ok(self) -> bool:
- """True iff the most recent poll attempt succeeded."""
- return self._last_poll_ok
-
- @property
- def seconds_since_last_success(self) -> float | None:
- """Monotonic seconds since the last successful poll.
-
- ``None`` before the first success — useful as a "never polled"
- signal on /metrics scrapes during startup or sustained outage.
- """
- if self._last_success_at is None:
- return None
- return time.monotonic() - self._last_success_at
-
- def stop(self) -> None:
- """Signal the polling loop to exit at the next iteration boundary."""
- self._stop_event.set()
diff --git a/switchyard/lib/backends/latency_service_llm_backend.py b/switchyard/lib/backends/latency_service_llm_backend.py
deleted file mode 100644
index 1942df790..000000000
--- a/switchyard/lib/backends/latency_service_llm_backend.py
+++ /dev/null
@@ -1,1098 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""``LLMBackend`` that routes across many endpoints by Latency Service verdicts.
-
-This is the usage case for Inference Hub deployments where a central
-Latency Service owns heartbeat probing and statistical profiling. The
-backend holds a pool of ``OpenAILLMClient`` instances keyed by model ID,
-reads health verdicts from a locally-cached map maintained by a
-:class:`HealthPoller` daemon thread, and picks a healthy endpoint on
-every request.
-
-Chain integration::
-
- request-side work → LatencyServiceLLMBackend → response-side work → TranslationEngine
-
-Request-format translation is delegated to
-:class:`switchyard_rust.translation.TranslationEngine` after an endpoint is
-selected, so Chat endpoints keep receiving Chat Completions while
-Responses-mode endpoints receive the OpenAI Responses API natively.
-"""
-
-import json
-import logging
-import random
-import threading
-import time
-import uuid
-from collections.abc import Mapping
-from dataclasses import dataclass
-
-from openai import APIStatusError, AsyncStream
-
-from switchyard.lib.affinity_pin_store import AffinityPinStore
-from switchyard.lib.backends.health_poller import (
- EndpointHealth,
- EndpointHealthStatus,
- HealthPoller,
-)
-from switchyard.lib.chat_response.openai_chat import ResponseStream
-from switchyard.lib.chat_response.openai_responses import ResponsesApiStream
-from switchyard.lib.config.latency_service_backend_config import (
- LatencyServiceBackendConfig,
-)
-from switchyard.lib.endpoints import outcome_metrics, prometheus_emitter
-from switchyard.lib.endpoints.upstream_error_log import log_upstream_attempt_failure
-from switchyard.lib.llm_client import OpenAILLMClient
-from switchyard.lib.prometheus_exposition import format_number, render_labels
-from switchyard.lib.proxy_context import (
- CTX_CALLER_API_KEY,
- CTX_ERROR_SOURCE,
- CTX_ROUTE_SELECTION,
- CTX_UPSTREAM_ATTEMPTS_RECORDED,
- CTX_UPSTREAM_HTTP_BODY,
- CTX_UPSTREAM_HTTP_STATUS,
- CTX_UPSTREAM_MODEL,
- ERROR_SOURCE_PROVIDER,
- ERROR_SOURCE_SWITCHYARD,
- ProxyContext,
-)
-from switchyard.lib.roles import LLMBackend
-from switchyard.lib.session_affinity import SessionAffinity
-from switchyard.lib.stats_accumulator import StatsAccumulator
-from switchyard.lib.tracing import routing_span, set_tags
-from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse, request_type_matches
-from switchyard_rust.translation import TranslationEngine
-
-log = logging.getLogger(__name__)
-
-#: Request header read by a LiteLLM front proxy; its JSON value is copied into
-#: the provider spend-log DB row, tying that row back to the Switchyard route
-#: that selected it (see :data:`CTX_ROUTE_SELECTION` for the payload contract).
-SPEND_LOGS_METADATA_HEADER = "x-litellm-spend-logs-metadata"
-
-#: ``router_strategy`` value identifying this backend's selection algorithm.
-ROUTER_STRATEGY_LATENCY = "latency"
-
-
-@dataclass(frozen=True)
-class _RouteDecision:
- """One endpoint-selection outcome, carried to the ``route_decision`` span.
-
- ``candidates`` is the set the picker chose among (the winning health tier),
- and ``was_fastest`` is whether ``selected`` had the lowest known latency in
- that set. ``affinity`` is ``True`` when ``selected`` came from a session
- pin (reused endpoint) rather than the latency-aware picker.
- """
-
- selected: str
- candidates: tuple[str, ...]
- was_fastest: bool
- affinity: bool = False
-
-
-class LatencyServiceLLMBackend(LLMBackend):
- """Routes to healthy endpoints based on Latency Service health verdicts.
-
- On construction, builds a pool of ``OpenAILLMClient`` instances
- (one per configured endpoint, keyed by ``model``) and starts a
- :class:`HealthPoller` daemon thread that refreshes the in-memory
- health cache every ``poll_interval_s`` seconds.
-
- On each ``call()``:
-
- 1. Pick an endpoint from the health cache — ``HEALTHY`` preferred,
- then ``UNKNOWN``, then ``DEGRADED``. Within the chosen tier,
- selection is weighted by inverse ``last_latency_ms`` when every
- candidate has a known sample; otherwise it falls back to uniform
- random.
- 2. Translate the request to that endpoint's configured OpenAI API
- surface and override the body's ``model`` with the selected endpoint ID.
- 3. Call ``OpenAILLMClient`` on the configured API surface. On a transient error
- (429 / 408 / 5xx / network), retry with a different endpoint
- (dedup prevents re-selecting the same one within a single
- request). On a 4xx client error the request is rejected
- identically by every replica, so the loop fails fast and passes
- the upstream status through.
- 4. Wrap the response into a Rust-owned ``ChatResponse``.
- """
-
- def __init__(
- self,
- config: LatencyServiceBackendConfig,
- *,
- stats_accumulator: StatsAccumulator | None = None,
- ) -> None:
- if not config.endpoints:
- raise ValueError("At least one endpoint must be configured")
-
- self._config = config
- self._translation = TranslationEngine()
- self._clients: dict[str, OpenAILLMClient] = {}
- self._upstream_models: dict[str, str] = {}
- self._request_types: dict[str, ChatRequestType] = {}
- self._health_cache: dict[str, EndpointHealth] = {}
- self._cache_lock = threading.Lock()
- # When provided, the backend records success/error/latency directly
- # into the accumulator on each attempt. This mirrors what the Rust
- # ``StatsLlmBackend`` does for native backends; the Python-only
- # latency-service backend can't be wrapped by it, so we record
- # in-place to keep ``/metrics`` populated.
- self._stats = stats_accumulator
-
- # Session affinity: pins a conversation to the endpoint that last served
- # it so its upstream prompt/KV cache stays warm. Shared coordinator with
- # the classifier router; the latency-specific reuse policy (health-gated,
- # failover-aware) lives in ``_select_endpoint_decision``. An optional
- # shared L2 (Redis) lets pins outlive this worker and be read by peers,
- # so a conversation stays pinned across replicas and pod churn.
- self._affinity = SessionAffinity(
- enabled=config.session_affinity,
- max_sessions=config.affinity_max_sessions,
- l2=_build_affinity_l2(config),
- )
- # Cumulative warm-reuse counters, published on /metrics when affinity is
- # enabled. A "hit" is a turn served by an existing pin; a "miss" is a
- # first/unpinnable turn routed by the latency-aware picker. Counted once
- # per request (first attempt only) so failover retries don't inflate
- # them. Incremented and read only on the event loop, so no lock needed.
- self._affinity_hits = 0
- self._affinity_misses = 0
-
- # Per-model upstream-attempt counts for the latency-service-scoped
- # /metrics breakdown, keyed by (requested model, selected upstream model,
- # outcome, HTTP code). Kept separate from the layer-aggregate
- # ``switchyard_upstream_attempts_total`` so that counter stays model-free.
- # Cardinality is bounded: ``upstream_model`` comes from config, and
- # ``requested_model`` is normalized to a config-derived id (route id or
- # endpoint id) or the ``"other"`` sentinel in ``_record_model_attempt``
- # so a caller-supplied model string can't grow the series set. Mutated
- # and read only on the event loop (like the affinity counters), so no
- # lock.
- self._upstream_attempts_by_model: dict[tuple[str, str, str, str], int] = {}
-
- for ep_cfg in config.endpoints:
- model_id = ep_cfg.model
- if not model_id:
- raise ValueError(
- "Every endpoint must have a 'model' field — "
- "this is the key used by the Latency Service"
- )
- if model_id in self._clients:
- raise ValueError(f"Duplicate model ID: {model_id}")
-
- self._clients[model_id] = OpenAILLMClient(
- api_key=ep_cfg.api_key,
- base_url=ep_cfg.base_url,
- timeout=ep_cfg.timeout,
- # The ``call`` retry loop already retries on a *different*
- # endpoint, which is what a health-aware router wants. Letting
- # the SDK also retry (default 2) on the *same* endpoint stacks
- # multiplicatively (up to (1+max_retries) x 3 attempts) and adds
- # exponential-backoff sleeps that hold the request — and its
- # buffered body — alive longer, amplifying connection-pool
- # pressure during an upstream incident. Disable it here.
- max_retries=0,
- )
- self._upstream_models[model_id] = ep_cfg.upstream_model or model_id
- self._request_types[model_id] = _latency_endpoint_request_type(
- ep_cfg.request_type
- )
- self._health_cache[model_id] = EndpointHealth(
- status=EndpointHealthStatus.UNKNOWN,
- )
- log.info(
- "LatencyServiceLLMBackend endpoint: model=%s upstream_model=%s "
- "request_type=%s base_url=%s",
- model_id,
- self._upstream_models[model_id],
- ep_cfg.request_type,
- ep_cfg.base_url,
- )
-
- # Bounded ``requested_model`` label set: the configured endpoint ids
- # plus the client-facing route id when the deployment supplied one
- # (IH clients request the route key, e.g. ``nvidia/switchyard/gpt-5.4``,
- # never an endpoint id). Both sources are config-derived, so metric
- # cardinality stays bounded.
- self._requested_model_ids = frozenset(self._clients) | (
- frozenset((config.route_model,)) if config.route_model else frozenset()
- )
-
- self._poller = HealthPoller(
- latency_service_url=config.latency_service_url,
- model_ids=list(self._clients.keys()),
- health_cache=self._health_cache,
- cache_lock=self._cache_lock,
- poll_interval_s=config.poll_interval_s,
- poll_timeout_s=config.poll_timeout_s,
- )
- # Publish LS verdicts + poll health on /metrics. Registering here
- # (and unregistering in ``shutdown``) ties emitter lifetime to the
- # backend's lifetime, so a re-built chain doesn't leak a closure
- # over a torn-down cache.
- prometheus_emitter.register(self._render_prometheus_lines)
- self._poller.start()
-
- @property
- def supported_request_types(self) -> list[ChatRequestType]:
- """OpenAI request types accepted by the configured endpoints."""
- ordered = [ChatRequestType.OPENAI_CHAT, ChatRequestType.OPENAI_RESPONSES]
- configured = set(self._request_types.values())
- return [request_type for request_type in ordered if request_type in configured]
-
- # -- Endpoint selection (reads from cache, never blocks on network) -----
-
- def _select_endpoint(self) -> str:
- """Pick a model ID — see :meth:`_select_endpoint_decision`."""
- return self._select_endpoint_decision().selected
-
- def _select_endpoint_decision(
- self,
- pinned_endpoint: str | None = None,
- tried: frozenset[str] = frozenset(),
- ) -> _RouteDecision:
- """Pick a model ID and report the candidate set it chose among.
-
- When ``pinned_endpoint`` is set (this conversation's session-affinity
- pin) and still usable — HEALTHY/UNKNOWN and not already failed this
- request — it is reused directly, bypassing the latency-aware pick so the
- conversation sticks to one endpoint and its upstream cache stays warm.
-
- Otherwise tier priority is HEALTHY > UNKNOWN > DEGRADED. Within the
- chosen tier, selection is **inverse-latency weighted** when every
- candidate has a known ``last_latency_ms`` — endpoints that the Latency
- Service most recently saw as faster receive proportionally more
- traffic. If any candidate's latency is unknown, the picker falls back
- to uniform random for that tier; this keeps behaviour predictable while
- the poller is warming up or when the upstream reports nulls.
-
- Returns the selected endpoint plus the candidates considered and
- whether the pick was the lowest-latency one — the signals the
- ``switchyard.route_decision`` span reports.
- """
- with self._cache_lock:
- snapshot = dict(self._health_cache)
-
- # -- Affinity fast-path: reuse the pinned endpoint while it stays
- # healthy and hasn't already failed this request. Skips latency
- # weighting entirely — the whole point is to not chase a marginally
- # faster endpoint at the cost of cache locality.
- if (
- pinned_endpoint is not None
- and pinned_endpoint not in tried
- and _affinity_usable(snapshot.get(pinned_endpoint))
- ):
- return _RouteDecision(
- selected=pinned_endpoint,
- candidates=(pinned_endpoint,),
- was_fastest=False,
- affinity=True,
- )
-
- by_health: dict[EndpointHealthStatus, list[str]] = {
- h: [] for h in EndpointHealthStatus
- }
- for mid, hp in snapshot.items():
- by_health[hp.status].append(mid)
-
- for tier in (
- EndpointHealthStatus.HEALTHY,
- EndpointHealthStatus.UNKNOWN,
- EndpointHealthStatus.DEGRADED,
- ):
- if by_health[tier]:
- candidates = by_health[tier]
- selected = self._pick_by_latency(candidates, snapshot)
- return _RouteDecision(
- selected=selected,
- candidates=tuple(candidates),
- was_fastest=self._is_fastest(selected, candidates, snapshot),
- )
-
- candidates = list(self._clients.keys())
- selected = random.choice(candidates)
- return _RouteDecision(
- selected=selected,
- candidates=tuple(candidates),
- was_fastest=self._is_fastest(selected, candidates, snapshot),
- )
-
- @staticmethod
- def _is_fastest(
- selected: str,
- candidates: list[str],
- snapshot: dict[str, EndpointHealth],
- ) -> bool:
- """Whether *selected* had the lowest known ``last_latency_ms``.
-
- ``False`` when no candidate has a usable latency sample (selection was
- uniform, so "fastest" is undefined). Over many requests the fraction of
- ``True`` shows how often inverse-latency weighting picked the front-runner.
- """
- known: dict[str, float] = {}
- for mid in candidates:
- latency = snapshot[mid].last_latency_ms
- if latency is not None and latency > 0:
- known[mid] = latency
- if not known or selected not in known:
- return False
- return known[selected] == min(known.values())
-
- def _poll_age_ms(self) -> float | None:
- """Milliseconds since the last successful Latency Service poll, or ``None``."""
- age_s = self._poller.seconds_since_last_success
- return age_s * 1000.0 if age_s is not None else None
-
- @staticmethod
- def _pick_by_latency(
- candidates: list[str],
- snapshot: dict[str, EndpointHealth],
- ) -> str:
- """Inverse-latency weighted random pick across ``candidates``.
-
- Returns a uniform random pick when only one candidate exists, when
- any candidate's ``last_latency_ms`` is unknown, or when any sample
- is non-positive (treated as bogus — the Latency Service should
- never report 0 or negative, but be defensive).
- """
- if len(candidates) == 1:
- return candidates[0]
- latencies = [snapshot[c].last_latency_ms for c in candidates]
- if any(lat is None or lat <= 0 for lat in latencies):
- return random.choice(candidates)
- weights = [1.0 / lat for lat in latencies] # type: ignore[operator]
- return random.choices(candidates, weights=weights, k=1)[0]
-
- # -- Request processing (hot path — no Latency Service call) ------------
-
- async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
- """Serve *request* on the best healthy endpoint, failing over on error.
-
- Candidates are ordered by health and observed latency (with optional
- session affinity); every upstream attempt is stamped with the
- spend-logs metadata header, and the attempt that succeeds records its
- route selection on *ctx* (see :data:`CTX_ROUTE_SELECTION`). Exhausting
- all candidates re-raises the last upstream error, annotated for the
- client-facing error envelope.
- """
- # This backend records its own per-attempt ``outcome_metrics`` counters
- # below (one per failover attempt). Claim attempt accounting for this
- # request so the endpoint-layer fallback in ``dispatch`` /
- # ``handle_chain_exception`` does not double-count the retry fan-out.
- ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] = True
- api_key_override = self._api_key_override_for_policy(
- ctx.metadata.get(CTX_CALLER_API_KEY)
- )
- # ``caller_required`` never falls back to the configured endpoint key:
- # reject before any upstream call so the service credential can't
- # authenticate caller inference — and before the affinity lookup, so a
- # doomed request never pays a pin resolution (possibly a shared-store
- # round-trip). The rejection deliberately stays invisible to
- # ``switchyard_upstream_attempts_total`` (the accounting flag above is
- # already claimed): no upstream attempt happened, and counting a
- # synthetic 401 would skew the direct-to-endpoint baseline error rate.
- # ``api_key_override`` is ``None`` here only when no usable caller key
- # was supplied.
- if self._config.credential_policy == "caller_required" and api_key_override is None:
- _reject_missing_caller_api_key(ctx)
-
- # Captured before the per-attempt ``body["model"]`` override so the span
- # records the model the client asked for, not the selected endpoint.
- incoming_model = request.model
- # One correlation id per client request, shared by every failover
- # attempt's spend-logs header and by the response headers — the join
- # key between a front proxy's spend-log row and the provider rows.
- correlation_id = str(uuid.uuid4())
- # Resolve the session-affinity pin once (keyed on the stable conversation
- # prefix). ``None`` when affinity is disabled or the conversation isn't
- # pinned yet, so every attempt routes purely by health + latency.
- pinned_endpoint = await self._affinity.pinned(ctx, request)
-
- last_exc: Exception | None = None
- # Upstream model of the last failed attempt, surfaced on the error
- # envelope/span/log so operators can tell *which* endpoint the
- # provider-originated failure came from.
- last_upstream_model: str | None = None
- tried: set[str] = set()
-
- for attempt in range(1 + self._config.max_retries):
- # -- Route decision span: which endpoint, out of which candidates --
- with routing_span("switchyard.route_decision") as route_span:
- decision = self._select_endpoint_decision(pinned_endpoint, frozenset(tried))
- model_id = decision.selected
- candidates = decision.candidates
- was_fastest = decision.was_fastest
- if model_id in tried and len(tried) < len(self._clients):
- remaining = [m for m in self._clients if m not in tried]
- model_id = random.choice(remaining)
- # A forced dedup pick, not the latency-weighted choice.
- candidates = tuple(remaining)
- was_fastest = False
- tried.add(model_id)
- # Count the warm-reuse outcome once per request (the first
- # decision); later attempts are failover, not an affinity signal.
- if self._affinity.enabled and attempt == 0:
- if decision.affinity:
- self._affinity_hits += 1
- else:
- self._affinity_misses += 1
- set_tags(route_span, {
- "switchyard.model": incoming_model,
- "switchyard.candidate_endpoints": ",".join(candidates),
- "switchyard.selected_endpoint": model_id,
- "switchyard.was_fastest_selected": was_fastest,
- "switchyard.affinity_hit": decision.affinity,
- "switchyard.latency_service_poll_age_ms": self._poll_age_ms(),
- })
-
- upstream_model = self._upstream_models[model_id]
- target_request_type = self._request_types[model_id]
- body = self._body_for_endpoint_request_type(ctx, request, target_request_type)
- body["model"] = upstream_model
- # Per-attempt route-selection record: each upstream call is stamped
- # with the endpoint it actually hits, so a failover retry's
- # spend-log row doesn't inherit the failed attempt's selection.
- route_selection = self._route_selection(
- incoming_model=incoming_model,
- model_id=model_id,
- upstream_model=upstream_model,
- correlation_id=correlation_id,
- )
- log.debug(
- "LatencyServiceLLMBackend: attempt=%d model=%s upstream=%s "
- "request_type=%s stream=%s",
- attempt + 1,
- model_id,
- upstream_model,
- target_request_type.value,
- body.get("stream"),
- )
-
- # -- Upstream attempt span: outcome of this one upstream call -----
- with routing_span("switchyard.upstream_attempt") as attempt_span:
- set_tags(attempt_span, {
- "switchyard.model": incoming_model,
- "switchyard.selected_endpoint": model_id,
- "switchyard.retry_count": attempt,
- })
- started_at = time.monotonic()
- try:
- result = await self._call_endpoint(
- model_id,
- target_request_type,
- api_key=api_key_override,
- body=body,
- extra_headers={
- SPEND_LOGS_METADATA_HEADER: json.dumps(route_selection),
- },
- )
- except APIStatusError as exc:
- set_tags(attempt_span, {
- "switchyard.outcome": outcome_metrics.classify(exc.status_code),
- "switchyard.upstream_status_code": exc.status_code,
- "switchyard.error_code": outcome_metrics.code_label(exc.status_code),
- "switchyard.error_source": ERROR_SOURCE_PROVIDER,
- "switchyard.upstream_model": upstream_model,
- })
- if self._stats is not None:
- await self._stats.record_error(model_id)
- outcome_metrics.record_upstream_attempt(exc.status_code)
- self._record_model_attempt(
- incoming_model,
- upstream_model,
- outcome_metrics.classify(exc.status_code),
- outcome_metrics.code_label(exc.status_code),
- )
- # Per-event structured log (Loki) — the timestamped complement
- # to the aggregate outcome counter recorded above.
- log_upstream_attempt_failure(
- model=model_id,
- attempt=attempt + 1,
- status_code=exc.status_code,
- error=exc,
- upstream_model=upstream_model,
- )
- last_exc = exc
- last_upstream_model = upstream_model
- # A client error (4xx other than 408/429) is deterministic —
- # replicas reject the same payload identically — so fail fast
- # instead of burning attempts; the post-loop passthrough
- # surfaces the upstream status to the client.
- if not _is_retryable_status(exc.status_code):
- break
- continue
- except Exception as exc:
- set_tags(attempt_span, {
- "switchyard.outcome": "retryable_error",
- "switchyard.error_code": outcome_metrics.NO_STATUS_CODE,
- "switchyard.error_source": ERROR_SOURCE_PROVIDER,
- "switchyard.upstream_model": upstream_model,
- })
- if self._stats is not None:
- await self._stats.record_error(model_id)
- # Non-HTTP failure (network, pre-status timeout, SDK error) —
- # treated as a retryable_error in the outcome ratio: those
- # are exactly the faults a health-aware router should absorb
- # by selecting a different endpoint on the next attempt.
- outcome_metrics.record_upstream_attempt(None)
- self._record_model_attempt(
- incoming_model,
- upstream_model,
- "retryable_error",
- outcome_metrics.NO_STATUS_CODE,
- )
- # status_code=None → logged as code="none" (no HTTP status).
- log_upstream_attempt_failure(
- model=model_id,
- attempt=attempt + 1,
- status_code=None,
- error=exc,
- upstream_model=upstream_model,
- )
- last_exc = exc
- last_upstream_model = upstream_model
- continue
-
- backend_latency_ms = (time.monotonic() - started_at) * 1000.0
- set_tags(attempt_span, {
- "switchyard.outcome": "success",
- "switchyard.upstream_status_code": 200,
- })
- if self._stats is not None:
- await self._stats.record_success(model_id, backend_latency_ms)
- outcome_metrics.record_upstream_attempt(200)
- self._record_model_attempt(
- incoming_model, upstream_model, "success", "200",
- )
- # A successful attempt after at least one failure is direct
- # evidence the steering logic rescued this request. Counted
- # once per client request, not per recovered retry.
- if attempt > 0:
- outcome_metrics.record_retry_recovered()
-
- # ``ctx.selected_model`` is the cross-language hook the Rust
- # ``StatsResponseProcessor`` reads to attribute token usage and
- # end-to-end latency per endpoint. Without it, the response
- # processor sees no ``BackendSelection`` and buckets every call
- # to ``model="<unknown>"`` on /metrics.
- ctx.selected_model = model_id
- # ``backend_call_latency_ms`` lets the Rust ``StatsResponseProcessor``
- # compute ``routing_overhead_ms = total_latency - backend_latency``
- # for this Python-only backend, which can't be wrapped by the
- # Rust ``StatsLlmBackend`` that normally publishes this signal.
- ctx.backend_call_latency_ms = backend_latency_ms
-
- # The selection that actually served the request, surfaced to
- # the endpoint layer as ``x-switchyard-*`` response headers so
- # a front proxy can enrich its own spend-log row with the same
- # correlation id the provider row received.
- ctx.metadata[CTX_ROUTE_SELECTION] = route_selection
-
- # Pin this conversation to the endpoint that served it so later
- # turns reuse it (warm cache). Re-pinning on every success also
- # follows a recovery: if the previous pin degraded and we
- # re-routed, the endpoint that worked becomes the new pin.
- # (No-op when affinity is disabled.)
- await self._affinity.pin(ctx, request, model_id)
-
- return _chat_response_for_request_type(target_request_type, result)
-
- # All attempts failed. If the last failure was an upstream HTTP
- # error (e.g. 401 from a bad API key), record the status code and
- # body on ctx so the endpoint can passthrough the upstream status
- # rather than masking it as a 500. The exception itself still
- # propagates — the chain is errored — but the endpoint reads ctx
- # to decide the response code.
- if isinstance(last_exc, APIStatusError):
- ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = last_exc.status_code
- upstream_body = _extract_upstream_body(last_exc)
- if upstream_body is not None:
- ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = upstream_body
-
- # Failure-origin annotation for the client-facing error envelope:
- # every exhausted attempt failed at a selected upstream, so what the
- # endpoint surfaces — the HTTP passthrough above, or the 500 for
- # status-less network failures — is provider-originated. (The stash
- # helpers below mark Switchyard-originated rejections instead.)
- ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_PROVIDER
- if last_upstream_model is not None:
- ctx.metadata[CTX_UPSTREAM_MODEL] = last_upstream_model
-
- raise last_exc # type: ignore[misc]
-
- def _body_for_endpoint_request_type(
- self,
- ctx: ProxyContext,
- request: ChatRequest,
- target_request_type: ChatRequestType,
- ) -> dict[str, object]:
- try:
- normalized = self._translation.request_to_any_of(
- request, [target_request_type],
- )
- except ValueError as exc:
- # Transparent-router contract: a payload the upstream provider would
- # reject (e.g. an unsupported message role like "api") must surface
- # as a provider-compatible 400, not a generic 500. The Rust
- # translation layer raises a ``ValueError`` whose message is
- # prefixed with the stable error kind; an invalid-value rejection is
- # recorded as an upstream 400 so the endpoint passes it through.
- _stash_invalid_request_error(ctx, exc)
- raise
- if not request_type_matches(normalized, target_request_type):
- raise TypeError(
- "LatencyServiceLLMBackend expected request type "
- f"{target_request_type.value} after translation"
- )
- return dict(normalized.body)
-
- def _route_selection(
- self,
- *,
- incoming_model: str | None,
- model_id: str,
- upstream_model: str,
- correlation_id: str,
- ) -> dict[str, str | None]:
- """Build the route-selection payload for one upstream attempt.
-
- This is the JSON value of the outbound ``x-litellm-spend-logs-metadata``
- header and, for the attempt that succeeds, the
- :data:`CTX_ROUTE_SELECTION` record behind the ``x-switchyard-*``
- response headers. ``router_model`` falls back to the configured route id
- when the client body carried no model; ``router_selected_provider`` is
- the upstream model's leading path segment (IH/LiteLLM naming, e.g.
- ``"openai/openai/gpt-5.4"`` → ``"openai"``).
- """
- return {
- "router_model": incoming_model or self._config.route_model,
- "router_strategy": ROUTER_STRATEGY_LATENCY,
- "router_selected_endpoint": model_id,
- "router_selected_model": upstream_model,
- "router_selected_provider": upstream_model.split("/", 1)[0],
- "router_correlation_id": correlation_id,
- }
-
- async def _call_endpoint(
- self,
- model_id: str,
- target_request_type: ChatRequestType,
- *,
- api_key: str | None,
- body: dict[str, object],
- extra_headers: dict[str, str],
- ) -> object:
- """Make one upstream SDK call with the protected *extra_headers* stamped on.
-
- ``extra_headers`` rides the client wrapper's ``**kwargs`` into the
- OpenAI SDK's per-request header merge (over ``default_headers``). A
- passthrough body may itself carry an SDK-style ``extra_headers`` field
- (shape-preserving translation keeps unknown keys); it is still
- forwarded — it used to ride ``**body`` into the same SDK parameter —
- but the protected headers must win the merge so callers can't spoof
- them, and header names are case-insensitive on the wire while dict
- keys are not, so every case variant of a protected name is stripped
- from the client mapping (a client-cased duplicate would otherwise
- reach the wire alongside ours and win first-match parsing on the
- receiving proxy). A non-mapping value is dropped: it could only ever
- have broken the SDK call.
- """
- client_extra = body.pop("extra_headers", None)
- protected = {name.lower() for name in extra_headers}
- headers: dict[str, object] = (
- {
- name: value
- for name, value in client_extra.items()
- if not (isinstance(name, str) and name.lower() in protected)
- }
- if isinstance(client_extra, Mapping)
- else {}
- )
- headers.update(extra_headers)
- if target_request_type == ChatRequestType.OPENAI_RESPONSES:
- return await self._clients[model_id].aresponses(
- api_key=api_key,
- extra_headers=headers,
- **body,
- )
- return await self._clients[model_id].acompletion(
- api_key=api_key,
- extra_headers=headers,
- **body,
- )
-
- def _api_key_override_for_policy(self, caller_api_key: object) -> str | None:
- """Per-request key to forward upstream, or ``None`` to use the endpoint key.
-
- ``configured_endpoint`` never forwards a caller key. ``caller_override``
- and ``caller_required`` forward a usable caller key; they differ only in
- the no-key case, which ``call`` handles before any upstream attempt
- (``caller_override`` falls back to the endpoint key, ``caller_required``
- returns 401).
- """
- if self._config.credential_policy == "configured_endpoint":
- return None
- if isinstance(caller_api_key, str) and caller_api_key.strip():
- return caller_api_key
- return None
-
- def _record_model_attempt(
- self,
- requested_model: str | None,
- upstream_model: str,
- outcome: str,
- code: str,
- ) -> None:
- """Count one upstream attempt for the per-model /metrics breakdown.
-
- Event-loop only (no lock). ``requested_model`` is the client-supplied
- model; it is bounded to a config-derived id — the route id
- (``config.route_model``) or a configured endpoint id — with the
- ``"other"`` sentinel as fallback before becoming a Prometheus label, so
- caller-controlled input can't create unbounded metric-series
- cardinality.
- """
- requested = (
- requested_model if requested_model in self._requested_model_ids else "other"
- )
- key = (requested, upstream_model, outcome, code)
- self._upstream_attempts_by_model[key] = (
- self._upstream_attempts_by_model.get(key, 0) + 1
- )
-
- # -- Lifecycle ----------------------------------------------------------
-
- async def shutdown(self) -> None:
- """Stop the background :class:`HealthPoller` daemon and release the
- shared session-affinity store, when one is configured.
-
- Picked up automatically by the server's component teardown hook
- (``_run_lifecycle_method``), which awaits awaitable ``shutdown``
- results. Safe to call multiple times.
- """
- prometheus_emitter.unregister(self._render_prometheus_lines)
- self._poller.stop()
- await self._affinity.aclose()
-
- def is_ready(self) -> bool:
- """True once the background poller has completed at least one successful poll."""
- return self._poller.has_polled
-
- # -- Metrics emitter ----------------------------------------------------
-
- def _render_prometheus_lines(self) -> list[str]:
- """Emit per-endpoint health verdicts and poll-loop health gauges.
-
- Snapshotted under ``self._cache_lock`` so a poll concurrent with a
- scrape can't produce a mixed view across endpoints. Output is
- Prometheus exposition lines without trailing newline — composed
- by :mod:`switchyard.lib.endpoints.prometheus_emitter`.
- """
- with self._cache_lock:
- snapshot = dict(self._health_cache)
-
- lines: list[str] = []
- lines.append(
- "# HELP switchyard_endpoint_status "
- "Latency-Service verdict per endpoint (1 = current status; "
- "exactly one status row per model is non-zero)."
- )
- lines.append("# TYPE switchyard_endpoint_status gauge")
- for model_id, health in sorted(snapshot.items()):
- current = health.status.value
- for status in EndpointHealthStatus:
- value = 1 if status.value == current else 0
- labels = render_labels({"model": model_id, "status": status.value})
- lines.append(f"switchyard_endpoint_status{labels} {value}")
-
- lines.append(
- "# HELP switchyard_endpoint_last_latency_ms "
- "Last latency sample (ms) reported by the Latency Service per endpoint. "
- "Absent until the first poll publishes a non-null sample."
- )
- lines.append("# TYPE switchyard_endpoint_last_latency_ms gauge")
- for model_id, health in sorted(snapshot.items()):
- if health.last_latency_ms is None:
- continue
- labels = render_labels({"model": model_id})
- lines.append(
- f"switchyard_endpoint_last_latency_ms{labels} "
- f"{format_number(health.last_latency_ms)}"
- )
-
- lines.append(
- "# HELP switchyard_latency_service_poll_ok "
- "1 when the last poll succeeded, 0 when it failed or has not yet run."
- )
- lines.append("# TYPE switchyard_latency_service_poll_ok gauge")
- last_age = self._poller.seconds_since_last_success
- # ``poll_ok`` reflects the latest poll attempt. Combined with
- # ``poll_age_seconds``, scrapers can tell "never polled" (no age line)
- # from "polled, but the latest attempt failed" (age present, ok=0).
- poll_ok = 1 if self._poller.last_poll_ok else 0
- lines.append(f"switchyard_latency_service_poll_ok {poll_ok}")
-
- lines.append(
- "# HELP switchyard_latency_service_poll_age_seconds "
- "Monotonic seconds since the last successful poll. Absent before the "
- "first success."
- )
- lines.append("# TYPE switchyard_latency_service_poll_age_seconds gauge")
- if last_age is not None:
- lines.append(
- "switchyard_latency_service_poll_age_seconds "
- f"{format_number(last_age)}"
- )
-
- lines.append(
- "# HELP switchyard_latency_service_polls_total "
- "Total successful health polls since the poller started."
- )
- lines.append("# TYPE switchyard_latency_service_polls_total counter")
- lines.append(
- f"switchyard_latency_service_polls_total {self._poller.poll_successes}"
- )
-
- lines.append(
- "# HELP switchyard_latency_service_poll_failures_total "
- "Total failed health polls; each failure resets every endpoint to "
- "UNKNOWN."
- )
- lines.append("# TYPE switchyard_latency_service_poll_failures_total counter")
- lines.append(
- "switchyard_latency_service_poll_failures_total "
- f"{self._poller.poll_failures}"
- )
-
- # Warm-reuse counters — only meaningful (and only emitted) when session
- # affinity is enabled, so the metric surface stays clean for the common
- # per-turn-routing case. Reuse rate = hits / (hits + misses).
- if self._config.session_affinity:
- lines.append(
- "# HELP switchyard_affinity_hits_total "
- "Conversation turns served by an existing session-affinity pin "
- "(warm endpoint reuse)."
- )
- lines.append("# TYPE switchyard_affinity_hits_total counter")
- lines.append(f"switchyard_affinity_hits_total {self._affinity_hits}")
-
- lines.append(
- "# HELP switchyard_affinity_misses_total "
- "First or unpinnable turns routed by the latency-aware picker "
- "while session affinity was enabled."
- )
- lines.append("# TYPE switchyard_affinity_misses_total counter")
- lines.append(f"switchyard_affinity_misses_total {self._affinity_misses}")
-
- # Shared-store (L2) counters — emitted only when a shared pin store
- # is configured. Hits measure cross-worker/churn pin reuse; errors
- # count fail-open store operations (the alerting signal for a store
- # that is degraded while requests keep succeeding).
- if self._affinity.l2_enabled:
- lines.append(
- "# HELP switchyard_affinity_l2_hits_total "
- "Pins resolved from the shared (L2) affinity store after an "
- "in-process (L1) miss — cross-worker warm reuse."
- )
- lines.append("# TYPE switchyard_affinity_l2_hits_total counter")
- lines.append(
- f"switchyard_affinity_l2_hits_total {self._affinity.l2_hits}"
- )
-
- lines.append(
- "# HELP switchyard_affinity_l2_errors_total "
- "Shared (L2) affinity-store operations that failed open "
- "(get or put); routing fell back to in-process pins only."
- )
- lines.append("# TYPE switchyard_affinity_l2_errors_total counter")
- lines.append(
- f"switchyard_affinity_l2_errors_total {self._affinity.l2_errors}"
- )
-
- lines.append(
- "# HELP switchyard_affinity_l2_breaker_open "
- "1 while the shared-store circuit breaker is open "
- "(operations skipped without a network attempt); 0 when closed."
- )
- lines.append("# TYPE switchyard_affinity_l2_breaker_open gauge")
- lines.append(
- "switchyard_affinity_l2_breaker_open "
- f"{int(self._affinity.l2_breaker_open)}"
- )
-
- # Per-model upstream-attempt breakdown — the latency-service-scoped
- # complement to the layer-aggregate ``switchyard_upstream_attempts_total``.
- # Series are created lazily per (requested_model, upstream_model, outcome,
- # code), so the surface stays empty until traffic flows. Snapshotted to a
- # plain dict for a stable view across the emission loop.
- attempts_by_model = dict(self._upstream_attempts_by_model)
- if attempts_by_model:
- lines.append(
- "# HELP switchyard_latency_upstream_attempts_total "
- "Upstream call attempts from the latency-service backend, labelled "
- "by requested (route) model, selected upstream model, outcome, and "
- "HTTP status code (code=\"none\" for non-HTTP failures). Per-model "
- "complement to the layer-aggregate switchyard_upstream_attempts_total; "
- "cardinality is bounded by the configured endpoint set."
- )
- lines.append("# TYPE switchyard_latency_upstream_attempts_total counter")
- # Sorted for deterministic exposition order across scrapes.
- for (requested_model, upstream_model, outcome, code), count in sorted(
- attempts_by_model.items()
- ):
- labels = render_labels({
- "requested_model": requested_model,
- "upstream_model": upstream_model,
- "outcome": outcome,
- "code": code,
- })
- lines.append(
- f"switchyard_latency_upstream_attempts_total{labels} {count}"
- )
- return lines
-
-
-def _latency_endpoint_request_type(value: str) -> ChatRequestType:
- if value == "openai_responses":
- return ChatRequestType.OPENAI_RESPONSES
- return ChatRequestType.OPENAI_CHAT
-
-
-def _chat_response_for_request_type(
- request_type: ChatRequestType,
- result: object,
-) -> ChatResponse:
- if request_type == ChatRequestType.OPENAI_RESPONSES:
- # Streaming yields raw SSE frame strings (``RawSSEFrameStream``) so the
- # upstream events pass through verbatim; anything async-iterable is a
- # stream, a plain mapping is the exact non-streaming JSON body.
- if isinstance(result, AsyncStream) or hasattr(result, "__aiter__"):
- return ChatResponse.openai_responses_stream(ResponsesApiStream(result))
- return ChatResponse.openai_responses_completion(result)
- if isinstance(result, AsyncStream):
- return ChatResponse.openai_stream(ResponseStream(result))
- return ChatResponse.openai_completion(result)
-
-
-def _is_retryable_status(status_code: int) -> bool:
- """Whether an upstream status warrants failing over to a different endpoint.
-
- Retries 5xx + 408 + 429 (transient/capacity/server-side); other 4xx are the
- client's payload, which every replica rejects identically, so fail fast.
- Deliberately broader than ``outcome_metrics.RETRYABLE_STATUSES`` (metrics
- bucketing) — e.g. 502/503 must fail over but aren't metrics-retryable.
- """
- return status_code >= 500 or status_code in (408, 429)
-
-
-def _affinity_usable(health: EndpointHealth | None) -> bool:
- """Whether a pinned endpoint may still serve its session.
-
- The pin holds while the endpoint is HEALTHY or UNKNOWN — the same tiers the
- latency-aware picker trusts. A DEGRADED verdict (or an endpoint that
- vanished from the health cache) breaks the pin so the next turn re-routes:
- locality must yield to health, or sticky routing would funnel a session
- into a failing endpoint.
- """
- return health is not None and health.status in (
- EndpointHealthStatus.HEALTHY,
- EndpointHealthStatus.UNKNOWN,
- )
-
-
-def _build_affinity_l2(config: LatencyServiceBackendConfig) -> AffinityPinStore | None:
- """Build the optional shared L2 pin store from config (``None`` = L1-only).
-
- ``affinity_store="memory"`` (the default) keeps pins per process. ``"redis"``
- imports :class:`RedisPinStore` lazily so the ``redis`` dependency stays
- optional. Config validation guarantees a URL when the store is Redis.
- """
- if config.affinity_store != "redis":
- return None
- from switchyard.lib.redis_pin_store import RedisPinStore
-
- assert config.affinity_store_url is not None # enforced by config validator
- return RedisPinStore(
- config.affinity_store_url,
- ttl_seconds=config.affinity_store_ttl_seconds,
- key_prefix=config.affinity_key_prefix,
- )
-
-
-# Prefix of the stringified translation error for a client-invalid payload.
-# ``TranslationError::kind()`` is the stable FFI signal for the error variant
-# (``crates/switchyard-translation/src/error.rs``); ``py_translation_error``
-# formats translation errors as ``"{kind}: {message}"``, so an invalid-value
-# rejection (e.g. an unsupported message role) is recognizable by this prefix.
-_TRANSLATION_INVALID_VALUE_PREFIX = "InvalidValue:"
-
-
-def _stash_invalid_request_error(ctx: ProxyContext, error: Exception) -> None:
- """Record a translation invalid-value error as an upstream HTTP 400 on ``ctx``.
-
- Mirrors the upstream-status passthrough used for provider HTTP errors: the
- Compatibility execution flattens the raised exception to a generic error
- string, but ``ctx.metadata`` survives the error path, so the endpoint's
- ``upstream_response_from_ctx`` can return a provider-compatible 400. This is
- a no-op for any other error (it keeps propagating and FastAPI maps it to a
- 500 as before).
- """
- message = str(error)
- if not message.startswith(_TRANSLATION_INVALID_VALUE_PREFIX):
- return
- detail = message[len(_TRANSLATION_INVALID_VALUE_PREFIX) :].strip()
- ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 400
- ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = {
- "error": {
- "message": detail or "invalid request payload",
- "type": "invalid_request_error",
- "code": "invalid_value",
- }
- }
- # This 400 rides the upstream-status channel but is Switchyard's own
- # translation rejection — label it so the error-source header is truthful.
- ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_SWITCHYARD
-
-
-def _reject_missing_caller_api_key(ctx: ProxyContext) -> None:
- """Reject a ``caller_required`` request that carries no caller API key.
-
- Stashes an HTTP 401 and a provider-compatible error envelope on ``ctx`` the
- same way upstream-status passthrough does, then raises so the chain errors
- out before any upstream call. No upstream is contacted or counted;
- ``upstream_response_from_ctx`` reads the stashed status to return a 401. This
- is what keeps the configured endpoint key from ever authenticating caller
- inference under the ``caller_required`` policy.
- """
- ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] = 401
- ctx.metadata[CTX_UPSTREAM_HTTP_BODY] = {
- "error": {
- "message": "missing caller API key; supply it via the x-switchyard-api-key header",
- "type": "invalid_request_error",
- "code": "missing_caller_api_key",
- }
- }
- # Rides the upstream-status channel but no upstream was contacted — this
- # is Switchyard's own credential-policy rejection.
- ctx.metadata[CTX_ERROR_SOURCE] = ERROR_SOURCE_SWITCHYARD
- raise PermissionError("caller_required policy: missing caller API key")
-
-
-def _extract_upstream_body(error: APIStatusError) -> object | None:
- """Best-effort extraction of the upstream response body for passthrough.
-
- Prefers the structured ``body`` attribute (set by the OpenAI SDK for
- JSON responses). Falls back to ``response.text`` for responses the
- SDK couldn't decode. Returns ``None`` when nothing usable is present
- — the endpoint then synthesizes a generic error envelope.
- """
- body: object | None = getattr(error, "body", None)
- if body is not None:
- return body
- response = getattr(error, "response", None)
- if response is None:
- return None
- text: object | None = getattr(response, "text", None)
- return text if text else None
diff --git a/switchyard/lib/config/__init__.py b/switchyard/lib/config/__init__.py
index f21a9bf7d..1c74cba12 100644
--- a/switchyard/lib/config/__init__.py
+++ b/switchyard/lib/config/__init__.py
@@ -7,14 +7,8 @@ from switchyard.lib.config.intake_sink_config import (
IntakeQueueFullPolicy,
IntakeSinkConfig,
)
-from switchyard.lib.config.latency_service_backend_config import (
- LatencyServiceBackendConfig,
- LatencyServiceEndpoint,
-)
__all__ = [
"IntakeQueueFullPolicy",
"IntakeSinkConfig",
- "LatencyServiceBackendConfig",
- "LatencyServiceEndpoint",
]
diff --git a/switchyard/lib/config/latency_service_backend_config.py b/switchyard/lib/config/latency_service_backend_config.py
deleted file mode 100644
index f52da2c39..000000000
--- a/switchyard/lib/config/latency_service_backend_config.py
+++ /dev/null
@@ -1,168 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Configuration models for the Latency Service usage case.
-
-``LatencyServiceEndpoint`` describes one LLM backend monitored by the
-Latency Service. ``LatencyServiceBackendConfig`` bundles the full
-backend configuration — URL of the Latency Service, the endpoint list,
-and polling/retry parameters.
-
-The ``model`` field on each endpoint doubles as the endpoint ID used by
-the Latency Service's health API — mirroring the routing-by-model-name
-convention the rest of the library already follows.
-"""
-
-from typing import Literal, Self
-
-from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
-
-LatencyServiceRequestType = Literal["openai_chat", "openai_responses"]
-LatencyServiceCredentialPolicy = Literal[
- "configured_endpoint", "caller_override", "caller_required"
-]
-
-
-class LatencyServiceEndpoint(BaseModel):
- """One LLM backend registered with the Latency Service.
-
- The ``model`` field is the endpoint ID used by the Latency Service —
- it must be unique across the endpoint list and it is the value the
- Latency Service returns health verdicts under. By default it is also
- the value written into ``body["model"]`` when calling the upstream;
- set ``upstream_model`` when the upstream expects a different name
- (e.g. routing the latency-service key ``"openai/gpt-5.5"`` through an
- IH gateway that expects ``"openai/openai/gpt-5.5"``).
-
- Attributes:
- model: Latency-service lookup key. Must be unique across the
- endpoint list. Also used as ``body["model"]`` unless
- ``upstream_model`` is set.
- upstream_model: Optional override for ``body["model"]`` sent to
- the upstream LLM. Defaults to ``model`` when ``None``.
- api_key: API key for the backing LLM API.
- base_url: Base URL for the backing LLM API (include ``/v1``).
- timeout: Request timeout in seconds, forwarded to the underlying
- ``OpenAILLMClient``. ``None`` uses the client default.
- request_type: Upstream OpenAI API surface used for this endpoint.
- ``"openai_chat"`` sends ``/v1/chat/completions``; ``"openai_responses"``
- sends ``/v1/responses`` for Responses-only upstream models.
- """
-
- model_config = ConfigDict(frozen=True)
-
- model: str
- upstream_model: str | None = None
- api_key: str | None = None
- base_url: str | None = None
- timeout: float | None = None
- request_type: LatencyServiceRequestType = "openai_chat"
-
- @field_validator("request_type", mode="before")
- @classmethod
- def _normalize_request_type(cls, value: object) -> object:
- if value == "chat":
- return "openai_chat"
- if value == "responses":
- return "openai_responses"
- return value
-
-
-class LatencyServiceBackendConfig(BaseModel):
- """Configuration for :class:`LatencyServiceLLMBackend`.
-
- Attributes:
- latency_service_url: Base URL of the Latency Service
- (e.g. ``"http://latency-service.inference-hub.svc:8080"``).
- endpoints: LLM backends to route across. Each must have a
- unique ``model`` — this is the routing + health-lookup key.
- poll_interval_s: How often the background poller refreshes
- health from the Latency Service. Health is cached between
- polls; the request hot path never blocks on a network call.
- poll_timeout_s: Timeout for the health API call.
- max_retries: On error, retry on a different endpoint up to this
- many times. Dedup prevents re-selecting an endpoint that
- already failed for the same request.
- route_model: Client-facing route id this backend serves (the
- route-table / YAML route key, e.g.
- ``"nvidia/switchyard/gpt-5.4"``). Metrics-only: it joins the
- bounded ``requested_model`` label set on
- ``switchyard_latency_upstream_attempts_total`` so route-key
- traffic is attributed instead of collapsing to ``"other"``.
- Has no effect on routing.
- credential_policy: Which credential authenticates the upstream call.
- The caller key is read from the ``x-switchyard-api-key`` header
- (preferred — it survives proxies such as LiteLLM that strip
- ``Authorization``), then ``Authorization: Bearer`` / ``x-api-key``, on
- every ingress path (``/chat/completions``, ``/responses``,
- ``/messages``). ``"configured_endpoint"`` always uses each endpoint's
- configured ``api_key`` and ignores any caller key. ``"caller_override"``
- opts into BYO-key forwarding: a caller key is used when present, else
- the call falls back to the configured ``api_key``. ``"caller_required"``
- forwards the caller key but never falls back — a request with no caller
- key is rejected with HTTP 401 and the configured ``api_key`` is never
- used for upstream inference (use this for per-user spend attribution,
- e.g. multi-tenant gateway routes).
- session_affinity: When ``True``, pin each conversation to the endpoint
- that first served it (cache stays warm); a pin is broken only when
- its endpoint degrades or the call fails. Per process. Default off.
- affinity_max_sessions: Bounded-LRU cap on pinned conversations; ignored
- when ``session_affinity`` is off.
- affinity_store: Shared L2 pin store behind the in-process LRU. ``"memory"``
- (default) keeps pins per process; ``"redis"`` shares them across
- workers/pods and persists them across pod churn. The store is
- best-effort — an L2 error never fails a request.
- affinity_store_url: Connection URL for the shared store (e.g.
- ``"redis://host:6379/0"``). Required when ``affinity_store`` is
- ``"redis"``.
- affinity_store_ttl_seconds: Expiry for a shared pin. The backend re-pins
- on every successful turn, so an active conversation slides its TTL.
- affinity_key_prefix: Namespace prefix for shared-store keys.
- enable_stats: When ``True`` (default), the factory wires a
- :class:`StatsRequestProcessor` + :class:`StatsResponseProcessor`
- pair sharing one :class:`StatsAccumulator` and wraps the
- backend in :class:`StatsLlmBackend`, so the chain contributes
- ``GET /metrics``, ``GET /v1/stats``, and the legacy
- ``GET /v1/routing/stats`` aliases via the standard
- ``get_endpoint()`` mechanism in :func:`build_switchyard_app`.
- """
-
- model_config = ConfigDict(frozen=True, extra="forbid")
-
- latency_service_url: str = ""
- endpoints: list[LatencyServiceEndpoint] = Field(default_factory=list)
- route_model: str | None = None
- poll_interval_s: float = 10.0
- poll_timeout_s: float = 5.0
- max_retries: int = 2
- credential_policy: LatencyServiceCredentialPolicy = "configured_endpoint"
- session_affinity: bool = False
- affinity_max_sessions: int = Field(default=10_000, ge=0)
- affinity_store: Literal["memory", "redis"] = "memory"
- affinity_store_url: str | None = None
- affinity_store_ttl_seconds: int = Field(default=3_600, gt=0)
- affinity_key_prefix: str = "swyd:pin:"
- enable_stats: bool = True
-
- @model_validator(mode="after")
- def _affinity_capacity_nonzero_when_enabled(self) -> Self:
- # A zero-capacity affinity store retains nothing — silently non-sticky.
- if self.session_affinity and self.affinity_max_sessions == 0:
- raise ValueError(
- "affinity_max_sessions must be > 0 when session_affinity is enabled"
- )
- return self
-
- @model_validator(mode="after")
- def _redis_store_requires_url_and_affinity(self) -> Self:
- # A shared store is dead config unless affinity is on and reachable.
- if self.affinity_store == "redis":
- if not self.session_affinity:
- raise ValueError(
- 'affinity_store="redis" requires session_affinity to be enabled'
- )
- if not self.affinity_store_url:
- raise ValueError(
- 'affinity_store="redis" requires affinity_store_url to be set'
- )
- return self
diff --git a/switchyard/lib/endpoints/prometheus_emitter.py b/switchyard/lib/endpoints/prometheus_emitter.py
index e49299285..aafe0274c 100644
--- a/switchyard/lib/endpoints/prometheus_emitter.py
+++ b/switchyard/lib/endpoints/prometheus_emitter.py
@@ -6,9 +6,8 @@
The accumulator-derived exposition rendered by
:func:`switchyard.lib.prometheus_exposition.render_prometheus` covers
request flow (counts, tokens, latency). Components that own state which
-is not request-derived — Latency Service verdicts, poll-loop health,
-endpoint-level live samples — register an emitter here so their lines
-appear on the same ``/metrics`` scrape rather than a sidecar URL.
+is not request-derived register an emitter here so their lines appear on
+the same ``/metrics`` scrape rather than a sidecar URL.
Single-process table by design: a Switchyard server is one process,
emitters are owned by component lifetimes, and the table is
diff --git a/switchyard/lib/endpoints/stats_endpoint.py b/switchyard/lib/endpoints/stats_endpoint.py
index 31d96f94b..3ffdc2674 100644
--- a/switchyard/lib/endpoints/stats_endpoint.py
+++ b/switchyard/lib/endpoints/stats_endpoint.py
@@ -68,10 +68,9 @@ class StatsEndpoint(NemoSwitchyardEndpoint):
async def get_metrics() -> Response:
"""Prometheus text-format exposition of the shared stats snapshot.
- Components that own non-request-derived state (Latency Service
- verdicts, poll-loop health) contribute extra lines via
- :mod:`switchyard.lib.endpoints.prometheus_emitter` so a single
- ``/metrics`` scrape carries both surfaces.
+ Components that own non-request-derived state contribute extra
+ lines via :mod:`switchyard.lib.endpoints.prometheus_emitter` so a
+ single ``/metrics`` scrape carries both surfaces.
"""
snapshot = await stats.snapshot()
outcome_block = "\n".join(outcome_metrics.render_lines()) + "\n"
diff --git a/switchyard/lib/endpoints/upstream_error.py b/switchyard/lib/endpoints/upstream_error.py
index 285eda79b..66de361d2 100644
--- a/switchyard/lib/endpoints/upstream_error.py
+++ b/switchyard/lib/endpoints/upstream_error.py
@@ -3,9 +3,9 @@
"""Endpoint-side helper: convert chain exceptions into upstream-status responses.
-Python LLM backends (e.g. :class:`LatencyServiceLLMBackend`) stash the
-upstream HTTP status / body into ``ctx.metadata`` before raising the
-upstream provider's exception. Rust backends attach typed
+Python LLM backends stash the upstream HTTP status / body into
+``ctx.metadata`` before raising the upstream provider's exception. Rust
+backends attach typed
``status_code`` and ``body`` attributes to ``SwitchyardUpstreamError`` so
endpoints can preserve provider failures without parsing exception text.
"""
@@ -53,9 +53,8 @@ def record_upstream_attempt_success(ctx: ProxyContext) -> None:
to one successful attempt observable here.
No-op when a backend already counted its own attempts
- (:data:`CTX_UPSTREAM_ATTEMPTS_RECORDED` set) — e.g.
- :class:`LatencyServiceLLMBackend`, whose retry fan-out must not be
- double-counted here.
+ (:data:`CTX_UPSTREAM_ATTEMPTS_RECORDED` set) — its retry fan-out must not
+ be double-counted here.
"""
if ctx.metadata.get(CTX_UPSTREAM_ATTEMPTS_RECORDED):
return
diff --git a/switchyard/lib/profiles/__init__.py b/switchyard/lib/profiles/__init__.py
index b7d3de6dc..1b06d983f 100644
--- a/switchyard/lib/profiles/__init__.py
+++ b/switchyard/lib/profiles/__init__.py
@@ -21,7 +21,6 @@ from switchyard.lib.profiles.header_routing import (
HeaderRoutingDecision,
HeaderRoutingProfile,
)
-from switchyard.lib.profiles.latency_service import LatencyServiceProfileConfig
from switchyard.lib.profiles.noop import NoopProfile, NoopProfileConfig
from switchyard.lib.profiles.passthrough import PassthroughProfileConfig
from switchyard.lib.profiles.protocols import (
@@ -61,7 +60,6 @@ __all__ = [
"HeaderRoutingConfig",
"HeaderRoutingDecision",
"HeaderRoutingProfile",
- "LatencyServiceProfileConfig",
"NoopProfile",
"NoopProfileConfig",
"PassthroughProfileConfig",
diff --git a/switchyard/lib/profiles/escalation_router_profile_config.py b/switchyard/lib/profiles/escalation_router_profile_config.py
index 96c59db29..f6f5b4ce8 100644
--- a/switchyard/lib/profiles/escalation_router_profile_config.py
+++ b/switchyard/lib/profiles/escalation_router_profile_config.py
@@ -138,9 +138,9 @@ class EscalationRouterProfileConfig:
def _build_affinity_l2(config: EscalationRouterConfig) -> AffinityPinStore | None:
"""Build the optional shared L2 latch store (``None`` = L1-only).
- Mirrors the latency route's ``affinity_store`` semantics; ``"redis"``
- imports :class:`RedisPinStore` lazily so the ``redis`` dependency stays
- optional. Config validation guarantees a URL when the store is Redis.
+ Uses the shared ``affinity_store`` semantics; ``"redis"`` imports
+ :class:`RedisPinStore` lazily so the ``redis`` dependency stays optional.
+ Config validation guarantees a URL when the store is Redis.
"""
if config.affinity_store != "redis":
return None
diff --git a/switchyard/lib/profiles/latency_service.py b/switchyard/lib/profiles/latency_service.py
deleted file mode 100644
index abdb144df..000000000
--- a/switchyard/lib/profiles/latency_service.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Profile-owned latency-service routing construction."""
-
-from __future__ import annotations
-
-from typing import Self
-
-from switchyard.lib.config import LatencyServiceBackendConfig
-from switchyard.lib.profiles.chain import ComponentChainProfile
-from switchyard.lib.profiles.table import profile_config
-
-
-@profile_config("latency_service")
-class LatencyServiceProfileConfig:
- """Profile config wrapper for health-aware latency-service profiles."""
-
- config: LatencyServiceBackendConfig
-
- @classmethod
- def from_config(cls, config: LatencyServiceBackendConfig) -> Self:
- """Create a profile config from the validated parsing model."""
- return cls(config=config)
-
- def build(self) -> ComponentChainProfile:
- """Build the latency-service profile runtime."""
- from switchyard.lib.backends import LatencyServiceLLMBackend
-
- backend = LatencyServiceLLMBackend(self.config)
- return ComponentChainProfile(
- backend=backend,
- )
-
-
-__all__ = ["LatencyServiceProfileConfig"]
diff --git a/switchyard/lib/proxy_context.py b/switchyard/lib/proxy_context.py
index 864ddfb61..01c9b78a4 100644
--- a/switchyard/lib/proxy_context.py
+++ b/switchyard/lib/proxy_context.py
@@ -58,8 +58,8 @@ CTX_UPSTREAM_HTTP_BODY = "_upstream_http_body"
#: Set truthy by a backend that records its own per-attempt
#: ``switchyard.lib.endpoints.outcome_metrics`` upstream-attempt counters
-#: (e.g. :class:`LatencyServiceLLMBackend`, which retries across endpoints and
-#: must count each attempt). When present, the endpoint layer skips its
+#: (e.g. a backend that retries across endpoints and must count each
+#: attempt). When present, the endpoint layer skips its
#: single-attempt fallback recording for this request so retry fan-out is not
#: double-counted. Absent for the Rust native / passthrough / multi backends,
#: which issue exactly one upstream attempt per call and have no Python retry
diff --git a/switchyard/lib/redis_pin_store.py b/switchyard/lib/redis_pin_store.py
index 0cc193540..0b1b24910 100644
--- a/switchyard/lib/redis_pin_store.py
+++ b/switchyard/lib/redis_pin_store.py
@@ -6,8 +6,8 @@
Implements :class:`~switchyard.lib.affinity_pin_store.AffinityPinStore` against a
standalone Redis so pins are visible to every Switchyard worker/pod and survive
pod churn. Keys are namespaced ``{key_prefix}{session_key}`` and expire after
-``ttl_seconds``; the latency backend re-pins on every successful turn, so an
-active conversation slides its own TTL.
+``ttl_seconds``; callers re-pin on every successful turn, so an active
+conversation slides its own TTL.
The ``redis`` dependency is optional (``switchyard[affinity-redis]``) and is
imported lazily, so the default install never pulls it in. Socket timeouts are
diff --git a/switchyard/lib/session_affinity.py b/switchyard/lib/session_affinity.py
index a4ef6e571..1d5f48d74 100644
--- a/switchyard/lib/session_affinity.py
+++ b/switchyard/lib/session_affinity.py
@@ -71,8 +71,8 @@ class SessionAffinity:
stickiness: derive a stable conversation key (system prompt + first user
message, memoized on the request context) and store/look up a pinned value
in a bounded LRU. The pinned value is opaque — a tier label for the
- classifier router, an endpoint id for the latency backend. Each caller keeps
- its own *policy* (when to write a pin, whether to honor one) on top.
+ classifier or escalation router. Each caller keeps its own *policy*
+ (when to write a pin, whether to honor one) on top.
The in-process ``SessionCache`` is the L1. An optional ``l2`` implementing
:class:`~switchyard.lib.affinity_pin_store.AffinityPinStore` is a shared,
diff --git a/switchyard/lib/tracing.py b/switchyard/lib/tracing.py
index ff2954873..5f80a7f15 100644
--- a/switchyard/lib/tracing.py
+++ b/switchyard/lib/tracing.py
@@ -64,8 +64,8 @@ def routing_span(name: str) -> Iterator[Span]:
def set_tags(span: Span, tags: Mapping[str, Any]) -> None:
"""Set each non-``None`` tag on *span*.
- ``None`` values are skipped so an unavailable signal (e.g. no latency-service
- poll has completed yet) leaves no empty/misleading tag on the span.
+ ``None`` values are skipped so an unavailable signal (e.g. a signal that
+ has not been recorded yet) leaves no empty/misleading tag on the span.
"""
for key, value in tags.items():
if value is not None:
diff --git a/tests/e2e/test_latency_service_llm_backend.py b/tests/e2e/test_latency_service_llm_backend.py
deleted file mode 100644
index fbd93dc50..000000000
--- a/tests/e2e/test_latency_service_llm_backend.py
+++ /dev/null
@@ -1,427 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Integration tests for :class:`LatencyServiceLLMBackend`.
-
-These tests make real LLM calls through the latency-service backend
-to validate end-to-end routing against an external health source.
-
-A lightweight mock Latency Service runs in-process to supply health
-verdicts. The LLM calls go to a real backend (OpenRouter by default,
-with NVIDIA Inference Hub as a fallback).
-
-Prerequisites:
- - OPENROUTER_API_KEY or NVIDIA_API_KEY environment variable
-
-Run with:
- OPENROUTER_API_KEY=sk-or-... pytest tests/e2e/test_latency_service_llm_backend.py -v
-"""
-
-import json
-import os
-import threading
-import time
-from http.server import BaseHTTPRequestHandler, HTTPServer
-from urllib.parse import parse_qs, urlparse
-
-import pytest
-
-from switchyard.lib.backends.health_poller import (
- EndpointHealthStatus,
-)
-from switchyard.lib.backends.latency_service_llm_backend import (
- LatencyServiceLLMBackend,
-)
-from switchyard.lib.config.latency_service_backend_config import (
- LatencyServiceBackendConfig,
- LatencyServiceEndpoint,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.core import (
- ChatRequest,
- ChatResponseType,
- response_type_matches,
-)
-
-from .conftest import find_free_port, get_nvidia_config
-
-pytestmark = pytest.mark.integration
-
-_nvidia = get_nvidia_config()
-_skip_reason = "OPENROUTER_API_KEY or NVIDIA_API_KEY not set"
-
-BACKEND_BASE_URL = _nvidia["base_url"]
-
-if _nvidia["provider"] == "openrouter":
- MODEL_A = _nvidia["model"]
- MODEL_B = os.environ.get("OPENROUTER_MODEL_B") or "anthropic/claude-opus-4.7"
-else:
- MODEL_A = "openai/openai/gpt-5.2"
- MODEL_B = "azure/openai/gpt-5.2"
-
-SIMPLE_MESSAGES = [
- {"role": "system", "content": "You are a helpful assistant."},
- {"role": "user", "content": "Hello! Can you help me?"},
-]
-
-
-# ---------------------------------------------------------------------------
-# Mock Latency Service
-# ---------------------------------------------------------------------------
-
-
-class _HealthState:
- """Thread-safe mutable health state for the mock Latency Service."""
-
- def __init__(self) -> None:
- self._lock = threading.Lock()
- self._health: dict[str, str] = {}
- self.request_count = 0
-
- def set_health(self, model_id: str, status: str) -> None:
- with self._lock:
- self._health[model_id] = status
-
- def get_response(self, requested_ids: list[str]) -> dict:
- with self._lock:
- self.request_count += 1
- return {
- "endpoint_health": {
- mid: {
- "status": self._health.get(mid, "unknown"),
- "last_latency_ms": None,
- }
- for mid in requested_ids
- }
- }
-
-
-def _make_handler(state: _HealthState):
- """Create an HTTP request handler class bound to the given health state."""
-
- class _Handler(BaseHTTPRequestHandler):
- def do_GET(self) -> None:
- parsed = urlparse(self.path)
- if parsed.path == "/v1/endpoints/health":
- qs = parse_qs(parsed.query)
- endpoint_ids = qs.get("endpoint_ids", [])
- body = json.dumps(state.get_response(endpoint_ids))
- self.send_response(200)
- self.send_header("Content-Type", "application/json")
- self.end_headers()
- self.wfile.write(body.encode())
- else:
- self.send_response(404)
- self.end_headers()
-
- def log_message(self, format, *args) -> None: # noqa: A002
- pass # Suppress request logging in test output
-
- return _Handler
-
-
-class MockLatencyService:
- """Manages a mock Latency Service HTTP server for testing."""
-
- def __init__(self) -> None:
- self.state = _HealthState()
- self.port = find_free_port()
- handler = _make_handler(self.state)
- self._server = HTTPServer(("127.0.0.1", self.port), handler)
- self._thread = threading.Thread(
- target=self._server.serve_forever,
- daemon=True,
- name="mock-latency-service",
- )
- self._thread.start()
-
- @property
- def url(self) -> str:
- return f"http://127.0.0.1:{self.port}"
-
- def set_health(self, model_id: str, status: str) -> None:
- self.state.set_health(model_id, status)
-
- def shutdown(self) -> None:
- self._server.shutdown()
- self._thread.join(timeout=5.0)
-
-
-def mock_latency_service():
- """Provide a mock Latency Service for the duration of a test."""
- svc = MockLatencyService()
- yield svc
- svc.shutdown()
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _endpoint(model: str) -> LatencyServiceEndpoint:
- return LatencyServiceEndpoint(
- model=model,
- api_key=_nvidia["api_key"],
- base_url=BACKEND_BASE_URL,
- )
-
-
-def _make_backend(
- latency_service_url: str,
- models: list[str],
- poll_interval_s: float = 1.0,
- **kwargs,
-) -> LatencyServiceLLMBackend:
- config = LatencyServiceBackendConfig(
- latency_service_url=latency_service_url,
- endpoints=[_endpoint(m) for m in models],
- poll_interval_s=poll_interval_s,
- **kwargs,
- )
- return LatencyServiceLLMBackend(config)
-
-
-def _chat_request(stream: bool = False) -> ChatRequest:
- # ``max_tokens`` is deliberately generous so reasoning-capable
- # backends (e.g. the GPT-5 / Qwen 3.5 families) have room to spend
- # the first several hundred tokens on internal reasoning before
- # emitting user-visible ``content``. Matches the value used by
- # the sibling passthrough e2e suites.
- body: dict = {
- "messages": SIMPLE_MESSAGES,
- "max_tokens": 2048,
- "stream": stream,
- }
- return ChatRequest.openai_chat(body) # type: ignore[arg-type]
-
-
-def _wait_for_first_poll(
- backend: LatencyServiceLLMBackend,
- timeout: float = 10.0,
-) -> None:
- """Block until the background poller completes at least one poll."""
- deadline = time.monotonic() + timeout
- while not backend.is_ready():
- if time.monotonic() > deadline:
- raise TimeoutError("Poller did not complete first poll within timeout")
- time.sleep(0.05)
-
-
-# ================================================================== #
-# Non-streaming tests
-# ================================================================== #
-
-
[email protected](not _nvidia["api_key"], reason=_skip_reason)
-class TestLatencyServiceBackendNonStreaming:
- """Non-streaming requests through the latency-service backend."""
-
- async def test_healthy_endpoint_serves_request(self, mock_latency_service):
- """A HEALTHY endpoint should successfully serve a non-streaming request."""
- mock_latency_service.set_health(MODEL_A, "healthy")
- backend = _make_backend(mock_latency_service.url, [MODEL_A])
- try:
- _wait_for_first_poll(backend)
-
- ctx = ProxyContext()
- response = await backend.call(ctx, _chat_request())
-
- assert response_type_matches(response, ChatResponseType.OPENAI_COMPLETION)
- assert response.body["choices"][0]["message"]["content"]
- assert response.body["choices"][0]["message"]["role"] == "assistant"
- assert ctx.selected_model == MODEL_A
- finally:
- await backend.shutdown()
-
- async def test_response_has_usage(self, mock_latency_service):
- """Response should include token usage statistics."""
- mock_latency_service.set_health(MODEL_A, "healthy")
- backend = _make_backend(mock_latency_service.url, [MODEL_A])
- try:
- _wait_for_first_poll(backend)
-
- ctx = ProxyContext()
- response = await backend.call(ctx, _chat_request())
-
- assert response_type_matches(response, ChatResponseType.OPENAI_COMPLETION)
- assert response.body["usage"] is not None
- assert response.body["usage"]["prompt_tokens"] > 0
- assert response.body["usage"]["completion_tokens"] > 0
- finally:
- await backend.shutdown()
-
-
-# ================================================================== #
-# Streaming tests
-# ================================================================== #
-
-
[email protected](not _nvidia["api_key"], reason=_skip_reason)
-class TestLatencyServiceBackendStreaming:
- """Streaming requests through the latency-service backend."""
-
- async def test_streaming_yields_chunks(self, mock_latency_service):
- """Streaming request should yield content chunks."""
- mock_latency_service.set_health(MODEL_A, "healthy")
- backend = _make_backend(mock_latency_service.url, [MODEL_A])
- try:
- _wait_for_first_poll(backend)
-
- ctx = ProxyContext()
- response = await backend.call(ctx, _chat_request(stream=True))
- assert response_type_matches(response, ChatResponseType.OPENAI_STREAM)
-
- chunks = [chunk async for chunk in response.stream]
- assert len(chunks) >= 2, "Expected at least one content chunk + stop chunk"
-
- text_parts = [
- c.choices[0].delta.content
- for c in chunks
- if c.choices and c.choices[0].delta and c.choices[0].delta.content
- ]
- full_text = "".join(text_parts)
- assert len(full_text) > 0
- finally:
- await backend.shutdown()
-
-
-# ================================================================== #
-# Health-based routing
-# ================================================================== #
-
-
[email protected](not _nvidia["api_key"], reason=_skip_reason)
-class TestLatencyServiceBackendHealth:
- """Verify health-based endpoint selection with real LLM calls."""
-
- async def test_healthy_preferred_over_degraded(self, mock_latency_service):
- """When one endpoint is HEALTHY and another DEGRADED, all traffic
- should go to the HEALTHY one."""
- mock_latency_service.set_health(MODEL_A, "healthy")
- mock_latency_service.set_health(MODEL_B, "degraded")
- backend = _make_backend(mock_latency_service.url, [MODEL_A, MODEL_B])
- try:
- _wait_for_first_poll(backend)
-
- selected_models = set()
- for _ in range(3):
- ctx = ProxyContext()
- await backend.call(ctx, _chat_request())
- selected_models.add(ctx.selected_model)
-
- assert selected_models == {MODEL_A}, (
- f"Expected all traffic to HEALTHY model, got: {selected_models}"
- )
- finally:
- await backend.shutdown()
-
- async def test_unknown_falls_back_to_random(self, mock_latency_service):
- """When all endpoints are UNKNOWN, the backend should distribute
- traffic randomly across all endpoints."""
- mock_latency_service.set_health(MODEL_A, "unknown")
- mock_latency_service.set_health(MODEL_B, "unknown")
- backend = _make_backend(mock_latency_service.url, [MODEL_A, MODEL_B])
- try:
- _wait_for_first_poll(backend)
-
- selected_models = set()
- for _ in range(6):
- ctx = ProxyContext()
- await backend.call(ctx, _chat_request())
- selected_models.add(ctx.selected_model)
-
- assert len(selected_models) > 1, (
- f"Expected random distribution across models, got: {selected_models}"
- )
- finally:
- await backend.shutdown()
-
- async def test_poller_receives_health_updates(self, mock_latency_service):
- """The background poller should pick up health changes from the
- Latency Service and reflect them in the health cache."""
- mock_latency_service.set_health(MODEL_A, "healthy")
- backend = _make_backend(
- mock_latency_service.url, [MODEL_A], poll_interval_s=0.5,
- )
- try:
- _wait_for_first_poll(backend)
- assert backend._health_cache[MODEL_A].status == EndpointHealthStatus.HEALTHY
-
- mock_latency_service.set_health(MODEL_A, "degraded")
- time.sleep(1.5) # Wait for at least one poll cycle
-
- assert backend._health_cache[MODEL_A].status == EndpointHealthStatus.DEGRADED
- finally:
- await backend.shutdown()
-
-
-# ================================================================== #
-# Readiness
-# ================================================================== #
-
-
[email protected](not _nvidia["api_key"], reason=_skip_reason)
-class TestLatencyServiceBackendReadiness:
- """Verify is_ready() behavior with a real poller."""
-
- async def test_ready_after_first_poll(self, mock_latency_service):
- """is_ready() should return True once the poller has fetched health."""
- mock_latency_service.set_health(MODEL_A, "healthy")
- backend = _make_backend(mock_latency_service.url, [MODEL_A])
- try:
- assert not backend.is_ready()
- _wait_for_first_poll(backend)
- assert backend.is_ready()
- finally:
- await backend.shutdown()
-
-
-# ================================================================== #
-# Session affinity (sticky routing)
-# ================================================================== #
-
-
[email protected](not _nvidia["api_key"], reason=_skip_reason)
-class TestLatencyServiceBackendStickiness:
- """Verify session affinity pins a multi-turn conversation to one endpoint."""
-
- async def test_conversation_sticks_to_one_endpoint(self, mock_latency_service):
- """With ``session_affinity`` on, every turn of one conversation routes
- to the same endpoint — even though both endpoints are HEALTHY with no
- latency sample and would otherwise be picked at random (the foil is
- ``test_unknown_falls_back_to_random``, which spreads without affinity)."""
- mock_latency_service.set_health(MODEL_A, "healthy")
- mock_latency_service.set_health(MODEL_B, "healthy")
- backend = _make_backend(
- mock_latency_service.url, [MODEL_A, MODEL_B], session_affinity=True,
- )
- try:
- _wait_for_first_poll(backend)
-
- # One conversation that grows each turn but keeps a stable prefix
- # (same system + first user message → same session key).
- messages = list(SIMPLE_MESSAGES)
- selected: list[str] = []
- for turn in range(6):
- ctx = ProxyContext()
- request = ChatRequest.openai_chat(
- {"messages": messages, "max_tokens": 128}, # type: ignore[arg-type]
- )
- response = await backend.call(ctx, request)
- assert response_type_matches(
- response, ChatResponseType.OPENAI_COMPLETION
- )
- selected.append(ctx.selected_model)
- messages = messages + [
- {"role": "assistant", "content": "Sure."},
- {"role": "user", "content": f"Follow-up {turn + 1}?"},
- ]
-
- assert len(set(selected)) == 1, (
- f"affinity should pin the conversation to one endpoint, got {selected}"
- )
- assert selected[0] in {MODEL_A, MODEL_B}
- finally:
- await backend.shutdown()
diff --git a/tests/readme/test_readme.py b/tests/readme/test_readme.py
index 6af37699c..b1686e62e 100644
--- a/tests/readme/test_readme.py
+++ b/tests/readme/test_readme.py
@@ -71,10 +71,9 @@ def test_python_snippet_tripwire(readme_text: str) -> None:
def _validate_route_blocks(text: str, source: Path) -> int:
# Schema/key validation, not a full chain build: building documented routes
# is NOT hermetic — `passthrough` with `discover: true` does a live catalog
- # fetch and `latency_service` polls. The
- # schema layer (route type + per-type key allowlist) is what we can check
- # offline, and it catches the likeliest drift: a renamed `type:` or a key
- # that no longer exists on that type.
+ # fetch. The schema layer (route type + per-type key allowlist) is what we
+ # can check offline, and it catches the likeliest drift: a renamed `type:`
+ # or a key that no longer exists on that type.
blocks = _code_blocks(text, "yaml")
validated_routes = 0
for idx, block in enumerate(blocks):
diff --git a/tests/test_error_source_annotation.py b/tests/test_error_source_annotation.py
index 6613fd256..57710da9f 100644
--- a/tests/test_error_source_annotation.py
+++ b/tests/test_error_source_annotation.py
@@ -5,9 +5,9 @@
Every client-facing error carries ``x-switchyard-error-source`` naming the
layer that originated it (``switchyard`` | ``provider``), plus
-``x-switchyard-upstream-model`` when a routing selection had happened. The
-backend-side ctx stamps are covered in ``test_latency_service_llm_backend.py``;
-these tests cover the endpoint layer that renders them.
+``x-switchyard-upstream-model`` when a routing selection had happened. These
+tests cover the endpoint layer that renders those annotations from the ctx
+stamps a backend sets.
"""
from __future__ import annotations
diff --git a/tests/test_inference_e2e.py b/tests/test_inference_e2e.py
index 2164ede1c..f8420b46d 100644
--- a/tests/test_inference_e2e.py
+++ b/tests/test_inference_e2e.py
@@ -13,10 +13,12 @@ from __future__ import annotations
import json
from collections.abc import AsyncIterator
+from unittest.mock import AsyncMock, patch
import httpx
import pytest
from fastapi import FastAPI
+from fastapi.testclient import TestClient
from openai.types.chat import ChatCompletion, ChatCompletionChunk
from openai.types.chat.chat_completion import Choice
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
@@ -93,6 +95,23 @@ class _RaisingMockLLMBackend(LLMBackend):
raise self._exc
+class _TranslatingMockLLMBackend(LLMBackend):
+ """Chat-only backend that translates the inbound request to Chat at the top of
+ ``call`` — mirroring any real chat-completions backend (native / passthrough).
+
+ An unsupported inbound field (e.g. a bad message role) is rejected by the
+ translation engine here, before any upstream call, exactly as it would be for
+ a production chat backend.
+ """
+
+ def supported_request_types(self) -> list[ChatRequestType]:
+ return [ChatRequestType.OPENAI_CHAT]
+
+ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
+ request = TranslationEngine().request_to_any_of(request, [ChatRequestType.OPENAI_CHAT])
+ return ChatResponse.openai_completion(_make_completion())
+
+
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@@ -257,6 +276,17 @@ async def raising_client(raising_app: FastAPI) -> AsyncIterator[httpx.AsyncClien
yield c
+def translating_app() -> FastAPI:
+ """App whose chat-only backend translates the inbound request to Chat, so an
+ invalid inbound role is rejected during translation before any upstream call."""
+ switchyard = Switchyard(
+ backend=_TranslatingMockLLMBackend(),
+ translator=TranslationEngine(),
+ )
+ return build_switchyard_app(switchyard)
+
+
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
@@ -502,3 +532,68 @@ class TestToolCallRoundTrip:
block = tool_use_blocks[0]
assert block["name"] == "get_weather"
assert block["input"] == {"city": "Paris"}
+
+
+class TestEndpointErrorContract:
+ """Endpoint error-mapping contracts previously covered only through the
+ removed latency-service backend, re-vehicled onto in-process backends."""
+
+ def test_post_dispatch_exception_returns_json_500(self, app: FastAPI) -> None:
+ """An exception raised AFTER dispatch (e.g. during result serialization)
+ must surface as a JSON 500 envelope, not FastAPI's plain-text 500."""
+
+ class _Unserializable:
+ def model_dump(self) -> None:
+ raise RuntimeError("serialization exploded")
+
+ with patch(
+ "switchyard.lib.endpoints.openai_chat_endpoint.dispatch_chat_request",
+ new_callable=AsyncMock,
+ return_value=_Unserializable(),
+ ):
+ with TestClient(app, raise_server_exceptions=False) as client:
+ response = client.post(
+ "/v1/chat/completions",
+ json={"model": "any-model", "messages": [{"role": "user", "content": "hi"}]},
+ )
+
+ assert response.status_code == 500
+ assert response.headers["content-type"].startswith("application/json")
+ body = response.json()
+ assert body["error"]["type"] == "internal_error"
+ assert body["error"]["code"] == "internal_chain_error"
+ assert "serialization exploded" in body["error"]["message"]
+
+ @pytest.mark.parametrize(
+ ("path", "payload"),
+ [
+ (
+ "/v1/responses",
+ {"model": "m", "input": [{"type": "message", "role": "api", "content": "ping"}]},
+ ),
+ (
+ "/v1/messages",
+ {"model": "m", "max_tokens": 16, "messages": [{"role": "api", "content": "ping"}]},
+ ),
+ ],
+ )
+ def test_invalid_inbound_role_surfaces_as_internal_error(
+ self, translating_app: FastAPI, path: str, payload: dict[str, object]
+ ) -> None:
+ """An unsupported inbound message role is rejected by the translation engine
+ when a chat-only backend translates the request, before any upstream call.
+
+ NOTE: this currently surfaces as a 500 ``internal_error``. The removed
+ latency-service backend special-cased translation ``invalid_value`` errors
+ into a provider-compatible 400; no surviving backend reproduces that, so
+ the standalone-server contract is now a 500. Update this test (and the
+ endpoint/translation layer) if a 400 is restored.
+ """
+ with TestClient(translating_app, raise_server_exceptions=False) as client:
+ response = client.post(path, json=payload)
+
+ assert response.status_code == 500
+ body = response.json()
+ assert body["error"]["type"] == "internal_error"
+ assert '"api"' in body["error"]["message"]
+ assert "role" in body["error"]["message"]
diff --git a/tests/test_latency_service_health_metrics.py b/tests/test_latency_service_health_metrics.py
deleted file mode 100644
index 9329137e4..000000000
--- a/tests/test_latency_service_health_metrics.py
+++ /dev/null
@@ -1,311 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""End-to-end gates for latency-service routing-state metrics.
-
-The Prometheus exposition rendered from the accumulator covers request
-flow but says nothing about *why* a request landed on a given endpoint.
-The latency-service backend now contributes per-endpoint verdict gauges
-and poll-loop health gauges so dashboards can see the routing inputs,
-not just outputs.
-"""
-
-from __future__ import annotations
-
-from unittest.mock import MagicMock, patch
-
-import pytest
-from fastapi.testclient import TestClient
-
-from switchyard.lib.backends.health_poller import (
- EndpointHealth,
- EndpointHealthStatus,
- HealthPoller,
-)
-from switchyard.lib.backends.latency_service_llm_backend import (
- LatencyServiceLLMBackend,
-)
-from switchyard.lib.config.latency_service_backend_config import (
- LatencyServiceBackendConfig,
- LatencyServiceEndpoint,
-)
-from switchyard.lib.endpoints import prometheus_emitter
-from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard
-from switchyard.server.switchyard_app import build_switchyard_app
-
-
[email protected](autouse=True)
-def _clean_table():
- prometheus_emitter._clear_for_tests()
- yield
- prometheus_emitter._clear_for_tests()
-
-
-def _config(*models: str) -> LatencyServiceBackendConfig:
- return LatencyServiceBackendConfig(
- latency_service_url="http://latency.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model=model,
- base_url=f"http://llm-{model}.test/v1",
- api_key="test-key",
- )
- for model in models
- ],
- )
-
-
-def _latency_service_switchyard(
- config: LatencyServiceBackendConfig,
-) -> ProfileSwitchyard:
- """Build the latency-service profile-backed serving adapter."""
- return ProfileSwitchyard(
- LatencyServiceProfileConfig.from_config(config)
- .build()
- .with_runtime_components(enable_stats=config.enable_stats)
- )
-
-
-def _make_backend(config: LatencyServiceBackendConfig) -> LatencyServiceLLMBackend:
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"):
- return LatencyServiceLLMBackend(config)
-
-
-def _emit(backend: LatencyServiceLLMBackend) -> str:
- """Render this backend's emitter output as a single string."""
- return "\n".join(backend._render_prometheus_lines())
-
-
-class TestEndpointStatusGauge:
- def test_emits_one_row_per_status_per_endpoint(self) -> None:
- """Three-state encoding lets `sum by (status)` give a clean histogram."""
- backend = _make_backend(_config("model-A", "model-B"))
- with backend._cache_lock:
- backend._health_cache["model-A"] = EndpointHealth(
- EndpointHealthStatus.HEALTHY, 100.0,
- )
- backend._health_cache["model-B"] = EndpointHealth(
- EndpointHealthStatus.DEGRADED, 800.0,
- )
-
- out = _emit(backend)
- assert 'switchyard_endpoint_status{model="model-A",status="healthy"} 1' in out
- assert 'switchyard_endpoint_status{model="model-A",status="degraded"} 0' in out
- assert 'switchyard_endpoint_status{model="model-A",status="unknown"} 0' in out
- assert 'switchyard_endpoint_status{model="model-B",status="degraded"} 1' in out
- assert 'switchyard_endpoint_status{model="model-B",status="healthy"} 0' in out
-
- def test_unknown_default_before_first_poll(self) -> None:
- """New backends start with every endpoint UNKNOWN."""
- backend = _make_backend(_config("model-A"))
- out = _emit(backend)
- assert 'switchyard_endpoint_status{model="model-A",status="unknown"} 1' in out
- assert 'switchyard_endpoint_status{model="model-A",status="healthy"} 0' in out
-
-
-class TestEndpointLatencyGauge:
- def test_emits_only_for_endpoints_with_sample(self) -> None:
- """Absence is meaningful: don't emit a zero where no sample exists."""
- backend = _make_backend(_config("with-sample", "no-sample"))
- with backend._cache_lock:
- backend._health_cache["with-sample"] = EndpointHealth(
- EndpointHealthStatus.HEALTHY, 250.5,
- )
- backend._health_cache["no-sample"] = EndpointHealth(
- EndpointHealthStatus.HEALTHY, None,
- )
-
- out = _emit(backend)
- assert (
- 'switchyard_endpoint_last_latency_ms{model="with-sample"} 250.5' in out
- )
- assert 'switchyard_endpoint_last_latency_ms{model="no-sample"}' not in out
-
-
-class TestPollHealthGauges:
- def test_before_first_poll_signals_never_polled(self) -> None:
- """poll_ok=0 + no poll_age line lets a scraper detect "never polled"."""
- backend = _make_backend(_config("model-A"))
- out = _emit(backend)
- assert "switchyard_latency_service_poll_ok 0" in out
- assert "switchyard_latency_service_poll_age_seconds" not in out.replace(
- "# HELP switchyard_latency_service_poll_age_seconds", ""
- ).replace(
- "# TYPE switchyard_latency_service_poll_age_seconds", ""
- )
- assert "switchyard_latency_service_polls_total 0" in out
- assert "switchyard_latency_service_poll_failures_total 0" in out
-
- def test_after_successful_poll_emits_age_and_ok(self) -> None:
- backend = _make_backend(_config("model-A"))
- backend._poller._poll_count = 3
- backend._poller._last_poll_ok = True
- backend._poller._last_success_at = __import__("time").monotonic() - 1.0
-
- out = _emit(backend)
- assert "switchyard_latency_service_poll_ok 1" in out
- assert "switchyard_latency_service_polls_total 3" in out
- # Age value is positive — exact value depends on test wall time.
- age_lines = [
- line for line in out.splitlines()
- if line.startswith("switchyard_latency_service_poll_age_seconds ")
- ]
- assert len(age_lines) == 1
- age_value = float(age_lines[0].split()[-1])
- assert age_value > 0
-
- def test_poll_failure_flips_ok_to_zero(self) -> None:
- """Even with prior successes, a recorded failure flips poll_ok off
- so dashboards alarm on the *latest* poll result, not history."""
- backend = _make_backend(_config("model-A"))
- backend._poller._poll_count = 5
- backend._poller._last_success_at = __import__("time").monotonic() - 1.0
- backend._poller._poll_failures = 1
- backend._poller._last_poll_ok = False
-
- out = _emit(backend)
- assert "switchyard_latency_service_poll_ok 0" in out
- assert "switchyard_latency_service_poll_failures_total 1" in out
-
- def test_success_after_prior_failure_emits_ok(self) -> None:
- """Prior failures are historical counters; poll_ok tracks latest outcome."""
- backend = _make_backend(_config("model-A"))
- backend._poller._poll_count = 5
- backend._poller._poll_failures = 1
- backend._poller._last_poll_ok = True
- backend._poller._last_success_at = __import__("time").monotonic() - 1.0
-
- out = _emit(backend)
- assert "switchyard_latency_service_poll_ok 1" in out
- assert "switchyard_latency_service_poll_failures_total 1" in out
-
-
-class TestEmitterLifecycle:
- async def test_construction_registers_emitter(self) -> None:
- assert prometheus_emitter._EMITTERS == []
- backend = _make_backend(_config("model-A"))
- assert len(prometheus_emitter._EMITTERS) == 1
- # Confirm table output includes this backend's lines.
- assert "switchyard_endpoint_status" in prometheus_emitter.render()
- await backend.shutdown()
- assert prometheus_emitter._EMITTERS == []
-
- async def test_shutdown_idempotent(self) -> None:
- """Lifespan tear-down may call shutdown more than once."""
- backend = _make_backend(_config("model-A"))
- await backend.shutdown()
- await backend.shutdown()
- assert prometheus_emitter._EMITTERS == []
-
-
-class TestRoutingOverhead:
- async def test_metrics_record_routing_overhead_for_python_backend(self) -> None:
- """End-to-end: a request through the latency-service chain must
- publish ``switchyard_routing_overhead_ms`` with non-zero count.
-
- Before ``ctx.backend_call_latency_ms`` existed, the Rust response
- processor had no backend-latency reading to subtract from total,
- and the summary stayed at ``count=0`` even under heavy load.
- """
- from unittest.mock import AsyncMock
-
- from openai.types.chat import ChatCompletion
- from openai.types.chat.chat_completion import Choice
- from openai.types.chat.chat_completion_message import ChatCompletionMessage
-
- from switchyard_rust.core import ChatRequest
-
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- switchyard = _latency_service_switchyard(
- LatencyServiceBackendConfig(
- latency_service_url="http://latency.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model="model-A",
- api_key="test-key",
- base_url="http://llm.test/v1",
- ),
- ],
- )
- )
-
- backend = next(
- component
- for component in switchyard.iter_components()
- if hasattr(component, "_clients")
- )
- completion = ChatCompletion(
- id="cmpl-test",
- object="chat.completion",
- created=1700000000,
- model="model-A",
- choices=[
- Choice(
- index=0,
- message=ChatCompletionMessage(role="assistant", content="ok"),
- finish_reason="stop",
- )
- ],
- )
- backend._clients["model-A"].acompletion = AsyncMock(return_value=completion)
-
- await switchyard.call(ChatRequest.openai_chat({
- "model": "model-A",
- "messages": [{"role": "user", "content": "hi"}],
- }))
-
- app = build_switchyard_app(switchyard)
- with TestClient(app, raise_server_exceptions=False) as client:
- metrics = client.get("/metrics")
-
- assert metrics.status_code == 200
- body = metrics.text
- count_line = next(
- line for line in body.splitlines()
- if line.startswith("switchyard_routing_overhead_ms_count")
- )
- count = int(count_line.split()[-1])
- assert count >= 1, f"expected non-zero routing_overhead samples, body=\n{body}"
-
-
-class TestEndToEnd:
- def test_metrics_endpoint_includes_health_lines(self) -> None:
- """Wire the full chain via the recipe and verify /metrics carries
- the new lines on top of the standard accumulator exposition."""
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- switchyard = _latency_service_switchyard(
- LatencyServiceBackendConfig(
- latency_service_url="http://latency.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model="model-A",
- api_key="test-key",
- base_url="http://llm.test/v1",
- ),
- ],
- )
- )
- app = build_switchyard_app(switchyard)
-
- with TestClient(app, raise_server_exceptions=False) as client:
- metrics = client.get("/metrics")
-
- assert metrics.status_code == 200
- body = metrics.text
- # Both surfaces coexist on the same scrape.
- assert "switchyard_requests_total" in body
- assert "switchyard_endpoint_status" in body
- assert "switchyard_latency_service_poll_ok" in body
- assert "switchyard_latency_service_polls_total" in body
diff --git a/tests/test_latency_service_llm_backend.py b/tests/test_latency_service_llm_backend.py
deleted file mode 100644
index 00e9addaf..000000000
--- a/tests/test_latency_service_llm_backend.py
+++ /dev/null
@@ -1,1976 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for :class:`LatencyServiceLLMBackend` (usage case)."""
-
-import json
-import random
-import threading
-import time
-import uuid
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import httpx
-import openai
-import pytest
-from openai.types.chat import ChatCompletion
-from openai.types.chat.chat_completion import Choice
-from openai.types.chat.chat_completion_message import ChatCompletionMessage
-from pydantic import ValidationError
-
-from switchyard.lib.backends.health_poller import (
- EndpointHealth,
- EndpointHealthStatus,
- HealthPoller,
-)
-from switchyard.lib.backends.latency_service_llm_backend import (
- SPEND_LOGS_METADATA_HEADER,
- LatencyServiceLLMBackend,
-)
-from switchyard.lib.config.latency_service_backend_config import (
- LatencyServiceBackendConfig,
- LatencyServiceEndpoint,
-)
-from switchyard.lib.proxy_context import (
- CTX_ERROR_SOURCE,
- CTX_ROUTE_SELECTION,
- CTX_UPSTREAM_HTTP_STATUS,
- CTX_UPSTREAM_MODEL,
- ProxyContext,
-)
-from switchyard.lib.switchyard import Switchyard
-from switchyard.server.switchyard_app import build_switchyard_app
-from switchyard_rust.core import (
- ChatRequest,
- ChatRequestType,
- ChatResponseType,
- response_type_matches,
-)
-from switchyard_rust.translation import TranslationEngine
-
-LATENCY_SERVICE_URL = "http://latency-service.test:8080"
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _ep(
- model: str,
- base_url: str | None = None,
- request_type: str = "openai_chat",
-) -> LatencyServiceEndpoint:
- return LatencyServiceEndpoint(
- model=model,
- base_url=base_url or f"http://llm-{model}.test",
- api_key="test-key",
- request_type=request_type, # type: ignore[arg-type]
- )
-
-
-def _config(*models: str, **kwargs) -> LatencyServiceBackendConfig:
- request_type = str(kwargs.pop("request_type", "openai_chat"))
- return LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[_ep(m, request_type=request_type) for m in models],
- **kwargs,
- )
-
-
-def _make_backend(config: LatencyServiceBackendConfig) -> LatencyServiceLLMBackend:
- """Construct a backend with a mocked OpenAILLMClient and stopped poller."""
- with patch(
- "switchyard.lib.backends"
- ".latency_service_llm_backend.OpenAILLMClient"
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(
- name=f"client-{kw.get('base_url')}"
- )
- with patch.object(HealthPoller, "start"):
- backend = LatencyServiceLLMBackend(config)
- return backend
-
-
-def _set_health(
- backend: LatencyServiceLLMBackend,
- health_map: dict[str, EndpointHealthStatus | EndpointHealth],
-) -> None:
- """Write directly to the backend's health cache for deterministic tests.
-
- Accepts either a bare ``EndpointHealthStatus`` (auto-wrapped with no
- latency sample) or a full ``EndpointHealth`` snapshot.
- """
- with backend._cache_lock:
- for model_id, value in health_map.items():
- if isinstance(value, EndpointHealthStatus):
- value = EndpointHealth(status=value)
- backend._health_cache[model_id] = value
-
-
-def _make_completion(
- *, model: str = "test-model", content: str = "hello",
-) -> ChatCompletion:
- return ChatCompletion(
- id="chatcmpl-test",
- object="chat.completion",
- created=1700000000,
- model=model,
- choices=[
- Choice(
- index=0,
- message=ChatCompletionMessage(role="assistant", content=content),
- finish_reason="stop",
- )
- ],
- )
-
-
-def _openai_request(**overrides) -> ChatRequest:
- body: dict = {
- "model": "incoming-model",
- "messages": [{"role": "user", "content": "hi"}],
- }
- body.update(overrides)
- return ChatRequest.openai_chat(body) # type: ignore[arg-type]
-
-
-def _api_status_error(status_code: int) -> openai.APIStatusError:
- """Synthetic APIStatusError carrying the given upstream status code."""
- response = httpx.Response(
- status_code,
- request=httpx.Request("POST", "http://llm.test/v1/chat/completions"),
- json={"error": {"message": "synthetic"}},
- )
- return openai.APIStatusError(
- "synthetic", response=response, body={"error": "synthetic"}
- )
-
-
-# ---------------------------------------------------------------------------
-# Health poller helpers
-# ---------------------------------------------------------------------------
-
-
-def _make_poller(
- model_ids: list[str],
- health_cache: dict[str, EndpointHealth],
- poll_interval_s: float = 100.0,
-) -> HealthPoller:
- cache_lock = threading.Lock()
- return HealthPoller(
- latency_service_url=LATENCY_SERVICE_URL,
- model_ids=model_ids,
- health_cache=health_cache,
- cache_lock=cache_lock,
- poll_interval_s=poll_interval_s,
- poll_timeout_s=5.0,
- )
-
-
-def _health_response(status_code: int, **kwargs) -> httpx.Response:
- return httpx.Response(
- status_code,
- request=httpx.Request("GET", LATENCY_SERVICE_URL + "/v1/endpoints/health"),
- **kwargs,
- )
-
-
-def _mock_health_response(poller: HealthPoller, response: httpx.Response) -> None:
- mock_client = MagicMock()
- mock_client.get.return_value = response
- poller._http_client = mock_client
-
-
-def _run_one_poll(poller: HealthPoller) -> None:
- """Execute exactly one poll iteration and then exit the run loop."""
- original_wait = poller._stop_event.wait
- call_count = 0
-
- def _wait_then_stop(timeout=None):
- nonlocal call_count
- call_count += 1
- if call_count >= 1:
- poller._stop_event.set()
- return original_wait(timeout=0)
-
- poller._stop_event.wait = _wait_then_stop # type: ignore[method-assign]
- poller.run()
-
-
-# ---------------------------------------------------------------------------
-# Config validation
-# ---------------------------------------------------------------------------
-
-
-class TestConfigValidation:
- def test_supported_request_types(self):
- backend = _make_backend(_config("model-A"))
- assert backend.supported_request_types == [ChatRequestType.OPENAI_CHAT]
-
- def test_responses_endpoint_supported_request_types(self):
- backend = _make_backend(_config("model-A", request_type="openai_responses"))
- assert backend.supported_request_types == [ChatRequestType.OPENAI_RESPONSES]
-
- def test_mixed_endpoint_supported_request_types_are_stable(self):
- config = LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[
- _ep("chat-model"),
- _ep("responses-model", request_type="openai_responses"),
- ],
- )
- backend = _make_backend(config)
- assert backend.supported_request_types == [
- ChatRequestType.OPENAI_CHAT,
- ChatRequestType.OPENAI_RESPONSES,
- ]
-
- def test_request_type_aliases_normalize(self):
- assert LatencyServiceEndpoint(model="m", request_type="chat").request_type == "openai_chat"
- assert (
- LatencyServiceEndpoint(model="m", request_type="responses").request_type
- == "openai_responses"
- )
-
- def test_unknown_request_type_raises(self):
- with pytest.raises(ValidationError):
- LatencyServiceEndpoint(model="m", request_type="anthropic") # type: ignore[arg-type]
-
- def test_unknown_backend_config_field_raises(self):
- with pytest.raises(ValidationError):
- LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[_ep("model-A")],
- credential_polciy="caller_override",
- )
-
- def test_no_endpoints_raises(self):
- with pytest.raises(ValueError, match="At least one endpoint"):
- _make_backend(_config())
-
- def test_single_endpoint_ok(self):
- backend = _make_backend(_config("model-A"))
- assert "model-A" in backend._clients
-
- def test_missing_model_raises(self):
- # dataclass enforces ``model`` as required, but an empty string
- # should still be rejected by the backend as a routing key.
- config = LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[LatencyServiceEndpoint(model="")],
- )
- with pytest.raises(ValueError, match="must have a 'model' field"):
- _make_backend(config)
-
- def test_duplicate_model_raises(self):
- config = LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[_ep("model-A"), _ep("model-A", base_url="http://other")],
- )
- with pytest.raises(ValueError, match="Duplicate model ID"):
- _make_backend(config)
-
-
-# ---------------------------------------------------------------------------
-# Client construction
-# ---------------------------------------------------------------------------
-
-
-class TestClientConstruction:
- def test_disables_sdk_retries(self):
- """The SDK retry layer is disabled so the health-aware ``call`` loop is
- the single source of retries.
-
- The loop already retries on a *different* endpoint; letting the SDK also
- retry the *same* endpoint stacks multiplicatively and adds backoff
- sleeps that hold each request (and its buffered body) alive longer,
- amplifying connection-pool pressure during an upstream incident.
- """
- with patch(
- "switchyard.lib.backends"
- ".latency_service_llm_backend.OpenAILLMClient"
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock()
- with patch.object(HealthPoller, "start"):
- LatencyServiceLLMBackend(_config("model-A", "model-B"))
-
- assert mock_cls.call_args_list, "expected one OpenAILLMClient per endpoint"
- for call in mock_cls.call_args_list:
- assert call.kwargs.get("max_retries") == 0
-
-
-# ---------------------------------------------------------------------------
-# Tiered endpoint selection
-# ---------------------------------------------------------------------------
-
-
-class TestEndpointSelection:
- def test_healthy_preferred_over_unknown(self):
- backend = _make_backend(_config("model-A", "model-B"))
- _set_health(backend, {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.UNKNOWN,
- })
- picks = {backend._select_endpoint() for _ in range(50)}
- assert picks == {"model-A"}
-
- def test_unknown_preferred_over_degraded(self):
- backend = _make_backend(_config("model-A", "model-B"))
- _set_health(backend, {
- "model-A": EndpointHealthStatus.UNKNOWN,
- "model-B": EndpointHealthStatus.DEGRADED,
- })
- picks = {backend._select_endpoint() for _ in range(50)}
- assert picks == {"model-A"}
-
- def test_all_healthy_random_distribution(self):
- backend = _make_backend(_config("model-A", "model-B", "model-C"))
- _set_health(backend, {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.HEALTHY,
- "model-C": EndpointHealthStatus.HEALTHY,
- })
- picks = {backend._select_endpoint() for _ in range(200)}
- assert len(picks) > 1
-
- def test_all_degraded_still_picks(self):
- backend = _make_backend(_config("model-A", "model-B"))
- _set_health(backend, {
- "model-A": EndpointHealthStatus.DEGRADED,
- "model-B": EndpointHealthStatus.DEGRADED,
- })
- pick = backend._select_endpoint()
- assert pick in {"model-A", "model-B"}
-
- def test_initial_state_is_unknown(self):
- backend = _make_backend(_config("model-A", "model-B"))
- assert backend._health_cache["model-A"].status == EndpointHealthStatus.UNKNOWN
- assert backend._health_cache["model-B"].status == EndpointHealthStatus.UNKNOWN
- assert backend._health_cache["model-A"].last_latency_ms is None
-
-
-# ---------------------------------------------------------------------------
-# Inverse-latency weighted selection within a tier
-# ---------------------------------------------------------------------------
-
-
-class TestLatencyWeightedSelection:
- def test_picks_skew_toward_lower_latency(self):
- """Two HEALTHY endpoints; the faster one should attract most traffic."""
- backend = _make_backend(_config("fast", "slow"))
- _set_health(backend, {
- "fast": EndpointHealth(EndpointHealthStatus.HEALTHY, 50.0),
- "slow": EndpointHealth(EndpointHealthStatus.HEALTHY, 500.0),
- })
- random.seed(0)
- picks = [backend._select_endpoint() for _ in range(2000)]
- fast = picks.count("fast")
- slow = picks.count("slow")
- # weights are 1/50 vs 1/500 → expected 10:1; bound loosely to avoid flake.
- assert fast > slow * 5, f"fast={fast} slow={slow}"
-
- def test_unknown_latency_falls_back_to_uniform(self):
- """If any candidate's last_latency_ms is None, the tier picks uniformly."""
- backend = _make_backend(_config("model-A", "model-B"))
- _set_health(backend, {
- "model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 100.0),
- "model-B": EndpointHealth(EndpointHealthStatus.HEALTHY, None),
- })
- random.seed(0)
- picks = [backend._select_endpoint() for _ in range(2000)]
- a = picks.count("model-A")
- b = picks.count("model-B")
- # Uniform within ±15% of 50/50 over 2000 draws is comfortably non-flaky.
- assert 850 < a < 1150, f"model-A picked {a}/2000"
- assert 850 < b < 1150, f"model-B picked {b}/2000"
-
- def test_zero_latency_falls_back_to_uniform(self):
- """Non-positive samples are treated as bogus and trigger uniform fallback."""
- backend = _make_backend(_config("model-A", "model-B"))
- _set_health(backend, {
- "model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 0.0),
- "model-B": EndpointHealth(EndpointHealthStatus.HEALTHY, 100.0),
- })
- # Must not raise (no 1/0); just exercise selection a few times.
- picks = {backend._select_endpoint() for _ in range(50)}
- assert picks <= {"model-A", "model-B"}
-
- def test_weighting_only_within_winning_tier(self):
- """A faster DEGRADED endpoint must not beat a slower HEALTHY one."""
- backend = _make_backend(_config("slow-healthy", "fast-degraded"))
- _set_health(backend, {
- "slow-healthy": EndpointHealth(EndpointHealthStatus.HEALTHY, 500.0),
- "fast-degraded": EndpointHealth(EndpointHealthStatus.DEGRADED, 10.0),
- })
- picks = {backend._select_endpoint() for _ in range(50)}
- assert picks == {"slow-healthy"}
-
-
-# ---------------------------------------------------------------------------
-# Retry with dedup
-# ---------------------------------------------------------------------------
-
-
-class TestRetryDedup:
- async def test_retry_avoids_same_endpoint(self):
- backend = _make_backend(_config("model-A", "model-B", max_retries=1))
- _set_health(backend, {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.HEALTHY,
- })
-
- fail_mock = AsyncMock(side_effect=RuntimeError("down"))
- success_mock = AsyncMock(return_value=_make_completion())
- backend._clients["model-A"].acompletion = fail_mock
- backend._clients["model-B"].acompletion = success_mock
-
- ctx = ProxyContext()
- result = await backend.call(ctx, _openai_request())
-
- assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION)
- assert success_mock.called
-
- async def test_all_retries_exhausted_raises(self):
- backend = _make_backend(_config("model-A", "model-B", max_retries=1))
-
- for mid in backend._clients:
- backend._clients[mid].acompletion = AsyncMock(
- side_effect=RuntimeError("down")
- )
-
- ctx = ProxyContext()
- with pytest.raises(RuntimeError, match="down"):
- await backend.call(ctx, _openai_request())
-
-
-# ---------------------------------------------------------------------------
-# Retry policy: transient errors retry, 4xx client errors fail fast
-# ---------------------------------------------------------------------------
-
-
-class TestRetryPolicy:
- @pytest.mark.parametrize("status", [400, 401, 403, 404, 409, 413, 415, 422])
- def test_client_errors_are_not_retryable(self, status):
- from switchyard.lib.backends.latency_service_llm_backend import (
- _is_retryable_status,
- )
-
- assert _is_retryable_status(status) is False
-
- @pytest.mark.parametrize("status", [408, 429, 500, 502, 503, 504])
- def test_transient_errors_are_retryable(self, status):
- from switchyard.lib.backends.latency_service_llm_backend import (
- _is_retryable_status,
- )
-
- assert _is_retryable_status(status) is True
-
- async def test_400_fails_fast_without_failover(self):
- """A 400 on the first endpoint must not retry the second."""
- from switchyard.lib.proxy_context import CTX_UPSTREAM_HTTP_STATUS
-
- backend = _make_backend(_config("model-A", "model-B", max_retries=2))
- # model-A is the sole HEALTHY endpoint → tried first deterministically.
- _set_health(backend, {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.UNKNOWN,
- })
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(400)
- )
- backend._clients["model-B"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- ctx = ProxyContext()
- with pytest.raises(openai.APIStatusError):
- await backend.call(ctx, _openai_request())
-
- # Failed once on model-A, never failed over to model-B.
- assert backend._clients["model-A"].acompletion.call_count == 1
- assert backend._clients["model-B"].acompletion.call_count == 0
- # Upstream status is stashed so the endpoint passes the 400 through.
- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 400
-
- async def test_5xx_retries_to_another_endpoint(self):
- """A 503 on the first endpoint fails over and recovers on the second."""
- backend = _make_backend(_config("model-A", "model-B", max_retries=2))
- _set_health(backend, {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.UNKNOWN,
- })
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(503)
- )
- backend._clients["model-B"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- ctx = ProxyContext()
- result = await backend.call(ctx, _openai_request())
-
- assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION)
- assert backend._clients["model-B"].acompletion.call_count == 1
- assert ctx.selected_model == "model-B"
-
-
-# ---------------------------------------------------------------------------
-# Session affinity (pin a conversation to one endpoint)
-# ---------------------------------------------------------------------------
-
-
-def _conv_request(text: str) -> ChatRequest:
- """A request whose first user message anchors a distinct session."""
- return _openai_request(messages=[{"role": "user", "content": text}])
-
-
-class TestSessionAffinity:
- async def test_disabled_by_default_pins_nothing(self):
- """Without session_affinity, the affinity map stays empty."""
- backend = _make_backend(_config("model-A", "model-B"))
- _set_health(backend, {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.HEALTHY,
- })
- for mid in backend._clients:
- backend._clients[mid].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- await backend.call(ProxyContext(), _conv_request("task"))
- assert len(backend._affinity) == 0
-
- async def test_first_turn_latency_aware_then_sticks(self):
- """Turn 1 uses the latency-aware pick; later turns reuse that endpoint
- even after latencies flip to favour the other one."""
- backend = _make_backend(_config("fast", "slow", session_affinity=True))
- for mid in backend._clients:
- backend._clients[mid].acompletion = AsyncMock(
- return_value=_make_completion()
- )
- # Turn 1: only "fast" is selectable (slow is DEGRADED) → deterministic pin.
- _set_health(backend, {
- "fast": EndpointHealthStatus.HEALTHY,
- "slow": EndpointHealthStatus.DEGRADED,
- })
- req = _conv_request("the task")
- ctx1 = ProxyContext()
- await backend.call(ctx1, req)
- assert ctx1.selected_model == "fast"
-
- # Flip: "slow" is now healthy AND the latency winner. Affinity must
- # still keep the conversation on "fast".
- _set_health(backend, {
- "fast": EndpointHealth(EndpointHealthStatus.HEALTHY, 1000.0),
- "slow": EndpointHealth(EndpointHealthStatus.HEALTHY, 1.0),
- })
- for _ in range(20):
- ctx = ProxyContext()
- await backend.call(ctx, req)
- assert ctx.selected_model == "fast"
-
- async def test_distinct_conversations_pin_independently(self):
- """Two conversations resolve to independent pins."""
- backend = _make_backend(_config("model-A", "model-B", session_affinity=True))
- for mid in backend._clients:
- backend._clients[mid].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- # Conversation 1 pins to model-A (only A selectable).
- _set_health(backend, {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.DEGRADED,
- })
- req_a = _conv_request("conversation one")
- ctx_a = ProxyContext()
- await backend.call(ctx_a, req_a)
- assert ctx_a.selected_model == "model-A"
-
- # Conversation 2 pins to model-B (only B selectable).
- _set_health(backend, {
- "model-A": EndpointHealthStatus.DEGRADED,
- "model-B": EndpointHealthStatus.HEALTHY,
- })
- req_b = _conv_request("conversation two")
- ctx_b = ProxyContext()
- await backend.call(ctx_b, req_b)
- assert ctx_b.selected_model == "model-B"
-
- # Independent pins: each conversation resolves to its own endpoint.
- assert len(backend._affinity) == 2
- assert await backend._affinity.pinned(ProxyContext(), req_a) == "model-A"
- assert await backend._affinity.pinned(ProxyContext(), req_b) == "model-B"
-
- async def test_degraded_pin_reroutes_and_repins(self):
- """When the pinned endpoint degrades, the next turn re-routes to a
- healthy endpoint and the pin follows."""
- backend = _make_backend(_config("model-A", "model-B", session_affinity=True))
- for mid in backend._clients:
- backend._clients[mid].acompletion = AsyncMock(
- return_value=_make_completion()
- )
- _set_health(backend, {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.DEGRADED,
- })
- req = _conv_request("task")
- await backend.call(ProxyContext(), req)
- assert await backend._affinity.pinned(ProxyContext(), req) == "model-A"
-
- # model-A degrades; model-B becomes the only healthy endpoint.
- _set_health(backend, {
- "model-A": EndpointHealthStatus.DEGRADED,
- "model-B": EndpointHealthStatus.HEALTHY,
- })
- ctx2 = ProxyContext()
- await backend.call(ctx2, req)
- assert ctx2.selected_model == "model-B"
- assert await backend._affinity.pinned(ProxyContext(), req) == "model-B"
-
- async def test_pin_follows_recovery_after_call_failure(self):
- """If the first-turn endpoint fails the call, the pin records the
- endpoint that actually served the request."""
- backend = _make_backend(
- _config("model-A", "model-B", max_retries=1, session_affinity=True)
- )
- _set_health(backend, {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.HEALTHY,
- })
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=RuntimeError("down")
- )
- backend._clients["model-B"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- req = _conv_request("task")
- ctx = ProxyContext()
- await backend.call(ctx, req)
- # Regardless of which endpoint was tried first, only model-B succeeds.
- assert ctx.selected_model == "model-B"
- assert await backend._affinity.pinned(ProxyContext(), req) == "model-B"
-
- async def test_lru_eviction_bounds_map(self):
- """The affinity map never exceeds affinity_max_sessions."""
- backend = _make_backend(
- _config("model-A", session_affinity=True, affinity_max_sessions=2)
- )
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
-
- for i in range(3):
- await backend.call(ProxyContext(), _conv_request(f"conversation {i}"))
-
- assert len(backend._affinity) == 2
-
- def test_affinity_config_reaches_backend(self):
- """A config dict's affinity settings reach the backend's pin map."""
- config = LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[{"model": "model-A", "base_url": "http://a.test", "api_key": "k"}],
- session_affinity=True,
- affinity_max_sessions=5,
- )
- assert config.session_affinity is True
- assert config.affinity_max_sessions == 5
-
- backend = _make_backend(config)
- assert backend._affinity.max_sessions == 5
-
- async def test_affinity_counters_rendered_on_metrics(self):
- """hits/misses counters appear on /metrics when affinity is enabled."""
- backend = _make_backend(_config("model-A", session_affinity=True))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
-
- req = _conv_request("the task")
- await backend.call(ProxyContext(), req) # first turn → miss (no pin yet)
- await backend.call(ProxyContext(), req) # reuse → hit
- await backend.call(ProxyContext(), req) # reuse → hit
-
- out = "\n".join(backend._render_prometheus_lines())
- assert "switchyard_affinity_hits_total 2" in out
- assert "switchyard_affinity_misses_total 1" in out
-
- async def test_affinity_counters_absent_when_disabled(self):
- """The warm-reuse counters stay off the metric surface by default."""
- backend = _make_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
-
- await backend.call(ProxyContext(), _conv_request("task"))
-
- out = "\n".join(backend._render_prometheus_lines())
- assert "switchyard_affinity_hits_total" not in out
- assert "switchyard_affinity_misses_total" not in out
-
- def test_affinity_l2_counters_rendered_when_shared_store_configured(self):
- """L2 hit/error counters appear on /metrics when a shared store is on.
-
- Constructing the Redis store never connects (the client is lazy), so
- this stays hermetic.
- """
- backend = _make_backend(_config(
- "model-A",
- session_affinity=True,
- affinity_store="redis",
- affinity_store_url="redis://cache.test:6379/0",
- ))
-
- out = "\n".join(backend._render_prometheus_lines())
- assert "switchyard_affinity_l2_hits_total 0" in out
- assert "switchyard_affinity_l2_errors_total 0" in out
- assert "switchyard_affinity_l2_breaker_open 0" in out
-
- def test_affinity_l2_breaker_gauge_reads_1_while_open(self):
- """The breaker gauge flips to 1 once the failure streak opens it."""
- backend = _make_backend(_config(
- "model-A",
- session_affinity=True,
- affinity_store="redis",
- affinity_store_url="redis://cache.test:6379/0",
- ))
- affinity = backend._affinity
- for _ in range(affinity._l2_breaker_threshold):
- affinity._record_l2_failure("test-induced failure")
-
- out = "\n".join(backend._render_prometheus_lines())
- assert "switchyard_affinity_l2_breaker_open 1" in out
-
- def test_affinity_l2_counters_absent_for_memory_store(self):
- """Without a shared store the L2 counters stay off the metric surface."""
- backend = _make_backend(_config("model-A", session_affinity=True))
-
- out = "\n".join(backend._render_prometheus_lines())
- assert "switchyard_affinity_l2_hits_total" not in out
- assert "switchyard_affinity_l2_errors_total" not in out
- assert "switchyard_affinity_l2_breaker_open" not in out
-
- def test_negative_affinity_max_rejected_at_construction(self):
- """A negative cap is rejected when the config is built — otherwise the
- LRU eviction loop in SessionCache.put would pop past an empty map."""
- from pydantic import ValidationError
-
- with pytest.raises(ValidationError):
- LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[_ep("model-A")],
- session_affinity=True,
- affinity_max_sessions=-1,
- )
-
- def test_zero_affinity_max_rejected_when_enabled(self):
- """A zero cap with affinity on is rejected — it would retain nothing."""
- from pydantic import ValidationError
-
- with pytest.raises(ValidationError):
- LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[_ep("model-A")],
- session_affinity=True,
- affinity_max_sessions=0,
- )
-
-
-# ---------------------------------------------------------------------------
-# Request processing
-# ---------------------------------------------------------------------------
-
-
-class TestCall:
- async def test_non_streaming_returns_completion_chat_response(self):
- backend = _make_backend(_config("model-A"))
- completion = _make_completion(content="world")
- backend._clients["model-A"].acompletion = AsyncMock(return_value=completion)
-
- ctx = ProxyContext()
- result = await backend.call(ctx, _openai_request())
-
- assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION)
- assert result.body["choices"][0]["message"]["content"] == "world"
- # ``selected_model`` is the cross-language ctx field the Rust
- # ``StatsResponseProcessor`` reads to attribute tokens and
- # end-to-end latency per endpoint on /metrics.
- assert ctx.selected_model == "model-A"
- # ``backend_call_latency_ms`` is the second cross-language hook —
- # the response processor uses it to compute ``routing_overhead_ms``.
- # Mocked upstream returns instantly so latency is small but non-None.
- assert ctx.backend_call_latency_ms is not None
- assert ctx.backend_call_latency_ms >= 0.0
-
- async def test_responses_endpoint_dispatches_responses_natively(self):
- backend = _make_backend(_config("model-A", request_type="openai_responses"))
- backend._clients["model-A"].aresponses = AsyncMock(
- return_value={
- "id": "resp-test",
- "object": "response",
- "model": "model-A",
- "output": [],
- }
- )
- backend._clients["model-A"].acompletion = AsyncMock()
-
- ctx = ProxyContext()
- result = await backend.call(
- ctx,
- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),
- )
-
- assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_COMPLETION)
- backend._clients["model-A"].aresponses.assert_awaited_once()
- backend._clients["model-A"].acompletion.assert_not_called()
- call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs
- assert call_kwargs["model"] == "model-A"
- assert call_kwargs["input"] == "hi"
- assert "messages" not in call_kwargs
- assert ctx.selected_model == "model-A"
-
- async def test_responses_endpoint_preserves_exact_upstream_body(self):
- """Same-format Responses passthrough returns the upstream JSON untouched.
-
- Provider-specific extras and explicit-null fields must survive the
- wrap into ``ChatResponse`` — nothing may be normalized,
- re-synthesized, or dropped on this path.
- """
- upstream_body = {
- "id": "resp-raw",
- "object": "response",
- "created_at": 1719890000,
- "model": "model-A",
- "status": "completed",
- "output": [],
- "store": False,
- "temperature": 1.0,
- "top_p": 0.9,
- "previous_response_id": None,
- "provider_meta": {"region": "eastus"},
- }
- backend = _make_backend(_config("model-A", request_type="openai_responses"))
- backend._clients["model-A"].aresponses = AsyncMock(return_value=dict(upstream_body))
-
- result = await backend.call(
- ProxyContext(),
- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),
- )
-
- assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_COMPLETION)
- assert result.body == upstream_body
-
- async def test_chat_endpoint_translates_responses_to_chat_fallback(self):
- backend = _make_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
- backend._clients["model-A"].aresponses = AsyncMock()
-
- result = await backend.call(
- ProxyContext(),
- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),
- )
-
- assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION)
- backend._clients["model-A"].acompletion.assert_awaited_once()
- backend._clients["model-A"].aresponses.assert_not_called()
- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs
- assert call_kwargs["model"] == "model-A"
- assert call_kwargs["messages"][0]["content"] == "hi"
-
- async def test_streaming_wraps_into_streaming_chat_response(self):
- backend = _make_backend(_config("model-A"))
- stream_mock = MagicMock(spec=openai.AsyncStream)
- backend._clients["model-A"].acompletion = AsyncMock(return_value=stream_mock)
-
- result = await backend.call(ProxyContext(), _openai_request(stream=True))
- assert response_type_matches(result, ChatResponseType.OPENAI_STREAM)
-
- async def test_responses_streaming_wraps_into_responses_stream(self):
- backend = _make_backend(_config("model-A", request_type="openai_responses"))
- stream_mock = MagicMock(spec=openai.AsyncStream)
- backend._clients["model-A"].aresponses = AsyncMock(return_value=stream_mock)
-
- result = await backend.call(
- ProxyContext(),
- ChatRequest.openai_responses({
- "model": "incoming-model",
- "input": "hi",
- "stream": True,
- }),
- )
-
- assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_STREAM)
-
- async def test_model_id_overrides_incoming_model(self):
- """The selected endpoint model ID should replace the request body's model."""
- backend = _make_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- await backend.call(ProxyContext(), _openai_request(model="incoming-model"))
-
- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs
- assert call_kwargs["model"] == "model-A"
-
- async def test_accepts_anthropic_request_via_translation(self):
- """Non-OpenAI inbound formats go through TranslationEngine."""
- backend = _make_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- anthropic_req = ChatRequest.anthropic({
- "model": "claude-sonnet-4-20250514",
- "messages": [{"role": "user", "content": "Hi"}],
- "max_tokens": 1024,
- })
- result = await backend.call(ProxyContext(), anthropic_req)
-
- assert response_type_matches(result, ChatResponseType.OPENAI_COMPLETION)
- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs
- assert call_kwargs["model"] == "model-A"
- assert "messages" in call_kwargs
-
- async def test_upstream_model_override_used_in_body(self):
- """When upstream_model is set, it's the value sent in body['model']."""
- config = LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[
- LatencyServiceEndpoint(
- model="openai/gpt-5.5",
- upstream_model="openai/openai/gpt-5.5",
- base_url="https://inference-api.nvidia.com/v1",
- api_key="nvapi-test",
- ),
- ],
- )
- backend = _make_backend(config)
- backend._clients["openai/gpt-5.5"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- ctx = ProxyContext()
- await backend.call(ctx, _openai_request(model="incoming-model"))
-
- call_kwargs = backend._clients["openai/gpt-5.5"].acompletion.call_args.kwargs
- assert call_kwargs["model"] == "openai/openai/gpt-5.5"
- assert ctx.selected_model == "openai/gpt-5.5"
-
- async def test_upstream_model_default_falls_back_to_model(self):
- """Without upstream_model, body['model'] should be the lookup key."""
- config = LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[
- LatencyServiceEndpoint(
- model="openai/gpt-5.5",
- base_url="http://llm.test",
- api_key="k",
- ),
- ],
- )
- backend = _make_backend(config)
- backend._clients["openai/gpt-5.5"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- await backend.call(ProxyContext(), _openai_request())
-
- call_kwargs = backend._clients["openai/gpt-5.5"].acompletion.call_args.kwargs
- assert call_kwargs["model"] == "openai/gpt-5.5"
-
-
-# ---------------------------------------------------------------------------
-# Route-selection spend-logs metadata (tokenomics attribution)
-# ---------------------------------------------------------------------------
-
-
-def _spend_logs_payload(mock: AsyncMock, call_index: int = -1) -> dict[str, object]:
- """Parse the spend-logs metadata header stamped on one recorded SDK call."""
- call_kwargs = mock.call_args_list[call_index].kwargs
- header_value = call_kwargs["extra_headers"][SPEND_LOGS_METADATA_HEADER]
- payload = json.loads(header_value)
- assert isinstance(payload, dict)
- return payload
-
-
-class TestRouteSelectionSpendLogs:
- """Every upstream attempt is stamped with route-selection metadata.
-
- The ``x-litellm-spend-logs-metadata`` request header ties a LiteLLM
- provider spend-log row back to the Switchyard route that selected it, and
- the successful attempt's selection is recorded on
- ``ctx.metadata[CTX_ROUTE_SELECTION]`` so the endpoint layer can return the
- matching ``x-switchyard-*`` response headers.
- """
-
- async def test_outbound_header_carries_route_selection(self):
- """The outbound header records the full selection for the chosen endpoint."""
- config = LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[
- LatencyServiceEndpoint(
- model="openai/gpt-5.5",
- upstream_model="openai/openai/gpt-5.5",
- base_url="https://inference-api.test/v1",
- api_key="k",
- ),
- ],
- )
- backend = _make_backend(config)
- backend._clients["openai/gpt-5.5"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- await backend.call(ProxyContext(), _openai_request(model="nvidia/switchyard/gpt-5.5"))
-
- payload = _spend_logs_payload(backend._clients["openai/gpt-5.5"].acompletion)
- assert payload["router_model"] == "nvidia/switchyard/gpt-5.5"
- assert payload["router_strategy"] == "latency"
- assert payload["router_selected_endpoint"] == "openai/gpt-5.5"
- assert payload["router_selected_model"] == "openai/openai/gpt-5.5"
- assert payload["router_selected_provider"] == "openai"
- # Must parse as a UUID — the join key between spend-log rows.
- uuid.UUID(str(payload["router_correlation_id"]))
-
- async def test_ctx_records_selection_matching_outbound_header(self):
- """ctx records exactly the payload the wire header carried."""
- backend = _make_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- ctx = ProxyContext()
- await backend.call(ctx, _openai_request())
-
- payload = _spend_logs_payload(backend._clients["model-A"].acompletion)
- assert ctx.metadata[CTX_ROUTE_SELECTION] == payload
- # Endpoint id without a provider prefix: the provider segment
- # degenerates to the whole id (split on the first "/").
- assert payload["router_selected_provider"] == "model-A"
-
- async def test_failover_restamps_selection_and_keeps_correlation_id(self):
- """Each attempt is stamped with its own endpoint under one correlation id."""
- backend = _make_backend(_config("model-A", "model-B"))
- _set_health(
- backend,
- {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.DEGRADED,
- },
- )
- failing = AsyncMock(side_effect=_api_status_error(500))
- succeeding = AsyncMock(return_value=_make_completion())
- backend._clients["model-A"].acompletion = failing
- backend._clients["model-B"].acompletion = succeeding
-
- ctx = ProxyContext()
- await backend.call(ctx, _openai_request())
-
- first = _spend_logs_payload(failing)
- second = _spend_logs_payload(succeeding)
- # Each attempt records the endpoint it actually hit…
- assert first["router_selected_endpoint"] == "model-A"
- assert second["router_selected_endpoint"] == "model-B"
- # …while sharing one correlation id per client request.
- assert first["router_correlation_id"] == second["router_correlation_id"]
- # ctx carries the selection that served the request, not the failure.
- assert ctx.metadata[CTX_ROUTE_SELECTION] == second
-
- async def test_correlation_id_is_fresh_per_request(self):
- """Two client requests never share a correlation id."""
- backend = _make_backend(_config("model-A"))
- mock = AsyncMock(return_value=_make_completion())
- backend._clients["model-A"].acompletion = mock
-
- await backend.call(ProxyContext(), _openai_request())
- await backend.call(ProxyContext(), _openai_request())
-
- first = _spend_logs_payload(mock, call_index=0)
- second = _spend_logs_payload(mock, call_index=1)
- assert first["router_correlation_id"] != second["router_correlation_id"]
-
- async def test_router_model_falls_back_to_configured_route_model(self):
- """A model-less client body attributes to the configured route id."""
- backend = _make_backend(
- _config("model-A", route_model="nvidia/switchyard/gpt-5.5")
- )
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- body: dict = {"messages": [{"role": "user", "content": "hi"}]}
- await backend.call(ProxyContext(), ChatRequest.openai_chat(body))
-
- payload = _spend_logs_payload(backend._clients["model-A"].acompletion)
- assert payload["router_model"] == "nvidia/switchyard/gpt-5.5"
-
- async def test_no_selection_recorded_when_all_attempts_fail(self):
- """No billed success → no selection recorded on ctx."""
- backend = _make_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(401),
- )
-
- ctx = ProxyContext()
- with pytest.raises(openai.APIStatusError):
- await backend.call(ctx, _openai_request())
-
- assert CTX_ROUTE_SELECTION not in ctx.metadata
-
- async def test_responses_surface_is_stamped_too(self):
- """The Responses-API surface is stamped like the Chat surface."""
- backend = _make_backend(_config("model-A", request_type="openai_responses"))
- backend._clients["model-A"].aresponses = AsyncMock(
- return_value={"id": "resp-test", "object": "response", "output": []}
- )
-
- await backend.call(
- ProxyContext(),
- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),
- )
-
- payload = _spend_logs_payload(backend._clients["model-A"].aresponses)
- assert payload["router_selected_endpoint"] == "model-A"
- assert payload["router_strategy"] == "latency"
-
- async def test_cased_spoof_of_spend_logs_header_is_stripped(self):
- """A differently-cased spoof key cannot ride client extra_headers to the wire.
-
- Header names are case-insensitive on the wire while dict merges are
- not: exactly one instance of the protected header — ours — may reach
- the SDK, while benign client headers still pass through.
- """
- backend = _make_backend(_config("model-A"))
- mock = AsyncMock(return_value=_make_completion())
- backend._clients["model-A"].acompletion = mock
-
- await backend.call(
- ProxyContext(),
- _openai_request(
- extra_headers={
- "X-LiteLLM-Spend-Logs-Metadata": "spoofed",
- "x-client-tag": "42",
- },
- ),
- )
-
- headers = mock.call_args.kwargs["extra_headers"]
- spoof_keys = [k for k in headers if k.lower() == SPEND_LOGS_METADATA_HEADER]
- assert spoof_keys == [SPEND_LOGS_METADATA_HEADER]
- payload = _spend_logs_payload(mock)
- assert payload["router_selected_endpoint"] == "model-A"
- assert headers["x-client-tag"] == "42"
-
-
-# ---------------------------------------------------------------------------
-# Credential policy
-# ---------------------------------------------------------------------------
-
-
-class TestCallerApiKey:
- """Per-request caller-key forwarding is opt-in for multi-tenant deployments.
-
- When the HTTP endpoint extracts an ``Authorization: Bearer ...`` header
- and writes the value into ``ctx.metadata[CTX_CALLER_API_KEY]``, the
- backend ignores that key by default. ``credential_policy="caller_override"``
- makes the caller key a per-call SDK override.
- """
-
- async def test_default_policy_ignores_caller_key_for_chat_sdk(self):
- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY
-
- backend = _make_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- ctx = ProxyContext()
- ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied"
- await backend.call(ctx, _openai_request())
-
- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs
- assert call_kwargs["api_key"] is None
-
- async def test_caller_override_policy_passes_caller_key_to_chat_sdk(self):
- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY
-
- backend = _make_backend(_config("model-A", credential_policy="caller_override"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- ctx = ProxyContext()
- ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied"
- await backend.call(ctx, _openai_request())
-
- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs
- assert call_kwargs["api_key"] == "nvapi-caller-supplied"
-
- async def test_default_policy_ignores_caller_key_for_responses_sdk(self):
- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY
-
- backend = _make_backend(_config("model-A", request_type="openai_responses"))
- backend._clients["model-A"].aresponses = AsyncMock(
- return_value={"id": "resp-test", "output": []}
- )
-
- ctx = ProxyContext()
- ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied"
- await backend.call(
- ctx,
- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),
- )
-
- call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs
- assert call_kwargs["api_key"] is None
-
- async def test_caller_override_policy_passes_caller_key_to_responses_sdk(self):
- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY
-
- backend = _make_backend(_config(
- "model-A",
- request_type="openai_responses",
- credential_policy="caller_override",
- ))
- backend._clients["model-A"].aresponses = AsyncMock(
- return_value={"id": "resp-test", "output": []}
- )
-
- ctx = ProxyContext()
- ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied"
- await backend.call(
- ctx,
- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),
- )
-
- call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs
- assert call_kwargs["api_key"] == "nvapi-caller-supplied"
-
- async def test_missing_caller_key_falls_back_to_config(self):
- """ctx without a caller key sends api_key=None to OpenAILLMClient.
-
- ``OpenAILLMClient.acompletion`` then uses the construction-time
- config key (no ``with_options`` override). The test only asserts
- the contract at the seam — that we pass ``api_key=None`` so the
- client falls back, rather than passing a stale or empty string.
- """
- backend = _make_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- await backend.call(ProxyContext(), _openai_request())
-
- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs
- assert call_kwargs["api_key"] is None
-
- async def test_missing_caller_key_falls_back_to_config_for_responses(self):
- backend = _make_backend(_config("model-A", request_type="openai_responses"))
- backend._clients["model-A"].aresponses = AsyncMock(
- return_value={"id": "resp-test", "output": []}
- )
-
- await backend.call(
- ProxyContext(),
- ChatRequest.openai_responses({"model": "incoming-model", "input": "hi"}),
- )
-
- call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs
- assert call_kwargs["api_key"] is None
-
- async def test_caller_required_policy_passes_caller_key_to_chat_sdk(self):
- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY
-
- backend = _make_backend(_config("model-A", credential_policy="caller_required"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- ctx = ProxyContext()
- ctx.metadata[CTX_CALLER_API_KEY] = "nvapi-caller-supplied"
- await backend.call(ctx, _openai_request())
-
- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs
- assert call_kwargs["api_key"] == "nvapi-caller-supplied"
-
- async def test_caller_required_policy_rejects_missing_caller_key(self):
- """No caller key under caller_required → 401 stashed, no upstream call."""
- from switchyard.lib.proxy_context import (
- CTX_UPSTREAM_ATTEMPTS_RECORDED,
- CTX_UPSTREAM_HTTP_STATUS,
- )
-
- backend = _make_backend(_config("model-A", credential_policy="caller_required"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- ctx = ProxyContext()
- with pytest.raises(PermissionError):
- await backend.call(ctx, _openai_request())
-
- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401
- backend._clients["model-A"].acompletion.assert_not_called()
- # Attempt accounting stays claimed: no upstream attempt happened, so
- # the endpoint fallback must not count this 401 in
- # ``switchyard_upstream_attempts_total``.
- assert ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] is True
- assert "switchyard_latency_upstream_attempts_total" not in "\n".join(
- backend._render_prometheus_lines()
- )
-
- async def test_caller_required_policy_rejects_blank_caller_key(self):
- """A blank caller key is unusable → 401, never the configured service key."""
- from switchyard.lib.proxy_context import (
- CTX_CALLER_API_KEY,
- CTX_UPSTREAM_ATTEMPTS_RECORDED,
- CTX_UPSTREAM_HTTP_STATUS,
- )
-
- backend = _make_backend(_config("model-A", credential_policy="caller_required"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- ctx = ProxyContext()
- ctx.metadata[CTX_CALLER_API_KEY] = " "
- with pytest.raises(PermissionError):
- await backend.call(ctx, _openai_request())
-
- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401
- backend._clients["model-A"].acompletion.assert_not_called()
- assert ctx.metadata[CTX_UPSTREAM_ATTEMPTS_RECORDED] is True
-
- async def test_caller_required_rejection_skips_affinity_lookup(self):
- """The credential check runs before pin resolution, so a doomed
- request never touches the session-affinity store."""
- backend = _make_backend(_config(
- "model-A", credential_policy="caller_required", session_affinity=True,
- ))
- backend._affinity.pinned = AsyncMock()
-
- with pytest.raises(PermissionError):
- await backend.call(ProxyContext(), _openai_request())
-
- backend._affinity.pinned.assert_not_awaited()
-
-
- ("path", "body"),
- [
- ("/v1/messages", {"model": "incoming-model", "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]}),
- ("/v1/responses", {"model": "incoming-model", "input": "hi"}),
- ],
-)
-async def test_non_chat_http_endpoints_default_to_endpoint_credentials(
- path: str,
- body: dict[str, object],
-) -> None:
- """HTTP ingress may attach caller keys, but the default backend policy ignores them."""
- backend = _make_backend(_config("model-A"))
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion(model="model-A")
- )
- app = build_switchyard_app(Switchyard(backend=backend, translator=TranslationEngine()))
-
- async with httpx.AsyncClient(
- transport=httpx.ASGITransport(app=app, raise_app_exceptions=False),
- base_url="http://test",
- ) as client:
- response = await client.post(
- path,
- json=body,
- headers={"Authorization": "Bearer CALLER-BYO-KEY"},
- )
-
- assert response.status_code == 200, response.text
- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs
- assert call_kwargs["api_key"] is None
-
-
- ("path", "body"),
- [
- ("/v1/messages", {"model": "incoming-model", "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]}),
- ("/v1/responses", {"model": "incoming-model", "input": "hi"}),
- ],
-)
-async def test_non_chat_http_endpoints_forward_caller_key_in_byo_mode(
- path: str,
- body: dict[str, object],
-) -> None:
- """Opt-in BYO mode preserves caller keys across Messages/Responses ingress."""
- backend = _make_backend(_config("model-A", credential_policy="caller_override"))
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion(model="model-A")
- )
- app = build_switchyard_app(Switchyard(backend=backend, translator=TranslationEngine()))
-
- async with httpx.AsyncClient(
- transport=httpx.ASGITransport(app=app, raise_app_exceptions=False),
- base_url="http://test",
- ) as client:
- response = await client.post(
- path,
- json=body,
- headers={"Authorization": "Bearer CALLER-BYO-KEY"},
- )
-
- assert response.status_code == 200, response.text
- call_kwargs = backend._clients["model-A"].acompletion.call_args.kwargs
- assert call_kwargs["api_key"] == "CALLER-BYO-KEY"
-
-
-async def test_responses_http_endpoint_can_use_native_latency_responses_mode() -> None:
- backend = _make_backend(_config("model-A", request_type="openai_responses"))
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
- backend._clients["model-A"].aresponses = AsyncMock(
- return_value={
- "id": "resp-test",
- "object": "response",
- "model": "model-A",
- "output": [],
- }
- )
- backend._clients["model-A"].acompletion = AsyncMock()
- app = build_switchyard_app(Switchyard(backend=backend, translator=TranslationEngine()))
-
- async with httpx.AsyncClient(
- transport=httpx.ASGITransport(app=app, raise_app_exceptions=False),
- base_url="http://test",
- ) as client:
- response = await client.post(
- "/v1/responses",
- json={"model": "incoming-model", "input": "hi"},
- headers={"Authorization": "Bearer CALLER-BYO-KEY"},
- )
-
- assert response.status_code == 200, response.text
- backend._clients["model-A"].aresponses.assert_awaited_once()
- backend._clients["model-A"].acompletion.assert_not_called()
- call_kwargs = backend._clients["model-A"].aresponses.call_args.kwargs
- assert call_kwargs["api_key"] is None
- assert call_kwargs["input"] == "hi"
-
-
-class TestEndpointApiKeyReachesUpstream:
- """End-to-end auth: which credential the upstream HTTP call carries.
-
- The seam test above only asserts the ``api_key`` value handed to
- ``acompletion``; it cannot catch a regression where a blank caller key
- clobbers the configured endpoint key and the upstream call goes out
- unauthenticated. These tests run the real
- ``OpenAILLMClient`` against a mocked HTTP transport and inspect the
- ``Authorization`` header that actually reaches the wire.
- """
-
- @staticmethod
- def _backend_with_captured_auth(
- captured: dict[str, str | None],
- credential_policy: str = "configured_endpoint",
- ) -> LatencyServiceLLMBackend:
- """Real-client backend whose upstream auth header is captured."""
-
- def handler(request: httpx.Request) -> httpx.Response:
- captured["authorization"] = request.headers.get("authorization")
- return httpx.Response(
- 200,
- json={
- "id": "chatcmpl-test",
- "object": "chat.completion",
- "created": 1700000000,
- "model": "model-A",
- "choices": [{
- "index": 0,
- "message": {"role": "assistant", "content": "ok"},
- "finish_reason": "stop",
- }],
- },
- )
-
- config = LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[LatencyServiceEndpoint(
- model="model-A",
- base_url="http://llm.test/v1",
- api_key="ENDPOINT-CONFIGURED-KEY",
- )],
- credential_policy=credential_policy,
- )
- with patch.object(HealthPoller, "start"):
- backend = LatencyServiceLLMBackend(config)
- backend._clients["model-A"].async_client._client = httpx.AsyncClient(
- transport=httpx.MockTransport(handler)
- )
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
- return backend
-
- async def test_no_caller_key_uses_configured_endpoint_key(self):
- """No caller key → the endpoint's configured key authenticates upstream."""
- captured: dict[str, str | None] = {}
- backend = self._backend_with_captured_auth(captured)
-
- await backend.call(ProxyContext(), _openai_request())
-
- assert captured["authorization"] == "Bearer ENDPOINT-CONFIGURED-KEY"
-
- async def test_default_policy_ignores_caller_key(self):
- """Default policy keeps using the endpoint key even when a caller key exists."""
- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY
-
- captured: dict[str, str | None] = {}
- backend = self._backend_with_captured_auth(captured)
-
- ctx = ProxyContext()
- ctx.metadata[CTX_CALLER_API_KEY] = "CALLER-BYO-KEY"
- await backend.call(ctx, _openai_request())
-
- assert captured["authorization"] == "Bearer ENDPOINT-CONFIGURED-KEY"
-
- async def test_caller_override_policy_uses_caller_key(self):
- """BYO mode lets a caller-supplied key override the endpoint key."""
- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY
-
- captured: dict[str, str | None] = {}
- backend = self._backend_with_captured_auth(
- captured,
- credential_policy="caller_override",
- )
-
- ctx = ProxyContext()
- ctx.metadata[CTX_CALLER_API_KEY] = "CALLER-BYO-KEY"
- await backend.call(ctx, _openai_request())
-
- assert captured["authorization"] == "Bearer CALLER-BYO-KEY"
-
- async def test_blank_caller_key_does_not_clobber_endpoint_key(self):
- """A blank caller key in BYO mode still falls back to the configured key."""
- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY
-
- captured: dict[str, str | None] = {}
- backend = self._backend_with_captured_auth(
- captured,
- credential_policy="caller_override",
- )
-
- ctx = ProxyContext()
- ctx.metadata[CTX_CALLER_API_KEY] = " "
- await backend.call(ctx, _openai_request())
-
- assert captured["authorization"] == "Bearer ENDPOINT-CONFIGURED-KEY"
-
- async def test_caller_required_policy_uses_caller_key(self):
- """caller_required forwards the caller key to the upstream wire."""
- from switchyard.lib.proxy_context import CTX_CALLER_API_KEY
-
- captured: dict[str, str | None] = {}
- backend = self._backend_with_captured_auth(
- captured,
- credential_policy="caller_required",
- )
-
- ctx = ProxyContext()
- ctx.metadata[CTX_CALLER_API_KEY] = "CALLER-BYO-KEY"
- await backend.call(ctx, _openai_request())
-
- assert captured["authorization"] == "Bearer CALLER-BYO-KEY"
-
- async def test_caller_required_rejects_missing_key_without_upstream_call(self):
- """caller_required with no caller key 401s before reaching the upstream.
-
- The configured ``ENDPOINT-CONFIGURED-KEY`` must never authenticate the
- call — the mock transport handler should not run at all.
- """
- from switchyard.lib.proxy_context import CTX_UPSTREAM_HTTP_STATUS
-
- captured: dict[str, str | None] = {}
- backend = self._backend_with_captured_auth(
- captured,
- credential_policy="caller_required",
- )
-
- ctx = ProxyContext()
- with pytest.raises(PermissionError):
- await backend.call(ctx, _openai_request())
-
- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401
- assert captured == {}
-
-
-# ---------------------------------------------------------------------------
-# Per-model upstream-attempt metrics
-# ---------------------------------------------------------------------------
-
-
-class TestPerModelUpstreamMetrics:
- """``switchyard_latency_upstream_attempts_total`` breaks attempts out by
- requested route model, selected upstream model, outcome, and HTTP code —
- the per-model complement to the layer-aggregate counter."""
-
- async def test_success_attempt_recorded_per_model(self):
- backend = _make_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
-
- # Request a configured endpoint id so ``requested_model`` is preserved.
- await backend.call(ProxyContext(), _openai_request(model="model-A"))
-
- out = "\n".join(backend._render_prometheus_lines())
- assert (
- 'switchyard_latency_upstream_attempts_total{'
- 'requested_model="model-A",upstream_model="model-A",'
- 'outcome="success",code="200"} 1'
- ) in out
-
- async def test_error_attempt_recorded_per_model(self):
- backend = _make_backend(_config("model-A", max_retries=0))
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(500)
- )
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
-
- with pytest.raises(openai.APIStatusError):
- await backend.call(ProxyContext(), _openai_request(model="model-A"))
-
- out = "\n".join(backend._render_prometheus_lines())
- assert (
- 'switchyard_latency_upstream_attempts_total{'
- 'requested_model="model-A",upstream_model="model-A",'
- 'outcome="retryable_error",code="500"} 1'
- ) in out
-
- async def test_route_model_preserved_as_requested_model_label(self):
- # Clients of a deployed route send the route key (e.g.
- # ``nvidia/switchyard/gpt-5.4``), never an endpoint id. The route id is
- # config-derived, so it joins the bounded label set instead of
- # collapsing to the ``"other"`` sentinel.
- config = LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[_ep("azure/openai/gpt-5.4")],
- route_model="nvidia/switchyard/gpt-5.4",
- )
- backend = _make_backend(config)
- backend._clients["azure/openai/gpt-5.4"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
- _set_health(backend, {"azure/openai/gpt-5.4": EndpointHealthStatus.HEALTHY})
-
- await backend.call(
- ProxyContext(), _openai_request(model="nvidia/switchyard/gpt-5.4")
- )
-
- out = "\n".join(backend._render_prometheus_lines())
- assert (
- 'switchyard_latency_upstream_attempts_total{'
- 'requested_model="nvidia/switchyard/gpt-5.4",'
- 'upstream_model="azure/openai/gpt-5.4",'
- 'outcome="success",code="200"} 1'
- ) in out
- assert 'requested_model="other"' not in out
-
- async def test_unknown_requested_model_normalized_to_sentinel(self):
- # A client-supplied model that is not a configured endpoint id must not
- # become a raw Prometheus label (unbounded cardinality); it collapses to
- # the ``"other"`` sentinel.
- backend = _make_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
-
- await backend.call(ProxyContext(), _openai_request()) # model="incoming-model"
-
- out = "\n".join(backend._render_prometheus_lines())
- assert 'requested_model="other"' in out
- assert 'requested_model="incoming-model"' not in out
-
- async def test_upstream_model_override_labels_the_upstream_name(self):
- config = LatencyServiceBackendConfig(
- latency_service_url=LATENCY_SERVICE_URL,
- endpoints=[LatencyServiceEndpoint(
- model="model-A",
- base_url="http://a.test",
- api_key="k",
- upstream_model="azure/openai/gpt-5.1-codex-mini",
- )],
- )
- backend = _make_backend(config)
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
-
- await backend.call(ProxyContext(), _openai_request())
-
- out = "\n".join(backend._render_prometheus_lines())
- assert 'upstream_model="azure/openai/gpt-5.1-codex-mini"' in out
-
- async def test_metric_absent_until_traffic(self):
- backend = _make_backend(_config("model-A"))
-
- out = "\n".join(backend._render_prometheus_lines())
-
- assert "switchyard_latency_upstream_attempts_total" not in out
-
-
-# ---------------------------------------------------------------------------
-# Failure-source annotation
-# ---------------------------------------------------------------------------
-
-
-class TestErrorSourceAnnotation:
- """The backend stamps the failure origin + upstream model on ctx."""
-
- async def test_http_failure_stamps_provider_source_and_upstream_model(self):
- backend = _make_backend(_config("model-A"))
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(401)
- )
-
- ctx = ProxyContext()
- with pytest.raises(openai.APIStatusError):
- await backend.call(ctx, _openai_request())
-
- assert ctx.metadata[CTX_ERROR_SOURCE] == "provider"
- assert ctx.metadata[CTX_UPSTREAM_MODEL] == "model-A"
- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401
-
- async def test_network_failure_stamps_provider_without_status(self):
- """A status-less upstream fault is still provider-originated, so the
- endpoint's 500 envelope can say so instead of implying an internal bug."""
- backend = _make_backend(_config("model-A"))
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=RuntimeError("connection reset")
- )
-
- ctx = ProxyContext()
- with pytest.raises(RuntimeError):
- await backend.call(ctx, _openai_request())
-
- assert ctx.metadata[CTX_ERROR_SOURCE] == "provider"
- assert ctx.metadata[CTX_UPSTREAM_MODEL] == "model-A"
- assert CTX_UPSTREAM_HTTP_STATUS not in ctx.metadata
-
- async def test_caller_required_rejection_is_switchyard_source(self):
- """The caller_required 401 rides the upstream-status channel but is
- Switchyard's own rejection — the source label must say so."""
- backend = _make_backend(
- _config("model-A", credential_policy="caller_required")
- )
-
- ctx = ProxyContext()
- with pytest.raises(PermissionError):
- await backend.call(ctx, _openai_request())
-
- assert ctx.metadata[CTX_ERROR_SOURCE] == "switchyard"
- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401
- assert CTX_UPSTREAM_MODEL not in ctx.metadata
-
- def test_translation_rejection_is_switchyard_source(self):
- """The translation invalid-value 400 is likewise Switchyard-originated."""
- from switchyard.lib.backends.latency_service_llm_backend import (
- _stash_invalid_request_error,
- )
-
- ctx = ProxyContext()
- _stash_invalid_request_error(
- ctx, ValueError("InvalidValue: unsupported message role 'api'")
- )
-
- assert ctx.metadata[CTX_ERROR_SOURCE] == "switchyard"
- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 400
-
-
-# ---------------------------------------------------------------------------
-# Readiness and shutdown
-# ---------------------------------------------------------------------------
-
-
-class TestReadinessAndShutdown:
- def test_not_ready_before_first_poll(self):
- backend = _make_backend(_config("model-A"))
- assert backend.is_ready() is False
-
- def test_ready_after_first_poll(self):
- backend = _make_backend(_config("model-A"))
- backend._poller._poll_count = 1
- assert backend.is_ready() is True
-
- async def test_shutdown_stops_poller(self):
- backend = _make_backend(_config("model-A"))
- await backend.shutdown()
- assert backend._poller._stop_event.is_set()
-
- async def test_shutdown_closes_shared_affinity_store(self):
- """Backend teardown releases the shared (L2) pin store's connections."""
-
- class _ClosableL2:
- def __init__(self) -> None:
- self.closed = False
-
- async def get(self, key: str) -> str | None:
- return None
-
- async def put(self, key: str, value: str) -> None:
- return None
-
- async def aclose(self) -> None:
- self.closed = True
-
- backend = _make_backend(_config("model-A", session_affinity=True))
- l2 = _ClosableL2()
- backend._affinity._l2 = l2
-
- await backend.shutdown()
- assert l2.closed is True
-
-
-# ---------------------------------------------------------------------------
-# Health poller
-# ---------------------------------------------------------------------------
-
-
-class TestHealthPoller:
- def test_poll_updates_cache(self):
- health_cache: dict[str, EndpointHealth] = {
- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),
- "model-B": EndpointHealth(EndpointHealthStatus.UNKNOWN),
- }
- poller = _make_poller(["model-A", "model-B"], health_cache)
-
- response_data = {
- "endpoint_health": {
- "model-A": {"status": "healthy", "last_latency_ms": 100.0},
- "model-B": {"status": "degraded", "last_latency_ms": 1500.0},
- }
- }
- _mock_health_response(poller, _health_response(200, json=response_data))
- _run_one_poll(poller)
-
- assert health_cache["model-A"].status == EndpointHealthStatus.HEALTHY
- assert health_cache["model-A"].last_latency_ms == 100.0
- assert health_cache["model-B"].status == EndpointHealthStatus.DEGRADED
- assert health_cache["model-B"].last_latency_ms == 1500.0
- assert poller.has_polled
-
- def test_poll_captures_null_latency(self):
- """``last_latency_ms`` is nullable in the spec; absence becomes None."""
- health_cache: dict[str, EndpointHealth] = {
- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),
- }
- poller = _make_poller(["model-A"], health_cache)
-
- response_data = {
- "endpoint_health": {
- "model-A": {"status": "healthy", "last_latency_ms": None},
- }
- }
- _mock_health_response(poller, _health_response(200, json=response_data))
- _run_one_poll(poller)
-
- assert health_cache["model-A"].status == EndpointHealthStatus.HEALTHY
- assert health_cache["model-A"].last_latency_ms is None
-
- def test_poll_failure_resets_to_unknown(self):
- """If the Latency Service is unreachable, all endpoints reset to UNKNOWN
- so the backend falls back to random routing rather than stale data."""
- health_cache: dict[str, EndpointHealth] = {
- "model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 100.0),
- "model-B": EndpointHealth(EndpointHealthStatus.DEGRADED, 800.0),
- }
- poller = _make_poller(["model-A", "model-B"], health_cache)
- _mock_health_response(poller, _health_response(500))
- _run_one_poll(poller)
-
- assert health_cache["model-A"].status == EndpointHealthStatus.UNKNOWN
- assert health_cache["model-A"].last_latency_ms is None
- assert health_cache["model-B"].status == EndpointHealthStatus.UNKNOWN
- assert health_cache["model-B"].last_latency_ms is None
- assert not poller.has_polled
-
- def test_poll_ignores_unknown_model_ids(self):
- health_cache: dict[str, EndpointHealth] = {
- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),
- }
- poller = _make_poller(["model-A"], health_cache)
-
- response_data = {
- "endpoint_health": {
- "model-A": {"status": "healthy", "last_latency_ms": 100.0},
- "model-UNKNOWN": {"status": "degraded", "last_latency_ms": 999.0},
- }
- }
- _mock_health_response(poller, _health_response(200, json=response_data))
- _run_one_poll(poller)
-
- assert health_cache["model-A"].status == EndpointHealthStatus.HEALTHY
- assert "model-UNKNOWN" not in health_cache
-
- def test_success_increments_polls_and_records_timestamp(self):
- """``/metrics`` reads these fields to expose poll-loop health."""
- health_cache: dict[str, EndpointHealth] = {
- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),
- }
- poller = _make_poller(["model-A"], health_cache)
- _mock_health_response(
- poller,
- _health_response(200, json={"endpoint_health": {
- "model-A": {"status": "healthy", "last_latency_ms": 50.0},
- }}),
- )
- _run_one_poll(poller)
-
- assert poller.poll_successes == 1
- assert poller.poll_failures == 0
- assert poller.last_poll_ok is True
- assert poller.seconds_since_last_success is not None
- assert poller.seconds_since_last_success >= 0
-
- def test_failure_increments_failures_and_leaves_age_unset(self):
- """Pre-success failure leaves ``seconds_since_last_success`` None so
- scrapers can distinguish "never polled" from "polled but stale"."""
- health_cache: dict[str, EndpointHealth] = {
- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),
- }
- poller = _make_poller(["model-A"], health_cache)
- _mock_health_response(poller, _health_response(500))
- _run_one_poll(poller)
-
- assert poller.poll_successes == 0
- assert poller.poll_failures == 1
- assert poller.last_poll_ok is False
- assert poller.seconds_since_last_success is None
-
- def test_stop_event_terminates_poller(self):
- health_cache: dict[str, EndpointHealth] = {
- "model-A": EndpointHealth(EndpointHealthStatus.UNKNOWN),
- }
- poller = _make_poller(["model-A"], health_cache, poll_interval_s=0.05)
- _mock_health_response(
- poller,
- _health_response(200, json={"endpoint_health": {
- "model-A": {"status": "healthy", "last_latency_ms": 50.0},
- }}),
- )
-
- poller.start()
- time.sleep(0.15)
- poller.stop()
- poller.join(timeout=2.0)
-
- assert not poller.is_alive()
- assert poller.has_polled
diff --git a/tests/test_latency_service_spans.py b/tests/test_latency_service_spans.py
deleted file mode 100644
index 09d7be99b..000000000
--- a/tests/test_latency_service_spans.py
+++ /dev/null
@@ -1,273 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Datadog routing spans/tags emitted by ``LatencyServiceLLMBackend``.
-
-The backend emits a ``switchyard.route_decision`` span around endpoint selection
-and a ``switchyard.upstream_attempt`` span around each upstream call. These tests
-install a fake tracer (so no ddtrace install is needed) and assert the tag set.
-"""
-
-from __future__ import annotations
-
-import time
-from typing import Any
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import httpx
-import openai
-import pytest
-from openai.types.chat import ChatCompletion
-from openai.types.chat.chat_completion import Choice
-from openai.types.chat.chat_completion_message import ChatCompletionMessage
-
-from switchyard.lib import tracing
-from switchyard.lib.backends.health_poller import (
- EndpointHealth,
- EndpointHealthStatus,
- HealthPoller,
-)
-from switchyard.lib.backends.latency_service_llm_backend import (
- LatencyServiceLLMBackend,
-)
-from switchyard.lib.config.latency_service_backend_config import (
- LatencyServiceBackendConfig,
- LatencyServiceEndpoint,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.core import ChatRequest
-
-# ---------------------------------------------------------------------------
-# Fake tracer (stands in for ddtrace so no install is required)
-# ---------------------------------------------------------------------------
-
-
-class _FakeSpan:
- def __init__(self, name: str) -> None:
- self.name = name
- self.tags: dict[str, Any] = {}
-
- def set_tag(self, key: str, value: Any) -> None:
- self.tags[key] = value
-
- def __enter__(self) -> _FakeSpan:
- return self
-
- def __exit__(self, *_args: object) -> bool:
- return False
-
-
-class _FakeTracer:
- def __init__(self) -> None:
- self.spans: list[_FakeSpan] = []
-
- def trace(self, name: str) -> _FakeSpan:
- span = _FakeSpan(name)
- self.spans.append(span)
- return span
-
- def named(self, name: str) -> list[_FakeSpan]:
- return [span for span in self.spans if span.name == name]
-
-
-def tracer(monkeypatch: pytest.MonkeyPatch) -> _FakeTracer:
- fake = _FakeTracer()
- monkeypatch.setattr(tracing, "_dd_tracer", fake)
- return fake
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _config(*models: str, **kwargs: Any) -> LatencyServiceBackendConfig:
- return LatencyServiceBackendConfig(
- latency_service_url="http://ls.test:8080",
- endpoints=[
- LatencyServiceEndpoint(model=m, base_url=f"http://{m}.test", api_key="k")
- for m in models
- ],
- **kwargs,
- )
-
-
-def _make_backend(config: LatencyServiceBackendConfig) -> LatencyServiceLLMBackend:
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient"
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock()
- with patch.object(HealthPoller, "start"):
- return LatencyServiceLLMBackend(config)
-
-
-def _set_health(
- backend: LatencyServiceLLMBackend,
- mapping: dict[str, EndpointHealthStatus | EndpointHealth],
-) -> None:
- with backend._cache_lock:
- for mid, value in mapping.items():
- if isinstance(value, EndpointHealthStatus):
- value = EndpointHealth(status=value)
- backend._health_cache[mid] = value
-
-
-def _completion(content: str = "ok") -> ChatCompletion:
- return ChatCompletion(
- id="chatcmpl-test",
- object="chat.completion",
- created=1700000000,
- model="m",
- choices=[
- Choice(
- index=0,
- message=ChatCompletionMessage(role="assistant", content=content),
- finish_reason="stop",
- )
- ],
- )
-
-
-def _request(**overrides: Any) -> ChatRequest:
- body: dict[str, Any] = {"model": "incoming", "messages": [{"role": "user", "content": "hi"}]}
- body.update(overrides)
- return ChatRequest.openai_chat(body) # type: ignore[arg-type]
-
-
-def _api_status_error(status_code: int) -> openai.APIStatusError:
- response = httpx.Response(
- status_code, request=httpx.Request("POST", "http://upstream.test/v1/chat/completions")
- )
- return openai.APIStatusError("upstream error", response=response, body=None)
-
-
-# ---------------------------------------------------------------------------
-# Tests
-# ---------------------------------------------------------------------------
-
-
-async def test_success_emits_route_and_attempt_spans(tracer: _FakeTracer) -> None:
- backend = _make_backend(_config("model-A"))
- _set_health(backend, {"model-A": EndpointHealth(EndpointHealthStatus.HEALTHY, 50.0)})
- backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion())
-
- await backend.call(ProxyContext(), _request(model="incoming"))
-
- route = tracer.named("switchyard.route_decision")
- attempt = tracer.named("switchyard.upstream_attempt")
- assert len(route) == 1
- assert len(attempt) == 1
-
- # poll-age is absent (poller never recorded a success) — None tags are skipped.
- assert route[0].tags == {
- "switchyard.model": "incoming",
- "switchyard.candidate_endpoints": "model-A",
- "switchyard.selected_endpoint": "model-A",
- "switchyard.was_fastest_selected": True,
- "switchyard.affinity_hit": False,
- }
-
- assert attempt[0].tags == {
- "switchyard.model": "incoming",
- "switchyard.selected_endpoint": "model-A",
- "switchyard.retry_count": 0,
- "switchyard.outcome": "success",
- "switchyard.upstream_status_code": 200,
- }
-
-
-async def test_poll_age_tag_present_after_a_poll(tracer: _FakeTracer) -> None:
- backend = _make_backend(_config("model-A"))
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
- backend._poller._last_success_at = time.monotonic()
- backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion())
-
- await backend.call(ProxyContext(), _request())
-
- route = tracer.named("switchyard.route_decision")[0]
- assert route.tags["switchyard.latency_service_poll_age_ms"] >= 0.0
-
-
-async def test_api_status_error_tags(tracer: _FakeTracer) -> None:
- backend = _make_backend(_config("model-A"))
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(429)
- )
-
- with pytest.raises(openai.APIStatusError):
- await backend.call(ProxyContext(), _request())
-
- attempt = tracer.named("switchyard.upstream_attempt")[0]
- assert attempt.tags["switchyard.upstream_status_code"] == 429
- assert attempt.tags["switchyard.outcome"] == "retryable_error"
- assert attempt.tags["switchyard.error_code"] == "429"
- # Failure-source annotation: the attempt failed upstream.
- assert attempt.tags["switchyard.error_source"] == "provider"
- assert attempt.tags["switchyard.upstream_model"] == "model-A"
-
-
-async def test_generic_error_tags(tracer: _FakeTracer) -> None:
- backend = _make_backend(_config("model-A"))
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
- backend._clients["model-A"].acompletion = AsyncMock(side_effect=RuntimeError("down"))
-
- with pytest.raises(RuntimeError, match="down"):
- await backend.call(ProxyContext(), _request())
-
- attempt = tracer.named("switchyard.upstream_attempt")[0]
- assert attempt.tags["switchyard.outcome"] == "retryable_error"
- assert attempt.tags["switchyard.error_code"] == "none"
- # A non-HTTP failure has no status line.
- assert "switchyard.upstream_status_code" not in attempt.tags
- # A network-level fault is still an upstream-side failure.
- assert attempt.tags["switchyard.error_source"] == "provider"
- assert attempt.tags["switchyard.upstream_model"] == "model-A"
-
-
-async def test_retry_count_is_sequential_and_ends_success(tracer: _FakeTracer) -> None:
- backend = _make_backend(_config("model-A", "model-B", max_retries=1))
- _set_health(backend, {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.HEALTHY,
- })
- backend._clients["model-A"].acompletion = AsyncMock(side_effect=RuntimeError("down"))
- backend._clients["model-B"].acompletion = AsyncMock(return_value=_completion())
-
- await backend.call(ProxyContext(), _request())
-
- attempts = tracer.named("switchyard.upstream_attempt")
- # retry_count is the 0-based attempt index, monotonically increasing.
- assert [a.tags["switchyard.retry_count"] for a in attempts] == list(range(len(attempts)))
- assert attempts[-1].tags["switchyard.outcome"] == "success"
-
-
-async def test_was_fastest_false_when_latency_unknown(tracer: _FakeTracer) -> None:
- backend = _make_backend(_config("model-A", "model-B"))
- # Both HEALTHY but no latency samples -> uniform pick, "fastest" undefined.
- _set_health(backend, {
- "model-A": EndpointHealthStatus.HEALTHY,
- "model-B": EndpointHealthStatus.HEALTHY,
- })
- selected = {"model-A", "model-B"}
- backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion())
- backend._clients["model-B"].acompletion = AsyncMock(return_value=_completion())
-
- await backend.call(ProxyContext(), _request())
-
- route = tracer.named("switchyard.route_decision")[0]
- assert route.tags["switchyard.was_fastest_selected"] is False
- assert route.tags["switchyard.selected_endpoint"] in selected
-
-
-async def test_call_works_without_tracer() -> None:
- # No ``tracer`` fixture: _dd_tracer is None (ddtrace absent in dev), so the
- # spans are no-ops and call() must proceed normally.
- backend = _make_backend(_config("model-A"))
- _set_health(backend, {"model-A": EndpointHealthStatus.HEALTHY})
- backend._clients["model-A"].acompletion = AsyncMock(return_value=_completion())
-
- ctx = ProxyContext()
- await backend.call(ctx, _request())
- assert ctx.selected_model == "model-A"
diff --git a/tests/test_latency_service_stats_metrics.py b/tests/test_latency_service_stats_metrics.py
deleted file mode 100644
index ec011c917..000000000
--- a/tests/test_latency_service_stats_metrics.py
+++ /dev/null
@@ -1,273 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""End-to-end gate that a latency-service-routed chain exposes ``/metrics``.
-
-Pins the fix for the production 404: the profile-backed latency service chain
-must contribute a stats response processor so Datadog / OTel scrapers do not
-see 404 on every scrape.
-"""
-
-from __future__ import annotations
-
-from unittest.mock import MagicMock, patch
-
-from fastapi.testclient import TestClient
-
-from switchyard.lib.backends.health_poller import HealthPoller
-from switchyard.lib.backends.latency_service_llm_backend import (
- LatencyServiceLLMBackend,
-)
-from switchyard.lib.config.latency_service_backend_config import (
- LatencyServiceBackendConfig,
- LatencyServiceEndpoint,
-)
-from switchyard.lib.processors.stats_request_processor import (
- StatsRequestProcessor,
-)
-from switchyard.lib.processors.stats_response_processor_accumulator import (
- StatsResponseProcessor,
-)
-from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard
-from switchyard.lib.stats_accumulator import StatsAccumulator
-from switchyard.server.switchyard_app import build_switchyard_app
-
-
-def _config(*, enable_stats: bool = True) -> LatencyServiceBackendConfig:
- return LatencyServiceBackendConfig(
- latency_service_url="http://latency-service.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model="model-A",
- api_key="test-key",
- base_url="http://llm.test/v1",
- ),
- ],
- enable_stats=enable_stats,
- )
-
-
-def _build_switchyard(
- config: LatencyServiceBackendConfig,
- *,
- stats_accumulator: StatsAccumulator | None = None,
-) -> ProfileSwitchyard:
- """Build a latency-service profile without starting real poller threads."""
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- return ProfileSwitchyard(
- LatencyServiceProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats_accumulator,
- enable_stats=config.enable_stats,
- )
- )
-
-
-# ---------------------------------------------------------------------------
-# Profile component assembly
-# ---------------------------------------------------------------------------
-
-
-def test_profile_wires_stats_processors_by_default() -> None:
- switchyard = _build_switchyard(_config())
-
- components = list(switchyard.iter_components())
-
- assert any(isinstance(p, StatsRequestProcessor) for p in components), (
- "expected StatsRequestProcessor in the latency-service profile"
- )
- assert any(isinstance(p, StatsResponseProcessor) for p in components), (
- "expected StatsResponseProcessor in the latency-service profile"
- )
-
-
-def test_profile_skips_stats_when_disabled() -> None:
- switchyard = _build_switchyard(_config(enable_stats=False))
- components = list(switchyard.iter_components())
-
- assert not any(isinstance(p, StatsRequestProcessor) for p in components)
- assert not any(isinstance(p, StatsResponseProcessor) for p in components)
-
-
-async def test_profile_shares_one_accumulator_across_components() -> None:
- """Response processor and backend must record into the same accumulator.
-
- The shared bucket is how ``/metrics`` ends up with both per-call counts
- (recorded by the backend) and token usage (recorded by the response
- processor) under the same model label. The Rust binding's
- ``accumulator`` getter returns a fresh Python wrapper each time, so
- we verify shared identity by writing through one surface and reading
- through the other.
- """
- switchyard = _build_switchyard(_config())
- components = list(switchyard.iter_components())
-
- stats_processor = next(
- p for p in components
- if isinstance(p, StatsResponseProcessor)
- )
- backend = next(
- p for p in components
- if isinstance(p, LatencyServiceLLMBackend)
- )
-
- # Write through the backend's accumulator handle, then read through the
- # response processor's accumulator handle. If construction wired
- # two separate accumulators, the read would not see the write.
- assert backend._stats is not None
- await backend._stats.record_success("model-A", 12.5)
-
- snapshot = await stats_processor.accumulator.snapshot()
- assert snapshot["total_requests"] == 1
- assert snapshot["models"]["model-A"]["calls"] == 1
-
- assert stats_processor.get_endpoint() is not None
-
-
-def test_profile_honors_externally_provided_stats_accumulator() -> None:
- shared = StatsAccumulator()
- switchyard = _build_switchyard(_config(), stats_accumulator=shared)
-
- stats_processor = next(
- p for p in switchyard.iter_components()
- if isinstance(p, StatsResponseProcessor)
- )
- # The processor's stats source must be the shared accumulator so an
- # externally-mounted StatsEndpoint surfaces the same data.
- assert stats_processor.get_endpoint() is not None
-
-
-# ---------------------------------------------------------------------------
-# End-to-end: build_switchyard_app exposes /metrics + /v1/stats
-# ---------------------------------------------------------------------------
-
-
-def _build_app(*, enable_stats: bool):
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- switchyard = ProfileSwitchyard(
- LatencyServiceProfileConfig.from_config(_config(enable_stats=enable_stats))
- .build()
- .with_runtime_components(enable_stats=enable_stats)
- )
- return build_switchyard_app(switchyard)
-
-
-def test_recipe_app_exposes_metrics_endpoint() -> None:
- """Regression: latency-service deployments must serve ``/metrics``.
-
- Before this fix, Datadog/OTel scrapers got 404 on every poll because the
- latency-service chain contributed no ``StatsEndpoint`` via
- :func:`build_switchyard_app`'s component iteration.
- """
- app = _build_app(enable_stats=True)
-
- with TestClient(app, raise_server_exceptions=False) as client:
- metrics = client.get("/metrics")
- stats = client.get("/v1/stats")
- routing_stats = client.get("/v1/routing/stats")
-
- assert metrics.status_code == 200, (
- f"/metrics should be 200 on a latency-service chain, got {metrics.status_code}"
- )
- assert metrics.headers["content-type"].startswith("text/plain")
- # Prometheus exposition includes the canonical counter family even at zero.
- assert "switchyard_requests_total" in metrics.text
-
- assert stats.status_code == 200
- assert routing_stats.status_code == 200
-
-
-def test_recipe_app_omits_metrics_when_stats_disabled() -> None:
- """``enable_stats=False`` keeps ``/metrics`` unmounted — opt-out is intentional."""
- app = _build_app(enable_stats=False)
-
- with TestClient(app, raise_server_exceptions=False) as client:
- metrics = client.get("/metrics")
- stats = client.get("/v1/stats")
-
- assert metrics.status_code == 404
- assert stats.status_code == 404
-
-
-# ---------------------------------------------------------------------------
-# YAML route-bundle path
-# ---------------------------------------------------------------------------
-
-
-def test_route_bundle_latency_service_exposes_metrics() -> None:
- """Deployment path (YAML bundle) must also surface ``/metrics``."""
- from switchyard.cli.route_bundle import build_route_bundle_table
-
- bundle = {
- "routes": {
- "ls-route": {
- "type": "latency_service",
- "latency_service_url": "http://latency-service.test:8080",
- "endpoints": [
- {
- "model": "model-A",
- "api_key": "test-key",
- "base_url": "http://llm.test/v1",
- },
- ],
- },
- },
- }
-
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- table = build_route_bundle_table(bundle)
- app = build_switchyard_app(table)
-
- with TestClient(app, raise_server_exceptions=False) as client:
- metrics = client.get("/metrics")
-
- assert metrics.status_code == 200
- assert "switchyard_requests_total" in metrics.text
-
-
-def test_route_bundle_latency_service_honors_enable_stats_false() -> None:
- """YAML ``enable_stats: false`` opts out of the metrics surface."""
- from switchyard.cli.route_bundle import build_route_bundle_table
-
- bundle = {
- "routes": {
- "ls-route": {
- "type": "latency_service",
- "latency_service_url": "http://latency-service.test:8080",
- "enable_stats": False,
- "endpoints": [
- {
- "model": "model-A",
- "api_key": "test-key",
- "base_url": "http://llm.test/v1",
- },
- ],
- },
- },
- }
-
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- table = build_route_bundle_table(bundle)
- app = build_switchyard_app(table)
-
- with TestClient(app, raise_server_exceptions=False) as client:
- metrics = client.get("/metrics")
-
- assert metrics.status_code == 404
diff --git a/tests/test_outcome_metrics.py b/tests/test_outcome_metrics.py
index 9cb27d23b..3d8397785 100644
--- a/tests/test_outcome_metrics.py
+++ b/tests/test_outcome_metrics.py
@@ -5,39 +5,33 @@
from __future__ import annotations
-import json
-import logging
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import openai
import pytest
from fastapi.testclient import TestClient
from openai.types.chat import ChatCompletion
from openai.types.chat.chat_completion import Choice
from openai.types.chat.chat_completion_message import ChatCompletionMessage
+from openai.types.completion_usage import CompletionUsage
-from switchyard.lib.backends.health_poller import (
- EndpointHealth,
- EndpointHealthStatus,
- HealthPoller,
-)
-from switchyard.lib.config.latency_service_backend_config import (
- LatencyServiceBackendConfig,
- LatencyServiceEndpoint,
-)
from switchyard.lib.endpoints import outcome_metrics
from switchyard.lib.endpoints.upstream_error import (
record_upstream_attempt_failure,
record_upstream_attempt_success,
)
-from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard
from switchyard.lib.proxy_context import (
CTX_UPSTREAM_ATTEMPTS_RECORDED,
CTX_UPSTREAM_HTTP_STATUS,
ProxyContext,
)
+from switchyard.lib.roles import LLMBackend
+from switchyard.lib.switchyard import Switchyard
from switchyard.server.switchyard_app import build_switchyard_app
-from switchyard_rust.core import ChatRequest, SwitchyardUpstreamError
+from switchyard_rust.core import (
+ ChatRequest,
+ ChatRequestType,
+ ChatResponse,
+ SwitchyardUpstreamError,
+)
+from switchyard_rust.translation import TranslationEngine
@pytest.fixture(autouse=True)
@@ -47,17 +41,6 @@ def _reset_counters():
outcome_metrics._reset_for_tests()
-def _latency_service_switchyard(
- config: LatencyServiceBackendConfig,
-) -> ProfileSwitchyard:
- """Build the latency-service profile-backed serving adapter."""
- return ProfileSwitchyard(
- LatencyServiceProfileConfig.from_config(config)
- .build()
- .with_runtime_components(enable_stats=config.enable_stats)
- )
-
-
# ---------------------------------------------------------------------------
# Classification
# ---------------------------------------------------------------------------
@@ -194,295 +177,6 @@ class TestRender:
)
-# ---------------------------------------------------------------------------
-# Latency-service backend wiring
-# ---------------------------------------------------------------------------
-
-
-def _config(*models: str) -> LatencyServiceBackendConfig:
- return LatencyServiceBackendConfig(
- latency_service_url="http://latency.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model=model,
- base_url=f"http://llm-{model}.test/v1",
- api_key="test-key",
- )
- for model in models
- ],
- max_retries=2,
- )
-
-
-def _build_backend(config: LatencyServiceBackendConfig):
- from switchyard.lib.backends.latency_service_llm_backend import (
- LatencyServiceLLMBackend,
- )
-
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"):
- return LatencyServiceLLMBackend(config)
-
-
-def _make_completion() -> ChatCompletion:
- return ChatCompletion(
- id="cmpl-test",
- object="chat.completion",
- created=1700000000,
- model="any",
- choices=[
- Choice(
- index=0,
- message=ChatCompletionMessage(role="assistant", content="ok"),
- finish_reason="stop",
- )
- ],
- )
-
-
-def _api_status_error(status_code: int) -> openai.APIStatusError:
- """Build a synthetic APIStatusError carrying the given status code.
-
- Mirrors what ``OpenAILLMClient.acompletion`` raises when the upstream
- HTTP call returns a non-2xx response.
- """
- import httpx
-
- response = httpx.Response(
- status_code,
- request=httpx.Request("POST", "http://llm.test/v1/chat/completions"),
- json={"error": {"message": "synthetic"}},
- )
- return openai.APIStatusError(
- "synthetic", response=response, body={"error": "synthetic"}
- )
-
-
-class TestBackendCounters:
- async def test_success_records_one_attempt_success(self) -> None:
- backend = _build_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion()
- )
-
- await backend.call(ProxyContext(), ChatRequest.openai_chat({
- "model": "model-A",
- "messages": [{"role": "user", "content": "hi"}],
- }))
-
- out = "\n".join(outcome_metrics.render_lines())
- assert 'switchyard_upstream_attempts_total{outcome="success",code="200"} 1' in out
- assert (
- 'switchyard_upstream_attempts_total{outcome="retryable_error",code="429"} 0'
- in out
- )
- assert "switchyard_router_retry_recovered_total 0" in out
-
- async def test_429_then_success_increments_recovered(self) -> None:
- """First attempt 429, retry succeeds — the steering signal we care about."""
- backend = _build_backend(_config("model-A", "model-B"))
- # Pin model-A as the sole HEALTHY endpoint so it is tried first
- # deterministically; otherwise selection is a random coin between two
- # UNKNOWN endpoints and the 429-then-recover path only fires by chance.
- with backend._cache_lock:
- backend._health_cache["model-A"] = EndpointHealth(
- status=EndpointHealthStatus.HEALTHY,
- )
- backend._health_cache["model-B"] = EndpointHealth(
- status=EndpointHealthStatus.UNKNOWN,
- )
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(429),
- )
- backend._clients["model-B"].acompletion = AsyncMock(
- return_value=_make_completion(),
- )
- with backend._cache_lock:
- backend._health_cache["model-A"] = EndpointHealth(
- EndpointHealthStatus.HEALTHY,
- 10.0,
- )
- backend._health_cache["model-B"] = EndpointHealth(
- EndpointHealthStatus.DEGRADED,
- 100.0,
- )
-
- await backend.call(ProxyContext(), ChatRequest.openai_chat({
- "model": "x",
- "messages": [{"role": "user", "content": "hi"}],
- }))
-
- out = "\n".join(outcome_metrics.render_lines())
- assert (
- 'switchyard_upstream_attempts_total{outcome="retryable_error",code="429"} 1'
- in out
- )
- assert 'switchyard_upstream_attempts_total{outcome="success",code="200"} 1' in out
- assert "switchyard_router_retry_recovered_total 1" in out
-
- async def test_401_does_not_count_as_retryable(self) -> None:
- """A 401 (bad key) is ``other_error`` and is not retried — fail fast."""
- backend = _build_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(401),
- )
-
- import pytest as _pytest
- with _pytest.raises(openai.APIStatusError):
- await backend.call(ProxyContext(), ChatRequest.openai_chat({
- "model": "x",
- "messages": [{"role": "user", "content": "hi"}],
- }))
-
- # A 4xx client error is deterministic, so the loop fails fast: exactly
- # one attempt, no failover retries.
- assert backend._clients["model-A"].acompletion.call_count == 1
- out = "\n".join(outcome_metrics.render_lines())
- assert (
- 'switchyard_upstream_attempts_total{outcome="other_error",code="401"} 1'
- in out
- )
- assert (
- 'switchyard_upstream_attempts_total{outcome="retryable_error",code="429"} 0'
- in out
- )
- assert "switchyard_router_retry_recovered_total 0" in out
-
- async def test_network_error_counts_as_retryable(self) -> None:
- """Non-HTTP exceptions (network, pre-status timeout) map to retryable_error."""
- backend = _build_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=RuntimeError("connection refused"),
- )
-
- import pytest as _pytest
- with _pytest.raises(RuntimeError):
- await backend.call(ProxyContext(), ChatRequest.openai_chat({
- "model": "x",
- "messages": [{"role": "user", "content": "hi"}],
- }))
-
- out = "\n".join(outcome_metrics.render_lines())
- assert (
- 'switchyard_upstream_attempts_total{outcome="retryable_error",code="none"} 3'
- in out
- )
-
- async def test_failure_emits_structured_error_log(
- self, caplog: pytest.LogCaptureFixture
- ) -> None:
- """Every failed attempt emits one per-event structured log for Loki.
-
- Single endpoint that always 429s — deterministic, unlike a
- multi-endpoint setup whose first pick is a random choice.
- """
- backend = _build_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(429),
- )
-
- with caplog.at_level(logging.WARNING, logger="switchyard.upstream_errors"):
- with pytest.raises(openai.APIStatusError):
- await backend.call(ProxyContext(), ChatRequest.openai_chat({
- "model": "x",
- "messages": [{"role": "user", "content": "hi"}],
- }))
-
- records = [
- json.loads(r.getMessage())
- for r in caplog.records
- if r.name == "switchyard.upstream_errors"
- ]
- # 3 attempts (1 + max_retries=2), all 429 → 3 structured records,
- # attempt numbers 1-based and increasing.
- assert len(records) == 3
- assert all(r["event"] == "upstream_attempt_failed" for r in records)
- assert all(r["status_code"] == 429 and r["code"] == "429" for r in records)
- assert all(r["model"] == "model-A" for r in records)
- assert [r["attempt"] for r in records] == [1, 2, 3]
-
-
-# ---------------------------------------------------------------------------
-# FastAPI middleware — client-side counter
-# ---------------------------------------------------------------------------
-
-
-class TestClientResponseMiddleware:
- def test_successful_chat_completion_counts_as_success(self) -> None:
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- switchyard = _latency_service_switchyard(
- LatencyServiceBackendConfig(
- latency_service_url="http://latency.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model="model-A",
- api_key="test-key",
- base_url="http://llm.test/v1",
- ),
- ],
- )
- )
- backend = next(
- component for component in switchyard.iter_components()
- if hasattr(component, "_clients")
- )
- backend._clients["model-A"].acompletion = AsyncMock(
- return_value=_make_completion(),
- )
-
- app = build_switchyard_app(switchyard)
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/chat/completions",
- json={
- "model": "model-A",
- "messages": [{"role": "user", "content": "hi"}],
- },
- )
- assert response.status_code == 200
- metrics = client.get("/metrics").text
-
- assert 'switchyard_client_responses_total{outcome="success"} 1' in metrics
- assert 'switchyard_client_responses_total{outcome="retryable_error"} 0' in metrics
-
- def test_metrics_route_is_not_counted_as_client_response(self) -> None:
- """Only the LLM routes feed the client outcome counter. Otherwise a
- scraper polling /metrics every 10s would inflate the success count."""
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- switchyard = _latency_service_switchyard(
- LatencyServiceBackendConfig(
- latency_service_url="http://latency.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model="model-A",
- api_key="test-key",
- base_url="http://llm.test/v1",
- ),
- ],
- )
- )
- app = build_switchyard_app(switchyard)
- with TestClient(app, raise_server_exceptions=False) as client:
- for _ in range(5):
- client.get("/metrics")
- client.get("/health")
- client.get("/v1/models")
- metrics = client.get("/metrics").text
-
- assert 'switchyard_client_responses_total{outcome="success"} 0' in metrics
-
-
# ---------------------------------------------------------------------------
# Endpoint-layer fallback — wires the upstream-attempt counter for backends
# (Rust native / passthrough / multi) that issue one attempt and can't reach
@@ -540,17 +234,71 @@ class TestEndpointUpstreamAttemptFallback:
assert f"{_upstream_count(out, 'success', '200')} 0" in out
assert f"{_upstream_count(out, 'retryable_error', 'none')} 0" in out
- async def test_latency_service_backend_sets_dedup_flag(self) -> None:
- """The latency-service backend claims attempt accounting on its ctx."""
- backend = _build_backend(_config("model-A"))
- backend._clients["model-A"].acompletion = AsyncMock(return_value=_make_completion())
- ctx = ProxyContext()
- await backend.call(ctx, ChatRequest.openai_chat({
- "model": "model-A",
- "messages": [{"role": "user", "content": "hi"}],
- }))
- assert ctx.metadata.get(CTX_UPSTREAM_ATTEMPTS_RECORDED) is True
- # It recorded exactly one attempt itself — and the flag would stop the
- # endpoint fallback from adding a second.
- out = "\n".join(outcome_metrics.render_lines())
- assert f"{_upstream_count(out, 'success', '200')} 1" in out
+
+# ---------------------------------------------------------------------------
+# Client-response middleware — only the /v1/* LLM routes feed the client
+# outcome counter; the operational routes (/metrics, /health, /v1/models) do
+# not, so a scraper polling them cannot inflate the success count.
+# ---------------------------------------------------------------------------
+
+
+class _CannedBackend(LLMBackend):
+ """Passthrough-style backend: returns one canned OpenAI completion per call."""
+
+ def supported_request_types(self) -> list[ChatRequestType]:
+ return [
+ ChatRequestType.OPENAI_CHAT,
+ ChatRequestType.OPENAI_RESPONSES,
+ ChatRequestType.ANTHROPIC,
+ ]
+
+ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
+ return ChatResponse.openai_completion(
+ ChatCompletion(
+ id="chatcmpl-test",
+ object="chat.completion",
+ created=1700000000,
+ model="mock-model",
+ choices=[
+ Choice(
+ index=0,
+ message=ChatCompletionMessage(role="assistant", content="hi"),
+ finish_reason="stop",
+ )
+ ],
+ usage=CompletionUsage(prompt_tokens=5, completion_tokens=3, total_tokens=8),
+ )
+ )
+
+
+def _canned_app() -> object:
+ return build_switchyard_app(
+ Switchyard(backend=_CannedBackend(), translator=TranslationEngine())
+ )
+
+
+class TestClientResponseMiddleware:
+ def test_successful_chat_completion_counts_as_success(self) -> None:
+ """A served /v1/chat/completions success increments the client outcome counter."""
+ with TestClient(_canned_app(), raise_server_exceptions=False) as client:
+ response = client.post(
+ "/v1/chat/completions",
+ json={"model": "mock-model", "messages": [{"role": "user", "content": "hi"}]},
+ )
+ assert response.status_code == 200
+
+ metrics = "\n".join(outcome_metrics.render_lines())
+ assert 'switchyard_client_responses_total{outcome="success"} 1' in metrics
+ assert 'switchyard_client_responses_total{outcome="retryable_error"} 0' in metrics
+
+ def test_operational_routes_are_not_counted_as_client_responses(self) -> None:
+ """Polling /metrics, /health, /v1/models must not feed the client counter —
+ otherwise a scraper on a fixed cadence would inflate the success count."""
+ with TestClient(_canned_app(), raise_server_exceptions=False) as client:
+ for _ in range(5):
+ client.get("/metrics")
+ client.get("/health")
+ client.get("/v1/models")
+
+ metrics = "\n".join(outcome_metrics.render_lines())
+ assert 'switchyard_client_responses_total{outcome="success"} 0' in metrics
diff --git a/tests/test_passthrough_profile_e2e.py b/tests/test_passthrough_profile_e2e.py
index 1e96a2878..3a42ee280 100644
--- a/tests/test_passthrough_profile_e2e.py
+++ b/tests/test_passthrough_profile_e2e.py
@@ -399,7 +399,7 @@ class TestPassthroughProfileBackendErrors:
class TestPassthroughProfileUpstreamAttemptCounters:
- """`switchyard_upstream_attempts_total` must populate for non-latency chains.
+ """`switchyard_upstream_attempts_total` must populate for passthrough chains.
The endpoint-layer fallback records one upstream attempt per request for
backends (here the Rust ``OpenAiPassthroughBackend``) that issue exactly
diff --git a/tests/test_redis_pin_store.py b/tests/test_redis_pin_store.py
index 49a7269d0..1ccf5fafa 100644
--- a/tests/test_redis_pin_store.py
+++ b/tests/test_redis_pin_store.py
@@ -14,11 +14,8 @@ from __future__ import annotations
import pytest
from pydantic import ValidationError
-from switchyard.lib.backends.latency_service_llm_backend import _build_affinity_l2
-from switchyard.lib.config.latency_service_backend_config import (
- LatencyServiceBackendConfig,
- LatencyServiceEndpoint,
-)
+from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig
+from switchyard.lib.profiles.escalation_router_profile_config import _build_affinity_l2
from switchyard.lib.redis_pin_store import RedisPinStore
@@ -86,15 +83,17 @@ async def test_aclose_before_first_use_is_noop() -> None:
# --- config validation + backend wiring -------------------------------------
-def _redis_config(**overrides: object) -> LatencyServiceBackendConfig:
+def _redis_config(**overrides: object) -> EscalationRouterConfig:
base: dict[str, object] = {
- "endpoints": [LatencyServiceEndpoint(model="m")],
- "session_affinity": True,
+ "strong": {"model": "strong-model"},
+ "weak": {"model": "weak-model"},
+ "judge": {"model": "judge-model"},
+ "fallback_target_on_evict": "weak",
"affinity_store": "redis",
"affinity_store_url": "redis://cache:6379/0",
}
base.update(overrides)
- return LatencyServiceBackendConfig(**base) # type: ignore[arg-type]
+ return EscalationRouterConfig(**base) # type: ignore[arg-type]
def test_redis_store_requires_url() -> None:
@@ -102,13 +101,13 @@ def test_redis_store_requires_url() -> None:
_redis_config(affinity_store_url=None)
-def test_redis_store_requires_session_affinity() -> None:
- with pytest.raises(ValidationError):
- _redis_config(session_affinity=False)
-
-
def test_memory_is_the_default_store() -> None:
- cfg = LatencyServiceBackendConfig(endpoints=[LatencyServiceEndpoint(model="m")])
+ cfg = EscalationRouterConfig(
+ strong={"model": "strong-model"},
+ weak={"model": "weak-model"},
+ judge={"model": "judge-model"},
+ fallback_target_on_evict="weak",
+ )
assert cfg.affinity_store == "memory"
assert _build_affinity_l2(cfg) is None
diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py
index b1a7e68a1..2f7365544 100644
--- a/tests/test_route_bundle.py
+++ b/tests/test_route_bundle.py
@@ -50,18 +50,6 @@ def _random_processor(chain: object) -> Any:
)
-def _latency_backend(chain: object) -> Any:
- from switchyard.lib.backends.latency_service_llm_backend import (
- LatencyServiceLLMBackend,
- )
-
- return next(
- component
- for component in chain.iter_components()
- if isinstance(component, LatencyServiceLLMBackend)
- )
-
-
class _NoopRequestProcessor:
async def process(self, _ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
return request
@@ -1486,115 +1474,15 @@ _AFFINITY_DET_ROUTE = {
"strong": {"model": "s", "api_key": "k", "base_url": "https://ls.test/v1"},
"weak": {"model": "w", "api_key": "k", "base_url": "https://ls.test/v1"},
}
-_AFFINITY_LAT_ROUTE = {
- "type": "latency_service",
- "latency_service_url": "http://ls.test:8080",
- "session_affinity": True,
- "affinity_max_sessions": 50,
- "endpoints": [
- {"model": "s", "api_key": "k", "base_url": "https://ls.test/v1"},
- {"model": "w", "api_key": "k", "base_url": "https://ls.test/v1"},
- ],
-}
[email protected]("route", [_AFFINITY_DET_ROUTE, _AFFINITY_LAT_ROUTE])
[email protected]("route", [_AFFINITY_DET_ROUTE])
def test_session_affinity_keys_accepted_by_route_bundle(route: dict[str, Any]) -> None:
- """session_affinity / affinity_max_sessions are valid keys on both route types."""
+ """session_affinity / affinity_max_sessions are valid keys on the deterministic route."""
table = build_route_bundle_table({"routes": {"r": route}})
assert isinstance(table, RouteTable)
-def test_latency_service_credential_policy_defaults_to_endpoint_keys() -> None:
- """Omitting credential_policy keeps the server-configured endpoint keys authoritative."""
- table = build_route_bundle_table({"routes": {"r": _AFFINITY_LAT_ROUTE}})
-
- backend = _latency_backend(table.lookup_switchyard("r"))
-
- assert backend._config.credential_policy == "configured_endpoint"
-
-
-def test_latency_service_credential_policy_reaches_backend_config() -> None:
- """YAML credential_policy opts into BYO-key caller overrides."""
- table = build_route_bundle_table({
- "routes": {
- "r": {
- **_AFFINITY_LAT_ROUTE,
- "credential_policy": "caller_override",
- },
- },
- })
-
- backend = _latency_backend(table.lookup_switchyard("r"))
-
- assert backend._config.credential_policy == "caller_override"
-
-
-def test_latency_service_invalid_credential_policy_rejected_via_bundle() -> None:
- """Invalid latency-service credential policy values fail closed."""
- from pydantic import ValidationError
-
- with pytest.raises(ValidationError):
- build_route_bundle_table({
- "routes": {
- "r": {
- **_AFFINITY_LAT_ROUTE,
- "credential_policy": "caller-overrides",
- },
- },
- })
-
-
-def test_latency_endpoint_request_type_reaches_backend_config() -> None:
- """Endpoint-level request_type in YAML selects the upstream API surface."""
- table = build_route_bundle_table({
- "routes": {
- "r": {
- "type": "latency_service",
- "latency_service_url": "http://ls.test:8080",
- "endpoints": [
- {
- "model": "codex-mini",
- "api_key": "k",
- "base_url": "https://ls.test/v1",
- "request_type": "openai_responses",
- },
- {"model": "w", "api_key": "k", "base_url": "https://ls.test/v1"},
- ],
- },
- },
- })
-
- backend = _latency_backend(table.lookup_switchyard("r"))
-
- by_model = {endpoint.model: endpoint for endpoint in backend._config.endpoints}
- assert by_model["codex-mini"].request_type == "openai_responses"
- assert by_model["w"].request_type == "openai_chat"
-
-
-def test_latency_route_key_reaches_backend_as_route_model() -> None:
- """The YAML route key becomes the backend's metrics route_model id."""
- table = build_route_bundle_table({
- "routes": {
- "nvidia/switchyard/gpt-5.4": {
- "type": "latency_service",
- "latency_service_url": "http://ls.test:8080",
- "endpoints": [
- {
- "model": "azure/openai/gpt-5.4",
- "api_key": "k",
- "base_url": "https://ls.test/v1",
- },
- ],
- },
- },
- })
-
- backend = _latency_backend(table.lookup_switchyard("nvidia/switchyard/gpt-5.4"))
-
- assert backend._config.route_model == "nvidia/switchyard/gpt-5.4"
-
-
def test_deterministic_affinity_warmup_turns_accepted_by_route_bundle() -> None:
"""affinity_warmup_turns is a deterministic-route knob."""
table = build_route_bundle_table({
@@ -1603,7 +1491,7 @@ def test_deterministic_affinity_warmup_turns_accepted_by_route_bundle() -> None:
assert isinstance(table, RouteTable)
[email protected]("route", [_AFFINITY_DET_ROUTE, _AFFINITY_LAT_ROUTE])
[email protected]("route", [_AFFINITY_DET_ROUTE])
def test_zero_capacity_affinity_rejected_via_bundle(route: dict[str, Any]) -> None:
"""A zero cap with affinity on is rejected — proving the keys reach the config."""
from pydantic import ValidationError
@@ -1612,37 +1500,6 @@ def test_zero_capacity_affinity_rejected_via_bundle(route: dict[str, Any]) -> No
build_route_bundle_table({"routes": {"r": {**route, "affinity_max_sessions": 0}}})
-def test_latency_affinity_redis_reaches_backend_via_bundle() -> None:
- """Redis L2 keys parse and construct a RedisPinStore on the latency backend."""
- from switchyard.lib.redis_pin_store import RedisPinStore
-
- table = build_route_bundle_table({
- "routes": {
- "r": {
- **_AFFINITY_LAT_ROUTE,
- "affinity_store": "redis",
- "affinity_store_url": "redis://cache:6379/0",
- "affinity_store_ttl_seconds": 120,
- "affinity_key_prefix": "k:",
- },
- },
- })
- backend = _latency_backend(table.lookup_switchyard("r"))
- assert backend._config.affinity_store == "redis"
- assert backend._config.affinity_store_url == "redis://cache:6379/0"
- assert isinstance(backend._affinity._l2, RedisPinStore)
-
-
-def test_latency_affinity_redis_requires_url_via_bundle() -> None:
- """affinity_store=redis without a URL fails closed at config load."""
- from pydantic import ValidationError
-
- with pytest.raises(ValidationError):
- build_route_bundle_table({
- "routes": {"r": {**_AFFINITY_LAT_ROUTE, "affinity_store": "redis"}},
- })
-
-
def test_negative_affinity_warmup_turns_rejected_via_bundle() -> None:
"""The deterministic config validates affinity_warmup_turns as non-negative."""
from pydantic import ValidationError
diff --git a/tests/test_route_selection_headers.py b/tests/test_route_selection_headers.py
index 350498f46..4b1dabacd 100644
--- a/tests/test_route_selection_headers.py
+++ b/tests/test_route_selection_headers.py
@@ -1,43 +1,22 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""End-to-end gate for route-selection spend-attribution headers.
+"""Route-selection spend-attribution response headers.
For tokenomics reporting, a front proxy (e.g. LiteLLM) must be able to tie
its provider spend-log rows back to the Switchyard logical route that
-selected them:
-
-* outbound — every upstream attempt carries an ``x-litellm-spend-logs-metadata``
- request header whose JSON payload records the route selection plus a
- per-request correlation id;
-* inbound — the Switchyard response returns ``x-switchyard-*`` headers with
- the successful attempt's selection and the same correlation id, so the
- parent spend-log row can be enriched to match the provider row.
-
-The app-level tests run the real ``OpenAILLMClient`` against a loopback HTTP
-stub, so the outbound header is asserted on the wire — not on a mock.
+selected them: the Switchyard response returns ``x-switchyard-*`` headers with
+the successful attempt's selection plus a per-request correlation id, so the
+parent spend-log row can be enriched to match the provider row.
"""
from __future__ import annotations
import json
-from collections.abc import Iterator
from typing import Any
-from fastapi import FastAPI
from fastapi.responses import JSONResponse
-from fastapi.testclient import TestClient
-from pytest import fixture
-from pytest_mock import MockerFixture
-from switchyard.lib.backends.health_poller import HealthPoller
-from switchyard.lib.backends.latency_service_llm_backend import (
- SPEND_LOGS_METADATA_HEADER,
-)
-from switchyard.lib.config.latency_service_backend_config import (
- LatencyServiceBackendConfig,
- LatencyServiceEndpoint,
-)
from switchyard.lib.endpoints.dispatch import serialize_chain_result
from switchyard.lib.endpoints.route_selection import (
ROUTER_CORRELATION_ID_HEADER,
@@ -47,10 +26,7 @@ from switchyard.lib.endpoints.route_selection import (
route_selection_headers,
)
from switchyard.lib.endpoints.upstream_error import handle_chain_exception
-from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard
from switchyard.lib.proxy_context import CTX_ROUTE_SELECTION, ProxyContext
-from switchyard.server.switchyard_app import build_switchyard_app
-from tests._chain_test_helpers import _backend_payload, _OpenAICompatStub, _sse_body, _stream_chunk
ROUTE_MODEL = "nvidia/switchyard/test-route"
ENDPOINT_ID = "openai/test-model"
@@ -58,10 +34,9 @@ UPSTREAM_MODEL = "openai/openai/test-model"
def _selection(**overrides: object) -> dict[str, object]:
- """Selection record as the latency backend writes it, with *overrides* applied."""
+ """Recorded route selection, with *overrides* applied."""
payload: dict[str, object] = {
"router_model": ROUTE_MODEL,
- "router_strategy": "latency",
"router_selected_endpoint": ENDPOINT_ID,
"router_selected_model": UPSTREAM_MODEL,
"router_selected_provider": "openai",
@@ -256,240 +231,3 @@ class TestErrorPathSelectionHeaders:
assert response.status_code == 500
assert ROUTER_CORRELATION_ID_HEADER not in response.headers
-
-
-# ---------------------------------------------------------------------------
-# End-to-end: HTTP in → wire header out → response headers back
-# ---------------------------------------------------------------------------
-
-
-@fixture
-def latency_app(mocker: MockerFixture) -> Iterator[tuple[FastAPI, _OpenAICompatStub]]:
- """Latency-service app whose single endpoint targets a loopback stub.
-
- The backend uses the real ``OpenAILLMClient``/OpenAI SDK, so the stub
- records the actual HTTP headers the upstream would receive. Only the
- health poller is stubbed out (no Latency Service in tests).
- """
- with _OpenAICompatStub() as stub:
- config = LatencyServiceBackendConfig(
- latency_service_url="http://latency-service.test:8080",
- route_model=ROUTE_MODEL,
- endpoints=[
- LatencyServiceEndpoint(
- model=ENDPOINT_ID,
- upstream_model=UPSTREAM_MODEL,
- base_url=stub.base_url,
- api_key="test-key",
- ),
- ],
- )
- mocker.patch.object(HealthPoller, "start")
- mocker.patch.object(HealthPoller, "stop")
- switchyard = ProfileSwitchyard(
- LatencyServiceProfileConfig.from_config(config)
- .build()
- .with_runtime_components(enable_stats=config.enable_stats)
- )
- yield build_switchyard_app(switchyard), stub
-
-
-def _wire_spend_logs_payload(stub: _OpenAICompatStub) -> dict[str, object]:
- """Parse the spend-logs metadata header the stub received on the wire."""
- headers = stub.requests[-1]["headers"]
- assert isinstance(headers, dict)
- payload = json.loads(str(headers[SPEND_LOGS_METADATA_HEADER]))
- assert isinstance(payload, dict)
- return payload
-
-
-def test_chat_completion_roundtrips_selection_and_correlation_id(
- latency_app: tuple[FastAPI, _OpenAICompatStub],
-) -> None:
- """The wire spend-logs header and the response headers share one selection."""
- app, stub = latency_app
- stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL))
-
- with TestClient(app) as client:
- response = client.post(
- "/v1/chat/completions",
- json={
- "model": ROUTE_MODEL,
- "messages": [{"role": "user", "content": "hi"}],
- },
- )
-
- assert response.status_code == 200
- payload = _wire_spend_logs_payload(stub)
- assert payload["router_model"] == ROUTE_MODEL
- assert payload["router_strategy"] == "latency"
- assert payload["router_selected_endpoint"] == ENDPOINT_ID
- assert payload["router_selected_model"] == UPSTREAM_MODEL
- assert payload["router_selected_provider"] == "openai"
- assert response.headers[ROUTER_MODEL_HEADER] == ROUTE_MODEL
- assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL
- assert response.headers[SELECTED_PROVIDER_HEADER] == "openai"
- # The join key: the response header must equal the id the provider row got.
- assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"]
-
-
-def test_streaming_chat_completion_carries_selection_headers(
- latency_app: tuple[FastAPI, _OpenAICompatStub],
-) -> None:
- """A streaming completion returns the same selection headers before the body."""
- app, stub = latency_app
- stub.respond_sse(_sse_body([_stream_chunk(content="hel"), _stream_chunk(finish="stop")]))
-
- with TestClient(app) as client:
- response = client.post(
- "/v1/chat/completions",
- json={
- "model": ROUTE_MODEL,
- "messages": [{"role": "user", "content": "hi"}],
- "stream": True,
- },
- )
-
- assert response.status_code == 200
- assert response.headers["content-type"].startswith("text/event-stream")
- assert "data:" in response.text
- payload = _wire_spend_logs_payload(stub)
- assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"]
- assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL
-
-
-def test_anthropic_messages_endpoint_carries_selection_headers(
- latency_app: tuple[FastAPI, _OpenAICompatStub],
-) -> None:
- """Anthropic inbound rides the same dispatch path and gets the same headers."""
- app, stub = latency_app
- stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL))
-
- with TestClient(app) as client:
- response = client.post(
- "/v1/messages",
- json={
- "model": ROUTE_MODEL,
- "max_tokens": 64,
- "messages": [{"role": "user", "content": "hi"}],
- },
- )
-
- assert response.status_code == 200
- payload = _wire_spend_logs_payload(stub)
- assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"]
- assert response.headers[SELECTED_PROVIDER_HEADER] == "openai"
-
-
-def test_responses_endpoint_carries_selection_headers(
- latency_app: tuple[FastAPI, _OpenAICompatStub],
-) -> None:
- """OpenAI Responses inbound rides the same dispatch path and gets the same headers."""
- app, stub = latency_app
- stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL))
-
- with TestClient(app) as client:
- response = client.post(
- "/v1/responses",
- json={"model": ROUTE_MODEL, "input": "hi"},
- )
-
- assert response.status_code == 200
- payload = _wire_spend_logs_payload(stub)
- assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"]
- assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL
-
-
-def test_upstream_error_response_has_no_selection_headers(
- latency_app: tuple[FastAPI, _OpenAICompatStub],
-) -> None:
- """No upstream call succeeded — the error response must not claim a selection.
-
- The failed attempt itself is still stamped on the wire (the provider row
- for the failure keeps its attribution); the client-facing headers are
- reserved for a recorded selection, i.e. a billed upstream success — which
- never happened here. (A failure *after* a billed success does carry them;
- see TestErrorPathSelectionHeaders.)
- """
- app, stub = latency_app
- stub.respond_json({"error": {"message": "bad key"}}, status=401)
-
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/chat/completions",
- json={
- "model": ROUTE_MODEL,
- "messages": [{"role": "user", "content": "hi"}],
- },
- )
-
- assert response.status_code == 401
- assert ROUTER_CORRELATION_ID_HEADER not in response.headers
- assert SPEND_LOGS_METADATA_HEADER in stub.requests[-1]["headers"] # type: ignore[operator]
-
-
-def test_hostile_model_string_never_breaks_the_response(
- latency_app: tuple[FastAPI, _OpenAICompatStub],
-) -> None:
- """A model string that is not legal header material must not 500 or inject.
-
- Single-chain profile serving forwards any client model string, so
- router_model is client-controlled; the billed 200 must survive, with the
- unsafe header skipped and the config-derived headers intact.
- """
- app, stub = latency_app
-
- with TestClient(app) as client:
- for hostile_model in ("gpt-4中文", "gpt\r\nx-evil: 1"):
- stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL))
- response = client.post(
- "/v1/chat/completions",
- json={
- "model": hostile_model,
- "messages": [{"role": "user", "content": "hi"}],
- },
- )
-
- assert response.status_code == 200
- assert ROUTER_MODEL_HEADER not in response.headers
- assert "x-evil" not in response.headers
- assert response.headers[SELECTED_MODEL_HEADER] == UPSTREAM_MODEL
- # The outbound header is JSON (ensure_ascii escapes control and
- # non-ASCII chars), so the wire payload still records the model.
- assert _wire_spend_logs_payload(stub)["router_model"] == hostile_model
-
-
-def test_client_body_extra_headers_forwarded_not_collided(
- latency_app: tuple[FastAPI, _OpenAICompatStub],
-) -> None:
- """A passthrough body carrying an SDK-style extra_headers field keeps working.
-
- Pre-spend-logs it rode ``**body`` into the SDK's header kwarg; it must
- still reach the wire — and must not be able to spoof the spend-logs
- header under any key casing, since header names are case-insensitive on
- the wire.
- """
- app, stub = latency_app
- stub.respond_json(_backend_payload(content="hello", model=UPSTREAM_MODEL))
-
- with TestClient(app) as client:
- response = client.post(
- "/v1/chat/completions",
- json={
- "model": ROUTE_MODEL,
- "messages": [{"role": "user", "content": "hi"}],
- "extra_headers": {
- "x-client-tag": "42",
- SPEND_LOGS_METADATA_HEADER: "spoofed",
- "X-LiteLLM-Spend-Logs-Metadata": "spoofed-cased",
- },
- },
- )
-
- assert response.status_code == 200
- headers = stub.requests[-1]["headers"]
- assert isinstance(headers, dict)
- assert headers["x-client-tag"] == "42"
- payload = _wire_spend_logs_payload(stub)
- assert payload["router_selected_endpoint"] == ENDPOINT_ID # not "spoofed"
- assert response.headers[ROUTER_CORRELATION_ID_HEADER] == payload["router_correlation_id"]
diff --git a/tests/test_session_affinity.py b/tests/test_session_affinity.py
index 26cd88975..d06a8ac57 100644
--- a/tests/test_session_affinity.py
+++ b/tests/test_session_affinity.py
@@ -3,7 +3,7 @@
"""Unit tests for SessionAffinity's L1 (in-process) + optional L2 (shared) tiering.
-The latency backend and classifier router integration tests cover affinity
+The classifier and escalation router integration tests cover affinity
*policy*; these tests cover the store mechanics in isolation: write-through,
read-through with L1 warming, and best-effort (fail-open) degradation when the
L2 errors.
diff --git a/tests/test_stream_close_chain.py b/tests/test_stream_close_chain.py
index 134844998..81b6e085e 100644
--- a/tests/test_stream_close_chain.py
+++ b/tests/test_stream_close_chain.py
@@ -3,7 +3,7 @@
"""Close ownership survives the Python -> Rust-core -> Python round trip.
-The latency-router production path is: a Python backend returns
+The production path is: a Python backend returns
``ChatResponse.openai_stream(ResponseStream(sdk_stream))``; the backend adapter
converts it to a Rust-core stream via ``take_core``; response processors run in
core; the terminal translator rebuilds a Python ``ChatResponseStream`` via
diff --git a/tests/test_stream_leak_repro.py b/tests/test_stream_leak_repro.py
index ec00b6f98..22c1f571c 100644
--- a/tests/test_stream_leak_repro.py
+++ b/tests/test_stream_leak_repro.py
@@ -195,7 +195,7 @@ async def test_our_close_releases_pool_connection() -> None:
class _RealSdkStreamingBackend(LLMBackend):
"""Backend that issues a real streaming SDK call and wraps the AsyncStream.
- Mirrors the latency-service production shape: ``call`` returns
+ Mirrors the production shape: ``call`` returns
``ChatResponse.openai_stream(ResponseStream(sdk_stream))`` where
``sdk_stream`` is a genuine OpenAI ``AsyncStream`` holding a pooled httpx
connection. Running this through ``Switchyard`` exercises the real
diff --git a/tests/test_upstream_error_passthrough.py b/tests/test_upstream_error_passthrough.py
index e2be77f59..5952906da 100644
--- a/tests/test_upstream_error_passthrough.py
+++ b/tests/test_upstream_error_passthrough.py
@@ -19,99 +19,23 @@ Switchyard error envelope instead of FastAPI's default plain-text 500.
from __future__ import annotations
import json
-from collections.abc import AsyncIterator
-from unittest.mock import AsyncMock, MagicMock, patch
-import httpx
-import openai
-import pytest
-from fastapi import FastAPI
from fastapi.testclient import TestClient
from switchyard.cli.route_bundle import build_route_bundle_table
-from switchyard.lib.backends.health_poller import HealthPoller
-from switchyard.lib.backends.latency_service_llm_backend import (
- LatencyServiceLLMBackend,
-)
-from switchyard.lib.config.latency_service_backend_config import (
- LatencyServiceBackendConfig,
- LatencyServiceEndpoint,
-)
-from switchyard.lib.endpoints.error_envelope import (
- ERROR_SOURCE_HEADER,
- UPSTREAM_MODEL_HEADER,
-)
+from switchyard.lib.endpoints.error_envelope import ERROR_SOURCE_HEADER
from switchyard.lib.endpoints.upstream_error import (
internal_chain_error_response,
upstream_response_from_ctx,
)
-from switchyard.lib.profiles import LatencyServiceProfileConfig, ProfileSwitchyard
from switchyard.lib.proxy_context import (
CTX_UPSTREAM_HTTP_BODY,
CTX_UPSTREAM_HTTP_STATUS,
ProxyContext,
)
from switchyard.server.switchyard_app import build_switchyard_app
-from switchyard_rust.core import ChatRequest
from tests._chain_test_helpers import _OpenAICompatStub
-# ---------------------------------------------------------------------------
-# Helper
-# ---------------------------------------------------------------------------
-
-
-def _api_status_error(status: int, body: object = None) -> openai.APIStatusError:
- """Build an ``openai.APIStatusError`` with a realistic response object."""
- request = httpx.Request("POST", "http://upstream.test/v1/chat/completions")
- response = httpx.Response(
- status_code=status,
- json=body if body is not None else {"error": {"message": f"upstream {status}"}},
- request=request,
- )
- return openai.APIStatusError(
- message=f"upstream returned {status}",
- response=response,
- body=body,
- )
-
-
-def _latency_service_switchyard(
- config: LatencyServiceBackendConfig,
-) -> ProfileSwitchyard:
- """Build the latency-service profile-backed serving adapter."""
- return ProfileSwitchyard(
- LatencyServiceProfileConfig.from_config(config)
- .build()
- .with_runtime_components(enable_stats=config.enable_stats)
- )
-
-
-def _make_backend() -> LatencyServiceLLMBackend:
- config = LatencyServiceBackendConfig(
- latency_service_url="http://latency-service.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model="model-A",
- api_key="bad-key",
- base_url="http://llm.test/v1",
- ),
- ],
- )
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"):
- return LatencyServiceLLMBackend(config)
-
-
-def _openai_request() -> ChatRequest:
- return ChatRequest.openai_chat({
- "model": "model-A",
- "messages": [{"role": "user", "content": "hi"}],
- })
-
-
# ---------------------------------------------------------------------------
# Helper unit tests
# ---------------------------------------------------------------------------
@@ -228,266 +152,11 @@ class TestUpstreamResponseFromCtx:
assert len(body["error"]["message"]) <= 200
-# ---------------------------------------------------------------------------
-# Backend records status on ctx before raising
-# ---------------------------------------------------------------------------
-
-
-class TestLatencyServiceBackendStashesStatus:
- async def test_401_recorded_on_ctx_before_raise(self) -> None:
- backend = _make_backend()
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(401, {"error": {"message": "bad key"}}),
- )
-
- ctx = ProxyContext()
- with pytest.raises(openai.APIStatusError):
- await backend.call(ctx, _openai_request())
-
- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 401
- body = ctx.metadata[CTX_UPSTREAM_HTTP_BODY]
- assert isinstance(body, dict)
- assert body["error"]["message"] == "bad key"
-
- async def test_429_recorded_on_ctx_before_raise(self) -> None:
- backend = _make_backend()
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(429, {"error": {"message": "slow down"}}),
- )
-
- ctx = ProxyContext()
- with pytest.raises(openai.APIStatusError):
- await backend.call(ctx, _openai_request())
-
- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 429
-
- async def test_non_http_error_leaves_ctx_clean(self) -> None:
- """Network errors etc shouldn't trip the upstream-status passthrough."""
- backend = _make_backend()
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=RuntimeError("connection refused"),
- )
-
- ctx = ProxyContext()
- with pytest.raises(RuntimeError, match="connection refused"):
- await backend.call(ctx, _openai_request())
-
- assert CTX_UPSTREAM_HTTP_STATUS not in ctx.metadata
- assert CTX_UPSTREAM_HTTP_BODY not in ctx.metadata
-
-
-# ---------------------------------------------------------------------------
-# End-to-end: HTTP request through the recipe → upstream 401 → client 401
-# ---------------------------------------------------------------------------
-
-
-async def latency_service_app() -> AsyncIterator[tuple[FastAPI, LatencyServiceLLMBackend]]:
- """Build a latency-service-backed FastAPI app sharing one backend instance.
-
- Returns ``(app, backend)`` so each test can patch ``backend._clients[...].acompletion``
- to simulate a specific upstream response.
- """
- config = LatencyServiceBackendConfig(
- latency_service_url="http://latency-service.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model="model-A",
- api_key="bad-key",
- base_url="http://llm.test/v1",
- ),
- ],
- )
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- switchyard = _latency_service_switchyard(config)
- app = build_switchyard_app(switchyard)
- # Reach into the chain to find the LatencyServiceLLMBackend
- # instance so tests can stub the upstream call.
- backend = _find_latency_backend(switchyard)
- yield app, backend
-
-
-def _find_latency_backend(switchyard: object) -> LatencyServiceLLMBackend:
- iter_components = getattr(switchyard, "iter_components", None)
- assert iter_components is not None
- for component in iter_components():
- # The latency-service backend may be wrapped (it isn't today, but
- # be tolerant) — match by isinstance, with attribute-walk fallback.
- if isinstance(component, LatencyServiceLLMBackend):
- return component
- inner = getattr(component, "_inner", None) or getattr(component, "inner", None)
- if isinstance(inner, LatencyServiceLLMBackend):
- return inner
- raise AssertionError("LatencyServiceLLMBackend not found in chain")
-
-
-def test_upstream_401_passes_through_as_401(
- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],
-) -> None:
- """A 401 from upstream must become a 401 to the client, not a 500."""
- app, backend = latency_service_app
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(
- 401, {"error": {"message": "bad key", "type": "invalid_api_key"}},
- ),
- )
-
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/chat/completions",
- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},
- )
-
- assert response.status_code == 401, (
- f"expected upstream 401 to pass through; got {response.status_code}, "
- f"body={response.text!r}"
- )
- body = response.json()
- assert body["error"]["message"] == "bad key"
- assert body["error"]["type"] == "invalid_api_key"
- assert body["error"]["code"] == "invalid_api_key"
-
-
-def test_upstream_429_passes_through_as_429(
- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],
-) -> None:
- app, backend = latency_service_app
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(
- 429, {"error": {"message": "rate limit", "type": "rate_limit_exceeded"}},
- ),
- )
-
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/chat/completions",
- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},
- )
-
- assert response.status_code == 429
- assert response.json() == {
- "error": {
- "message": "rate limit",
- "type": "rate_limit_exceeded",
- "code": "rate_limit_exceeded",
- }
- }
-
-
-def test_non_http_error_returns_wrapped_500(
- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],
-) -> None:
- """Errors without an upstream HTTP status still return a JSON error envelope."""
- app, backend = latency_service_app
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=RuntimeError("connection refused"),
- )
-
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/chat/completions",
- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},
- )
-
- assert response.status_code == 500
- assert response.headers["content-type"].startswith("application/json")
- body = response.json()
- assert body["error"]["type"] == "internal_error"
- assert body["error"]["code"] == "internal_chain_error"
- assert "connection refused" in body["error"]["message"]
-
-
-def test_post_dispatch_exception_returns_json_500(
- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],
-) -> None:
- """Exception raised after dispatch (e.g. during result serialization) must not
- fall through to FastAPI's plain-text 500 handler."""
- app, _ = latency_service_app
-
- class _UnserializableResult:
- def model_dump(self) -> None:
- raise RuntimeError("serialization exploded")
-
- with patch(
- "switchyard.lib.endpoints.openai_chat_endpoint.dispatch_chat_request",
- new_callable=AsyncMock,
- return_value=_UnserializableResult(),
- ):
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/chat/completions",
- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},
- )
-
- assert response.status_code == 500
- assert response.headers["content-type"].startswith("application/json")
- body = response.json()
- assert body["error"]["type"] == "internal_error"
- assert body["error"]["code"] == "internal_chain_error"
- assert "serialization exploded" in body["error"]["message"]
-
-
# ---------------------------------------------------------------------------
# Anthropic and Responses endpoints share the same helper
# ---------------------------------------------------------------------------
-def test_anthropic_endpoint_passes_through_upstream_401(
- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],
-) -> None:
- app, backend = latency_service_app
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(401, {"error": {"message": "bad key"}}),
- )
-
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/messages",
- json={
- "model": "model-A",
- "max_tokens": 16,
- "messages": [{"role": "user", "content": "ping"}],
- },
- )
-
- assert response.status_code == 401
- assert response.json() == {
- "error": {
- "message": "bad key",
- "type": "upstream_error",
- "code": "upstream_error",
- }
- }
-
-
-def test_responses_endpoint_passes_through_upstream_401(
- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],
-) -> None:
- app, backend = latency_service_app
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(401, {"error": {"message": "bad key"}}),
- )
-
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/responses",
- json={"model": "model-A", "input": "ping"},
- )
-
- assert response.status_code == 401
- assert response.json() == {
- "error": {
- "message": "bad key",
- "type": "upstream_error",
- "code": "upstream_error",
- }
- }
-
-
def test_rust_openai_route_upstream_401_returns_structured_openai_error() -> None:
"""Rust OpenAI-native backend errors must not fall through to FastAPI's 500."""
with _OpenAICompatStub() as upstream:
@@ -609,111 +278,6 @@ def test_rust_openai_route_upstream_401_returns_same_error_shape_for_responses_i
}
-# ---------------------------------------------------------------------------
-# Invalid request payloads
-# ---------------------------------------------------------------------------
-# A transparent router must reject the same payloads the upstream provider
-# would. An unsupported message role (e.g. "api") was previously coerced to
-# "user" and returned 200; it now surfaces as a provider-compatible 400 via the
-# same ctx-stash → endpoint-passthrough mechanism used for upstream HTTP errors.
-
-
-class TestInvalidRoleRejection:
- async def test_invalid_role_stashes_400_without_upstream_call(self) -> None:
- """An unsupported role is rejected at translation, before any upstream call."""
- backend = _make_backend()
- backend._clients["model-A"].acompletion = AsyncMock()
-
- ctx = ProxyContext()
- request = ChatRequest.openai_responses({
- "model": "model-A",
- "input": [{"type": "message", "role": "api", "content": "hi"}],
- })
- with pytest.raises(ValueError):
- await backend.call(ctx, request)
-
- assert ctx.metadata[CTX_UPSTREAM_HTTP_STATUS] == 400
- body = ctx.metadata[CTX_UPSTREAM_HTTP_BODY]
- assert isinstance(body, dict)
- assert body["error"]["code"] == "invalid_value"
- # The upstream must never be called for an invalid payload.
- backend._clients["model-A"].acompletion.assert_not_awaited()
-
- async def test_valid_role_passes_translation_and_reaches_upstream(self) -> None:
- """A valid role must not be over-rejected — translation succeeds and the
- request proceeds to the upstream call (no invalid-value 400 stashed)."""
- backend = _make_backend()
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=RuntimeError("reached upstream"),
- )
-
- ctx = ProxyContext()
- request = ChatRequest.openai_responses({
- "model": "model-A",
- "input": [{"type": "message", "role": "user", "content": "hi"}],
- })
- with pytest.raises(RuntimeError, match="reached upstream"):
- await backend.call(ctx, request)
-
- assert CTX_UPSTREAM_HTTP_STATUS not in ctx.metadata
-
-
-def test_responses_invalid_role_returns_400(
- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],
-) -> None:
- """role:"api" on the Responses endpoint must return 400, not a coerced 200/500."""
- app, backend = latency_service_app
- # No upstream stub: the request must be rejected before any upstream call.
- backend._clients["model-A"].acompletion = AsyncMock()
-
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/responses",
- json={
- "model": "model-A",
- "input": [{"type": "message", "role": "api", "content": "ping"}],
- },
- )
-
- assert response.status_code == 400, (
- f"expected invalid role to be rejected with 400; got {response.status_code}, "
- f"body={response.text!r}"
- )
- body = response.json()
- assert body["error"]["code"] == "invalid_value"
- backend._clients["model-A"].acompletion.assert_not_awaited()
-
-
-def test_anthropic_invalid_role_returns_400(
- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],
-) -> None:
- """role:"api" on the Anthropic endpoint is rejected with a 400.
-
- The Anthropic inbound format must be translated to OpenAI Chat for the
- latency backend, so the request is decoded and the unsupported role is
- rejected before any upstream call. (A native OpenAI-Chat request is a
- passthrough for this backend, so its contract is enforced upstream.)
- """
- app, backend = latency_service_app
- backend._clients["model-A"].acompletion = AsyncMock()
-
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/messages",
- json={
- "model": "model-A",
- "max_tokens": 16,
- "messages": [{"role": "api", "content": "ping"}],
- },
- )
-
- assert response.status_code == 400, (
- f"expected invalid role to be rejected with 400; got {response.status_code}, "
- f"body={response.text!r}"
- )
- backend._clients["model-A"].acompletion.assert_not_awaited()
-
-
# ---------------------------------------------------------------------------
# Failure-source headers on the wire
# ---------------------------------------------------------------------------
@@ -722,71 +286,6 @@ def test_anthropic_invalid_role_returns_400(
# the HTTP response a client receives.
-def test_upstream_401_header_labels_provider_on_the_wire(
- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],
-) -> None:
- app, backend = latency_service_app
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=_api_status_error(401, {"error": {"message": "bad key"}}),
- )
-
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/chat/completions",
- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},
- )
-
- assert response.status_code == 401
- assert response.headers[ERROR_SOURCE_HEADER] == "provider"
- assert response.headers[UPSTREAM_MODEL_HEADER] == "model-A"
- # The provider body still passes through unmodified — annotation is header-only.
- assert response.json()["error"]["message"] == "bad key"
-
-
-def test_network_failure_500_header_labels_provider_on_the_wire(
- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],
-) -> None:
- """A status-less upstream fault renders as the internal 500 envelope but
- the header says provider — clients can tell it wasn't a Switchyard bug."""
- app, backend = latency_service_app
- backend._clients["model-A"].acompletion = AsyncMock(
- side_effect=RuntimeError("connection refused"),
- )
-
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/chat/completions",
- json={"model": "model-A", "messages": [{"role": "user", "content": "ping"}]},
- )
-
- assert response.status_code == 500
- assert response.json()["error"]["code"] == "internal_chain_error"
- assert response.headers[ERROR_SOURCE_HEADER] == "provider"
- assert response.headers[UPSTREAM_MODEL_HEADER] == "model-A"
-
-
-def test_invalid_role_400_header_labels_switchyard_on_the_wire(
- latency_service_app: tuple[FastAPI, LatencyServiceLLMBackend],
-) -> None:
- """The translation rejection rides the upstream-status channel but must
- surface as switchyard-originated on the wire."""
- app, backend = latency_service_app
- backend._clients["model-A"].acompletion = AsyncMock()
-
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/responses",
- json={
- "model": "model-A",
- "input": [{"type": "message", "role": "api", "content": "ping"}],
- },
- )
-
- assert response.status_code == 400
- assert response.json()["error"]["code"] == "invalid_value"
- assert response.headers[ERROR_SOURCE_HEADER] == "switchyard"
-
-
def test_model_not_found_404_header_labels_switchyard_on_the_wire() -> None:
"""RouteTable dispatch 404s carry the switchyard source header."""
with _OpenAICompatStub() as upstream:
@@ -816,232 +315,3 @@ def test_model_not_found_404_header_labels_switchyard_on_the_wire() -> None:
assert response.status_code == 404
assert response.json()["error"]["code"] == "model_not_found"
assert response.headers[ERROR_SOURCE_HEADER] == "switchyard"
-
-
-def test_caller_required_401_header_labels_switchyard_on_the_wire() -> None:
- """The caller_required rejection is Switchyard's own 401: same status as a
- provider auth failure, distinguishable by the source header on the wire."""
- config = LatencyServiceBackendConfig(
- latency_service_url="http://latency-service.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model="model-A",
- api_key="svc-key", # pragma: allowlist secret
- base_url="http://llm.test/v1",
- ),
- ],
- credential_policy="caller_required",
- )
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- app = build_switchyard_app(_latency_service_switchyard(config))
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/chat/completions",
- json={
- "model": "model-A",
- "messages": [{"role": "user", "content": "ping"}],
- },
- )
-
- assert response.status_code == 401
- assert response.json()["error"]["code"] == "missing_caller_api_key"
- assert response.headers[ERROR_SOURCE_HEADER] == "switchyard"
- assert UPSTREAM_MODEL_HEADER not in response.headers
-
-
-# ---------------------------------------------------------------------------
-# Responses API fidelity on the wire
-# ---------------------------------------------------------------------------
-
-
-def test_responses_body_returned_exactly_on_the_wire() -> None:
- """A Responses request through the full app returns the upstream JSON
- byte-for-byte: provider extras and explicit-null fields included.
-
- This is the composed fidelity proof — backend raw wrap, the
- terminal translation short-circuit, and endpoint serialization together.
- """
- upstream_body = {
- "id": "resp-e2e",
- "object": "response",
- "created_at": 1719890000,
- "model": "model-A",
- "status": "completed",
- "output": [
- {
- "type": "message",
- "id": "msg-1",
- "role": "assistant",
- "status": "completed",
- "content": [{"type": "output_text", "text": "hello", "annotations": []}],
- }
- ],
- "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5},
- "store": False,
- "temperature": 1.0,
- "top_p": 0.9,
- "text": {"format": {"type": "text"}},
- "reasoning": {"effort": None, "summary": None},
- "previous_response_id": None,
- "provider_meta": {"azure_region": "eastus"},
- }
- config = LatencyServiceBackendConfig(
- latency_service_url="http://latency-service.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model="model-A",
- api_key="k",
- base_url="http://llm.test/v1",
- request_type="openai_responses",
- ),
- ],
- )
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- switchyard = _latency_service_switchyard(config)
- backend = _find_latency_backend(switchyard)
- backend._clients["model-A"].aresponses = AsyncMock(
- return_value=dict(upstream_body)
- )
- app = build_switchyard_app(switchyard)
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/responses",
- json={"model": "model-A", "input": "ping"},
- )
-
- assert response.status_code == 200, response.text
- assert response.json() == upstream_body
-
-
-def _latency_responses_app(frames: list[str]) -> FastAPI:
- """Full app over one ``openai_responses`` endpoint streaming *frames*."""
- config = LatencyServiceBackendConfig(
- latency_service_url="http://latency-service.test:8080",
- endpoints=[
- LatencyServiceEndpoint(
- model="model-A",
- api_key="k",
- base_url="http://llm.test/v1",
- request_type="openai_responses",
- ),
- ],
- )
- with patch(
- "switchyard.lib.backends.latency_service_llm_backend.OpenAILLMClient",
- ) as mock_cls:
- mock_cls.side_effect = lambda **kw: MagicMock(name=f"client-{kw.get('base_url')}")
- with patch.object(HealthPoller, "start"), patch.object(HealthPoller, "stop"):
- switchyard = _latency_service_switchyard(config)
- backend = _find_latency_backend(switchyard)
-
- async def _frame_iter() -> AsyncIterator[str]:
- for frame in frames:
- yield frame
-
- backend._clients["model-A"].aresponses = AsyncMock(
- return_value=_frame_iter()
- )
- return build_switchyard_app(switchyard)
-
-
-def test_responses_stream_forwarded_verbatim_on_the_wire() -> None:
- """A same-format streaming Responses request returns the upstream SSE
- frames byte-for-byte: unknown provider fields, explicit nulls, event
- names, and comment keep-alives all survive (streaming fidelity leg)."""
- frames = [
- (
- 'event: response.created\n'
- 'data: {"type":"response.created","response":{"id":"resp_s1","store":false,'
- '"temperature":1.0,"reasoning":{"effort":null,"summary":null}}}\n\n'
- ),
- ": keep-alive\n\n",
- (
- 'event: response.output_text.delta\n'
- 'data: {"type":"response.output_text.delta","delta":"hello","vendor_extra":1}\n\n'
- ),
- (
- 'event: response.completed\n'
- 'data: {"type":"response.completed","response":{"id":"resp_s1","usage":'
- '{"input_tokens":3,"output_tokens":2}}}\n\n'
- ),
- ]
- app = _latency_responses_app(frames)
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/responses",
- json={"model": "model-A", "input": "ping", "stream": True},
- )
-
- assert response.status_code == 200, response.text
- assert response.text == "".join(frames)
-
-
-def test_azure_flavored_responses_stream_preserved_on_the_wire() -> None:
- """Azure-shaped Responses events (content-filter annotations and other
- provider extras) pass through the streaming path unmodified."""
- frames = [
- (
- 'event: response.output_text.delta\n'
- 'data: {"type":"response.output_text.delta","delta":"hi",'
- '"content_filter_results":{"hate":{"filtered":false,"severity":"safe"}}}\n\n'
- ),
- (
- 'event: response.completed\n'
- 'data: {"type":"response.completed","response":{"id":"resp_az",'
- '"prompt_filter_results":[{"prompt_index":0}],"usage":'
- '{"input_tokens":1,"output_tokens":1},"previous_response_id":null}}\n\n'
- ),
- ]
- app = _latency_responses_app(frames)
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/responses",
- json={"model": "model-A", "input": "ping", "stream": True},
- )
-
- assert response.status_code == 200, response.text
- assert response.text == "".join(frames)
-
-
-def test_responses_raw_frames_translate_for_chat_client_on_the_wire() -> None:
- """Cross-format regression: a chat client served by a Responses endpoint
- still gets translated chat chunks — raw frame strings are parsed back into
- events for the translator instead of passing through."""
- frames = [
- (
- 'event: response.created\n'
- 'data: {"type":"response.created","response":{"id":"resp_x","model":"m"}}\n\n'
- ),
- (
- 'event: response.output_text.delta\n'
- 'data: {"type":"response.output_text.delta","delta":"hello"}\n\n'
- ),
- (
- 'event: response.completed\n'
- 'data: {"type":"response.completed","response":{"id":"resp_x","usage":'
- '{"input_tokens":3,"output_tokens":2}}}\n\n'
- ),
- ]
- app = _latency_responses_app(frames)
- with TestClient(app, raise_server_exceptions=False) as client:
- response = client.post(
- "/v1/chat/completions",
- json={
- "model": "model-A",
- "messages": [{"role": "user", "content": "ping"}],
- "stream": True,
- },
- )
-
- assert response.status_code == 200, response.text
- assert "chat.completion.chunk" in response.text
- assert "hello" in response.text
- assert "response.output_text.delta" not in response.text