Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(python): remove legacy routing profiles (#268)

nachiketb-nvidiaf30498d31b6d
diff --git a/.agents/skills/switchyard-stage-router-scorer/SKILL.md b/.agents/skills/switchyard-stage-router-scorer/SKILL.md
index 46c3f3c5c..b11cea0e8 100644
--- a/.agents/skills/switchyard-stage-router-scorer/SKILL.md
+++ b/.agents/skills/switchyard-stage-router-scorer/SKILL.md
@@ -34,12 +34,12 @@ trajectory step (tool_use + tool_result)
→ dc.process(ctx, request) # DimensionCollector — one per task, accumulates state
→ get_tool_result_signal(ctx) # read signal from ctx
→ stage_score_signal(signal) # raw (score, confidence) from the Rust scorer (for analysis)
- → pick_capable_first(ctx, threshold) # actual cf decision (same ctx, no re-process)
- → pick_efficient_first(ctx, threshold) # actual ef decision (same ctx, no re-process)
+ → stage_pick_tier(signal, "capable_first", threshold) # actual cf decision
+ → stage_pick_tier(signal, "efficient_first", threshold) # actual ef decision
```
-**Key:** `dc.process()` is called **once per turn** on a single ctx. Both pickers read the signal
-already stored in that ctx — no duplicate processing.
+**Key:** `dc.process()` is called **once per turn** on a single context. Both picker modes use the
+same extracted signal, with no duplicate processing.
**What the picker does beyond raw score:**
- **escalate** (`should_escalate`): `compacted` OR `severity >= 1.0` → force CAPABLE
@@ -78,7 +78,7 @@ opus_pct_cf, nemotron_pct_cf, opus_pct_ef, nemotron_pct_ef`
|------|-------------|------------|------------|
| strong_clear | ≥ T | Opus | Opus |
| strong_uncertain | (0, T) | Opus (fall_open) | Nemotron (fall_open) |
-| weak_uncertain | (-T, 0) | Nemotron (fall_open) | Nemotron (fall_open) |
+| weak_uncertain | (-T, 0) | Opus (fall_open) | Nemotron (fall_open) |
| weak_clear | ≤ -T | Nemotron | Nemotron |
Overrides can change any band. Use `pick_cf`/`pick_ef` for the true decision.
@@ -92,6 +92,6 @@ Overrides can change any band. Use `pick_cf`/`pick_ef` for the true decision.
## Anti-patterns
-- Don't call `dc.process()` multiple times per turn for different pickers — both pickers read from the same ctx. One `dc.process()` call per turn is correct.
+- Don't call `dc.process()` multiple times per turn for different picker modes. Extract one signal per turn and reuse it.
- Don't infer routing split from score bands alone — overrides and fall_open change the actual decision. Always use `pick_cf` / `pick_ef`.
- Don't compare cost directly across configs: Nemotron has ~39% cache hit rate vs ~92% for Opus.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index c67be6dec..bcbb7eed7 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -12,11 +12,11 @@ What happened? One or two sentences.
## Reproduction
-Minimal steps to reproduce. Include the command line, the inbound request shape, and the configured recipe.
+Minimal steps to reproduce. Include the command line, the inbound request shape, and the configured route.
```bash
# example
-switchyard serve --config profiles.yaml --port 4000
+switchyard serve --routes examples/route.yaml --port 4000
curl -s http://localhost:4000/v1/chat/completions -d '{"model":"...","messages":[...]}'
```
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4e96fa177..67e3ae3b9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -185,9 +185,6 @@ jobs:
uv run --isolated --no-project --python 3.14 \
--with "${SWITCHYARD_DEFAULT_PACKAGE}" \
python -c "import switchyard; print('import OK, version:', switchyard.__version__)"
- uv run --isolated --no-project --python 3.12 \
- --with "${SWITCHYARD_DEFAULT_PACKAGE}" \
- python -c "from switchyard import RandomRoutingPresets; print('preset import OK:', sorted(RandomRoutingPresets.PRESETS))"
- name: Assert heavy packages absent from slim install
working-directory: /tmp
run: |
diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml
index 59524629f..cd797d567 100644
--- a/.github/workflows/perf.yml
+++ b/.github/workflows/perf.yml
@@ -56,7 +56,7 @@ jobs:
# ------------------------------------------------------------------
# Start a local zero-latency mock OpenAI upstream. This replaces the
- # removed noop route: the proxy serves a real `type: model` chain that
+ # The proxy serves a real `type: passthrough` chain that
# forwards to this loopback stub, which returns a fixed completion
# instantly. The extra loopback hop adds a small constant overhead.
# ------------------------------------------------------------------
@@ -127,7 +127,7 @@ jobs:
echo "STUB_PID=$!" >> "$GITHUB_ENV"
# ------------------------------------------------------------------
- # Start the proxy: a `type: model` route pointed at the local stub.
+ # Start the proxy: a `type: passthrough` route pointed at the local stub.
# ------------------------------------------------------------------
- name: Start switchyard proxy
run: |
@@ -138,10 +138,10 @@ jobs:
format: openai
routes:
mock-model:
- type: model
- model: mock-model
+ type: passthrough
+ target: mock-model
YAML
- uv run switchyard serve --routing-profiles bench.yaml --port $PROXY_PORT &
+ uv run switchyard serve --routes bench.yaml --port $PROXY_PORT &
echo "PROXY_PID=$!" >> "$GITHUB_ENV"
- name: Wait for proxy to be ready
diff --git a/AGENTS.md b/AGENTS.md
index 3a542c3d1..cc046923e 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -167,9 +167,7 @@ switchyard/
│ ├── roles.py # Python LLMBackend re-export and translation aliases
│ ├── switchyard.py # Switchyard — chain executor
│ ├── proxy_context.py # ProxyContext — per-request state carrier
-│ ├── profiles/ # Profile configs/runtimes for pre-built routing behavior
│ ├── route_table.py # RouteTable — model-id dispatch to runnable chains
-│ ├── route_table_builders.py # Shared profile-backed table builders
│ ├── llm_client.py # OpenAILLMClient
│ ├── cost_estimator.py # Token-cost bookkeeping
│ ├── stats_accumulator.py # Stats accumulation helpers
@@ -189,7 +187,6 @@ switchyard/
│ │ └── backend_format_resolver.py # BackendFormatResolver
│ ├── processors/ # Request-side / response-side component implementations
│ │ ├── format_translate.py
-│ │ ├── random_routing_request_processor.py
│ │ ├── stats_request_processor.py
│ │ └── stats_response_processor_accumulator.py
│ ├── endpoints/ # FastAPI endpoint wrappers (require `nemo-switchyard[server]`)
@@ -245,8 +242,8 @@ and their transitives never appear in downstream vulnerability scans.
```bash
export OPENROUTER_API_KEY="sk-or-..."
-# Serve a routing bundle. Routes live in YAML; see docs/routing_algorithms/overview.md.
-switchyard serve --routing-profiles routes.yaml --port 4000
+# Serve the minimal Python YAML bundle (noop and passthrough only).
+switchyard serve --routes examples/route.yaml --port 4000
# Launch against the packaged OpenRouter deployment.
switchyard launch claude --model switchyard
@@ -264,7 +261,7 @@ uv run pytest tests/ -v
# Single test file / function
uv run pytest tests/test_switchyard.py -v
-uv run pytest tests/test_random_routing_llm_backend.py::test_server_config -v
+uv run pytest tests/test_route_bundle.py::test_noop_route_returns_ok_without_an_upstream -v
# Live end-to-end tests are not part of the public test suite; if you write
# one, set the provider key explicitly and run it directly, e.g.:
@@ -280,7 +277,7 @@ uv run mypy switchyard
1. Pick the right stage: request component (pre-call), response component (post-call), `LLMBackend` (rare), or Rust translation codec work.
2. Create a file with the explicit name (`snake_case` of the class name), one class per file.
3. Implement the async method for that stage (`process` for components, `call` for backends).
-4. Wire into the owning programmatic profile config or route-bundle builder.
+4. Wire it into the owning explicit chain or route-bundle builder.
5. Add tests under `tests/`.
6. Export from the relevant `__init__.py` and from `switchyard/__init__.py`'s `__all__`.
@@ -297,16 +294,24 @@ class MyRequestComponent:
return request
```
-### Profiles and app factory
+### App factory
```python
-from switchyard import PassthroughProfileConfig, ProfileSwitchyard, build_switchyard_app
+from switchyard import BackendFormat, LlmTarget, Switchyard, TranslationEngine
+from switchyard.lib.backends import OpenAiNativeBackend
+from switchyard import build_switchyard_app
import uvicorn
-switchyard = ProfileSwitchyard(PassthroughProfileConfig(
+target = LlmTarget(
+ model="gpt-4o",
+ format=BackendFormat.OPENAI,
api_key="sk-...",
base_url="https://api.openai.com/v1",
-).build())
+)
+switchyard = Switchyard(
+ backend=OpenAiNativeBackend(target),
+ translator=TranslationEngine(),
+)
uvicorn.run(build_switchyard_app(switchyard), port=4000)
```
diff --git a/INSTALLATION.md b/INSTALLATION.md
index 99298cdb4..b63ae8d58 100644
--- a/INSTALLATION.md
+++ b/INSTALLATION.md
@@ -111,13 +111,20 @@ pip install nemo-switchyard[all]
Embed Switchyard with minimal overhead:
```python
-from switchyard import PassthroughProfileConfig, ProfileSwitchyard
+from switchyard import LlmTarget, OpenAiNativeBackend, Switchyard, TranslationEngine
# Core library only — no server/CLI dependencies
-switchyard = ProfileSwitchyard(PassthroughProfileConfig(
+target = LlmTarget(
+ id="direct",
+ model="gpt-4o-mini",
+ format="openai",
api_key="sk-...",
base_url="https://api.openai.com/v1",
-).build())
+)
+switchyard = Switchyard(
+ backend=OpenAiNativeBackend(target),
+ translator=TranslationEngine(),
+)
```
## Troubleshooting
diff --git a/benchmark/score_staged_run.py b/benchmark/score_staged_run.py
index 7bfafba78..6e4c9a2f6 100644
--- a/benchmark/score_staged_run.py
+++ b/benchmark/score_staged_run.py
@@ -4,13 +4,13 @@
"""Score a benchmark run directory via the stage-router Rust scorer.
Reads trajectory.json from each completed task, feeds each tool-use turn through:
- DimensionCollector.process(ctx, request) → pick_capable_first / pick_efficient_first
+ DimensionCollector.process(ctx, request) → stage_pick_tier
The picker functions replicate live routing exactly: overrides, scorer, and fall_open.
Usage:
- uv run python benchmark/score_run.py --run benchmark/tb_runs/<run-name>
- uv run python benchmark/score_run.py --run benchmark/tb_runs/<run-name> \\
+ uv run python benchmark/score_staged_run.py --run benchmark/tb_runs/<run-name>
+ uv run python benchmark/score_staged_run.py --run benchmark/tb_runs/<run-name> \\
--output /tmp/scores.jsonl --threshold 0.20 --window 3
"""
import argparse
@@ -22,19 +22,16 @@ from collections import defaultdict
from pathlib import Path
from statistics import mean
-from switchyard.lib.processors.stage_router.picker import (
- CAPABLE,
- pick_capable_first,
- pick_efficient_first,
-)
from switchyard_rust.components import (
DimensionCollector,
get_tool_result_signal,
+ stage_pick_tier,
stage_score_signal,
)
from switchyard_rust.core import ChatRequest, ProxyContext
RECENT_WINDOW = 3
+CAPABLE = 1
def _tool_use_id(message: str) -> str:
@@ -119,10 +116,13 @@ async def score_trajectory(
# histogram shows the capable/efficient separation directly.
score, confidence = stage_score_signal(signal)
- # Actual picker decisions on the same ctx — both just read the signal,
- # no extra dc.process() calls needed
- tier_cf = await pick_capable_first(ctx, confidence_threshold)
- tier_ef = await pick_efficient_first(ctx, confidence_threshold)
+ # Apply both picker modes to the same signal without reprocessing the turn.
+ outcome_cf = stage_pick_tier(signal, "capable_first", confidence_threshold)
+ outcome_ef = stage_pick_tier(signal, "efficient_first", confidence_threshold)
+ tier_cf_name = outcome_cf.tier if outcome_cf.resolved else outcome_cf.default_tier
+ tier_ef_name = outcome_ef.tier if outcome_ef.resolved else outcome_ef.default_tier
+ tier_cf = int(tier_cf_name == "capable")
+ tier_ef = int(tier_ef_name == "capable")
rows.append({
"task_name": task_name,
diff --git a/crates/switchyard-components/src/backends/multi.rs b/crates/switchyard-components/src/backends/multi.rs
index 8f6482c4c..0180adea4 100644
--- a/crates/switchyard-components/src/backends/multi.rs
+++ b/crates/switchyard-components/src/backends/multi.rs
@@ -6,7 +6,7 @@
//! This backend owns the mechanical part of routing once a processor or caller
//! has selected a target: rewrite the request model, stamp typed context, and
//! delegate to the configured backend for that target. Selection policy stays
-//! outside this type; processors and profiles decide which target should run.
+//! outside this type; processors and algorithms decide which target should run.
use std::collections::HashSet;
use std::fmt;
diff --git a/crates/switchyard-components/src/stage_router.rs b/crates/switchyard-components/src/stage_router.rs
index a2dc048b6..04a845030 100644
--- a/crates/switchyard-components/src/stage_router.rs
+++ b/crates/switchyard-components/src/stage_router.rs
@@ -7,7 +7,7 @@
//! Its input, a [`ToolResultSignal`][crate::dimension_collector::ToolResultSignal],
//! is already libsy's `ToolSignals`, so no request adaptation is needed here — this
//! module simply re-exports the API so the crate's processors decide a turn's tier
-//! through the same implementation the libsy profile uses.
+//! through the same implementation as the libsy algorithm.
pub use switchyard_libsy::{
CodingAgentDimensions, DecisionSource, PickOutcome, PickerMode, ScoreResult, StageClassifier,
diff --git a/crates/switchyard-components/src/stats/accumulator.rs b/crates/switchyard-components/src/stats/accumulator.rs
index 19e925269..455c8cafe 100644
--- a/crates/switchyard-components/src/stats/accumulator.rs
+++ b/crates/switchyard-components/src/stats/accumulator.rs
@@ -138,7 +138,7 @@ impl StatsAccumulator {
/// Records usage after the corresponding success call was already attributed.
///
- /// `StatsLlmBackend` and profile runtimes record success before the response
+ /// `StatsLlmBackend` and routing runtimes record success before the response
/// processor records tokens. Those internal paths must not consume a legacy
/// pending untiered success that belongs to some other direct accumulator caller.
pub fn record_usage_after_success_attribution(
@@ -281,7 +281,7 @@ impl StatsAccumulator {
Ok(())
}
- /// Records one routing decision source for a profile family.
+ /// Records one routing decision source for a routing algorithm.
///
/// This is intentionally separate from model/tier accounting: a stage-router can
/// choose `efficient` because of an override, a dimensions score, an LLM-classifier
diff --git a/crates/switchyard-components/tests/adversarial_random_routing.rs b/crates/switchyard-components/tests/adversarial_random_routing.rs
index 746d70905..3cd0fcaed 100644
--- a/crates/switchyard-components/tests/adversarial_random_routing.rs
+++ b/crates/switchyard-components/tests/adversarial_random_routing.rs
@@ -35,7 +35,7 @@ fn openai_request(model: &str) -> ChatRequest {
}))
}
-// Runs one request through the engine and applies the profile-local model rewrite.
+// Runs one request through the engine and applies the route-local model rewrite.
fn route_once(engine: &RandomRoutingEngine, mut request: ChatRequest) -> Result<ChatRequest> {
let decision = engine.select(request.model().map(str::to_owned))?;
request.set_model(decision.selected_model.as_str());
diff --git a/crates/switchyard-py/src/component_bindings/stage_router.rs b/crates/switchyard-py/src/component_bindings/stage_router.rs
index f05df6381..33a33d3a0 100644
--- a/crates/switchyard-py/src/component_bindings/stage_router.rs
+++ b/crates/switchyard-py/src/component_bindings/stage_router.rs
@@ -5,9 +5,8 @@
//!
//! Exposes [`switchyard_components::stage_router::pick_tier`] as
//! `stage_pick_tier(signal, picker_mode, confidence_threshold) -> PickOutcome`
-//! so the Python `processor.py` runs the exact same routing decision as the Rust
-//! profile. The async classifier and the `no_signal` case stay in Python: this
-//! returns a resolved decision, or a request to consult the classifier.
+//! for Python analysis tools. This returns a resolved decision or a request to
+//! consult a classifier.
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
diff --git a/crates/switchyard-py/src/interop/subagent.rs b/crates/switchyard-py/src/interop/subagent.rs
index c9690d826..c74885cda 100644
--- a/crates/switchyard-py/src/interop/subagent.rs
+++ b/crates/switchyard-py/src/interop/subagent.rs
@@ -3,10 +3,10 @@
//! Python binding for the canonical sub-agent detection policy.
//!
-//! Profiles must not sniff lineage headers themselves: the fact (explicit
+//! Routing implementations must not sniff lineage headers themselves: the fact (explicit
//! `x-switchyard-is-subagent`, Claude Code agent lineage, Codex/relay markers) and the
//! work-vs-maintenance policy both live in the protocol crate, so every engine — the libsy
-//! classifier and the serve-path profile alike — answers "is this delegated work?"
+//! classifier and the server alike — answers "is this delegated work?"
//! identically.
use std::collections::HashMap;
diff --git a/docs/getting_started.md b/docs/getting_started.md
index b6cb82d8c..3dd0d039c 100644
--- a/docs/getting_started.md
+++ b/docs/getting_started.md
@@ -122,8 +122,8 @@ Cargo builds the release binary and installs it into `~/.cargo/bin` by default.
### Configure
-The Rust server reads an explicit TOML file. It does not use the legacy Python
-CLI's saved configuration or YAML routing profiles.
+The Rust server reads an explicit TOML file. It does not use the Python
+server's minimal YAML route bundle.
Create `routes.toml` with an LLM-classifier route:
diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md
index 6487d495f..aea807d9e 100644
--- a/docs/internal/metrics_reference.md
+++ b/docs/internal/metrics_reference.md
@@ -14,7 +14,8 @@ a drop-in scrape config and starter alert rules.
| Auth | None |
| Default scrape interval | 15s |
-`GET /metrics` is served by `switchyard-server`.
+`GET /metrics` is served by the Python route-bundle server started with
+`switchyard serve --routes PATH` and by the native Rust server.
A JSON summary of the same traffic lives at `GET /v1/stats`.
diff --git a/docs/internal/subagent_override.md b/docs/internal/subagent_override.md
deleted file mode 100644
index e8b59b268..000000000
--- a/docs/internal/subagent_override.md
+++ /dev/null
@@ -1,54 +0,0 @@
-# Sub-Agent Override
-
-Internal note. Not published: `mkdocs.yml` excludes `internal/**` from the site.
-
-`subagent_target` exists only on the Python route bundle loaded by
-`switchyard serve`. The `switchyard-server` TOML schema has no equivalent key.
-
-Any bundle route may name an optional `subagent_target` in its common envelope,
-alongside `type`. A request carrying a recognized sub-agent signal — Claude Code
-agent-lineage headers, Codex delegated-work kinds (`x-openai-subagent:
-collab_spawn` or `review`), or an explicit `x-switchyard-is-subagent: true` —
-bypasses the route's own chain and runs as a direct passthrough to that target:
-
-```yaml
-routes:
- assistant:
- type: model
- target:
- model: strong-model
- subagent_target:
- model: cheaper-worker-model
-```
-
-This keeps a sub-agent loop on one intentional, cache-compatible target instead
-of re-routing every worker turn. The worker may live on a different provider
-entirely — give it its own `base_url` and `api_key` and one route spans two
-upstreams:
-
-```yaml
- subagent_target:
- model: my-local-model
- base_url: http://localhost:8000/v1
- api_key: dummy
- format: openai
-```
-
-Detection is the protocol crate's canonical policy, shared with the
-`switchyard-libsy` `SubagentOverride` classifier, so both engines agree on what
-counts as delegated work. Harness-maintenance turns (`compact`,
-`memory_consolidation`) and unrecognized kinds stay on normal routing, as does
-everything else when the field is absent. A worker-target failure surfaces as a
-normal target error — it is never silently re-routed through the route's own
-chain.
-
-To suppress sub-agent routing for a request that carries a recognized signal,
-send `x-switchyard-is-subagent: false`. This explicit header overrides Codex and
-Claude Code lineage signals in either direction: `false` keeps the request on
-normal routing even when delegated-work headers are present, and `true` marks a
-request as a sub-agent even when no harness headers appear.
-
-The key applies to bundle routes of type `model`, `deterministic`,
-`escalation_router`, and `stage_router`. It is accepted on `random_routing`
-routes but has no effect, because they expand into their table entries on a
-separate path that never consumes it.
diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md
index 1051928a1..207586134 100644
--- a/docs/routing_algorithms/stage_router_routing.md
+++ b/docs/routing_algorithms/stage_router_routing.md
@@ -156,7 +156,7 @@ From the overlap tasks (those with both capable and efficient results):
Replay your runs through the real Rust scorer and picker with
`benchmark/score_staged_run.py` (the `switchyard-stage-router-scorer` skill). It emits
per-turn scores and per-task routing splits at a given threshold and window —
-the actual `pick_capable_first` / `pick_efficient_first` decisions, not a
+the actual `capable_first` / `efficient_first` picker decisions, not a
counterfactual:
```bash
diff --git a/examples/minimal.py b/examples/minimal.py
index 26555b07b..95b5a610b 100644
--- a/examples/minimal.py
+++ b/examples/minimal.py
@@ -8,41 +8,27 @@ Minimal Switchyard Example
This example demonstrates how to use Switchyard as a Python library
to route LLM requests through a backend.
-Configuration (in priority order):
- 1. Environment variables: OPENAI_API_KEY, OPENAI_BASE_URL
- 2. Secrets file: secrets/secrets.json
-
Usage:
- export OPENAI_API_KEY="sk-..."
+ export OPENROUTER_API_KEY="sk-or-..."
python examples/minimal.py
"""
import asyncio
-import os
import sys
from pathlib import Path
# Add package to path for development (not needed when installed via pip)
sys.path.insert(0, str(Path(__file__).parent.parent))
-from switchyard import ChatRequest, PassthroughProfileConfig, ProfileSwitchyard
+from switchyard import ChatRequest
+from switchyard.cli.route_bundle import load_route_bundle_table
-async def main():
+async def main() -> None:
"""Run a minimal Switchyard example."""
- # Get API key from environment or secrets file
- api_key = os.environ.get("OPENAI_API_KEY")
- if not api_key:
- print("Error: OPENAI_API_KEY not set")
- print("Set it with: export OPENAI_API_KEY='sk-...'")
- return
-
- # Create a passthrough proxy that forwards to OpenAI
- switchyard = ProfileSwitchyard(PassthroughProfileConfig(
- api_key=api_key,
- base_url="https://api.openai.com/v1",
- ).build())
+ routes = load_route_bundle_table(Path(__file__).with_name("route.yaml"))
+ switchyard = routes.lookup_switchyard("fast-kimi")
print("=" * 60)
print("Switchyard Minimal Example")
@@ -50,7 +36,7 @@ async def main():
# Create a chat request
request = ChatRequest.openai_chat({
- "model": "gpt-4o-mini",
+ "model": "fast-kimi",
"messages": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What is 2+2?"},
diff --git a/examples/route.yaml b/examples/route.yaml
index df468ce44..5b3ef858e 100644
--- a/examples/route.yaml
+++ b/examples/route.yaml
@@ -4,35 +4,9 @@ defaults:
format: openai
routes:
- fast-kimi: # `type: model` — single model
- type: model
+ fast-kimi:
+ type: passthrough
target: moonshotai/kimi-k2.6
- random-router: # `type: random_routing` — weighted coin
- type: random_routing
- strong: anthropic/claude-opus-4.7
- weak: moonshotai/kimi-k2.6
- strong_probability: 0.3
- fallback_target_on_evict: strong
-
- llm-classifier: # `type: deterministic` — LLM-as-classifier
- type: deterministic
- profile: coding_agent # general | coding_agent | openclaw
- fallback_target_on_evict: strong # tier to retry on context-window evict
-
- classifier:
- model: google/gemini-3.5-flash
- api_key: ${OPENROUTER_API_KEY}
- base_url: https://openrouter.ai/api/v1
- timeout_secs: 30.0
- min_confidence: 0.6 # tier-selector confidence floor
- fail_open: true # on classifier error, route to strong
- recent_turn_window: 4
- strong:
- model: anthropic/claude-opus-4.7
- api_key: ${OPENROUTER_API_KEY}
- base_url: https://openrouter.ai/api/v1
- weak:
- model: moonshotai/kimi-k2.6
- api_key: ${OPENROUTER_API_KEY}
- base_url: https://openrouter.ai/api/v1
+ smoke-test:
+ type: noop
diff --git a/pyproject.toml b/pyproject.toml
index f39163c35..30203d106 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -63,18 +63,9 @@ tracing = [
"ddtrace>=2.9,<4",
]
-# Shared session-affinity pin store. redis-py ships ``redis.asyncio`` natively,
-# so no separate async client is needed. Lazy-imported by
-# ``switchyard.lib.redis_pin_store`` — the default install never pulls it in.
-# Floor 5.0.1: first release with ``aclose()`` on async clients, which the
-# store's shutdown path calls.
-affinity-redis = [
- "redis>=5.0.1,<6",
-]
-
# Everything — all optional dependencies for full-featured deployment.
all = [
- "nemo-switchyard[server,cli,tracing,affinity-redis]",
+ "nemo-switchyard[server,cli,tracing]",
]
# Dev tooling lives in a PEP 735 dependency group rather than an optional
diff --git a/switchyard/__init__.py b/switchyard/__init__.py
index fdedd8bad..ae1574d2f 100644
--- a/switchyard/__init__.py
+++ b/switchyard/__init__.py
@@ -39,33 +39,6 @@ from switchyard.lib.chat_response import (
)
from switchyard.lib.processors.rl_logging_request_processor import RlLoggingRequestProcessor
from switchyard.lib.processors.rl_logging_response_processor import RlLoggingResponseProcessor
-from switchyard.lib.profiles import (
- ClassifierConfig,
- ContextAwareProfile,
- DeterministicRoutingConfig,
- DeterministicRoutingPresets,
- DeterministicRoutingProfileConfig,
- EscalationRouterConfig,
- EscalationRouterProfileConfig,
- PassthroughProfileConfig,
- Profile,
- ProfileConfig,
- ProfileConfigError,
- ProfileHooks,
- ProfileInput,
- ProfileLifecycle,
- ProfileRunner,
- ProfileSwitchyard,
- RandomRoutingConfig,
- RandomRoutingPresets,
- RandomRoutingProfileConfig,
- StageRouterConfig,
- StageRouterProfileConfig,
- TranslateProfileConfig,
- build_profile,
- profile_config,
- profile_config_type,
-)
from switchyard.lib.request_metadata import RequestMetadata
from switchyard.lib.roles import (
LLMBackend,
@@ -132,31 +105,6 @@ __all__ = [
# Chain infrastructure
"Switchyard",
"LLMBackend",
- "StageRouterConfig",
- "StageRouterProfileConfig",
- "ClassifierConfig",
- "ContextAwareProfile",
- "DeterministicRoutingConfig",
- "DeterministicRoutingProfileConfig",
- "DeterministicRoutingPresets",
- "EscalationRouterConfig",
- "EscalationRouterProfileConfig",
- "PassthroughProfileConfig",
- "Profile",
- "ProfileConfig",
- "ProfileConfigError",
- "ProfileHooks",
- "ProfileInput",
- "ProfileLifecycle",
- "ProfileRunner",
- "ProfileSwitchyard",
- "RandomRoutingConfig",
- "RandomRoutingPresets",
- "RandomRoutingProfileConfig",
- "TranslateProfileConfig",
- "build_profile",
- "profile_config",
- "profile_config_type",
"AnthropicNativeBackend",
"OpenAiNativeBackend",
"OpenAIChatEndpoint",
diff --git a/switchyard/cli/launchers/claude_code_launcher.py b/switchyard/cli/launchers/claude_code_launcher.py
index e3c17374a..4b69ab298 100644
--- a/switchyard/cli/launchers/claude_code_launcher.py
+++ b/switchyard/cli/launchers/claude_code_launcher.py
@@ -147,7 +147,7 @@ def _run_claude_with_switchyard(
display_model=display_model,
log_path=log_path,
strategy_summary=strategy_summary,
- profile_routes=[display_model],
+ routes=[display_model],
default_route=display_model,
)
if stdin_is_tty():
diff --git a/switchyard/cli/launchers/codex_cli_launcher.py b/switchyard/cli/launchers/codex_cli_launcher.py
index 204005399..973835d7d 100644
--- a/switchyard/cli/launchers/codex_cli_launcher.py
+++ b/switchyard/cli/launchers/codex_cli_launcher.py
@@ -178,7 +178,7 @@ def _run_codex_with_switchyard(
display_model=display_model,
log_path=log_path,
strategy_summary=strategy_summary,
- profile_routes=[display_model],
+ routes=[display_model],
default_route=display_model,
)
if stdin_is_tty():
diff --git a/switchyard/cli/launchers/launcher_runtime.py b/switchyard/cli/launchers/launcher_runtime.py
index e8410f957..4584d1235 100644
--- a/switchyard/cli/launchers/launcher_runtime.py
+++ b/switchyard/cli/launchers/launcher_runtime.py
@@ -113,59 +113,28 @@ def stdin_is_tty() -> bool:
return False
-def routing_profiles_strategy_summary(routing_profiles: str, default_model: str) -> str:
- """Return the strategy summary for the default route in a routing-profiles launch.
-
- Parses the YAML to find the default route (first key) and describes its type.
- Falls back to ``routing-profiles: <default_model>`` if parsing fails.
- """
+def route_bundle_strategy_summary(route_bundle: str, default_model: str) -> str:
+ """Describe the default route in a Python server bundle."""
try:
from collections.abc import Mapping as _Mapping
from importlib import import_module
yaml = import_module("yaml")
- raw = yaml.safe_load(Path(routing_profiles).read_text())
+ raw = yaml.safe_load(Path(route_bundle).read_text())
routes = raw.get("routes") if isinstance(raw, dict) else None
if isinstance(routes, _Mapping) and routes:
first_key = next(iter(routes))
route = routes[first_key]
route_type = route.get("type") if isinstance(route, _Mapping) else None
if isinstance(route_type, str):
- return _route_type_summary(route_type.lower().replace("-", "_"), route, first_key)
+ if route_type == "noop":
+ return "noop"
+ if route_type == "passthrough":
+ target = route.get("target")
+ model = target.get("model") if isinstance(target, _Mapping) else target
+ return f"passthrough: model={model or first_key}"
except Exception:
pass
- return f"routing-profiles: {default_model}"
-
-
-def _route_type_summary(route_type: str, route: object, route_key: str) -> str:
- """Describe a single route by type for the startup banner."""
- from collections.abc import Mapping as _Mapping
- r = route if isinstance(route, _Mapping) else {}
-
- def _model(tier: object) -> str:
- return tier.get("model", "") if isinstance(tier, _Mapping) else ""
-
- def _classifier_part(r: _Mapping) -> str: # type: ignore[type-arg]
- clf = r.get("classifier")
- m = _model(clf)
- return f", llm-classifier={m}" if m else ""
-
- if route_type == "stage_router":
- clf = _classifier_part(r)
- threshold = r.get("confidence_threshold")
- threshold_part = f", confidence_threshold={threshold}" if threshold is not None else ""
- return f"stage_router: strong={_model(r.get('strong'))}, weak={_model(r.get('weak'))}{clf}{threshold_part}"
- if route_type in ("deterministic", "llm_classifier"):
- clf = _classifier_part(r)
- profile = r.get("profile")
- profile_part = f", profile={profile}" if profile else ""
- return f"llm-classifier: strong={_model(r.get('strong'))}, weak={_model(r.get('weak'))}{clf}{profile_part}"
- if route_type == "random_routing":
- p = r.get("strong_probability", "")
- return f"random-routing: strong={_model(r.get('strong'))}, weak={_model(r.get('weak'))}, p_strong={p}"
- if route_type == "model":
- target = r.get("model") or r.get("target") or route_key
- return f"passthrough → {target}"
- return f"{route_type}: {route_key}"
+ return f"route: {default_model}"
# Keys that are abbreviated in the banner display.
@@ -183,7 +152,7 @@ def _format_strategy_lines(summary: str) -> list[str]:
"""Expand a strategy summary string into one or more banner lines.
``type: k1=v1, k2=v2`` → type on first line, each k=v pair indented.
- ``passthrough → model`` → single line (no key=value pairs to split).
+ ``noop`` → single line (no key=value pairs to split).
"""
col = " routing " # label column: 2 indent + 7 label + 3 gap = 12 chars
pad = " " * len(col)
@@ -227,7 +196,7 @@ def print_ready_banner(
display_model: str,
log_path: Path | None = None,
strategy_summary: str | None = None,
- profile_routes: list[str] | None = None,
+ routes: list[str] | None = None,
default_route: str | None = None,
) -> None:
"""Write proxy/stats/routing details to stderr before the child takes over."""
@@ -248,11 +217,11 @@ def print_ready_banner(
f" {bold('switchyard')} {green('ready')} → {bold(display_model)}",
]
- if profile_routes:
- col = " profiles "
+ if routes:
+ col = " routes "
pad = " " * len(col)
lines.append("")
- shown, rest = profile_routes[:5], profile_routes[5:]
+ shown, rest = routes[:5], routes[5:]
for i, route in enumerate(shown):
is_default = route == default_route
marker = amber("▶") + " " if is_default else dim("○") + " "
diff --git a/switchyard/cli/launchers/openclaw_launcher.py b/switchyard/cli/launchers/openclaw_launcher.py
index 321956f9a..da5bcbc25 100644
--- a/switchyard/cli/launchers/openclaw_launcher.py
+++ b/switchyard/cli/launchers/openclaw_launcher.py
@@ -221,7 +221,7 @@ def _run_openclaw_with_switchyard(
display_model=display_model,
log_path=log_path,
strategy_summary=strategy_summary,
- profile_routes=[display_model],
+ routes=[display_model],
default_route=display_model,
)
if stdin_is_tty():
diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py
index a1d3d214b..61f210fa8 100644
--- a/switchyard/cli/route_bundle.py
+++ b/switchyard/cli/route_bundle.py
@@ -1,356 +1,108 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Build a model-dispatch table from a YAML route bundle.
+"""Build a Python server route table from a minimal YAML bundle."""
-Each YAML route becomes one entry in a :class:`RouteTable`. Chains are
-built through the shared :mod:`switchyard.lib.route_table_builders` helpers — the
-same path the Claude/Codex launchers take — so the table has uniform shape
-no matter which front-end produced it.
-
-The flow is::
-
- raw dict (from YAML / programmatic)
- │
- ▼ _parse_route_bundle_dict
- RouteBundle (defaults + routes + optional pre/post processors)
- │
- ▼ build_table_from_bundle
- RouteTable
-
-"""
+from __future__ import annotations
import os
import re
+import time
from collections.abc import Mapping, Sequence
-from dataclasses import dataclass, field, replace
from importlib import import_module
from pathlib import Path
-from typing import Any, Protocol, cast, overload
-
-from pydantic import ValidationError
+from typing import Any, Protocol, cast
-from switchyard.cli.model_catalog.model_discovery import fetch_model_ids
from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target
-from switchyard.lib.processors.llm_classifier import DEFAULT_MAX_REQUEST_CHARS
-from switchyard.lib.processors.llm_classifier.presets import PROFILE_FACTORIES
-from switchyard.lib.profiles import (
- DeterministicRoutingProfileConfig,
- EscalationRouterProfileConfig,
- ProfileSwitchyard,
- StageRouterProfileConfig,
+from switchyard.lib.backends.multi_llm_backend import build_native_backend
+from switchyard.lib.backends.stats_llm_backend import StatsLlmBackend
+from switchyard.lib.processors.stats_request_processor import StatsRequestProcessor
+from switchyard.lib.processors.stats_response_processor_accumulator import (
+ StatsResponseProcessor,
)
-from switchyard.lib.profiles.deterministic_routing_config import DeterministicRoutingConfig
-from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig
-from switchyard.lib.profiles.random_routing import RandomRoutingConfig
-from switchyard.lib.profiles.stage_router_config import StageRouterConfig
-from switchyard.lib.profiles.subagent_override import SubagentOverrideRuntime
+from switchyard.lib.proxy_context import ProxyContext
+from switchyard.lib.roles import LLMBackend
from switchyard.lib.route_table import ChainRuntime, RouteTable
-from switchyard.lib.route_table_builders import (
- build_passthrough_table,
- build_random_routing_switchyard,
- build_random_routing_table,
- build_tier_passthrough_switchyard,
-)
from switchyard.lib.stats_accumulator import StatsAccumulator
-
-
-def _default_discovery_fn(base_url: str, api_key: str) -> list[str]:
- """Wrap provider catalog fetching for the lib-level callable shape.
-
- Same wrapping pattern the Claude/Codex launchers use, so a YAML route's
- tier catalogs hydrate the same way the launchers do. Failures intentionally
- raise so table builders can preserve non-fatal warning metadata for
- ``/v1/models``.
- """
- return fetch_model_ids(base_url, api_key)
-
-
-def _merge_table(
- table: RouteTable,
- sub_table: RouteTable,
-) -> None:
- """Merge entries, warnings, and listing defaults from *sub_table*."""
- was_empty = not table.registered_models()
- for sub_model, sub_chain, sub_metadata in sub_table.items():
- table.register(sub_model, sub_chain, metadata=sub_metadata)
- for warning in sub_table.model_listing_warnings():
- table.add_model_listing_warning(warning)
- if was_empty:
- default_model = sub_table.default_model()
- if default_model is not None and default_model in table.registered_models():
- table.set_default_model(default_model)
-
-
-@dataclass(frozen=True)
-class RouteBundle:
- """Parsed route-bundle config — input to :func:`build_table_from_bundle`.
-
- The YAML loader produces one of these. The ``routes`` map is keyed by
- inbound model id (the YAML route key) and each value is a ``dict``-shaped
- route description matching the YAML schema.
-
- ``stats_accumulator`` lets callers thread their own accumulator into the
- builder so a launcher that's merging a YAML table on top of its own
- can share the same instance with downstream readers (live stats footer,
- ``/v1/routing/stats``). ``None`` makes the builder create a fresh one.
-
- Per-chain processor injection (for example, custom hooks) is
- call-site runtime state, not bundle data. ``serve`` and launchers may pass
- processors through the table builder, but route YAML never declares
- those processors itself.
- """
-
- defaults: Mapping[str, Any] = field(default_factory=dict)
- routes: Mapping[str, Mapping[str, Any]] = field(default_factory=dict)
- stats_accumulator: StatsAccumulator | None = None
-
-
-def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]:
- """Convert an :class:`LlmTarget` into the dict shape a route bundle accepts.
-
- Used by launchers to construct :class:`RouteBundle` route entries from
- already-built tier targets without serializing through YAML. Only emits
- fields with non-default values so the loader's defaults cascade still
- applies.
- """
- data: dict[str, Any] = {
- "model": target.model,
- "format": str(target.format),
- }
- if target.endpoint.api_key:
- data["api_key"] = target.endpoint.api_key
- if target.endpoint.base_url:
- data["base_url"] = target.endpoint.base_url
- if target.endpoint.timeout_secs is not None:
- data["timeout_secs"] = target.endpoint.timeout_secs
- return data
+from switchyard.lib.switchyard import Switchyard
+from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse
+from switchyard_rust.translation import TranslationEngine
_ENV_REF_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
-_TARGET_DEFAULT_KEYS = frozenset({
- "api_key",
- "base_url",
- "format",
- "backend_format",
- "timeout",
- "timeout_secs",
- "extra_body",
- "extra_headers",
- "endpoint",
-})
-# Envelope keys valid on every route type, whatever its own schema. They configure the
-# wrapper around a route's chain rather than the chain itself, so they are accepted by
-# _validate_route_keys and excluded from the per-profile config in _route_config.
-_ROUTE_ENVELOPE_KEYS = frozenset({
- # Delegated sub-agent work is served by this target instead of the route's own
- # chain. Consumed in _build_switchyard_for_route.
- "subagent_target",
-})
-_COMMON_ROUTE_KEYS = frozenset({
- "type",
- "kind",
- "defaults",
- "display_name",
- "description",
-}) | _ROUTE_ENVELOPE_KEYS
-_ROUTE_METADATA_KEYS = frozenset({
- "type",
- "kind",
- "display_name",
- "description",
-})
-_TARGET_KEYS = _TARGET_DEFAULT_KEYS | frozenset({
- "id",
- "model",
- "tuning",
-})
-_TARGET_DEFAULT_ROUTE_KEYS = _TARGET_DEFAULT_KEYS | frozenset({"defaults"})
-_MODEL_ROUTE_KEYS = _ROUTE_METADATA_KEYS | _TARGET_DEFAULT_ROUTE_KEYS | frozenset({
- "target",
- "model",
-})
-_RANDOM_ROUTING_ROUTE_KEYS = _ROUTE_METADATA_KEYS | _TARGET_DEFAULT_ROUTE_KEYS | frozenset({
- "strong",
- "weak",
- "strong_probability",
- "enable_stats",
- "rng_seed",
- "preset",
- "fallback_target_on_evict",
-})
-_DETERMINISTIC_ROUTE_KEYS = (
- _ROUTE_METADATA_KEYS
- | _TARGET_DEFAULT_ROUTE_KEYS
- | frozenset({
- "classifier",
- "strong",
- "weak",
- "profile",
- "enable_stats",
- "fallback_target_on_evict",
- "tier_timeout_s",
- "session_affinity",
- "affinity_max_sessions",
- "affinity_warmup_turns",
- })
-)
-_STAGE_ROUTER_ROUTE_KEYS = (
- _ROUTE_METADATA_KEYS
- | _TARGET_DEFAULT_ROUTE_KEYS
- | frozenset({
- "strong",
- "weak",
- "picker",
- "confidence_threshold",
- "signal_recent_window",
- "classifier",
- "handoff_notes",
- "strong_system_prompt",
- "weak_system_prompt",
- "enable_stats",
- "fallback_target_on_evict",
- })
-)
-_ESCALATION_ROUTE_KEYS = (
- _ROUTE_METADATA_KEYS
- | _TARGET_DEFAULT_ROUTE_KEYS
- | frozenset({
- "judge",
- "strong",
- "weak",
- "enable_stats",
- "fallback_target_on_evict",
- "tier_timeout_s",
- "session_key_depth",
- "affinity_max_sessions",
- "affinity_store",
- "affinity_store_url",
- "affinity_store_ttl_seconds",
- "affinity_key_prefix",
- })
-)
-_ESCALATION_JUDGE_KEYS = frozenset({
- "model",
- "api_key",
- "base_url",
- "timeout",
- "timeout_secs",
- "min_turn",
- "confirmations",
- "confirmation_window",
- "disable_reasoning",
- "max_completion_tokens",
- "dump_verdicts",
- "recent_turn_window",
- "window_message_chars",
- "prompt",
- "prompt_path",
- "max_request_chars",
-})
-_DETERMINISTIC_CLASSIFIER_KEYS = frozenset({
- "model",
- "api_key",
- "base_url",
- "timeout",
- "timeout_secs",
- "min_confidence",
- "fail_open",
- "recent_turn_window",
- "prompt",
- "max_request_chars",
-})
-_STAGE_ROUTER_CLASSIFIER_KEYS = frozenset({
- "model",
- "api_key",
- "base_url",
- "timeout",
- "timeout_secs",
- "recent_turn_window",
-})
-_CLASSIFIER_DEFAULT_KEYS = frozenset({
- "api_key",
- "base_url",
- "timeout",
- "timeout_secs",
-})
-_ROUTE_KEYS_BY_TYPE: Mapping[str, frozenset[str]] = {
- "model": _MODEL_ROUTE_KEYS,
- "random_routing": _RANDOM_ROUTING_ROUTE_KEYS,
- "deterministic": _DETERMINISTIC_ROUTE_KEYS,
- "escalation_router": _ESCALATION_ROUTE_KEYS,
- "stage_router": _STAGE_ROUTER_ROUTE_KEYS,
-}
-_DEFAULT_KEYS_BY_TYPE: Mapping[str, frozenset[str]] = {
- "model": _TARGET_DEFAULT_KEYS,
- "random_routing": _TARGET_DEFAULT_KEYS,
- "deterministic": _TARGET_DEFAULT_KEYS,
- "escalation_router": _TARGET_DEFAULT_KEYS,
- "stage_router": _TARGET_DEFAULT_KEYS,
+_TOP_LEVEL_KEYS = frozenset({"defaults", "routes"})
+_ROUTE_METADATA_KEYS = frozenset({"display_name", "description"})
+_ROUTE_KEYS = {
+ "noop": frozenset({"type"}) | _ROUTE_METADATA_KEYS,
+ "passthrough": frozenset({"type", "target"}) | _ROUTE_METADATA_KEYS,
}
-def _deterministic_profile_factories() -> dict[
- str, Any,
-]:
- """LLM-classifier presets recognized by ``type: deterministic`` routes."""
- return dict(PROFILE_FACTORIES)
-
-
-_DETERMINISTIC_PROFILE_FACTORIES = _deterministic_profile_factories()
-
class RouteBundleConfigError(ValueError):
- """Raised when a YAML route bundle is malformed."""
+ """Raised when a Python server route bundle is invalid."""
class _YamlModule(Protocol):
- YAMLError: type[Exception]
-
def safe_load(self, stream: str) -> object: ...
-def parse_routing_profiles_file(path: str | Path) -> dict[str, object]:
- """Read *path* and return the parsed YAML dict (no env-var expansion)."""
- yaml = cast(_YamlModule, import_module("yaml"))
+class _NoopBackend(LLMBackend):
+ """Return a fixed response without making an upstream request."""
+
+ @property
+ def supported_request_types(self) -> list[ChatRequestType]:
+ return list(ChatRequestType)
+
+ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
+ model = request.model or "switchyard/noop"
+ ctx.selected_model = model
+ ctx.selected_target = model
+ return ChatResponse.openai_completion({
+ "id": "switchyard-noop",
+ "object": "chat.completion",
+ "created": int(time.time()),
+ "model": model,
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "OK"},
+ "finish_reason": "stop",
+ }],
+ "usage": {
+ "prompt_tokens": 0,
+ "completion_tokens": 0,
+ "total_tokens": 0,
+ },
+ })
+
+
+def parse_route_bundle_file(path: str | Path) -> dict[str, object]:
+ """Read a YAML route bundle and return its top-level mapping."""
resolved = Path(path)
try:
- raw = resolved.read_text(encoding="utf-8")
- loaded = yaml.safe_load(raw)
- except FileNotFoundError as exc:
- raise RouteBundleConfigError(f"{resolved}: file not found") from exc
- except (OSError, UnicodeError) as exc:
- raise RouteBundleConfigError(
- f"{resolved}: cannot read: {_format_exception_one_line(exc)}"
- ) from exc
- except yaml.YAMLError as exc:
- raise RouteBundleConfigError(
- f"{resolved}: invalid YAML: {_format_exception_one_line(exc)}"
- ) from exc
- return loaded if isinstance(loaded, dict) else {}
-
+ contents = resolved.read_text()
+ except FileNotFoundError as error:
+ raise RouteBundleConfigError(f"{resolved}: file not found") from error
+ except (OSError, UnicodeError) as error:
+ raise RouteBundleConfigError(f"{resolved}: cannot read: {error}") from error
-def _format_exception_one_line(exc: BaseException) -> str:
- return " ".join(str(exc).split())
+ try:
+ yaml = cast(_YamlModule, import_module("yaml"))
+ raw = yaml.safe_load(contents)
+ except Exception as error:
+ message = " ".join(str(error).splitlines())
+ raise RouteBundleConfigError(f"{resolved}: invalid YAML: {message}") from error
+ return _mapping(raw, "route bundle")
def load_route_bundle_table(
path: str | Path,
+ *,
stats_accumulator: StatsAccumulator | None = None,
pre_routing_request_processors: Sequence[Any] = (),
extra_response_processors: Sequence[Any] = (),
) -> RouteTable:
- """Load *path* and return a table keyed by route model id.
-
- Pass ``stats_accumulator`` when a caller is merging this YAML table into
- an existing runtime so YAML-declared chains
- record into the same accumulator surfaced at ``/v1/routing/stats`` and the
- live stats footer. ``None`` (default) creates a fresh accumulator — what
- standalone ``switchyard serve`` callers want.
-
- ``pre_routing_request_processors`` / ``extra_response_processors`` let
- callers attach process-level components to every YAML-declared route.
- """
+ """Load a YAML route bundle into a server dispatch table."""
return build_route_bundle_table(
- parse_routing_profiles_file(path),
+ parse_route_bundle_file(path),
stats_accumulator=stats_accumulator,
pre_routing_request_processors=pre_routing_request_processors,
extra_response_processors=extra_response_processors,
@@ -359,1014 +111,121 @@ def load_route_bundle_table(
def build_route_bundle_table(
raw: object,
+ *,
stats_accumulator: StatsAccumulator | None = None,
pre_routing_request_processors: Sequence[Any] = (),
extra_response_processors: Sequence[Any] = (),
) -> RouteTable:
- """Parse *raw* dict and build a :class:`RouteTable`.
-
- Thin entrypoint that the YAML loader and any other dict-driven caller
- uses. Parses + env-expands + validates the dict into a :class:`RouteBundle`,
- then delegates to :func:`build_table_from_bundle`. Optional
- ``stats_accumulator`` overrides the parsed bundle's default ``None``.
- """
- bundle = _parse_route_bundle_dict(raw)
- if stats_accumulator is not None:
- bundle = replace(bundle, stats_accumulator=stats_accumulator)
- return build_table_from_bundle(
- bundle,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
-
-
-def _parse_route_bundle_dict(raw: object) -> RouteBundle:
- """Validate *raw* and return a :class:`RouteBundle`.
-
- Performs env-var expansion, top-level-key validation, and the
- "routes must contain at least one route" check. Per-route schema
- validation runs inside :func:`build_table_from_bundle` so callers
- that construct a bundle programmatically still benefit.
- """
- return _validate_route_bundle_dict(_expand_env(raw))
-
-
-def _validate_route_bundle_dict(raw: object) -> RouteBundle:
- """Validate bundle structure without expanding environment references."""
- bundle = _require_mapping(raw, "route bundle")
- _validate_allowed_keys(bundle, frozenset({"defaults", "routes"}), "route bundle")
- defaults = _optional_mapping(bundle.get("defaults", {}), "defaults")
- _validate_allowed_keys(defaults, _TARGET_DEFAULT_KEYS, "defaults")
- routes_raw = _require_mapping(bundle.get("routes"), "routes")
- if not routes_raw:
+ """Build a table containing only noop and passthrough routes."""
+ bundle = _mapping(_expand_env(raw), "route bundle")
+ _reject_unknown_keys(bundle, _TOP_LEVEL_KEYS, "route bundle")
+ defaults = _mapping(bundle.get("defaults", {}), "defaults")
+ routes = _mapping(bundle.get("routes"), "routes")
+ if not routes:
raise RouteBundleConfigError("routes must contain at least one route")
- routes: dict[str, Mapping[str, Any]] = {
- name: _require_mapping(spec, f"routes.{name}")
- for name, spec in routes_raw.items()
- }
- return RouteBundle(defaults=defaults, routes=routes)
-
-def build_table_from_bundle(
- bundle: RouteBundle,
- pre_routing_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
-) -> RouteTable:
- """Build a :class:`RouteTable` from a :class:`RouteBundle`.
-
- Single assembly path for both the YAML loader and the launchers.
-
- If ``bundle.stats_accumulator`` is set, the same accumulator is shared
- across every produced chain so the live stats footer and
- ``/v1/routing/stats`` see consistent numbers.
-
- ``pre_routing_request_processors`` / ``extra_response_processors`` are
- call-time kwargs (not bundle data). ``serve`` and launchers pass
- process-level processors here; YAML routes never declare those processors
- themselves.
- """
+ stats = stats_accumulator or StatsAccumulator()
table = RouteTable()
- stats = bundle.stats_accumulator or StatsAccumulator()
- for model_id, route_raw in bundle.routes.items():
- if not isinstance(model_id, str) or not model_id:
- raise RouteBundleConfigError("routes keys must be non-empty strings")
- route = _normalize_route(model_id, route_raw)
- route_type = _route_type(model_id, route)
- _validate_route_keys(model_id, route, route_type)
- route_defaults = _target_defaults(bundle.defaults, route)
-
- # `random_routing` always expands into the launcher-shaped N+1 entries:
- # each tier's configured model registered as a direct passthrough +
- # a virtual routing-policy id under the route's YAML key, plus each
- # tier's GET /v1/models catalog hydrated into the same table. This
- # matches the Claude/Codex launcher behavior so client model pickers
- # always see strong/weak (and the rest of the catalog) as direct
- # overrides.
- if route_type == "random_routing":
- _merge_random_routing_route(
- table, model_id, route,
- target_defaults=route_defaults, stats=stats,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
+ for route_id, raw_route in routes.items():
+ if not route_id:
+ raise RouteBundleConfigError("route ids must be non-empty strings")
+ route = _mapping(raw_route, f"route {route_id!r}")
+ route_type = route.get("type")
+ if not isinstance(route_type, str):
+ raise RouteBundleConfigError(f"route {route_id!r}: missing string 'type'")
+ if route_type not in _ROUTE_KEYS:
+ raise RouteBundleConfigError(
+ f"route {route_id!r}: unsupported route type {route_type!r}; "
+ "expected 'noop' or 'passthrough'"
)
- continue
-
- # `stage_router` and `deterministic` routes register the routing-policy chain
- # at the route key AND hydrate each tier's catalog (`strong` + `weak`)
- # into the table as direct passthroughs — same client-facing
- # model-picker experience as random_routing's discovery path.
- if route_type in ("stage_router", "deterministic", "escalation_router"):
- _merge_multi_target_discovery(
- table,
- model_id,
- route,
- route_type=route_type,
- target_defaults=route_defaults,
- stats=stats,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
+ _reject_unknown_keys(route, _ROUTE_KEYS[route_type], f"route {route_id!r}")
+
+ if route_type == "noop":
+ runtime = _build_runtime(
+ _NoopBackend(),
+ stats,
+ pre_routing_request_processors,
+ extra_response_processors,
+ )
+ else:
+ target = _target(route_id, route.get("target"), defaults)
+ runtime = _build_runtime(
+ StatsLlmBackend(build_native_backend(target), stats),
+ stats,
+ pre_routing_request_processors,
+ extra_response_processors,
)
- continue
- switchyard = _build_switchyard_for_route(
- model_id,
- route,
- route_type=route_type,
- target_defaults=route_defaults,
- stats=stats,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
- table.register(
- model_id,
- switchyard,
- metadata=_route_metadata(model_id, route, route_type),
- )
+ metadata = {
+ key: value
+ for key in _ROUTE_METADATA_KEYS
+ if (value := route.get(key)) is not None
+ }
+ table.register(route_id, runtime, metadata=metadata, default=table.default_model() is None)
return table
-def _merge_random_routing_route(
- table: RouteTable,
- model_id: str,
- route: Mapping[str, object],
- target_defaults: Mapping[str, object],
+def _build_runtime(
+ backend: LLMBackend,
stats: StatsAccumulator,
- pre_routing_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
-) -> None:
- """Expand a ``random_routing`` route into virtual id + tier passthroughs.
-
- Always goes through :func:`build_random_routing_table` from
- :mod:`switchyard.lib.route_table_builders` — the same path the Claude/Codex
- launchers take. Registration order matches the unified rule (YAML route
- key first, discovered/tier entries after):
-
- - The route's YAML key registered as the virtual routing-policy id.
- - Each tier's configured model registered as a direct passthrough.
- - Each tier's ``GET /v1/models`` catalog hydrated alongside.
-
- Tier registration and catalog hydration both happen unconditionally so
- client model pickers see strong/weak (and the rest of the catalog) as
- direct overrides for the random-routing default.
- """
- was_empty = not table.registered_models()
- random_config = RandomRoutingConfig.model_validate(
- _route_config(route, target_defaults, ("strong", "weak"))
- )
- sub_table = build_random_routing_table(
- config=random_config,
- stats=stats,
- random_routing_switchyard=build_random_routing_switchyard(
- random_config,
- stats,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- ),
- routing_model=model_id,
- discovery_fn=_default_discovery_fn,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
- # Register the YAML key (the routing-policy virtual id) first so
- # `registered_models()[0]` matches the user's declared route key,
- # then the tier passthroughs + catalog entries the builder produced.
- for sub_model, sub_chain, sub_metadata in sorted(
- sub_table.items(), key=lambda item: item[0] != model_id,
- ):
- table.register(sub_model, sub_chain, metadata=sub_metadata)
- for warning in sub_table.model_listing_warnings():
- table.add_model_listing_warning(warning)
- if was_empty:
- default_model = sub_table.default_model()
- if default_model is not None and default_model in table.registered_models():
- table.set_default_model(default_model)
-
-
-def _merge_multi_target_discovery(
- table: RouteTable,
- model_id: str,
- route: Mapping[str, object],
- *,
- route_type: str,
- target_defaults: Mapping[str, object],
- stats: StatsAccumulator,
- pre_routing_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
-) -> None:
- """Expand a ``stage_router``/``deterministic`` route with catalog discovery.
-
- Registers two layers, route key first so ``registered_models()[0]`` is the
- user-declared YAML key:
-
- 1. The route's primary routing-policy chain at the route key (the regular
- stage_router/deterministic switchyard).
- 2. Each tier (``strong`` + ``weak``) registered as a direct passthrough with
- its catalog hydrated via :func:`_default_discovery_fn` — same shape the
- launcher's per-tier registration produces, so client model pickers see
- strong/weak as standalone choices alongside the routing policy.
-
- The ``classifier`` tier (when present on the route) is intentionally not
- discovered — it is an internal-only LLM call, not a user-facing target.
- """
- was_empty = not table.registered_models()
- switchyard = _build_switchyard_for_route(
- model_id,
- route,
- route_type=route_type,
- target_defaults=target_defaults,
- stats=stats,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
- table.register(
- model_id,
- switchyard,
- metadata=_route_metadata(model_id, route, route_type),
- )
-
- strong = _target_value(
- route.get("strong"), target_defaults, default_id="strong", where="strong",
- )
- weak = _target_value(
- route.get("weak"), target_defaults, default_id="weak", where="weak",
- )
- sub_table = build_passthrough_table(
- (strong, weak),
- stats,
- enable_stats=_optional_bool(route.get("enable_stats"), default=True),
- discovery_fn=_default_discovery_fn,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
- for sub_model, sub_chain, sub_metadata in sub_table.items():
- # The route key wins over any discovered/configured model id collision.
- if sub_model == model_id:
- continue
- table.register(sub_model, sub_chain, metadata=sub_metadata)
- for warning in sub_table.model_listing_warnings():
- table.add_model_listing_warning(warning)
- if was_empty and model_id in table.registered_models():
- table.set_default_model(model_id)
-
-
-def _build_switchyard_for_route(
- model_id: str,
- route: Mapping[str, object],
- route_type: str,
- target_defaults: Mapping[str, object],
- stats: StatsAccumulator,
- pre_routing_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
+ request_processors: Sequence[Any],
+ response_processors: Sequence[Any],
) -> ChainRuntime:
- """Build the route's chain, wrapping it when the route sets ``subagent_target``."""
- chain = _build_route_chain(
- model_id,
- route,
- route_type=route_type,
- target_defaults=target_defaults,
- stats=stats,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
- worker = _subagent_worker_runtime(
- model_id,
- route,
- target_defaults=target_defaults,
- stats=stats,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
+ return Switchyard(
+ request_processors=[StatsRequestProcessor(), *request_processors],
+ backend=backend,
+ response_processors=[StatsResponseProcessor(stats), *response_processors],
+ translator=TranslationEngine(),
)
- return chain if worker is None else SubagentOverrideRuntime(chain, worker)
-def _subagent_worker_runtime(
- model_id: str,
- route: Mapping[str, object],
- target_defaults: Mapping[str, object],
- stats: StatsAccumulator,
- pre_routing_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
-) -> ChainRuntime | None:
- """Build the route's sub-agent worker chain, or ``None`` when unset.
-
- The worker is a plain passthrough to one target: delegated work is served directly
- rather than re-routed through the route's own policy.
- """
- subagent_raw = route.get("subagent_target")
- if subagent_raw is None:
- return None
- return build_tier_passthrough_switchyard(
- _target_value(
- subagent_raw,
- target_defaults,
- default_id=f"{model_id}#subagent",
- where=f"route {model_id!r} subagent_target",
- ),
- stats,
- enable_stats=_optional_bool(route.get("enable_stats"), default=True),
- extra_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
-
-
-def _build_route_chain(
- model_id: str,
- route: Mapping[str, object],
- route_type: str,
- target_defaults: Mapping[str, object],
- stats: StatsAccumulator,
- pre_routing_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
-) -> ChainRuntime:
- if route_type == "model":
- # Resolves to a single-tier passthrough chain — same shape the launcher
- # produces via build_passthrough_table's per-tier registration.
- target = _passthrough_target(model_id, route, target_defaults)
- return build_tier_passthrough_switchyard(
- target,
- stats,
- enable_stats=_optional_bool(route.get("enable_stats"), default=True),
- extra_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
-
- if route_type == "deterministic":
- return _deterministic_switchyard(
- model_id,
- route,
- target_defaults=target_defaults,
- stats=stats,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
-
- if route_type == "escalation_router":
- return _escalation_router_switchyard(
- model_id,
- route,
- target_defaults=target_defaults,
- stats=stats,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
-
- if route_type == "stage_router":
- return _stage_router_switchyard(
- model_id,
- route,
- target_defaults=target_defaults,
- stats=stats,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
-
- raise RouteBundleConfigError(f"unsupported route type {route_type!r}")
-
-
-def _deterministic_switchyard(
- model_id: str,
- route: Mapping[str, object],
- target_defaults: Mapping[str, object],
- stats: StatsAccumulator,
- pre_routing_request_processors: Sequence[Any],
- extra_response_processors: Sequence[Any],
-) -> ChainRuntime:
- """Build the LLM-classifier deterministic-routing chain for a route.
-
- Wires the chain assembled from the LLM-classifier router primitives:
-
- StatsRequestProcessor
- → LLMClassifierRequestProcessor (real LLM call → RouteSignals)
- → SignalTierSelectorRequestProcessor (collapse to strong/weak)
- → DeterministicRoutingLLMBackend (per-tier OpenAI backend)
- → DefaultResponseTranslator
-
- YAML schema::
-
- type: deterministic
- profile: general # general | coding_agent | openclaw
- classifier:
- model: google/gemini-3.5-flash
- api_key: ${OPENROUTER_API_KEY}
- base_url: https://openrouter.ai/api/v1
- timeout_secs: 30.0
- min_confidence: 0.6 # tier-selector confidence floor
- fail_open: true # on classifier error, route to strong
- recent_turn_window: 4
- strong:
- model: anthropic/claude-opus-4.7
- api_key: ${OPENROUTER_API_KEY}
- base_url: https://openrouter.ai/api/v1
- weak:
- model: moonshotai/kimi-k2.6
- api_key: ${OPENROUTER_API_KEY}
- base_url: https://openrouter.ai/api/v1
- """
- classifier_raw = route.get("classifier")
- if not isinstance(classifier_raw, Mapping):
- raise RouteBundleConfigError(
- f"route {model_id!r}: type=deterministic requires a `classifier:` "
- "mapping with model/api_key/base_url",
- )
- classifier = _classifier_mapping(
- classifier_raw,
- target_defaults,
- allowed_keys=_DETERMINISTIC_CLASSIFIER_KEYS,
- where=f"{model_id}.classifier",
- )
-
- strong = _target_value(
- route.get("strong"), target_defaults, default_id="strong", where="strong",
- )
- weak = _target_value(
- route.get("weak"), target_defaults, default_id="weak", where="weak",
- )
-
- fallback_target_on_evict = _required_str(
- route.get("fallback_target_on_evict"),
- f"{model_id}.fallback_target_on_evict",
- )
- valid_ids = {strong.id, weak.id}
- if fallback_target_on_evict not in valid_ids:
- raise RouteBundleConfigError(
- f"route {model_id!r}: fallback_target_on_evict="
- f"{fallback_target_on_evict!r} must match one of {sorted(valid_ids)} "
- f"(the configured strong/weak target ids)",
- )
-
- profile_name = _optional_str(route.get("profile")) or "general"
- if profile_name not in _DETERMINISTIC_PROFILE_FACTORIES:
- raise RouteBundleConfigError(
- f"route {model_id!r}: unknown profile {profile_name!r}; "
- f"expected one of {sorted(_DETERMINISTIC_PROFILE_FACTORIES)}",
- )
-
- config_data: dict[str, object] = {
- "strong": strong,
- "weak": weak,
- "classifier": {
- "id": "classifier",
- "model": _required_str(
- classifier.get("model"), f"{model_id}.classifier.model"
- ),
- "api_key": _required_str(
- classifier.get("api_key"), f"{model_id}.classifier.api_key"
- ),
- "base_url": _required_str(
- classifier.get("base_url"), f"{model_id}.classifier.base_url"
- ),
- "timeout_secs": _optional_float(
- classifier.get("timeout_secs"),
- default=30.0,
- ),
- },
- "fallback_target_on_evict": fallback_target_on_evict,
- "profile_name": profile_name,
- "classifier_min_confidence": _optional_float(
- classifier.get("min_confidence"), default=0.6,
- ),
- "classifier_fail_open": _optional_bool(
- classifier.get("fail_open"), default=True
- ),
- "classifier_recent_turn_window": _optional_int(
- classifier.get("recent_turn_window"), default=4,
- ),
- "classifier_system_prompt": _optional_str(classifier.get("prompt")),
- "classifier_max_request_chars": _optional_int(
- classifier.get("max_request_chars"),
- default=DEFAULT_MAX_REQUEST_CHARS,
- ),
- "classifier_timeout_s": _optional_float(
- classifier.get("timeout_secs"), default=30.0
- ),
- "enable_stats": _optional_bool(route.get("enable_stats"), default=True),
- }
- if "tier_timeout_s" in route:
- config_data["tier_timeout_s"] = _optional_float(
- route.get("tier_timeout_s"),
- default=None,
- )
- if "session_affinity" in route:
- config_data["session_affinity"] = _optional_bool(
- route.get("session_affinity"), default=False
- )
- if "affinity_max_sessions" in route:
- config_data["affinity_max_sessions"] = _optional_int(
- route.get("affinity_max_sessions"), default=10_000
- )
- if "affinity_warmup_turns" in route:
- config_data["affinity_warmup_turns"] = _optional_int(
- route.get("affinity_warmup_turns"), default=0
- )
-
- config = DeterministicRoutingConfig.model_validate(config_data)
- return ProfileSwitchyard(
- DeterministicRoutingProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats,
- enable_stats=config.enable_stats,
- pre_request_processors=pre_routing_request_processors,
- response_processors=extra_response_processors,
- )
- )
-
-
-def _judge_prompt_value(judge: Mapping[str, object], model_id: str) -> str | None:
- """Resolve the judge prompt override from ``prompt`` or ``prompt_path``.
-
- ``prompt`` inlines the text; ``prompt_path`` reads it from a file
- (relative paths resolve against the server's working directory).
- ``None`` keeps the built-in default.
- """
- inline = _optional_str(judge.get("prompt"))
- path = _optional_str(judge.get("prompt_path"))
- if inline is not None and path is not None:
- raise RouteBundleConfigError(
- f"route {model_id!r}: judge.prompt and judge.prompt_path are "
- "mutually exclusive",
- )
- if path is None:
- return inline
- try:
- return Path(path).read_text(encoding="utf-8")
- except (OSError, UnicodeError) as exc:
- raise RouteBundleConfigError(
- f"route {model_id!r}: judge.prompt_path {path!r}: cannot read: "
- f"{_format_exception_one_line(exc)}"
- ) from exc
-
-
-def _escalation_router_switchyard(
- model_id: str,
- route: Mapping[str, object],
- target_defaults: Mapping[str, object],
- stats: StatsAccumulator,
- pre_routing_request_processors: Sequence[Any],
- extra_response_processors: Sequence[Any],
-) -> ChainRuntime:
- """Build the judge-latched escalation-routing chain for a route.
-
- Every conversation starts on ``weak``; an LLM judge watches the
- trajectory and, on a clear pattern of trouble, latches the conversation
- to ``strong`` for the rest of the task. Requires ``judge``/``strong``/
- ``weak`` targets plus ``fallback_target_on_evict``; the full YAML schema
- and knob reference live in ``docs/routing_algorithms/
- escalation_router_routing.md``.
- """
- judge_raw = route.get("judge")
- if not isinstance(judge_raw, Mapping):
- raise RouteBundleConfigError(
- f"route {model_id!r}: type=escalation_router requires a `judge:` "
- "mapping with model/api_key/base_url",
- )
- judge = _classifier_mapping(
- judge_raw,
- target_defaults,
- allowed_keys=_ESCALATION_JUDGE_KEYS,
- where=f"{model_id}.judge",
- )
-
- strong = _target_value(
- route.get("strong"), target_defaults, default_id="strong", where="strong",
- )
- weak = _target_value(
- route.get("weak"), target_defaults, default_id="weak", where="weak",
- )
-
- fallback_target_on_evict = _required_str(
- route.get("fallback_target_on_evict"),
- f"{model_id}.fallback_target_on_evict",
- )
- valid_ids = {strong.id, weak.id}
- if fallback_target_on_evict not in valid_ids:
- raise RouteBundleConfigError(
- f"route {model_id!r}: fallback_target_on_evict="
- f"{fallback_target_on_evict!r} must match one of {sorted(valid_ids)} "
- f"(the configured strong/weak target ids)",
- )
-
- config_data: dict[str, object] = {
- "strong": strong,
- "weak": weak,
- "judge": {
- "id": "judge",
- "model": _required_str(
- judge.get("model"), f"{model_id}.judge.model"
- ),
- "api_key": _required_str(
- judge.get("api_key"), f"{model_id}.judge.api_key"
- ),
- "base_url": _required_str(
- judge.get("base_url"), f"{model_id}.judge.base_url"
- ),
- },
- "fallback_target_on_evict": fallback_target_on_evict,
- "judge_system_prompt": _judge_prompt_value(judge, model_id),
- }
- # Optional knobs are forwarded verbatim and only when present, so
- # ``EscalationRouterConfig`` stays the single owner of every default and
- # of value validation; re-declaring defaults here is exactly the config
- # drift this compatibility path must not accumulate.
- judge_key_map = {
- "timeout_secs": "judge_timeout_s",
- "min_turn": "judge_min_turn",
- "confirmations": "judge_escalate_confirmations",
- "confirmation_window": "judge_confirmation_window",
- "disable_reasoning": "judge_disable_reasoning",
- "max_completion_tokens": "judge_max_completion_tokens",
- "dump_verdicts": "judge_dump_verdicts",
- "recent_turn_window": "judge_recent_turn_window",
- "window_message_chars": "judge_window_message_chars",
- "max_request_chars": "judge_max_request_chars",
- }
- for source_key, config_key in judge_key_map.items():
- if judge.get(source_key) is not None:
- config_data[config_key] = judge[source_key]
- for route_key in (
- "enable_stats",
- "tier_timeout_s",
- "session_key_depth",
- "affinity_max_sessions",
- "affinity_store",
- "affinity_store_url",
- "affinity_store_ttl_seconds",
- "affinity_key_prefix",
- ):
- if route.get(route_key) is not None:
- config_data[route_key] = route[route_key]
-
- try:
- config = EscalationRouterConfig.model_validate(config_data)
- except ValidationError as exc:
- # Surface config mistakes as the CLI's one-line diagnostic instead of
- # a raw pydantic traceback (see the RouteBundleConfigError handler in
- # switchyard_cli.main).
- raise RouteBundleConfigError(
- f"route {model_id!r}: invalid escalation_router config: {exc}"
- ) from exc
- return ProfileSwitchyard(
- EscalationRouterProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats,
- enable_stats=config.enable_stats,
- pre_request_processors=pre_routing_request_processors,
- response_processors=extra_response_processors,
- )
- )
-
-
-def _stage_router_switchyard(
- model_id: str,
- route: Mapping[str, object],
- target_defaults: Mapping[str, object],
- stats: StatsAccumulator,
- pre_routing_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
-) -> ChainRuntime:
- """Build a stage-router-routing Switchyard from a YAML ``type: stage_router`` route.
-
- Schema (mapped onto :class:`StageRouterConfig`)::
-
- route:
- type: stage_router
- picker: capable_first # or efficient_first
- confidence_threshold: 0.7 # default; range [0.0, 1.0]
- signal_recent_window: 3 # Rust sliding-window size
- strong: <target spec> # e.g. { id: strong, model: ..., api_key: ..., format: anthropic }
- weak: <target spec> # e.g. { id: weak, model: ..., api_key: ..., format: openai }
- classifier: # optional; omit to skip the LLM fallback
- model: google/gemini-3.5-flash
- api_key: ${SWITCHYARD_CLASSIFIER_API_KEY}
- base_url: https://openrouter.ai/api/v1
- timeout_secs: 30.0
- recent_turn_window: 3
- enable_stats: true
-
- Each tier spec accepts the same shapes as other route types
- (``{ id, model, api_key, base_url, ... }`` or a model-id string);
- per-target tuning fields are honoured via :func:`_target_value`.
- """
- if route.get("strong") is None or route.get("weak") is None:
- raise RouteBundleConfigError(
- f"route {model_id!r}: stage_router route requires both 'strong' and "
- f"'weak' target specs",
- )
- if route.get("classifier") is not None:
- route = dict(route)
- route["classifier"] = _classifier_mapping(
- route["classifier"],
- target_defaults,
- allowed_keys=_STAGE_ROUTER_CLASSIFIER_KEYS,
- where=f"{model_id}.classifier",
- )
- # The YAML schema shares strong/weak tier keys with deterministic routing;
- # map them onto StageRouterConfig's capable/efficient fields.
- resolved = _route_config(route, target_defaults, ("strong", "weak"))
- resolved["capable"] = resolved.pop("strong")
- resolved["efficient"] = resolved.pop("weak")
- stage_router_config = StageRouterConfig.model_validate(resolved)
- return ProfileSwitchyard(
- StageRouterProfileConfig.from_config(stage_router_config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats,
- enable_stats=stage_router_config.enable_stats,
- pre_request_processors=pre_routing_request_processors,
- response_processors=extra_response_processors,
- )
- )
-
-
-def _passthrough_target(
- model_id: str,
- route: Mapping[str, object],
- target_defaults: Mapping[str, object],
-) -> LlmTarget:
- """Resolve the LlmTarget for a ``model`` route.
-
- A ``model`` route requires an explicit ``target`` or ``model`` field; the
- target is built from that plus the route defaults and inline fields.
- """
- target_raw = route.get("target", route.get("model"))
- if target_raw is None:
- raise RouteBundleConfigError(f"route {model_id!r} requires target or model")
- return coerce_llm_target(
- _target_mapping(target_raw, target_defaults, default_id=model_id, where="target"),
- default_id=model_id,
- )
-
-
-def _route_config(
- route: Mapping[str, object],
- target_defaults: Mapping[str, object],
- target_fields: tuple[str, ...],
-) -> dict[str, object]:
- data = {
- key: value
- for key, value in route.items()
- if key not in _COMMON_ROUTE_KEYS
- and key not in _TARGET_DEFAULT_KEYS
- }
- for tier_field in target_fields:
- data[tier_field] = _target_value(
- route.get(tier_field),
- target_defaults,
- default_id=tier_field,
- where=tier_field,
- )
- return data
-
-
-def _target_value(
- raw: object,
- defaults: Mapping[str, object],
- default_id: str,
- where: str,
-) -> LlmTarget:
- if raw is None:
- raise RouteBundleConfigError(f"{where} target is required")
- return coerce_llm_target(
- _target_mapping(raw, defaults, default_id=default_id, where=where),
- default_id=default_id,
- )
-
-
-def _classifier_mapping(
- raw: object,
- defaults: Mapping[str, object],
- *,
- allowed_keys: frozenset[str],
- where: str,
-) -> dict[str, object]:
- classifier = {
- key: value
- for key, value in defaults.items()
- if key in _CLASSIFIER_DEFAULT_KEYS
- }
- classifier_mapping = _require_mapping(raw, where)
- _validate_allowed_keys(classifier_mapping, allowed_keys, where)
- classifier.update(classifier_mapping)
- if "timeout" in classifier_mapping and "timeout_secs" not in classifier_mapping:
- classifier["timeout_secs"] = classifier_mapping["timeout"]
- elif "timeout_secs" not in classifier and "timeout" in classifier:
- classifier["timeout_secs"] = classifier["timeout"]
- classifier.pop("timeout", None)
- return classifier
-
-
-def _target_mapping(
- raw: object,
- defaults: Mapping[str, object],
- default_id: str,
- where: str,
-) -> dict[str, object]:
- target = dict(defaults)
- if isinstance(raw, str):
- target["model"] = raw
- elif isinstance(raw, Mapping):
- target_mapping = _require_mapping(raw, where)
- _validate_allowed_keys(target_mapping, _TARGET_KEYS, where)
- target.update(target_mapping)
+def _target(route_id: str, value: object, defaults: Mapping[str, object]) -> LlmTarget:
+ if isinstance(value, str):
+ target: dict[str, object] = {"model": value}
else:
- raise RouteBundleConfigError(f"{where} must be a string or mapping")
- target.setdefault("id", default_id)
- return target
-
-
-def _target_defaults(
- bundle_defaults: Mapping[str, object],
- route: Mapping[str, object],
-) -> dict[str, object]:
- defaults = dict(bundle_defaults)
- nested = route.get("defaults")
- if nested is not None:
- defaults.update(_require_mapping(nested, "route.defaults"))
- for key in _TARGET_DEFAULT_KEYS:
- if key in route:
- defaults[key] = route[key]
- return defaults
-
-
-def _normalize_route(model_id: str, raw: object) -> Mapping[str, object]:
- if isinstance(raw, str):
- return {"type": "model", "target": raw}
- if isinstance(raw, Mapping):
- return _require_mapping(raw, f"route {model_id!r}")
- raise RouteBundleConfigError(f"route {model_id!r} must be a string or mapping")
-
-
-def _route_type(model_id: str, route: Mapping[str, object]) -> str:
- raw_type = route.get("type", route.get("kind"))
- if raw_type is None:
- if not route:
- raise RouteBundleConfigError(f"route {model_id!r}: missing 'type'")
- if "strong" in route and "weak" in route:
- return "random_routing"
- if "target" in route or "model" in route:
- return "model"
- raise RouteBundleConfigError(
- f"route {model_id!r} requires an explicit type or a recognizable route shape"
- )
- if not isinstance(raw_type, str):
- raise RouteBundleConfigError("route type must be a string")
-
- normalized = raw_type.lower().replace("-", "_")
- aliases = {
- "direct": "model",
- "llm_target": "model",
- "model": "model",
- "target": "model",
- "random": "random_routing",
- "random_routing": "random_routing",
- "deterministic": "deterministic",
- "llm_classifier": "deterministic",
- "llm_classifier_routing": "deterministic",
- "escalation": "escalation_router",
- "escalation_router": "escalation_router",
- "stage_router": "stage_router",
- "stage_router_routing": "stage_router",
- }
+ target = _mapping(value, f"route {route_id!r} target")
try:
- return aliases[normalized]
- except KeyError as exc:
- raise RouteBundleConfigError(f"unsupported route type {raw_type!r}") from exc
-
-
-def _validate_route_keys(
- model_id: str,
- route: Mapping[str, object],
- route_type: str,
-) -> None:
- where = f"route {model_id!r}"
- _validate_allowed_keys(route, _ROUTE_KEYS_BY_TYPE[route_type] | _ROUTE_ENVELOPE_KEYS, where)
- if "defaults" in route:
- defaults = _require_mapping(route["defaults"], f"{where}.defaults")
- _validate_allowed_keys(
- defaults,
- _DEFAULT_KEYS_BY_TYPE[route_type],
- f"{where}.defaults",
- )
-
-
-def _validate_allowed_keys(
- mapping: Mapping[str, object],
- allowed_keys: frozenset[str],
- where: str,
-) -> None:
- unknown = sorted(set(mapping) - allowed_keys)
- if unknown:
- raise RouteBundleConfigError(
- f"unknown key(s) for {where}: {', '.join(unknown)}"
- )
-
-
-def _route_metadata(
- model_id: str,
- route: Mapping[str, object],
- route_type: str,
-) -> dict[str, object]:
- metadata: dict[str, object] = {
- "display_name": _optional_str(route.get("display_name")) or model_id,
- "switchyard": {"profile": route_type},
- }
- description = _optional_str(route.get("description"))
- if description is not None:
- metadata["description"] = description
- return metadata
+ return coerce_llm_target({**defaults, **target}, default_id=route_id)
+ except (TypeError, ValueError) as error:
+ raise RouteBundleConfigError(f"route {route_id!r}: invalid target: {error}") from error
def _expand_env(value: object) -> object:
- if isinstance(value, str):
- return _expand_env_string(value)
+ if isinstance(value, dict):
+ return {key: _expand_env(item) for key, item in value.items()}
if isinstance(value, list):
return [_expand_env(item) for item in value]
- if isinstance(value, Mapping):
- result: dict[str, object] = {}
- for key, item in value.items():
- if not isinstance(key, str):
- raise RouteBundleConfigError("route bundle keys must be strings")
- result[key] = _expand_env(item)
- return result
- return value
+ if not isinstance(value, str):
+ return value
+ def replace(match: re.Match[str]) -> str:
+ name = match.group(1)
+ if name not in os.environ:
+ raise RouteBundleConfigError(f"environment variable {name} is not set")
+ return os.environ[name]
-def _expand_env_string(value: str) -> str:
- missing = [name for name in _ENV_REF_RE.findall(value) if name not in os.environ]
- if missing:
- raise RouteBundleConfigError(
- f"missing environment variable(s): {', '.join(sorted(set(missing)))}"
- )
- return os.path.expandvars(value)
+ return _ENV_REF_RE.sub(replace, value)
-def _require_mapping(value: object, where: str) -> dict[str, object]:
+def _mapping(value: object, where: str) -> dict[str, object]:
if not isinstance(value, Mapping):
raise RouteBundleConfigError(f"{where} must be a mapping")
- result: dict[str, object] = {}
- for key, item in value.items():
- if not isinstance(key, str):
- raise RouteBundleConfigError(f"{where} keys must be strings")
- result[key] = item
- return result
-
-
-def _optional_mapping(value: object, where: str) -> dict[str, object]:
- if value is None:
- return {}
- return _require_mapping(value, where)
-
-
-def _required_str(value: object, where: str) -> str:
- if not isinstance(value, str) or not value:
- raise RouteBundleConfigError(f"{where} must be a non-empty string")
- return value
+ if not all(isinstance(key, str) for key in value):
+ raise RouteBundleConfigError(f"{where} keys must be strings")
+ return {str(key): item for key, item in value.items()}
-def _optional_str(value: object) -> str | None:
- if value is None:
- return None
- if not isinstance(value, str):
- raise RouteBundleConfigError(f"expected string, got {type(value).__name__}")
- return value
-
-
-@overload
-def _optional_float(value: object, default: float) -> float: ...
-
-
-@overload
-def _optional_float(value: object, default: None = None) -> float | None: ...
-
-
-def _optional_float(value: object, default: float | None = None) -> float | None:
- if value is None:
- return default
- if isinstance(value, bool) or not isinstance(value, int | float):
- raise RouteBundleConfigError(f"expected number, got {type(value).__name__}")
- return float(value)
-
-
-def _optional_int(value: object, default: int) -> int:
- if value is None:
- return default
- if isinstance(value, bool) or not isinstance(value, int):
- raise RouteBundleConfigError(f"expected integer, got {type(value).__name__}")
- return value
-
-
-def _optional_bool(value: object, default: bool) -> bool:
- if value is None:
- return default
- if not isinstance(value, bool):
- raise RouteBundleConfigError(f"expected boolean, got {type(value).__name__}")
- return value
+def _reject_unknown_keys(
+ value: Mapping[str, object], allowed: frozenset[str], where: str
+) -> None:
+ unknown = sorted(set(value) - allowed)
+ if unknown:
+ raise RouteBundleConfigError(f"unknown key(s) for {where}: {', '.join(unknown)}")
__all__ = [
"RouteBundleConfigError",
"build_route_bundle_table",
"load_route_bundle_table",
- "parse_routing_profiles_file",
+ "parse_route_bundle_file",
]
diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.py
index 67279700d..3698e4aa9 100644
--- a/switchyard/cli/switchyard_cli.py
+++ b/switchyard/cli/switchyard_cli.py
@@ -28,7 +28,7 @@ from switchyard.server.server_util import (
logger = logging.getLogger(__name__)
def _cmd_serve(args: argparse.Namespace) -> None:
- """Serve an explicit routing-profile bundle."""
+ """Serve an explicit Python route bundle."""
request_processors, response_processors = build_rl_logging_processors(
resolve_rl_log_dir(args)
@@ -41,24 +41,24 @@ def _cmd_serve(args: argparse.Namespace) -> None:
response_processors.append(RoutingLogResponseProcessor(args.routing_log_file))
table = load_route_bundle_table(
- args.routing_profiles,
+ args.routes,
pre_routing_request_processors=request_processors,
extra_response_processors=response_processors,
)
logger.info(
"Switchyard route bundle loaded %d route(s) from %s",
len(table.registered_models()),
- args.routing_profiles,
+ args.routes,
)
strategy_summary: str | None = None
default_model = table.default_model()
if default_model:
from switchyard.cli.launchers.launcher_runtime import (
- routing_profiles_strategy_summary,
+ route_bundle_strategy_summary,
)
- strategy_summary = routing_profiles_strategy_summary(
- args.routing_profiles,
+ strategy_summary = route_bundle_strategy_summary(
+ args.routes,
default_model,
)
build_and_serve(
@@ -116,13 +116,13 @@ def _build_parser() -> argparse.ArgumentParser:
)
subparsers = parser.add_subparsers(dest="command")
- serve = subparsers.add_parser("serve", help="Serve a routing-profile bundle")
+ serve = subparsers.add_parser("serve", help="Serve a Python route bundle")
serve.add_argument(
- "--routing-profiles",
+ "--routes",
"-c",
required=True,
metavar="PATH",
- help="Routing-profile YAML bundle.",
+ help="YAML bundle containing noop and passthrough routes.",
)
serve.add_argument("--enable-rl-logging", action="store_true")
serve.add_argument("--rl-log-dir", default="./rl_data", metavar="DIR")
diff --git a/switchyard/lib/__init__.py b/switchyard/lib/__init__.py
index 237c5767c..b7032e225 100644
--- a/switchyard/lib/__init__.py
+++ b/switchyard/lib/__init__.py
@@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Core library — Rust-backed chat request/response values, routing, and profiles.
+"""Core library — Rust-backed chat values, translation, and Python serving.
This subpackage holds the protocol-agnostic building blocks used across the rest
of the library:
@@ -9,8 +9,7 @@ of the library:
- ``ChatRequest`` — Rust-backed request values (OpenAI, Responses, Anthropic)
- ``chat_response`` — Rust-backed response values plus Python stream adapters
- ``translation`` — pure format-conversion functions and typed translation engines
-- ``processors`` — reusable request/response components used by profiles
+- ``processors`` — reusable request/response components
- ``backends`` — LLM backend implementations (OpenAI, Anthropic, multi-tier routing)
-- ``profiles`` — profile-owned runtime and programmatic config abstractions
- ``roles`` — backend role definitions and translation response aliases
"""
diff --git a/switchyard/lib/affinity_pin_store.py b/switchyard/lib/affinity_pin_store.py
deleted file mode 100644
index a0a076913..000000000
--- a/switchyard/lib/affinity_pin_store.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Pluggable async L2 store for session-affinity pins.
-
-The in-process ``SessionCache`` stays the L1 inside
-:class:`switchyard.lib.session_affinity.SessionAffinity`. An optional object
-implementing this protocol is the L2: a shared, out-of-process store (e.g.
-Redis) that lets pins be read by every Switchyard worker/pod and survive pod
-churn, so a conversation keeps hitting the same upstream even when its turns
-land on different replicas.
-"""
-
-from __future__ import annotations
-
-from typing import Protocol, runtime_checkable
-
-
-@runtime_checkable
-class AffinityPinStore(Protocol):
- """Shared, out-of-process store for conversation→decision pins.
-
- The pinned value is opaque to the store (a tier label for the classifier
- or escalation router). Implementations are
- called on the request path, so they should be fast, and
- :class:`SessionAffinity` treats them as **best-effort**: an error or timeout
- must never break request routing (the caller falls back to L1 / unpinned).
- """
-
- async def get(self, key: str) -> str | None:
- """Return the value pinned to ``key``, or ``None`` if absent."""
- ...
-
- async def put(self, key: str, value: str) -> None:
- """Pin ``key`` to ``value``. May apply a TTL or eviction policy."""
- ...
-
-
-__all__ = ["AffinityPinStore"]
diff --git a/switchyard/lib/backends/anthropic_cache_breakpoint_backend.py b/switchyard/lib/backends/anthropic_cache_breakpoint_backend.py
deleted file mode 100644
index cbab05005..000000000
--- a/switchyard/lib/backends/anthropic_cache_breakpoint_backend.py
+++ /dev/null
@@ -1,263 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Anthropic ephemeral prompt-caching wrapper backend.
-
-When wired into a tier slot of
-:class:`switchyard.lib.backends.deterministic_routing_llm_backend.DeterministicRoutingLLMBackend` (or any
-multi-LLM-backend), this wrapper prepares each outbound call to its
-inner :class:`AnthropicNativeBackend` so that Anthropic / Bedrock
-honors prompt caching:
-
-1. Translates the inbound :class:`ChatRequest` body to Anthropic
- Messages format (no-op if already Anthropic).
-2. Injects ``cache_control: {"type": "ephemeral"}`` markers on (a)
- the system field's last text block and (b) the last message's
- last block — two breakpoints out of Anthropic's four-per-request
- budget. Injection is skipped when the inbound body already carries
- its own markers (an Anthropic-native client like Claude Code), so
- wrapping that path can never exceed the four-breakpoint cap.
-3. Builds a **local** Anthropic-typed :class:`ChatRequest` and
- delegates to the inner backend, which sees an already-Anthropic
- request and passes the body through verbatim (preserving the
- markers all the way to the wire).
-
-Why this lives in a wrapper instead of a request processor: mutating the
-chain-level :attr:`ChatRequest.request_type` before the backend call propagates
-through the rest of the chain.
-The terminal :class:`TranslationEngine` step then reads the modified
-type, decides "the client wanted Anthropic," and skips translation
-of the Anthropic response back to the client's original wire format.
-OpenAI Chat / OpenAI Responses clients (Harbor's LiteLLM, OpenAI
-SDK) receive an Anthropic-shape body and crash on missing
-``choices``.
-
-Keeping the translation **inside the backend** means the chain-level
-request object stays in its inbound shape; the terminal translator
-correctly converts the Anthropic response back to whatever the
-client expected. No translator or backend-Rust changes required —
-PR #82's removal of global cache_control injection stands.
-"""
-
-from __future__ import annotations
-
-import logging
-from typing import TYPE_CHECKING, Any
-
-from switchyard.lib.roles import LLMBackend
-
-log = logging.getLogger(__name__)
-
-if TYPE_CHECKING:
- from switchyard.lib.proxy_context import ProxyContext
- from switchyard_rust.core import ChatRequest, ChatResponse
- from switchyard_rust.translation import TranslationEngine
-
-
-def maybe_wrap_anthropic_cache(backend: LLMBackend, target: Any) -> LLMBackend:
- """Wrap ``backend`` for prompt caching iff ``target`` is Anthropic-format.
-
- Shared by every deterministic-routing construction path (the
- ``DeterministicRoutingFactory`` and the ``serve --routing-profiles`` YAML
- bundle builder) so an OpenAI-shaped harness (Codex, OpenAI SDK) routed onto
- a Claude tier still gets ``cache_control`` breakpoints injected.
-
- ``target`` must already be format-resolved (``BackendFormat.AUTO`` is gone
- by this point). Apply **outermost** — :class:`AnthropicCacheBreakpointBackend`
- is a Python-only backend, while a wrapping ``StatsLlmBackend`` requires a
- Rust-native inner.
- """
- from switchyard.lib.backends.llm_target import BackendFormat
-
- wrap = getattr(target, "format", None) == BackendFormat.ANTHROPIC
- log.info(
- "anthropic cache-wrap decision: model=%s resolved_format=%s wrapped=%s",
- getattr(target, "model", "?"), getattr(target, "format", "?"), wrap,
- )
- return AnthropicCacheBreakpointBackend(backend) if wrap else backend
-
-
-class AnthropicCacheBreakpointBackend(LLMBackend):
- """Wrap an Anthropic-format LLMBackend to inject ephemeral cache breakpoints.
-
- Drop-in around the inner backend. ``startup`` / ``shutdown`` /
- ``supported_request_types`` are forwarded; ``call`` is intercepted
- to prepare the body before delegation.
- """
-
- def __init__(self, inner: LLMBackend) -> None:
- self._inner = inner
- # Lazy: build the engine on first call, not at construction
- # time, so unit tests can construct the wrapper without
- # importing the Rust translation extension.
- self._engine: TranslationEngine | None = None
-
- async def startup(self) -> None:
- await self._inner.startup()
-
- async def shutdown(self) -> None:
- await self._inner.shutdown()
-
- async def call(
- self,
- ctx: ProxyContext,
- request: ChatRequest,
- ) -> ChatResponse:
- from switchyard_rust.core import ChatRequestType, request_with_type
-
- if request.request_type is ChatRequestType.ANTHROPIC:
- body = dict(request.body)
- # An Anthropic-native client (e.g. Claude Code) chooses its own
- # cache breakpoints. Respect them verbatim — re-marking could push
- # the request past Anthropic's 4-breakpoint cap — so only inject
- # when the client supplied none (e.g. a bare Anthropic SDK call).
- if _has_cache_control(body):
- return await self._inner.call(ctx, request)
- else:
- engine = self._engine
- if engine is None:
- from switchyard_rust.translation import TranslationEngine
- engine = TranslationEngine()
- self._engine = engine
- anthropic_request = engine.request_to("anthropic_messages", request)
- body = dict(anthropic_request.body)
-
- body = _inject_anthropic_cache_breakpoints(body)
- marked_request = request_with_type("anthropic", body)
- return await self._inner.call(ctx, marked_request)
-
-
-def _inject_anthropic_cache_breakpoints(body: dict[str, Any]) -> dict[str, Any]:
- """Add ``cache_control: {"type": "ephemeral"}`` markers to an Anthropic body.
-
- Places at most two breakpoints (Anthropic supports four):
-
- * One **stable** anchor on the ``system`` field — caches the
- terminus-2 / Harbor system prompt across every turn of the
- task, since it doesn't change.
- * One **rolling** anchor on the last message — extends the
- cached prefix as the conversation grows, so each subsequent
- turn re-uses the prior turn's full history.
-
- Both markers are placed on the *last text block* of their
- container, which is what Anthropic's API uses as the cache
- boundary. If the container is a plain string, it's converted to
- a single-block list with the marker — Anthropic accepts both
- string and list shapes for these fields, and the list shape is
- the only one that admits ``cache_control``.
-
- Returns a fresh dict; the input is not mutated.
- """
- body = dict(body)
- _mark_system_field(body)
- _mark_last_message(body)
- return body
-
-
-def _mark_system_field(body: dict[str, Any]) -> None:
- """In-place: annotate ``body["system"]`` with a cache breakpoint."""
- system = body.get("system")
- if isinstance(system, str) and system:
- body["system"] = [
- {
- "type": "text",
- "text": system,
- "cache_control": {"type": "ephemeral"},
- },
- ]
- elif isinstance(system, list) and system:
- new_system = [
- dict(blk) if isinstance(blk, dict) else blk for blk in system
- ]
- last_text = _last_text_block_index(new_system)
- if last_text is not None:
- new_system[last_text]["cache_control"] = {"type": "ephemeral"}
- body["system"] = new_system
-
-
-def _mark_last_message(body: dict[str, Any]) -> None:
- """In-place: annotate the last text block in the last message.
-
- Falls back to the last block of any type if the last message has
- no text block (e.g., a pure tool_result message) — Anthropic
- allows ``cache_control`` on non-text blocks too.
- """
- messages = body.get("messages")
- if not isinstance(messages, list) or not messages:
- return
-
- last_idx = len(messages) - 1
- last_msg = messages[last_idx]
- if not isinstance(last_msg, dict):
- return
-
- new_msg = dict(last_msg)
- content = new_msg.get("content")
-
- if isinstance(content, str) and content:
- new_msg["content"] = [
- {
- "type": "text",
- "text": content,
- "cache_control": {"type": "ephemeral"},
- },
- ]
- elif isinstance(content, list) and content:
- new_content = [
- dict(blk) if isinstance(blk, dict) else blk for blk in content
- ]
- last_text = _last_text_block_index(new_content)
- if last_text is None:
- for i in range(len(new_content) - 1, -1, -1):
- blk = new_content[i]
- if isinstance(blk, dict):
- blk["cache_control"] = {"type": "ephemeral"}
- last_text = i
- break
- else:
- new_content[last_text]["cache_control"] = {"type": "ephemeral"}
- if last_text is not None:
- new_msg["content"] = new_content
- else:
- return
-
- new_messages = list(messages)
- new_messages[last_idx] = new_msg
- body["messages"] = new_messages
-
-
-def _has_cache_control(body: dict[str, Any]) -> bool:
- """True if an Anthropic body already carries a ``cache_control`` marker.
-
- Only list-shaped ``system`` / message ``content`` blocks can hold one;
- the plain-string forms never do. Used to leave a client's own
- breakpoints untouched instead of stacking ours on top.
- """
- if _blocks_have_cache_control(body.get("system")):
- return True
- messages = body.get("messages")
- if isinstance(messages, list):
- return any(
- isinstance(msg, dict) and _blocks_have_cache_control(msg.get("content"))
- for msg in messages
- )
- return False
-
-
-def _blocks_have_cache_control(field: Any) -> bool:
- """True if ``field`` is a block list with any ``cache_control`` marker."""
- return isinstance(field, list) and any(
- isinstance(blk, dict) and "cache_control" in blk for blk in field
- )
-
-
-def _last_text_block_index(blocks: list[Any]) -> int | None:
- """Return the index of the last ``{"type": "text", ...}`` block, or None."""
- for i in range(len(blocks) - 1, -1, -1):
- blk = blocks[i]
- if isinstance(blk, dict) and blk.get("type") == "text":
- return i
- return None
-
-
-__all__ = ["AnthropicCacheBreakpointBackend", "maybe_wrap_anthropic_cache"]
diff --git a/switchyard/lib/backends/deterministic_routing_llm_backend.py b/switchyard/lib/backends/deterministic_routing_llm_backend.py
deleted file mode 100644
index b029161ba..000000000
--- a/switchyard/lib/backends/deterministic_routing_llm_backend.py
+++ /dev/null
@@ -1,166 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Deterministic router dispatch backend.
-
-This backend executes a pre-stamped routing decision. It does not score the
-request or duplicate routing rules; upstream experimental processors choose a
-tier and write it to ``ProxyContext.metadata``.
-"""
-
-from __future__ import annotations
-
-import logging
-from collections.abc import Mapping
-from typing import TYPE_CHECKING
-
-from switchyard.lib.backends.llm_target import LlmTarget
-from switchyard.lib.roles import LLMBackend
-from switchyard_rust.core import ChatRequestType
-
-if TYPE_CHECKING:
- from switchyard.lib.proxy_context import ProxyContext
- from switchyard_rust.core import ChatRequest, ChatResponse
-
-log = logging.getLogger(__name__)
-
-#: ``ProxyContext.metadata`` key under which the upstream processor stamps the
-#: chosen tier label. Read by :meth:`DeterministicRoutingLLMBackend._pick_tier`.
-CTX_DETERMINISTIC_ROUTING_TIER = "_deterministic_routing_tier"
-
-
-class DeterministicRoutingLLMBackend(LLMBackend):
- """Dispatch to the tier picked by the upstream router processor.
-
- Args:
- tiers: Mapping of tier label to ``(LLMBackend, model_name)``.
- Pass pre-built backends here when you want full control. For the
- common case, use :meth:`from_tiers` so the standard OpenAI /
- Anthropic backends get built from :class:`LlmTarget` config.
- default_tier: Tier label to dispatch to when the request processor
- did not run or stamped an unrecognised label. Must be a key of
- ``tiers``.
- """
-
- def __init__(
- self,
- *,
- tiers: dict[str, tuple[LLMBackend, str]],
- default_tier: str,
- ) -> None:
- if not tiers:
- raise ValueError(
- "DeterministicRoutingLLMBackend requires at least one tier",
- )
- if default_tier not in tiers:
- raise ValueError(
- f"default_tier {default_tier!r} not in tiers {sorted(tiers)}",
- )
- self._backends = {label: backend for label, (backend, _) in tiers.items()}
- self._models = {label: model for label, (_, model) in tiers.items()}
- self._default_tier = default_tier
-
- log.info(
- "DeterministicRoutingLLMBackend: tiers=%s default=%s",
- {label: model for label, (_, model) in tiers.items()},
- default_tier,
- )
-
- @classmethod
- def from_tiers(
- cls,
- *,
- tiers: Mapping[str, LlmTarget],
- default_tier: str,
- ) -> DeterministicRoutingLLMBackend:
- """Build a backend from a mapping of ``label -> LlmTarget``."""
- if not tiers:
- raise ValueError("from_tiers requires at least one tier")
- if default_tier not in tiers:
- raise ValueError(
- f"default_tier {default_tier!r} not in tiers {sorted(tiers)}",
- )
-
- built = {
- label: (_build_backend(target), target.model)
- for label, target in tiers.items()
- }
- return cls(tiers=built, default_tier=default_tier)
-
- @property
- def supported_request_types(self) -> list[ChatRequestType]:
- return [
- ChatRequestType.OPENAI_CHAT,
- ChatRequestType.OPENAI_RESPONSES,
- ChatRequestType.ANTHROPIC,
- ]
-
- async def startup(self) -> None:
- for backend in self._backends.values():
- await backend.startup()
-
- async def shutdown(self) -> None:
- for backend in reversed(list(self._backends.values())):
- await backend.shutdown()
-
- async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
- from switchyard_rust.components import set_stats_route_label
-
- tier = self._pick_tier(ctx, request)
- model = self._models[tier]
- request.set_model(model)
- ctx.selected_target = tier
- ctx.selected_model = model
- # Stamp the tier label so the inner StatsLlmBackend's
- # ``selected_stats_tier`` returns ``"strong"`` / ``"weak"`` and
- # the accumulator's snapshot populates the ``tiers`` block —
- # without this, the launcher's LiveStatsFooter tier rows stay
- # empty and ``GET /v1/routing/stats`` loses per-tier attribution.
- set_stats_route_label(ctx, tier)
- return await self._backends[tier].call(ctx, request)
-
- def _pick_tier(self, ctx: ProxyContext, request: ChatRequest) -> str: # noqa: ARG002
- """Read the tier label stamped upstream; fall back defensively.
-
- ``ctx.selected_target`` is the routing runtime's source of truth — it
- gets rewritten to the configured ``fallback_target_on_evict`` after a
- context-window overflow, so honouring it (over the upstream-stamped
- metadata key) is what lets evict-and-reroute actually reach this
- backend's strong tier on the retry.
- """
- rerouted = ctx.selected_target
- if isinstance(rerouted, str) and rerouted in self._backends:
- return rerouted
-
- picked = ctx.metadata.get(CTX_DETERMINISTIC_ROUTING_TIER)
- if isinstance(picked, str) and picked in self._backends:
- return picked
-
- log.warning(
- "DeterministicRoutingLLMBackend: no valid tier on ctx "
- "(found %r); defaulting to %r.",
- picked,
- self._default_tier,
- )
- return self._default_tier
-
-
-__all__ = [
- "CTX_DETERMINISTIC_ROUTING_TIER",
- "DeterministicRoutingLLMBackend",
-]
-
-
-def _build_backend(target: LlmTarget) -> LLMBackend:
- """Build a backend for the experimental deterministic router.
-
- This experimental path now uses the same Rust-owned native backend builder
- as production factories so routing behavior cannot drift.
- """
- from switchyard.lib.backends.multi_llm_backend import (
- build_native_backend,
- resolve_llm_target,
- )
-
- target = resolve_llm_target(target)
- return build_native_backend(target)
diff --git a/switchyard/lib/backends/multi_llm_backend.py b/switchyard/lib/backends/multi_llm_backend.py
index 546458b9d..48afdcb4f 100644
--- a/switchyard/lib/backends/multi_llm_backend.py
+++ b/switchyard/lib/backends/multi_llm_backend.py
@@ -1,27 +1,22 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Rust-owned multi-LLM backend helpers."""
+"""Build a native backend for a single LLM target."""
from __future__ import annotations
import logging
-from collections.abc import Iterable, Mapping
-from typing import cast
from switchyard.lib.backends.backend_format_resolver import BackendFormatResolver
from switchyard.lib.backends.llm_target import (
BackendFormat,
LlmTarget,
- coerce_llm_target,
llm_target_with_format,
llm_target_with_runtime_defaults,
)
from switchyard.lib.roles import LLMBackend
from switchyard_rust.components import (
AnthropicNativeBackend,
- LlmTargetBackend,
- MultiLlmBackend,
OpenAiNativeBackend,
)
@@ -53,38 +48,7 @@ def build_native_backend(target: LlmTarget) -> LLMBackend:
raise ValueError(f"Unsupported backend format: {target.format!r}")
-def build_target_backend(target: LlmTarget) -> LlmTargetBackend:
- """Build one target/backend pair for ``MultiLlmBackend``."""
- target = llm_target_with_runtime_defaults(resolve_llm_target(target))
- return LlmTargetBackend(target, build_native_backend(target))
-
-
-def build_multi_llm_backend(
- targets: Iterable[LlmTarget] | Mapping[str, LlmTarget],
- *,
- default_target_id: str | None = None,
-) -> MultiLlmBackend:
- """Build a Rust ``MultiLlmBackend`` from configured targets."""
- if isinstance(targets, Mapping):
- target_values: Iterable[LlmTarget] = [
- coerce_llm_target(target, default_id=str(target_id))
- for target_id, target in targets.items()
- ]
- else:
- target_values = targets
- return cast( # type: ignore[redundant-cast]
- MultiLlmBackend,
- MultiLlmBackend(
- [build_target_backend(target) for target in target_values],
- default_target_id=default_target_id,
- ),
- )
-
-
__all__ = [
- "MultiLlmBackend",
- "build_multi_llm_backend",
"build_native_backend",
- "build_target_backend",
"resolve_llm_target",
]
diff --git a/switchyard/lib/cost_estimator.py b/switchyard/lib/cost_estimator.py
index 19d747384..ed86c00f2 100644
--- a/switchyard/lib/cost_estimator.py
+++ b/switchyard/lib/cost_estimator.py
@@ -215,10 +215,7 @@ MODEL_PRICING: dict[str, ModelPriceData] = {
# --- Anthropic Claude on Azure (via NVIDIA Inference Hub) ---
# Same per-token pricing as the Bedrock tier at parity versions;
# listed separately so routing decisions that pick the Azure-hosted
- # variant don't silently report $0 in cost dashboards. Used by
- # ``RandomRoutingPresets.opus_kimi`` as the Bedrock-avoidance path
- # for Claude Code (Bedrock's 64-char ``toolSpec.name`` cap breaks
- # on auto-injected MCP tool names).
+ # variant doesn't silently report $0 in cost dashboards.
"azure/anthropic/claude-opus-4-6": ModelPriceData(
input=5.00, output=25.00, cached=0.50, cache_write=6.25,
),
diff --git a/switchyard/lib/processors/_structured_output.py b/switchyard/lib/processors/_structured_output.py
deleted file mode 100644
index bdf62aa39..000000000
--- a/switchyard/lib/processors/_structured_output.py
+++ /dev/null
@@ -1,120 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Strict-mode JSON-schema helpers for OpenAI Structured Outputs.
-
-Routing processors build their ``response_format`` payload via
-:func:`build_response_format` and rely on
-:func:`to_strict_openai_schema` to translate a pydantic
-``model_json_schema()`` into the subset OpenAI accepts under
-``strict: true``.
-"""
-
-from __future__ import annotations
-
-from typing import Any, Literal
-
-from pydantic import BaseModel
-
-# Keywords that pydantic emits but OpenAI Structured Outputs' strict subset
-# does not accept. We drop them from the wire schema.
-_STRICT_DROP_KEYS: frozenset[str] = frozenset(
- {
- "default",
- "minimum",
- "maximum",
- "exclusiveMinimum",
- "exclusiveMaximum",
- "multipleOf",
- "format",
- "pattern",
- "minLength",
- "maxLength",
- "minItems",
- "maxItems",
- "uniqueItems",
- "title",
- },
-)
-
-# JSON Schema keywords whose value is a name→schema mapping rather than a
-# normal schema-annotation dict. Their child keys are user-defined names
-# (property names, definition names) and must never be filtered against
-# ``_STRICT_DROP_KEYS`` — a Pydantic model with a field literally named
-# ``title`` would otherwise lose that property because ``title`` is also a
-# JSON Schema annotation we strip. Recurse into the values; leave the keys
-# untouched.
-_NAME_MAP_KEYS: frozenset[str] = frozenset(
- {
- "properties",
- "patternProperties",
- "$defs",
- "definitions",
- "dependentSchemas",
- },
-)
-
-
-def to_strict_openai_schema(model_cls: type[BaseModel]) -> dict[str, Any]:
- """Convert a pydantic JSON schema to the OpenAI Structured Outputs subset.
-
- Strict mode requires every object to set ``additionalProperties: false``
- and list ALL of its property keys under ``required`` (even ones that have
- defaults in the pydantic model). Several JSON-Schema keywords are not
- supported and must be stripped.
- """
- result = _strictify(model_cls.model_json_schema())
- assert isinstance(result, dict)
- return result
-
-
-def _strictify(node: Any) -> Any:
- if isinstance(node, dict):
- out: dict[str, Any] = {}
- for key, value in node.items():
- if key in _STRICT_DROP_KEYS:
- continue
- if key in _NAME_MAP_KEYS and isinstance(value, dict):
- # ``properties`` / ``$defs`` / etc.: child keys are
- # user-defined property/definition names. Recurse into
- # the values but preserve every key verbatim.
- out[key] = {k: _strictify(v) for k, v in value.items()}
- else:
- out[key] = _strictify(value)
- if out.get("type") == "object" and "properties" in out:
- out["additionalProperties"] = False
- out["required"] = list(out["properties"].keys())
- return out
- if isinstance(node, list):
- return [_strictify(item) for item in node]
- return node
-
-
-def build_response_format(
- schema: type[BaseModel] | None,
- mode: Literal["json_schema", "json_object"],
-) -> dict[str, Any]:
- """Build the ``response_format`` kwarg for ``acompletion(...)``.
-
- ``json_schema`` (preferred) returns the strict Structured Outputs
- payload built from ``schema``. Falls back to ``{"type":
- "json_object"}`` when ``mode="json_object"`` or no schema is
- supplied — useful for backends that don't advertise Structured
- Outputs support.
- """
- if mode == "json_object" or schema is None:
- return {"type": "json_object"}
- return {
- "type": "json_schema",
- "json_schema": {
- "name": schema.__name__,
- "strict": True,
- "schema": to_strict_openai_schema(schema),
- },
- }
-
-
-__all__ = [
- "build_response_format",
- "to_strict_openai_schema",
-]
diff --git a/switchyard/lib/processors/escalation_judge_request_processor.py b/switchyard/lib/processors/escalation_judge_request_processor.py
deleted file mode 100644
index 3cdf44f51..000000000
--- a/switchyard/lib/processors/escalation_judge_request_processor.py
+++ /dev/null
@@ -1,537 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Trajectory judge for the escalation router: weak until in trouble, then strong."""
-
-from __future__ import annotations
-
-import json
-import logging
-import sys
-import time
-from collections import OrderedDict
-from importlib.resources import files
-from typing import Any, Literal
-
-from pydantic import BaseModel, ConfigDict, Field
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- CTX_DETERMINISTIC_ROUTING_TIER,
-)
-from switchyard.lib.conversation_turn import conversation_turn_number
-from switchyard.lib.llm_client import OpenAILLMClient
-from switchyard.lib.processors.llm_classifier.request_processor import (
- LLMClassifierClient,
- OpenAIChatLLMClassifierClient,
-)
-from switchyard.lib.processors.message_condensing import (
- content_text,
- message_text,
- strip_markdown_fence,
- trim_messages,
- truncate_middle,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.session_affinity import (
- CTX_SESSION_KEY,
- SessionAffinity,
- resolve_session_key,
-)
-from switchyard.lib.stats_accumulator import StatsAccumulator
-from switchyard_rust.core import ChatRequest
-
-log = logging.getLogger(__name__)
-
-#: ``ProxyContext.metadata`` key holding the audit record of the judge's
-#: decision for this turn: ``{"escalate", "reason", "turn", "source",
-#: "session", "streak", "confirmed"}`` (the last three only on judged turns).
-CTX_ESCALATION_VERDICT = "_escalation_verdict"
-
-#: LRU cap on the consecutive-escalate streak store.
-_MAX_STREAK_SESSIONS = 10_000
-
-#: Default tier labels stamped into ``CTX_DETERMINISTIC_ROUTING_TIER``. The
-#: labels must match what the profile registers on
-#: ``DeterministicRoutingLLMBackend`` — profiles with custom target ids pass
-#: their own via ``strong_tier`` / ``weak_tier``.
-TIER_STRONG = "strong"
-TIER_WEAK = "weak"
-
-
-def _load_system_prompt() -> str:
- """Read the default judge system prompt from the prompts/ package-data file.
-
- The file carries the benchmarked line-of-record prompt byte-exact; its
- content is hash-pinned in tests, so edits require deliberate re-validation.
- """
- return (
- files("switchyard.lib.processors.prompts")
- .joinpath("escalation_judge.md")
- .read_text(encoding="utf-8")
- )
-
-
-ESCALATION_JUDGE_SYSTEM_PROMPT: str = _load_system_prompt()
-
-
-class EscalationVerdict(BaseModel):
- """Binary judge verdict: escalate to the strong tier or stay on weak."""
-
- model_config = ConfigDict(frozen=True)
-
- escalate: bool
- reason: str = ""
-
-
-class EscalationJudgeConfig(BaseModel):
- """Configuration for :class:`EscalationJudgeRequestProcessor`.
-
- The judge call is OpenAI-chat-compatible; tests inject any object
- implementing the classifier's ``LLMClassifierClient`` protocol.
- """
-
- model_config = ConfigDict(frozen=True)
-
- model: str = Field(min_length=1)
- api_key: str | None = None
- base_url: str | None = None
- timeout_s: float = Field(default=5.0, gt=0.0)
- """Judge wall-clock ceiling. The judge sits on the request path of every
- pre-escalation turn; a slow judge taxes the whole session, so keep this
- tight — any failure or timeout fails open to the weak tier."""
-
- max_completion_tokens: int = Field(default=128, ge=16)
- system_prompt: str = Field(default=ESCALATION_JUDGE_SYSTEM_PROMPT, min_length=1)
- structured_output_mode: Literal["json_schema", "json_object"] = "json_schema"
- disable_reasoning: bool = True
- extra_headers: dict[str, str] | None = None
-
- dump_verdicts_to_stderr: bool = False
- """Emit one ``escalation_verdict={...}`` JSON line to ``sys.stderr`` per
- judge call (verdict or fail-open).
-
- Benchmark-only diagnostics, **opt-in**: the payload includes a short
- first-user snippet (``task_hint``) and the line interferes with agent
- TUIs sharing stderr, so shipping profiles keep it off and production
- routing telemetry flows through the normal stats path. When enabled it
- is written directly (not via the logging module) so it lands in the
- benchmark server's captured log regardless of uvicorn's logger config.
- Grep ``escalation_verdict=`` to reconstruct per-turn judge decisions
- and escalation timing for a run."""
-
- min_judge_turn: int = Field(default=3, ge=1)
- """First conversation turn on which the judge runs. Earlier turns have no
- trajectory to judge and always route weak."""
-
- escalate_confirmations: int = Field(default=2, ge=1)
- """Consecutive ``escalate`` verdicts required before the latch fires.
-
- ``1`` pins on the first escalate verdict. ``2`` (default) requires the
- judge to confirm on the next judged turn, filtering one-shot eager
- verdicts (a single failed command misread as a pattern) while keeping
- recall on real trouble, which by definition persists across turns. A
- non-escalate verdict resets the streak; a fail-open leaves it
- unchanged (no evidence either way)."""
-
- confirmation_window: int = Field(default=1, ge=1)
- """How many judged turns an escalate verdict stays live for
- confirmation purposes.
-
- ``1`` (default) keeps the strict-consecutive behaviour: any decline
- resets the streak. ``N > 1`` lets a later escalate verdict confirm an
- earlier one across up to ``N - 1`` intervening declines — trouble that
- keeps *recurring* is trouble, even when a quiet turn separates the
- flare-ups."""
-
- recent_turn_window: int = Field(default=28, ge=1)
- """Trailing messages shown to the judge on top of the anchors. Wider than
- the classifier's default because loop detection needs to see the repeats:
- a cycle longer than the window is invisible."""
-
- max_request_chars: int = Field(default=18_000, ge=1_000)
- """Cap on the assembled judge transcript. When exceeded, the *oldest*
- window messages are dropped first — for a trajectory judge the newest
- evidence is strictly the most valuable."""
-
- system_chars: int = Field(default=1_000, ge=100)
- """Per-message cap for system/developer anchors. Coding-agent harnesses
- (Claude Code) inject very large boilerplate system prompts with no
- trajectory signal; without this cap they would crowd out the window."""
-
- first_user_chars: int = Field(default=2_000, ge=100)
- """Cap for the first user message — the task statement the judge needs
- for drift detection, so it gets the most generous anchor budget."""
-
- window_message_chars: int = Field(default=500, ge=50)
- """Per-message cap inside the trailing window. Error signatures and
- command shapes survive this easily; full file dumps do not need to."""
-
-
-class EscalationJudgeRequestProcessor:
- """Route weak until an LLM judge says the run is in trouble, then latch strong.
-
- Per-turn policy (the latch): a pinned conversation routes strong with no
- judge call; an unpinned one routes weak, and from ``min_judge_turn`` on the
- judge reads a condensed transcript. An ``escalate`` verdict pins the
- conversation to the strong tier via :class:`SessionAffinity` — one-way for
- the rest of the task. Weak is never pinned, and any judge failure fails
- open to weak without pinning, so an outage costs quality risk, never money.
- """
-
- def __init__(
- self,
- config: EscalationJudgeConfig,
- *,
- affinity: SessionAffinity,
- client: LLMClassifierClient | None = None,
- session_key_depth: int = 0,
- stats_accumulator: StatsAccumulator | None = None,
- strong_tier: str = TIER_STRONG,
- weak_tier: str = TIER_WEAK,
- ) -> None:
- if session_key_depth < 0:
- raise ValueError("session_key_depth must be >= 0")
- if strong_tier == weak_tier:
- raise ValueError("strong_tier and weak_tier must differ")
- self._config = config
- self._affinity = affinity
- self._session_key_depth = session_key_depth
- # The labels this processor stamps and pins; they key the tier dict on
- # the downstream DeterministicRoutingLLMBackend, so both must match
- # what the owning profile registered there.
- self._strong_tier = strong_tier
- self._weak_tier = weak_tier
- # Also attached post-construction by the profile chain's
- # ``with_runtime_components`` (same hook as the LLM classifier).
- self._stats_accumulator = stats_accumulator
- # Per-conversation (streak, declines_since_last_fire), only consulted
- # when ``escalate_confirmations > 1``. LRU-capped alongside the
- # affinity store so a long-lived server cannot grow it unbounded.
- self._escalate_streaks: OrderedDict[str, tuple[int, int]] = OrderedDict()
- self._client = client or OpenAIChatLLMClassifierClient(
- OpenAILLMClient(
- api_key=config.api_key,
- base_url=config.base_url,
- timeout=config.timeout_s,
- # Fail-open to weak is the retry surface; let the SDK fail fast.
- max_retries=0,
- ),
- signal_schema=EscalationVerdict,
- structured_output_mode=config.structured_output_mode,
- max_completion_tokens=config.max_completion_tokens,
- disable_reasoning=config.disable_reasoning,
- extra_headers=config.extra_headers,
- )
-
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- """Stamp the tier for this turn and maybe escalate. Leaves the request unchanged."""
- turn = conversation_turn_number(request)
- # With a deep key (session_key_depth > 0), affinity is untouchable
- # until the key prefix is complete — hashing a shorter prefix would
- # produce a key that later turns of the same conversation no longer
- # match. resolve_session_key returns None (memoizing nothing) until
- # then; once resolved, the memoized key is what affinity and streak
- # bookkeeping see for the rest of the request.
- affinity_ready = (
- resolve_session_key(ctx, request, depth=self._session_key_depth) is not None
- )
-
- if affinity_ready:
- pinned = await self._affinity.pinned(ctx, request)
- if pinned == self._strong_tier:
- ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = self._strong_tier
- ctx.metadata[CTX_ESCALATION_VERDICT] = {
- "escalate": True,
- "reason": "",
- "turn": turn,
- "source": "pinned",
- }
- return request
-
- ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = self._weak_tier
- if not affinity_ready or turn < self._config.min_judge_turn:
- return request
-
- # Lets verdict-dump consumers group per-turn judge decisions by
- # conversation.
- session = ctx.metadata.get(CTX_SESSION_KEY)
- summary = _summarize_for_judge(request, turn=turn, config=self._config)
- started_at = time.perf_counter()
- try:
- completion = await self._client.classify(
- model=self._config.model,
- system_prompt=self._config.system_prompt,
- request_summary=summary,
- )
- verdict = EscalationVerdict.model_validate_json(
- strip_markdown_fence(completion.content),
- )
- except Exception as exc:
- # Fail open to the current (weak) tier and never pin: a judge
- # outage must not silently burn strong-model tokens.
- log.warning(
- "EscalationJudgeRequestProcessor: judge failed; staying on weak tier: %s",
- exc,
- )
- if self._stats_accumulator is not None:
- # Record the failure into the classifier bucket so
- # ``/v1/routing/stats`` exposes the judge fail-open rate;
- # a silently failing judge reads as "router never
- # escalates" to the benchmark observer.
- await self._stats_accumulator.record_classifier_error(self._config.model)
- ctx.metadata[CTX_ESCALATION_VERDICT] = {
- "escalate": False,
- "reason": f"fail_open: {str(exc)[:200]}",
- "turn": turn,
- "source": "fail_open",
- "session": session,
- }
- self._dump_verdict(
- ctx.metadata[CTX_ESCALATION_VERDICT],
- latency_ms=(time.perf_counter() - started_at) * 1000,
- request=request,
- )
- return request
-
- if self._stats_accumulator is not None:
- await self._record_judge_call(
- usage=completion.usage,
- latency_ms=(time.perf_counter() - started_at) * 1000,
- )
-
- if verdict.escalate:
- streak = self._bump_streak(ctx, request)
- confirmed = streak >= self._config.escalate_confirmations
- else:
- streak = 0
- confirmed = False
- self._reset_streak(ctx, request)
-
- ctx.metadata[CTX_ESCALATION_VERDICT] = {
- "escalate": verdict.escalate,
- "reason": verdict.reason,
- "turn": turn,
- "source": "judge",
- "session": session,
- "streak": streak,
- "confirmed": confirmed,
- }
- self._dump_verdict(
- ctx.metadata[CTX_ESCALATION_VERDICT],
- latency_ms=(time.perf_counter() - started_at) * 1000,
- request=request,
- )
- if confirmed:
- log.info(
- "EscalationJudgeRequestProcessor: escalating to strong tier "
- "(turn %d): %s",
- turn,
- verdict.reason,
- )
- await self._affinity.pin(ctx, request, self._strong_tier)
- ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = self._strong_tier
- return request
-
- def _streak_key(self, ctx: ProxyContext, request: ChatRequest) -> str:
- """Conversation key for streak bookkeeping (seeded deep key or Rust default)."""
- return resolve_session_key(ctx, request)
-
- def _bump_streak(self, ctx: ProxyContext, request: ChatRequest) -> int:
- """Record one escalate verdict for this conversation; return the streak."""
- if self._config.escalate_confirmations <= 1:
- # Single-verdict latch: no bookkeeping needed, streak is per-call.
- return 1
- key = self._streak_key(ctx, request)
- streak, _ = self._escalate_streaks.get(key, (0, 0))
- streak += 1
- self._escalate_streaks[key] = (streak, 0)
- self._escalate_streaks.move_to_end(key)
- while len(self._escalate_streaks) > _MAX_STREAK_SESSIONS:
- self._escalate_streaks.popitem(last=False)
- return streak
-
- def _reset_streak(self, ctx: ProxyContext, request: ChatRequest) -> None:
- """Register a non-escalate verdict against the streak.
-
- With ``confirmation_window == 1`` any decline clears the streak
- (strict-consecutive). A larger window tolerates up to
- ``window - 1`` intervening declines before the streak expires, so
- recurring intermittent trouble can still confirm.
- """
- if self._config.escalate_confirmations <= 1:
- return
- key = self._streak_key(ctx, request)
- entry = self._escalate_streaks.get(key)
- if entry is None:
- return
- streak, declines = entry
- declines += 1
- if declines >= self._config.confirmation_window:
- self._escalate_streaks.pop(key, None)
- else:
- self._escalate_streaks[key] = (streak, declines)
- self._escalate_streaks.move_to_end(key)
-
- def _dump_verdict(
- self,
- verdict: dict[str, Any],
- *,
- latency_ms: float,
- request: ChatRequest | None = None,
- ) -> None:
- """Write one ``escalation_verdict={...}`` JSON line to stderr.
-
- Direct stderr (not the logging module) so benchmark server logs
- capture it — see :attr:`EscalationJudgeConfig.dump_verdicts_to_stderr`.
- Includes a short first-user-message snippet (``task_hint``) so
- offline analysis can attribute per-turn verdicts to tasks exactly,
- instead of guessing from overlapping trial time windows.
- """
- if not self._config.dump_verdicts_to_stderr:
- return
- payload = dict(verdict)
- payload["latency_ms"] = round(latency_ms, 1)
- if request is not None:
- hint = _first_user_text(request)
- if hint:
- payload["task_hint"] = hint[:48]
- sys.stderr.write(f"escalation_verdict={json.dumps(payload, sort_keys=True)}\n")
- sys.stderr.flush()
-
- async def _record_judge_call(self, *, usage: Any, latency_ms: float) -> None:
- """Record judge token spend and latency into the classifier stats bucket.
-
- The judge is a routing-overhead call, exactly like the LLM
- classifier: keeping it in the classifier bucket stops its spend
- from merging with a same-named tier model and keeps
- ``routing_stats_final.json`` cost math honest for the
- escalation-router benchmark.
- """
- assert self._stats_accumulator is not None
- prompt = 0
- completion = 0
- cached = 0
- if usage is not None:
- prompt = getattr(usage, "prompt_tokens", 0) or 0
- completion = getattr(usage, "completion_tokens", 0) or 0
- ptd = getattr(usage, "prompt_tokens_details", None)
- if ptd is not None:
- cached = getattr(ptd, "cached_tokens", 0) or 0
- await self._stats_accumulator.record_classifier_usage(
- model=self._config.model,
- prompt_tokens=prompt,
- completion_tokens=completion,
- cached_tokens=cached,
- latency_ms=latency_ms,
- )
-
-def _first_user_text(request: ChatRequest) -> str:
- """Flatten the first user message's task text for verdict dumps.
-
- Harness wrappers (``<system-reminder>``/``<environment_context>``
- blocks injected ahead of the actual task statement) are skipped so the
- snippet identifies the task rather than repeating boilerplate shared
- by every conversation.
- """
- body = getattr(request, "body", None)
- if not isinstance(body, dict):
- return ""
- messages = body.get("messages")
- if not isinstance(messages, list):
- messages = body.get("input")
- if not isinstance(messages, list):
- return ""
- for m in messages:
- if isinstance(m, dict) and m.get("role") == "user":
- text = message_text(m).strip()
- while True:
- for tag in ("system-reminder", "environment_context"):
- if text.startswith(f"<{tag}>"):
- end = text.find(f"</{tag}>")
- if end >= 0:
- text = text[end + len(tag) + 3 :].lstrip()
- break
- else:
- return text
- return ""
-
-
-def _summarize_for_judge(
- request: ChatRequest,
- *,
- turn: int,
- config: EscalationJudgeConfig,
-) -> str:
- """Render a compact role-labeled transcript for the judge.
-
- Anchors (system + first user) are individually capped; the trailing window
- keeps the last ``recent_turn_window`` messages at ``window_message_chars``
- each. A coverage header states how much history is *not* shown so the
- judge can reason about pace. If the global cap still binds, the oldest
- window lines are dropped first.
- """
- body = getattr(request, "body", {})
- messages: list[Any] = []
- if isinstance(body, dict):
- raw = body.get("messages")
- if not isinstance(raw, list):
- raw = body.get("input")
- if isinstance(raw, list):
- messages = raw
-
- trimmed = trim_messages(messages, recent_turn_window=config.recent_turn_window)
-
- anchor_lines: list[str] = []
- window_lines: list[str] = []
- if isinstance(body, dict):
- top_system = body.get("system")
- if isinstance(top_system, str | list):
- anchor_lines.append(
- "[system] " + truncate_middle(content_text(top_system), config.system_chars)
- )
- first_user_seen = False
- for m in trimmed:
- if not isinstance(m, dict):
- continue
- role = m.get("role")
- text = message_text(m)
- if role in ("system", "developer"):
- anchor_lines.append(f"[{role}] " + truncate_middle(text, config.system_chars))
- elif role == "user" and not first_user_seen:
- first_user_seen = True
- anchor_lines.append(
- "[user (task)] " + truncate_middle(text, config.first_user_chars)
- )
- else:
- label = role or m.get("type") or "message"
- window_lines.append(
- f"[{label}] " + truncate_middle(text, config.window_message_chars)
- )
-
- n_shown = len(window_lines)
- header = (
- f"Conversation turn {turn}; showing the last {n_shown} of "
- f"{len(messages)} messages after the task framing."
- )
-
- def _assemble() -> str:
- return "\n".join([header, *anchor_lines, *window_lines])
-
- text = _assemble()
- # Drop oldest window lines first: for a trajectory judge the newest
- # evidence is strictly the most valuable.
- while len(text) > config.max_request_chars and window_lines:
- window_lines.pop(0)
- text = _assemble()
- if len(text) > config.max_request_chars:
- text = text[: config.max_request_chars - 15] + "...<truncated>"
- return text
-
-
-__all__ = [
- "CTX_ESCALATION_VERDICT",
- "ESCALATION_JUDGE_SYSTEM_PROMPT",
- "EscalationJudgeConfig",
- "EscalationJudgeRequestProcessor",
- "EscalationVerdict",
-]
diff --git a/switchyard/lib/processors/llm_classifier/__init__.py b/switchyard/lib/processors/llm_classifier/__init__.py
deleted file mode 100644
index 999d81fd2..000000000
--- a/switchyard/lib/processors/llm_classifier/__init__.py
+++ /dev/null
@@ -1,88 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""LLM classifier signal types for experimental deterministic routing."""
-
-from switchyard.lib.processors.llm_classifier.presets import (
- CODING_AGENT_CLASSIFIER_SYSTEM_PROMPT,
- OPENCLAW_CLASSIFIER_SYSTEM_PROMPT,
- LLMClassifierPresets,
- LLMClassifierProfile,
-)
-from switchyard.lib.processors.llm_classifier.request_processor import (
- DEFAULT_CLASSIFIER_SYSTEM_PROMPT,
- DEFAULT_MAX_REQUEST_CHARS,
- ClassifierCompletion,
- LLMClassifierClient,
- LLMClassifierConfig,
- LLMClassifierError,
- LLMClassifierRequestProcessor,
- OpenAIChatLLMClassifierClient,
- parse_route_decision,
- parse_route_signals,
-)
-from switchyard.lib.processors.llm_classifier.signals import (
- CTX_DETERMINISTIC_ROUTE_SIGNALS,
- ChannelKind,
- CodeModificationScope,
- CodingAgentRouteDecision,
- CodingAgentTurnType,
- Complexity,
- ContextDependency,
- MemoryDependency,
- OpenClawRouteDecision,
- OpenClawTurnType,
- PrecisionRequirement,
- ReasonCode,
- ReasoningDepth,
- RiskLevel,
- RouteDecision,
- RouteSignals,
- RouteTier,
- TaskType,
-)
-from switchyard.lib.processors.llm_classifier.tier_selector_request_processor import (
- CTX_DETERMINISTIC_TIER_DECISION,
- SignalTierSelectorConfig,
- SignalTierSelectorRequestProcessor,
- TierSelectionDecision,
-)
-
-__all__ = [
- "CODING_AGENT_CLASSIFIER_SYSTEM_PROMPT",
- "CTX_DETERMINISTIC_ROUTE_SIGNALS",
- "CTX_DETERMINISTIC_TIER_DECISION",
- "ChannelKind",
- "ClassifierCompletion",
- "CodeModificationScope",
- "CodingAgentRouteDecision",
- "CodingAgentTurnType",
- "Complexity",
- "ContextDependency",
- "DEFAULT_CLASSIFIER_SYSTEM_PROMPT",
- "DEFAULT_MAX_REQUEST_CHARS",
- "LLMClassifierClient",
- "LLMClassifierConfig",
- "LLMClassifierError",
- "LLMClassifierPresets",
- "LLMClassifierProfile",
- "LLMClassifierRequestProcessor",
- "MemoryDependency",
- "OPENCLAW_CLASSIFIER_SYSTEM_PROMPT",
- "OpenAIChatLLMClassifierClient",
- "OpenClawRouteDecision",
- "OpenClawTurnType",
- "PrecisionRequirement",
- "ReasonCode",
- "ReasoningDepth",
- "RiskLevel",
- "RouteDecision",
- "RouteSignals",
- "RouteTier",
- "SignalTierSelectorConfig",
- "SignalTierSelectorRequestProcessor",
- "TaskType",
- "TierSelectionDecision",
- "parse_route_decision",
- "parse_route_signals",
-]
diff --git a/switchyard/lib/processors/llm_classifier/presets.py b/switchyard/lib/processors/llm_classifier/presets.py
deleted file mode 100644
index 9e603a353..000000000
--- a/switchyard/lib/processors/llm_classifier/presets.py
+++ /dev/null
@@ -1,446 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Profile bundles for the LLM-classifier routing chain.
-
-A "profile" is the indivisible unit that pairs a classifier system prompt with
-its expected JSON schema and a 2-tier routing policy. Pairing the wrong prompt
-with the wrong schema would produce JSON the validator rejects and degrade
-silently through fail-open — so the profile factories below ship them together.
-
-Three profiles are provided:
-
-- ``general_2_tier`` — mixed traffic, conservative SIMPLE→weak mapping.
-- ``coding_agent_2_tier`` — Claude Code / Codex / Cursor; SIMPLE+MEDIUM→weak.
-- ``openclaw_2_tier`` — OpenClaw personal-agent channels
- (Telegram / Discord / Slack / iMessage / WhatsApp);
- SIMPLE+MEDIUM→weak.
-
-The tier selector itself stays profile-agnostic — it consumes only the contract
-exposed on :class:`RouteDecision` (``policy_tier()``, ``confidence``,
-``abstain``). Per-profile schemas exist for prompting and analytics; routing
-reads the computed policy tier, not LLM-emitted hints.
-"""
-
-from __future__ import annotations
-
-import hashlib
-from collections.abc import Callable, Mapping
-from dataclasses import dataclass
-
-from switchyard.lib.processors.llm_classifier.request_processor import (
- DEFAULT_CLASSIFIER_SYSTEM_PROMPT,
- DEFAULT_MAX_REQUEST_CHARS,
- LLMClassifierConfig,
-)
-from switchyard.lib.processors.llm_classifier.signals import (
- CodingAgentRouteDecision,
- OpenClawRouteDecision,
- RouteDecision,
- RouteSignals,
- RouteTier,
-)
-from switchyard.lib.processors.llm_classifier.tier_selector_request_processor import (
- SignalTierSelectorConfig,
-)
-from switchyard.lib.processors.reasoning_hint import model_accepts_reasoning_hint
-
-CODING_AGENT_CLASSIFIER_SYSTEM_PROMPT = """\
-You are a routing classifier inside a coding-agent harness (Claude Code,
-Codex CLI, Cursor, or similar). You see one *turn* in a multi-turn session
-where a human and an agent collaborate on a real codebase. Rate this turn —
-not the overall session — and return exactly one JSON object matching this
-schema:
-
-{
- "recommended_tier": one of ["simple", "medium", "complex", "reasoning"],
- "confidence": number between 0 and 1,
- "abstain": boolean,
- "turn_type": one of ["chitchat", "planning", "exploration", "edit",
- "debug", "explanation", "clarification", "summarize"],
- "code_modification_scope": one of ["none", "single_line", "function",
- "file", "multi_file", "cross_module"],
- "tool_call_count_estimate": non-negative integer,
- "requires_codebase_context": boolean
-}
-
-# Tier rubric — RELATIVE TO THE WEAK MODEL, NOT ABSOLUTE DIFFICULTY
-
-You are routing between two near-frontier models. The **weak tier in 2026
-is itself a top-class model** (DeepSeek V4 Pro 1.6T MoE, Kimi K2.6 1T-class,
-or similar). It handles routine coding, file exploration, single-file
-edits, normal debugging, multi-tool sequences, and most refactors without
-meaningful loss vs the strong tier. The **strong tier** (Claude Opus 4.7
-on Bedrock, GPT-5.2 frontier, or similar) only earns its keep on the
-long-tail of subtle cross-module reasoning that the weak tier fumbles.
-
-**Ask yourself for every turn: would Opus 4.7 actually outperform V4 Pro
-here, or would both produce equivalent output?** If both would be fine,
-this is MEDIUM. Most agent-loop turns fall in this band.
-
-- simple — chitchat, single-line edits, simple explanations, status
- questions answerable from the latest message. Either tier
- is overkill.
-- medium — **DEFAULT for substantive coding work.** Function- or
- file-scope edits, exploration with several tool calls,
- planning a known approach, debugging localized errors,
- routine refactors, mechanical test or build fixes, running
- a build pipeline, setting up a project from a template.
- The weak tier handles all of these — picking strong here
- just burns money for no quality gain.
-- complex — Only turns where the weak tier would *meaningfully degrade*:
- cross-module refactors with subtle invariants, debugging
- across several files where the root cause requires reasoning
- the weak tier would miss (race conditions, type-system
- interactions, non-obvious data flow), architectural choices
- with non-obvious trade-offs, security-sensitive logic with
- easy-to-miss footguns.
-- reasoning — Frontier-only territory. Novel algorithm synthesis,
- correctness proofs, deep multi-step derivation that *no
- one-shot model* handles well. Rare in agent traffic; if
- you're tempted to pick this, it's almost always COMPLEX.
-
-Default to MEDIUM; reach for COMPLEX when the turn **plausibly** stresses
-cross-file reasoning, subtle invariants, or correctness-sensitive logic
-the weak tier could miss. You don't need certainty — a plausible failure
-mode is enough to escalate. Vague unease about "this is hard" still isn't
-a reason; vague unease about "this cluster of files has subtle invariants"
-*is*.
-
-``recommended_tier`` is your own best guess. The downstream router also
-computes a deterministic tier from the other fields, so focus on
-extracting accurate observable features; the tier vote is a sanity check.
-
-Counting ``tool_call_count_estimate``: count tool calls **this turn
-alone** will need, not anticipated future turns. A single `cat foo.py`
-is 1. A read-then-edit is 2. A grep-spread + multi-file patch is 4+.
-Conservative is fine — under-estimating only loses tier escalation;
-over-counting wastes the strong tier on simple turns.
-
-Worked examples (tier calibrated to weak-tier capability in 2026, none
-drawn from any benchmark task set):
-
-* "Read /etc/hosts and tell me what's in it." → SIMPLE, 1 tool,
- exploration, requires_codebase_context=false.
-* "List every file in this repo that imports a deprecated module." →
- MEDIUM, 2-3 tools, exploration. Multi-tool but mechanical; weak tier
- breezes through.
-* "Scaffold a Click-based CLI that wraps an existing function and adds
- --json output." → **MEDIUM** (not COMPLEX). Tool-heavy but procedural;
- the weak tier follows the recipe correctly. Escalate only if the
- wrapper has to preserve a non-obvious behavior contract.
-* "This unit test is failing — figure out why and fix it." → **MEDIUM**
- (not COMPLEX) when the failure is localized to one file. Escalate to
- COMPLEX only when the root cause genuinely spans modules or hinges on
- a subtle invariant the weak tier would miss.
-* "Update this single function so its error envelope matches the
- convention the rest of the package uses." → **COMPLEX**, edit,
- scope=file, requires_codebase_context=true. **Single-file edit but
- cross-codebase synthesis** — the model has to read elsewhere in the
- package to learn the convention, then apply it consistently here.
- Exactly the case where the weak tier's narrower synthesis loses
- ground; not all file-scope edits are MEDIUM.
-* "Move the retry-with-jitter logic out of the worker module and into
- a shared utility used by both the worker and the ingestion pipeline."
- → COMPLEX. Cross-module refactor; multiple call sites share contracts
- that must stay consistent.
-* "Derive a closed-form expression for the variance of this estimator."
- → REASONING. Multi-step formal derivation, rare in agent traffic.
-* Terminal output appearing alone (e.g. "New Terminal Output: $
- ls /app") with no question → look at the *first* user message
- (the task framing) to infer turn_type; rate against the original
- task's difficulty band, not the terminal echo.
-
-Set "abstain": true with low confidence when the turn is too ambiguous to
-classify (e.g. user message is "yes" and you cannot tell what was being
-agreed to). Do not emit markdown, commentary, or chain-of-thought.
-"""
-
-OPENCLAW_CLASSIFIER_SYSTEM_PROMPT = """\
-You are a routing classifier inside an OpenClaw personal-assistant agent.
-OpenClaw runs across messaging channels (Telegram, Discord, Slack, Signal,
-iMessage, WhatsApp). The agent has long-term memory in MEMORY.md, a fixed
-personality in SOUL.md, and a user profile in USER.md. You see one inbound
-*message* on one channel. Rate this message — not the broader conversation —
-and return exactly one JSON object matching this schema:
-
-{
- "recommended_tier": one of ["simple", "medium", "complex", "reasoning"],
- "confidence": number between 0 and 1,
- "abstain": boolean,
- "turn_type": one of ["chitchat", "lookup", "memory_recall", "planning",
- "tool_orchestration", "action", "explanation", "clarification"],
- "tool_call_count_estimate": non-negative integer (how many tools the
- response will likely need to call),
- "memory_dependency": one of ["none", "light", "heavy"] (does the
- response need to read across MEMORY.md / long history?),
- "external_action_required": boolean (will the response send a real
- message, post to a channel, hit a third-party API, or otherwise do
- something visible to the outside world?),
- "precision_requirement": one of ["low", "medium", "high"] (how costly
- is a small mistake?),
- "ambiguity": one of ["low", "medium", "high"] (chat is full of "yes"
- / "ok" / "sure" — is this one of those?),
- "channel_kind": one of ["casual", "deliberate"] (WhatsApp group chat
- is casual; a curated Telegram bot or business channel is deliberate)
-}
-
-Tier rubric (per message):
-
-- simple — chitchat, casual greetings, looking up a single fact from
- memory, clarifying questions, short status replies.
-- medium — planning a short response, explanations that need 1–3 tool
- calls, light cross-referencing of memory.
-- complex — multi-tool orchestration, irreversible external actions
- (sending messages, posting, calling third-party APIs),
- heavy memory reasoning, high-precision tasks.
-- reasoning — multi-step problem solving, careful derivation, anything
- where getting the right answer requires substantial thought.
-
-``recommended_tier`` is your own best guess. The downstream router will
-also compute a deterministic tier from the other fields, so focus on
-extracting accurate observable features; the tier vote is a sanity check.
-
-Set "abstain": true with low confidence when the message is too thin to
-classify (e.g. "ok", "sure", "👍" with no prior context). Do not emit
-markdown, commentary, or chain-of-thought.
-"""
-
-
-@dataclass(frozen=True)
-class LLMClassifierProfile:
- """A bundle pairing a classifier prompt with its schema and routing policy.
-
- Treat this as the indivisible unit. Don't mix-and-match a prompt from
- one profile with the schema from another — the JSON shapes diverge by
- design.
- """
-
- name: str
- system_prompt: str
- signal_schema: type[RouteDecision]
- tier_mapping: Mapping[RouteTier, str]
- default_tier: str
- min_confidence: float = 0.0
- escalate_on_tool_planning: bool = False
- """Bias the profile toward ``default_tier`` (typically strong) on
- multi-step tool-driven turns. See
- :attr:`SignalTierSelectorConfig.escalate_on_tool_planning`."""
-
- align_with_llm_recommendation: bool = False
- """Trust the classifier's own ``recommended_tier`` when it
- confidently says COMPLEX/REASONING but feature-based scoring
- landed on SIMPLE/MEDIUM. One-way bump. See
- :attr:`SignalTierSelectorConfig.align_with_llm_recommendation`."""
-
- alignment_min_confidence: float = 0.85
- """Floor on ``signals.confidence`` for the alignment bump to fire.
- See :attr:`SignalTierSelectorConfig.alignment_min_confidence`.
-
- Default ``0.85`` (was ``0.7`` through 2026-05-13): the bump is a
- one-way weak→strong override that bypasses the entire policy_tier
- scoring scheme, so the calibration bar should be high. Empirically
- DeepSeek V4 Flash emits confidence ≥0.7 on a substantial fraction
- of TB turns where the underlying features don't actually warrant
- COMPLEX — the 0.7 floor lets the LLM be a soft-escalation backdoor.
- 0.85 keeps the bump available for genuinely-clear COMPLEX reads
- while filtering out the chronic-pessimism case."""
-
- def make_classifier_config(
- self,
- *,
- model: str,
- api_key: str | None = None,
- base_url: str | None = None,
- timeout_s: float | None = None,
- max_request_chars: int = DEFAULT_MAX_REQUEST_CHARS,
- fail_open: bool = True,
- fallback_recommended_tier: RouteTier = RouteTier.MEDIUM,
- recent_turn_window: int = 0,
- extra_headers: dict[str, str] | None = None,
- disable_reasoning: bool | None = None,
- system_prompt: str | None = None,
- ) -> LLMClassifierConfig:
- """Return an :class:`LLMClassifierConfig` with this profile's prompt baked in.
-
- ``disable_reasoning=None`` (default) auto-detects from ``model`` via
- :func:`~switchyard.lib.processors.reasoning_hint.model_accepts_reasoning_hint`.
- Pass an explicit bool to override.
-
- ``system_prompt=None`` uses the profile's built-in prompt. Passing
- a string overrides only the prompt text; callers must still keep it
- aligned with this profile's ``signal_schema``.
- """
- if disable_reasoning is None:
- disable_reasoning = model_accepts_reasoning_hint(model)
- return LLMClassifierConfig(
- model=model,
- api_key=api_key,
- base_url=base_url,
- timeout_s=timeout_s,
- max_request_chars=max_request_chars,
- fail_open=fail_open,
- fallback_recommended_tier=fallback_recommended_tier,
- system_prompt=self.system_prompt if system_prompt is None else system_prompt,
- recent_turn_window=recent_turn_window,
- extra_headers=extra_headers,
- disable_reasoning=disable_reasoning,
- )
-
- def make_tier_selector_config(
- self,
- *,
- min_confidence: float | None = None,
- escalate_on_tool_planning: bool | None = None,
- align_with_llm_recommendation: bool | None = None,
- alignment_min_confidence: float | None = None,
- ) -> SignalTierSelectorConfig:
- """Return a :class:`SignalTierSelectorConfig` with this profile's mapping."""
- return SignalTierSelectorConfig(
- tier_mapping=self.tier_mapping,
- default_tier=self.default_tier,
- min_confidence=self.min_confidence if min_confidence is None else min_confidence,
- escalate_on_tool_planning=(
- self.escalate_on_tool_planning
- if escalate_on_tool_planning is None
- else escalate_on_tool_planning
- ),
- align_with_llm_recommendation=(
- self.align_with_llm_recommendation
- if align_with_llm_recommendation is None
- else align_with_llm_recommendation
- ),
- alignment_min_confidence=(
- self.alignment_min_confidence
- if alignment_min_confidence is None
- else alignment_min_confidence
- ),
- )
-
-
-class LLMClassifierPresets:
- """Factory for pre-built classifier profiles."""
-
- @staticmethod
- def general_2_tier(*, weak: str, strong: str) -> LLMClassifierProfile:
- """General-purpose mixed traffic, conservative SIMPLE→weak mapping.
-
- Default-tier on abstain / low-confidence is ``strong`` (fail safer,
- not cheaper).
- """
- return LLMClassifierProfile(
- name="general_2_tier",
- system_prompt=DEFAULT_CLASSIFIER_SYSTEM_PROMPT,
- signal_schema=RouteSignals,
- tier_mapping={
- RouteTier.SIMPLE: weak,
- RouteTier.MEDIUM: strong,
- RouteTier.COMPLEX: strong,
- RouteTier.REASONING: strong,
- },
- default_tier=strong,
- )
-
- @staticmethod
- def coding_agent_2_tier(*, weak: str, strong: str) -> LLMClassifierProfile:
- """Claude Code / Codex / Cursor traffic; SIMPLE+MEDIUM→weak.
-
- Routine turns inside an agentic session (single-line edits, file
- exploration, localized debugging) route to the weak tier; multi-file
- refactors and deep reasoning route to strong.
- """
- return LLMClassifierProfile(
- name="coding_agent_2_tier",
- system_prompt=CODING_AGENT_CLASSIFIER_SYSTEM_PROMPT,
- signal_schema=CodingAgentRouteDecision,
- tier_mapping={
- RouteTier.SIMPLE: weak,
- RouteTier.MEDIUM: weak,
- RouteTier.COMPLEX: strong,
- RouteTier.REASONING: strong,
- },
- default_tier=strong,
- # Coding-agent turns with 2+ tool calls escalate to strong.
- # SIMPLE+MEDIUM scope w/ multi-step tool sequences are
- # exactly where weak models fumble; the override pushes
- # those to strong without changing the base mapping.
- escalate_on_tool_planning=True,
- # Trust the LLM's gestalt read when it confidently flags
- # COMPLEX/REASONING but feature scoring undercounted. The
- # classifier sees session-level context (multi-turn
- # debugging, cross-file refactoring) that discrete schema
- # fields can't fully capture.
- align_with_llm_recommendation=True,
- )
-
- @staticmethod
- def openclaw_2_tier(*, weak: str, strong: str) -> LLMClassifierProfile:
- """OpenClaw personal-assistant traffic; SIMPLE+MEDIUM→weak.
-
- Casual chitchat, lookups, light memory recall route to the weak
- tier; tool orchestration, irreversible external actions, and
- heavy memory reasoning route to strong.
- """
- return LLMClassifierProfile(
- name="openclaw_2_tier",
- system_prompt=OPENCLAW_CLASSIFIER_SYSTEM_PROMPT,
- signal_schema=OpenClawRouteDecision,
- tier_mapping={
- RouteTier.SIMPLE: weak,
- RouteTier.MEDIUM: weak,
- RouteTier.COMPLEX: strong,
- RouteTier.REASONING: strong,
- },
- default_tier=strong,
- # Personal-assistant turns chain tools (messaging + memory
- # + actions); escalate multi-step tool sequences to strong
- # since irreversible external actions are unforgiving.
- escalate_on_tool_planning=True,
- # Same rationale as coding_agent — assistant turns carry
- # channel/memory context the schema doesn't fully express,
- # so a confident LLM COMPLEX/REASONING read deserves trust.
- align_with_llm_recommendation=True,
- )
-
-
-PROFILE_FACTORIES: Mapping[str, Callable[..., LLMClassifierProfile]] = {
- "general": LLMClassifierPresets.general_2_tier,
- "coding_agent": LLMClassifierPresets.coding_agent_2_tier,
- "openclaw": LLMClassifierPresets.openclaw_2_tier,
-}
-
-
-def profile_default_prompt(profile_name: str) -> str:
- """Return the built-in classifier prompt for ``profile_name``."""
- try:
- factory = PROFILE_FACTORIES[profile_name]
- except KeyError as exc:
- raise ValueError(
- f"unknown profile {profile_name!r}; expected one of "
- f"{sorted(PROFILE_FACTORIES)}"
- ) from exc
- return factory(weak="weak", strong="strong").system_prompt
-
-
-def resolve_classifier_prompt(profile_name: str, prompt_override: str | None) -> str:
- """Return the effective classifier prompt for a profile + optional override."""
- if prompt_override is not None:
- return prompt_override
- return profile_default_prompt(profile_name)
-
-
-def classifier_prompt_sha256(prompt: str) -> str:
- """Return a stable SHA-256 hex digest for a classifier prompt."""
- return hashlib.sha256(prompt.encode("utf-8")).hexdigest()
-
-
-__all__ = [
- "CODING_AGENT_CLASSIFIER_SYSTEM_PROMPT",
- "LLMClassifierPresets",
- "LLMClassifierProfile",
- "OPENCLAW_CLASSIFIER_SYSTEM_PROMPT",
- "PROFILE_FACTORIES",
- "classifier_prompt_sha256",
- "profile_default_prompt",
- "resolve_classifier_prompt",
-]
diff --git a/switchyard/lib/processors/llm_classifier/request_processor.py b/switchyard/lib/processors/llm_classifier/request_processor.py
deleted file mode 100644
index 4a8e292b9..000000000
--- a/switchyard/lib/processors/llm_classifier/request_processor.py
+++ /dev/null
@@ -1,581 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Request processor that asks an LLM to extract routing signals."""
-
-import json
-import logging
-import sys
-import time
-from dataclasses import dataclass
-from typing import Any, Literal, Protocol
-
-from pydantic import BaseModel, ConfigDict, Field, ValidationError
-
-from switchyard.lib.llm_client import OpenAILLMClient
-from switchyard.lib.processors._structured_output import build_response_format
-from switchyard.lib.processors.llm_classifier.signals import (
- CTX_DETERMINISTIC_ROUTE_SIGNALS,
- RouteDecision,
- RouteSignals,
- RouteTier,
-)
-from switchyard.lib.processors.message_condensing import strip_markdown_fence, trim_messages
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.session_affinity import SessionAffinity
-from switchyard.lib.stats_accumulator import StatsAccumulator
-from switchyard_rust.core import ChatRequest
-
-log = logging.getLogger(__name__)
-
-DEFAULT_CLASSIFIER_SYSTEM_PROMPT = """\
-You are a routing classifier for an LLM proxy.
-
-Inspect the incoming request summary and return exactly one JSON object matching
-this schema:
-
-{
- "task_type": one of ["chat", "summarization", "extraction", "translation",
- "coding", "debugging", "math", "planning", "creative_writing",
- "agentic_task", "research", "data_analysis", "other"],
- "complexity": one of ["simple", "medium", "complex", "reasoning"],
- "reasoning_depth": one of ["none", "light", "multi_step", "deep"],
- "tool_planning_required": boolean,
- "precision_requirement": one of ["low", "medium", "high"],
- "context_dependency": one of ["latest_message", "conversation",
- "external_context"],
- "structured_output_risk": one of ["low", "medium", "high"],
- "recommended_tier": one of ["simple", "medium", "complex", "reasoning"],
- "confidence": number between 0 and 1,
- "reason_code": one of ["simple_qa", "summarization", "extraction",
- "translation", "coding_simple", "coding_complex", "debugging",
- "math_reasoning", "tool_agentic", "long_context", "structured_output",
- "creative_generation", "research_synthesis", "ambiguous", "other"],
- "abstain": boolean
-}
-
-Do not include markdown, commentary, or chain-of-thought. Use "abstain": true
-and reason_code "ambiguous" when the request summary is insufficient.
-"""
-
-DEFAULT_MAX_REQUEST_CHARS = 16_000
-
-
-class LLMClassifierError(RuntimeError):
- """Raised when the classifier LLM cannot produce valid route signals."""
-
-
-@dataclass(frozen=True)
-class ClassifierCompletion:
- """Result of a single classifier LLM call.
-
- Carries the raw ``content`` (a JSON string to be parsed into
- :class:`RouteDecision`) and the raw SDK ``usage`` object so the
- processor can record token spend into a
- :class:`~switchyard.lib.stats_accumulator.StatsAccumulator` when one
- is wired in. ``usage`` is intentionally typed as ``Any`` — the
- OpenAI-Python SDK's ``CompletionUsage`` shape is what we have today;
- custom client implementations may return any duck-typed object with
- the same attribute set.
- """
-
- content: str
- usage: Any | None = None
-
-
-class LLMClassifierConfig(BaseModel):
- """Configuration for :class:`LLMClassifierRequestProcessor`.
-
- The classifier call is OpenAI-chat-compatible by default. Tests and custom
- hosts can inject any object implementing :class:`LLMClassifierClient`.
- """
-
- model_config = ConfigDict(frozen=True)
-
- model: str = Field(min_length=1)
- api_key: str | None = None
- base_url: str | None = None
- timeout_s: float | None = Field(default=None, gt=0.0)
- max_request_chars: int = Field(default=DEFAULT_MAX_REQUEST_CHARS, ge=256)
- fail_open: bool = True
- fallback_recommended_tier: RouteTier = RouteTier.MEDIUM
-
- recent_turn_window: int = Field(default=0, ge=0)
- """Number of trailing messages to keep in the classifier summary
- on top of the system + first-user anchors.
-
- ``0`` (default) preserves the historical "system + first user +
- last user" slice — small, cache-friendly, but leaves the
- classifier blind to recent assistant tool calls and tool results,
- which is exactly the state needed to estimate
- ``tool_call_count_estimate`` and identify DEBUG / EXPLORATION
- turn types. The classifier ends up guessing from a terse "Continue"
- last-user echo and tends to over-escalate.
-
- Set ``4`` for agent-loop traffic on a long-context classifier
- (DeepSeek V4 Flash 1M, GPT-5.2 1M, etc.). That window catches the
- last assistant turn + its tool result + the current user message,
- giving the classifier visibility into what the agent has actually
- been doing. The growing prefix is cache-friendly: each turn
- appends to a stable prefix the upstream can prompt-cache, so the
- extra tokens cost ~2% of their list price on cache-discounted
- backends (V4 Flash sees ~98% discount on cached input).
-
- Wider windows (8, 16) help further on dense multi-step debug
- sessions but plateau quickly; 4 is the sweet spot for TB-style
- workloads."""
- system_prompt: str = Field(
- default=DEFAULT_CLASSIFIER_SYSTEM_PROMPT,
- min_length=1,
- )
- structured_output_mode: Literal["json_schema", "json_object"] = "json_schema"
- """Wire-level structured-output mechanism.
-
- ``json_schema`` (default) sends the pydantic ``signal_schema`` to the
- backend with ``strict: true``, constraining the decoder to schema-valid
- output. Downgrade to ``json_object`` for OpenAI-compatible backends that
- do not advertise Structured Outputs support.
- """
-
- max_completion_tokens: int = Field(default=4096, ge=64)
- """Upper bound on ``max_tokens`` for the classifier completion call.
-
- Has to fit **both** internal reasoning content (for chain-of-thought
- models like DeepSeek V4 Flash / R1) **and** the structured JSON
- output. Setting this too low silently truncates the model
- mid-reasoning so the actual content comes back empty, the JSON
- parse fails, and the classifier abstains — fallback to default
- tier. 4096 is large enough for typical 200-500-token reasoning +
- 300-token JSON without ever binding the cap; for non-reasoning
- classifiers (Nemotron-3-super-v3 etc.) the model only emits ~300
- tokens, so the higher ceiling is a no-op for them. Lower to
- 1024-ish if you're paying for outputs by the token and your
- classifier is non-reasoning, but never below ~600 (small JSON
- schemas need at least that)."""
-
- extra_headers: dict[str, str] | None = None
- """Per-call HTTP headers forwarded on every classifier completion.
-
- Mirrors :attr:`~switchyard.lib.backends.llm_target.LlmTarget.extra_headers`
- for the classifier's own LLM call, which goes through a separate
- client (not :class:`LlmTarget`). Typical use: pin
- ``X-Inference-Priority: batch`` on benchmark deployments so the
- classifier's calls land on the same benchmarking gateway as the
- routed tier backends and inherit the relaxed proxy timeout.
- Unknown headers are silently ignored upstream, so passing them on
- non-NIH endpoints is benign."""
-
- dump_signals_to_stderr: bool = True
- """Emit one ``classifier_signals={...}`` JSON line to ``sys.stderr``
- per classifier call.
-
- Defaults ``True`` for the benchmark-server entrypoint, where the
- server's stderr is captured to a per-run log file and the lines are
- grepped post-hoc to tally tier-selection distributions. **Disabled
- by callers that share stderr with an interactive TUI** (e.g. the
- ``switchyard launch claude`` / ``launch codex`` paths (LLM-classifier
- routing is their default), where the proxy runs in
- the same process as the spawned coding agent and bare stderr writes
- bleed into the agent's terminal rendering). Toggled off in
- :class:`DeterministicRoutingFactory` for that reason."""
-
- disable_reasoning: bool = True
- """Pass ``chat_template_kwargs.enable_thinking=False`` via
- ``extra_body`` so reasoning models (DeepSeek V4 Flash / V4 Pro,
- R1-style) skip chain-of-thought and emit the structured JSON
- response directly.
-
- **Why this defaults True:** the classifier's job is fast
- structured-JSON extraction. When reasoning models are combined
- with strict ``response_format``, the JSON output gets misrouted
- into ``reasoning_content`` while ``content`` stays empty — a known
- vLLM regression (vllm-project/vllm#41132). The classifier then
- abstains (empty content fails JSON parse) and every request
- fallback-routes to the default tier. Disabling reasoning
- eliminates the misroute, restores ``content``, and keeps the
- classifier's wall-time tax low (10 vs 200+ completion tokens on
- a typical request).
-
- **Compatibility:** ``chat_template_kwargs`` is a vLLM-side hint
- on the NVIDIA Inference Hub. Non-reasoning models ignore the
- field as a no-op; OpenAI-direct backends are not known to reject
- unknown ``extra_body`` keys, so the default is safe. Set to
- ``False`` if a specific upstream rejects it, or if you
- deliberately want reasoning during classification."""
-
-
-class LLMClassifierClient(Protocol):
- """Protocol for the underlying LLM used by the classifier processor."""
-
- async def classify(
- self,
- *,
- model: str,
- system_prompt: str,
- request_summary: str,
- ) -> ClassifierCompletion:
- """Return the classifier's content + raw usage from one LLM call.
-
- Content must be a JSON string that validates as the configured
- :class:`RouteDecision` subclass. ``usage`` can be ``None`` for
- in-memory test doubles that don't simulate token spend.
- """
- ...
-
-
-class OpenAIChatLLMClassifierClient:
- """OpenAI-chat-compatible implementation of :class:`LLMClassifierClient`.
-
- Defaults to strict Structured Outputs (``response_format`` of type
- ``json_schema`` with ``strict: true``), which constrains the model's
- decoder to emit schema-valid JSON. Falls back to ``json_object`` mode
- when the configured backend does not advertise Structured Outputs.
- """
-
- def __init__(
- self,
- client: OpenAILLMClient,
- *,
- signal_schema: type[BaseModel] | None = None,
- structured_output_mode: Literal["json_schema", "json_object"] = "json_schema",
- max_completion_tokens: int = 4096,
- disable_reasoning: bool = True,
- extra_headers: dict[str, str] | None = None,
- ) -> None:
- if structured_output_mode == "json_schema" and signal_schema is None:
- raise ValueError(
- "OpenAIChatLLMClassifierClient requires signal_schema when "
- "structured_output_mode='json_schema'",
- )
- self._client = client
- self._signal_schema = signal_schema
- self._structured_output_mode = structured_output_mode
- self._max_completion_tokens = max_completion_tokens
- self._disable_reasoning = disable_reasoning
- self._extra_headers = extra_headers
- self._response_format = build_response_format(signal_schema, structured_output_mode)
-
- async def classify(
- self,
- *,
- model: str,
- system_prompt: str,
- request_summary: str,
- ) -> ClassifierCompletion:
- extra_body: dict[str, Any] | None = None
- if self._disable_reasoning:
- # Hint to vLLM (NVIDIA Inference Hub uses vLLM under LiteLLM)
- # to skip chain-of-thought for this call. Without it, DeepSeek
- # V4 Flash / V4 Pro misroute structured-JSON output into
- # ``reasoning_content`` and leave ``content`` empty — see
- # ``LLMClassifierConfig.disable_reasoning`` for the full
- # explanation.
- extra_body = {"chat_template_kwargs": {"enable_thinking": False}}
-
- result = await self._client.acompletion(
- model=model,
- messages=[
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": request_summary},
- ],
- temperature=0,
- # Wide enough to fit reasoning_content + JSON for chain-of-thought
- # classifiers (DeepSeek V4 Flash, R1-style) when reasoning is
- # enabled. Default 4096 is generous and never binds for
- # non-reasoning models that emit only ~300 tokens of JSON.
- max_tokens=self._max_completion_tokens,
- response_format=self._response_format,
- extra_body=extra_body,
- extra_headers=self._extra_headers,
- )
- return ClassifierCompletion(
- content=_completion_content(result),
- usage=getattr(result, "usage", None),
- )
-
-
-class LLMClassifierRequestProcessor:
- """Extract semantic routing signals by calling an underlying LLM.
-
- The processor leaves the inbound request unchanged. It validates the
- classifier's JSON output as :class:`RouteSignals` and stores it under
- ``ctx.metadata[CTX_DETERMINISTIC_ROUTE_SIGNALS]`` for a later deterministic
- picker to consume.
- """
-
- def __init__(
- self,
- config: LLMClassifierConfig,
- *,
- client: LLMClassifierClient | None = None,
- signal_schema: type[RouteDecision] = RouteSignals,
- stats_accumulator: StatsAccumulator | None = None,
- affinity: SessionAffinity | None = None,
- ) -> None:
- self._config = config
- self._signal_schema = signal_schema
- self._stats_accumulator = stats_accumulator
- # Shared tier-pin store. Once a conversation is pinned, its tier is
- # fixed, so the classifier verdict would be ignored — skip the LLM call.
- self._affinity = affinity
- self._client = client or OpenAIChatLLMClassifierClient(
- OpenAILLMClient(
- api_key=config.api_key,
- base_url=config.base_url,
- timeout=config.timeout_s,
- # Disable the OpenAI SDK's default 2-retry budget on the
- # classifier path. With slow / cold-start upstreams (e.g.
- # DeepSeek V4 Flash on NVIDIA Inference Hub) the SDK's
- # exponential-backoff retries compound to >60s wall time on
- # what's meant to be a 30s ceiling. Our own ``fail_open``
- # fallback is the right surface for "classifier didn't
- # answer in time" — let it fire cleanly at ``timeout_s``.
- max_retries=0,
- ),
- signal_schema=signal_schema,
- structured_output_mode=config.structured_output_mode,
- max_completion_tokens=config.max_completion_tokens,
- disable_reasoning=config.disable_reasoning,
- extra_headers=config.extra_headers,
- )
-
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- if self._affinity is not None and (await self._affinity.pinned(ctx, request)) is not None:
- # Already pinned: the selector reuses the pin and ignores any
- # verdict, so skip the LLM call — classify once per task, not per turn.
- return request
-
- request_summary = _summarize_request(
- request,
- max_chars=self._config.max_request_chars,
- recent_turn_window=self._config.recent_turn_window,
- )
- started_at = time.perf_counter()
- try:
- completion = await self._client.classify(
- model=self._config.model,
- system_prompt=self._config.system_prompt,
- request_summary=request_summary,
- )
- signals = parse_route_decision(completion.content, self._signal_schema)
- except Exception as exc:
- if not self._config.fail_open:
- raise LLMClassifierError(
- "LLM classifier failed to produce valid route signals",
- ) from exc
- log.warning(
- "LLMClassifierRequestProcessor: classifier failed; "
- "stamping abstain signals: %s",
- exc,
- )
- if self._stats_accumulator is not None:
- # Record the failure into the classifier bucket so
- # ``/v1/routing/stats`` reports ``classifier.total_errors``
- # alongside ``classifier.total_requests`` — otherwise a
- # silent fail-open hides the failure rate from the
- # benchmark observer. Mirrors the success-branch
- # ``_record_classifier_call`` below.
- await self._stats_accumulator.record_classifier_error(self._config.model)
- signals = self._signal_schema.make_abstain(
- self._config.fallback_recommended_tier,
- )
- _fail_open_exc: Exception | None = exc
- else:
- _fail_open_exc = None
- prompt_tokens, completion_tokens, cached_tokens = _classifier_token_counts(
- completion.usage,
- )
- if self._stats_accumulator is not None:
- await self._record_classifier_call(
- prompt_tokens=prompt_tokens,
- completion_tokens=completion_tokens,
- cached_tokens=cached_tokens,
- latency_ms=(time.perf_counter() - started_at) * 1000,
- )
-
- ctx.metadata[CTX_DETERMINISTIC_ROUTE_SIGNALS] = signals
- # One-line dump of the extracted signals + the deterministic
- # policy_tier the selector will route on. Written to stderr
- # directly (not via the logging module) so it lands in the
- # captured server log regardless of uvicorn's logger config,
- # which does not pick up non-uvicorn package loggers. Grep
- # ``classifier_signals=`` to tally the distribution of
- # turn_type / scope / tool_count / recommended_tier per run.
- # Disabled by interactive launcher callers — see
- # :attr:`LLMClassifierConfig.dump_signals_to_stderr`.
- if self._config.dump_signals_to_stderr:
- try:
- signals_payload: dict[str, Any] = signals.model_dump(mode="json")
- except Exception:
- signals_payload = {"abstain": True}
- signals_payload["policy_tier"] = signals.policy_tier().value
- if _fail_open_exc is not None:
- signals_payload["fail_open"] = True
- signals_payload["error"] = str(_fail_open_exc)[:200]
- sys.stderr.write(
- f"classifier_signals={json.dumps(signals_payload, sort_keys=True)}\n"
- )
- sys.stderr.flush()
- return request
-
- async def _record_classifier_call(
- self,
- prompt_tokens: int,
- completion_tokens: int,
- cached_tokens: int,
- latency_ms: float,
- ) -> None:
- """Record classifier token spend into the ``StatsAccumulator``.
-
- Records into the classifier bucket so the classifier model's token
- spend doesn't merge with the same-named backend tier (default TB-lite
- config has classifier and weak both pointing at Nemotron-3-Super-v3).
- Latency is recorded too so the per-request classifier-tax line shows up
- on the snapshot's classifier-models block.
- """
- assert self._stats_accumulator is not None
- await self._stats_accumulator.record_classifier_usage(
- model=self._config.model,
- prompt_tokens=prompt_tokens,
- completion_tokens=completion_tokens,
- cached_tokens=cached_tokens,
- latency_ms=latency_ms,
- )
-
-
-def _classifier_token_counts(usage: Any) -> tuple[int, int, int]:
- """Return ``(prompt, completion, cached)`` tokens from an SDK usage object."""
- if usage is None:
- return 0, 0, 0
- prompt = getattr(usage, "prompt_tokens", 0) or 0
- completion = getattr(usage, "completion_tokens", 0) or 0
- ptd = getattr(usage, "prompt_tokens_details", None)
- cached = (getattr(ptd, "cached_tokens", 0) or 0) if ptd is not None else 0
- return prompt, completion, cached
-
-
-def parse_route_decision(
- raw: str,
- schema: type[RouteDecision] = RouteSignals,
-) -> RouteDecision:
- """Parse classifier JSON output against the given decision schema."""
- stripped = strip_markdown_fence(raw)
- try:
- return schema.model_validate_json(stripped)
- except ValidationError as exc:
- raise LLMClassifierError(
- f"classifier JSON did not match {schema.__name__}",
- ) from exc
-
-
-def parse_route_signals(raw: str) -> RouteSignals:
- """Parse classifier JSON output into :class:`RouteSignals` (legacy alias)."""
- decision = parse_route_decision(raw, RouteSignals)
- assert isinstance(decision, RouteSignals)
- return decision
-
-
-def _summarize_request(
- request: ChatRequest, *, max_chars: int, recent_turn_window: int = 0
-) -> str:
- body = getattr(request, "body", {})
- summary_body = (
- _condense_body(body, recent_turn_window=recent_turn_window)
- if isinstance(body, dict)
- else body
- )
- payload = {
- "request_type": request.request_type.value,
- "body": summary_body,
- }
- text = json.dumps(payload, default=str, ensure_ascii=False, sort_keys=True)
- if len(text) <= max_chars:
- return text
- return text[: max_chars - 32] + "...<truncated>"
-
-
-# Bulk fields that we synthesize ourselves below — copy everything else from
-# the raw body so the classifier still sees model, temperature,
-# reasoning_effort, etc.
-_BULK_BODY_FIELDS = ("messages", "tools", "input", "tool_choice")
-
-
-def _condense_body(
- body: dict[str, Any], *, recent_turn_window: int = 0
-) -> dict[str, Any]:
- """Strip non-routing-signal bulk before serializing for the classifier.
-
- Keeps only the slice that reliably carries routing signal:
-
- * system / developer messages (and Anthropic's top-level ``system``);
- * tool *names* + *descriptions* (drops parameter / input_schema bodies);
- * the **first** user message (task framing on agent harnesses that
- bundle the task into ``role="user"`` rather than a ``system`` field);
- * the last ``recent_turn_window`` messages (assistant tool calls,
- tool results, intermediate user turns) — ``0`` (default) keeps
- only the last user message; raise on long-context classifiers
- that need visibility into recent agent activity.
-
- Drops bulk that doesn't carry signal: prior assistant turns outside
- the window, tool-result messages outside the window, and full tool
- parameter schemas. These dominate prompt size on multi-turn agent
- traffic (TB tasks, Claude Code) and contribute almost nothing to a
- tier decision relative to their token cost.
- """
- out: dict[str, Any] = {k: v for k, v in body.items() if k not in _BULK_BODY_FIELDS}
-
- tools = body.get("tools")
- if isinstance(tools, list):
- out["tools"] = [_condense_tool(t) for t in tools]
-
- messages = body.get("messages")
- if isinstance(messages, list):
- out["messages"] = trim_messages(messages, recent_turn_window=recent_turn_window)
-
- raw_input = body.get("input")
- if isinstance(raw_input, list):
- out["input"] = trim_messages(raw_input, recent_turn_window=recent_turn_window)
- elif isinstance(raw_input, str):
- out["input"] = raw_input
-
- return out
-
-
-def _condense_tool(tool: Any) -> Any:
- """Drop tool parameter schemas; keep ``name`` + ``description`` only.
-
- Handles both OpenAI Chat (``{"type": "function", "function": {...}}``)
- and Anthropic (``{"name", "description", "input_schema"}``).
- """
- if not isinstance(tool, dict):
- return tool
- fn = tool.get("function")
- if isinstance(fn, dict):
- slim_fn = {k: v for k, v in fn.items() if k != "parameters"}
- return {**{k: v for k, v in tool.items() if k != "function"}, "function": slim_fn}
- return {k: v for k, v in tool.items() if k != "input_schema"}
-
-
-def _completion_content(result: Any) -> str:
- choices = getattr(result, "choices", None)
- if not choices:
- raise LLMClassifierError("classifier completion had no choices")
-
- message = getattr(choices[0], "message", None)
- content = getattr(message, "content", None)
- if isinstance(content, str) and content.strip():
- return content
- raise LLMClassifierError("classifier completion had empty content")
-
-
-__all__ = [
- "DEFAULT_CLASSIFIER_SYSTEM_PROMPT",
- "ClassifierCompletion",
- "LLMClassifierClient",
- "LLMClassifierConfig",
- "LLMClassifierError",
- "LLMClassifierRequestProcessor",
- "OpenAIChatLLMClassifierClient",
- "parse_route_decision",
- "parse_route_signals",
-]
diff --git a/switchyard/lib/processors/llm_classifier/signals.py b/switchyard/lib/processors/llm_classifier/signals.py
deleted file mode 100644
index 08d1a4d09..000000000
--- a/switchyard/lib/processors/llm_classifier/signals.py
+++ /dev/null
@@ -1,568 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Strongly typed signal vocabulary for LLM classifier routing.
-
-Each routing profile (general chat, coding agent, OpenClaw personal agent)
-defines its own pydantic schema. All schemas share a minimal contract —
-:class:`RouteDecision` — so the downstream tier selector can stay
-profile-agnostic.
-
-The base contract exposes:
-
-* :attr:`RouteDecision.recommended_tier` — the LLM's own vote, captured
- verbatim from the classifier output. Kept for audit / comparison only;
- routing does **not** read it.
-* :attr:`RouteDecision.confidence` — calibration signal used by the
- tier-selector's confidence floor.
-* :attr:`RouteDecision.abstain` — explicit "too ambiguous to classify"
- marker that forces the selector to its default tier.
-* :meth:`RouteDecision.policy_tier` — a deterministic, code-derived
- tier computed from the profile-specific feature set. **This is what
- the tier selector routes on.** Each subclass implements it as a small
- scoring scheme over its observable features.
-
-Splitting "extract features" (LLM) from "decide tier" (Python) makes the
-policy testable, replayable from captured signals, and tunable without
-re-prompting. When ``policy_tier()`` disagrees with the LLM's
-``recommended_tier``, that's a signal for the operator to investigate —
-both values are persisted in :class:`TierSelectionDecision`.
-"""
-
-from __future__ import annotations
-
-from enum import Enum
-
-from pydantic import BaseModel, ConfigDict, Field
-
-#: ``ProxyContext.metadata`` key for the latest semantic route decision record.
-CTX_DETERMINISTIC_ROUTE_SIGNALS = "_deterministic_route_signals"
-
-
-class TaskType(str, Enum):
- """Coarse task family inferred from the incoming LLM request."""
-
- CHAT = "chat"
- SUMMARIZATION = "summarization"
- EXTRACTION = "extraction"
- TRANSLATION = "translation"
- CODING = "coding"
- DEBUGGING = "debugging"
- MATH = "math"
- PLANNING = "planning"
- CREATIVE_WRITING = "creative_writing"
- AGENTIC_TASK = "agentic_task"
- RESEARCH = "research"
- DATA_ANALYSIS = "data_analysis"
- OTHER = "other"
-
-
-class Complexity(str, Enum):
- """Estimated capability tier needed to answer well."""
-
- SIMPLE = "simple"
- MEDIUM = "medium"
- COMPLEX = "complex"
- REASONING = "reasoning"
-
-
-class ReasoningDepth(str, Enum):
- """How much multi-step reasoning the request appears to require."""
-
- NONE = "none"
- LIGHT = "light"
- MULTI_STEP = "multi_step"
- DEEP = "deep"
-
-
-class PrecisionRequirement(str, Enum):
- """How costly small correctness errors are likely to be."""
-
- LOW = "low"
- MEDIUM = "medium"
- HIGH = "high"
-
-
-class ContextDependency(str, Enum):
- """How much surrounding context the final model likely needs."""
-
- LATEST_MESSAGE = "latest_message"
- CONVERSATION = "conversation"
- EXTERNAL_CONTEXT = "external_context"
-
-
-class RiskLevel(str, Enum):
- """Small ordinal risk scale for policy-relevant routing signals."""
-
- LOW = "low"
- MEDIUM = "medium"
- HIGH = "high"
-
-
-class RouteTier(str, Enum):
- """Advisory abstract tier before policy maps it to a concrete model."""
-
- SIMPLE = "simple"
- MEDIUM = "medium"
- COMPLEX = "complex"
- REASONING = "reasoning"
-
-
-_TIER_ORDER: tuple[RouteTier, ...] = (
- RouteTier.SIMPLE,
- RouteTier.MEDIUM,
- RouteTier.COMPLEX,
- RouteTier.REASONING,
-)
-
-
-def _argmax_tier(scores: dict[RouteTier, int]) -> RouteTier:
- """Return the highest-scoring tier; ties resolve to the *stronger* tier.
-
- The "ties → stronger" rule is the safer side of the cost/quality
- trade-off, matching the chain's overall fail-open posture.
- """
- if not scores:
- return RouteTier.MEDIUM
- best_score = max(scores.values())
- for tier in reversed(_TIER_ORDER):
- if scores.get(tier, 0) == best_score:
- return tier
- return RouteTier.MEDIUM
-
-
-class ReasonCode(str, Enum):
- """Stable, operator-friendly reason code for the extracted signal set."""
-
- SIMPLE_QA = "simple_qa"
- SUMMARIZATION = "summarization"
- EXTRACTION = "extraction"
- TRANSLATION = "translation"
- CODING_SIMPLE = "coding_simple"
- CODING_COMPLEX = "coding_complex"
- DEBUGGING = "debugging"
- MATH_REASONING = "math_reasoning"
- TOOL_AGENTIC = "tool_agentic"
- LONG_CONTEXT = "long_context"
- STRUCTURED_OUTPUT = "structured_output"
- CREATIVE_GENERATION = "creative_generation"
- RESEARCH_SYNTHESIS = "research_synthesis"
- AMBIGUOUS = "ambiguous"
- OTHER = "other"
-
-
-# ---- Coding-agent profile vocabulary ----------------------------------------
-
-
-class CodingAgentTurnType(str, Enum):
- """Per-turn intent in a coding-agent harness like Claude Code or Cursor."""
-
- CHITCHAT = "chitchat"
- PLANNING = "planning"
- EXPLORATION = "exploration"
- EDIT = "edit"
- DEBUG = "debug"
- EXPLANATION = "explanation"
- CLARIFICATION = "clarification"
- SUMMARIZE = "summarize"
-
-
-class CodeModificationScope(str, Enum):
- """Estimated blast radius of code changes the turn is likely to emit."""
-
- NONE = "none"
- SINGLE_LINE = "single_line"
- FUNCTION = "function"
- FILE = "file"
- MULTI_FILE = "multi_file"
- CROSS_MODULE = "cross_module"
-
-
-# ---- OpenClaw profile vocabulary --------------------------------------------
-
-
-class OpenClawTurnType(str, Enum):
- """Per-message intent on an OpenClaw personal-assistant channel.
-
- OpenClaw runs across messaging platforms (Telegram, Discord, Slack,
- Signal, iMessage, WhatsApp) with workspace-file memory (SOUL.md,
- USER.md, MEMORY.md). Turns trend short and conversational; tool calls
- are common but usually narrow.
- """
-
- CHITCHAT = "chitchat"
- LOOKUP = "lookup"
- MEMORY_RECALL = "memory_recall"
- PLANNING = "planning"
- TOOL_ORCHESTRATION = "tool_orchestration"
- ACTION = "action"
- EXPLANATION = "explanation"
- CLARIFICATION = "clarification"
-
-
-class MemoryDependency(str, Enum):
- """How much OpenClaw memory the turn needs to read."""
-
- NONE = "none"
- LIGHT = "light"
- HEAVY = "heavy"
-
-
-class ChannelKind(str, Enum):
- """Posture of the underlying messaging channel.
-
- Per the OpenClaw docs: ``casual`` channels (e.g. WhatsApp group chat)
- bias toward fast/cheap models; ``deliberate`` channels (e.g. a curated
- Telegram bot) bias toward a stronger model.
- """
-
- CASUAL = "casual"
- DELIBERATE = "deliberate"
-
-
-# ---- Decision schemas -------------------------------------------------------
-
-
-class RouteDecision(BaseModel):
- """Minimal contract every profile schema must satisfy.
-
- The deterministic tier selector reads :meth:`policy_tier`, ``confidence``,
- and ``abstain`` — never ``recommended_tier`` directly. Profile schemas
- add their own observable feature set and override :meth:`policy_tier`
- to score against those features.
- """
-
- model_config = ConfigDict(frozen=True)
-
- recommended_tier: RouteTier
- confidence: float = Field(ge=0.0, le=1.0)
- abstain: bool = False
-
- def policy_tier(self) -> RouteTier:
- """Deterministic tier derived from the schema's features.
-
- Base implementation falls back to the LLM-emitted ``recommended_tier``
- because the bare :class:`RouteDecision` has no other features to score
- against. Subclasses with feature fields override this with their own
- scoring scheme.
- """
- return self.recommended_tier
-
- def requires_tool_planning(self) -> bool:
- """Return True when this turn needs careful tool orchestration.
-
- The deterministic tier selector consults this when
- :attr:`SignalTierSelectorConfig.escalate_on_tool_planning` is
- enabled — a True return value forces escalation to the
- ``default_tier`` (typically strong) even when ``policy_tier()``
- would otherwise route to a cheaper tier. Use for profiles whose
- weak-tier picks regress on multi-step tool-driven turns.
-
- Base implementation returns ``False`` — schemas without a
- tool-use signal can keep the override a no-op. Subclasses with
- relevant fields override per their own semantics.
- """
- return False
-
- @classmethod
- def make_abstain(cls, fallback_tier: RouteTier) -> RouteDecision:
- """Construct an abstain-stamped instance for fail-open behavior.
-
- The base implementation works for any subclass whose extra fields have
- safe defaults. Profiles with required-without-default fields override
- this to supply their own safe defaults.
- """
- return cls(
- recommended_tier=fallback_tier,
- confidence=0.0,
- abstain=True,
- )
-
-
-class RouteSignals(RouteDecision):
- """General-purpose signal record (default profile, backwards-compatible).
-
- Suited for mixed traffic — chat, summarization, extraction, generic coding.
- The coding-agent and OpenClaw profiles are better served by their dedicated
- profile schemas.
- """
-
- task_type: TaskType
- complexity: Complexity
- reasoning_depth: ReasoningDepth
- tool_planning_required: bool
- precision_requirement: PrecisionRequirement
- context_dependency: ContextDependency
- structured_output_risk: RiskLevel
- reason_code: ReasonCode
-
- def policy_tier(self) -> RouteTier:
- scores: dict[RouteTier, int] = dict.fromkeys(_TIER_ORDER, 0)
-
- if self.complexity is Complexity.SIMPLE:
- scores[RouteTier.SIMPLE] += 2
- elif self.complexity is Complexity.MEDIUM:
- scores[RouteTier.MEDIUM] += 2
- elif self.complexity is Complexity.COMPLEX:
- scores[RouteTier.COMPLEX] += 2
- elif self.complexity is Complexity.REASONING:
- scores[RouteTier.REASONING] += 2
-
- if self.reasoning_depth is ReasoningDepth.DEEP:
- scores[RouteTier.REASONING] += 2
- elif self.reasoning_depth is ReasoningDepth.MULTI_STEP:
- scores[RouteTier.COMPLEX] += 1
-
- if self.tool_planning_required:
- scores[RouteTier.COMPLEX] += 1
-
- if self.precision_requirement is PrecisionRequirement.HIGH:
- scores[RouteTier.COMPLEX] += 1
-
- if self.structured_output_risk is RiskLevel.HIGH:
- scores[RouteTier.COMPLEX] += 1
-
- return _argmax_tier(scores)
-
- def requires_tool_planning(self) -> bool:
- return self.tool_planning_required
-
- @classmethod
- def make_abstain(cls, fallback_tier: RouteTier) -> RouteSignals:
- return cls(
- task_type=TaskType.OTHER,
- complexity=Complexity.MEDIUM,
- reasoning_depth=ReasoningDepth.LIGHT,
- tool_planning_required=False,
- precision_requirement=PrecisionRequirement.MEDIUM,
- context_dependency=ContextDependency.CONVERSATION,
- structured_output_risk=RiskLevel.MEDIUM,
- recommended_tier=fallback_tier,
- confidence=0.0,
- reason_code=ReasonCode.AMBIGUOUS,
- abstain=True,
- )
-
-
-class CodingAgentRouteDecision(RouteDecision):
- """Decision schema for coding-agent harnesses (Claude Code, Codex, Cursor).
-
- These harnesses interleave conversational turns, exploration, planning,
- edits, and debugging, with a human in the loop. ``turn_type`` plus
- ``code_modification_scope`` carry most of the routing-meaningful signal.
- """
-
- turn_type: CodingAgentTurnType = CodingAgentTurnType.EXPLORATION
- code_modification_scope: CodeModificationScope = CodeModificationScope.NONE
- tool_call_count_estimate: int = Field(default=0, ge=0)
- requires_codebase_context: bool = False
-
- def policy_tier(self) -> RouteTier:
- scores: dict[RouteTier, int] = dict.fromkeys(_TIER_ORDER, 0)
-
- if self.turn_type in (
- CodingAgentTurnType.CHITCHAT,
- CodingAgentTurnType.CLARIFICATION,
- CodingAgentTurnType.SUMMARIZE,
- ):
- scores[RouteTier.SIMPLE] += 2
- elif self.turn_type in (
- CodingAgentTurnType.EXPLORATION,
- CodingAgentTurnType.EXPLANATION,
- CodingAgentTurnType.EDIT,
- ):
- scores[RouteTier.MEDIUM] += 2
- elif self.turn_type in (
- CodingAgentTurnType.PLANNING,
- CodingAgentTurnType.DEBUG,
- ):
- scores[RouteTier.COMPLEX] += 2
-
- if self.code_modification_scope in (
- CodeModificationScope.NONE,
- CodeModificationScope.SINGLE_LINE,
- ):
- scores[RouteTier.SIMPLE] += 1
- elif self.code_modification_scope in (
- CodeModificationScope.FUNCTION,
- CodeModificationScope.FILE,
- ):
- scores[RouteTier.MEDIUM] += 1
- elif self.code_modification_scope in (
- CodeModificationScope.MULTI_FILE,
- CodeModificationScope.CROSS_MODULE,
- ):
- scores[RouteTier.COMPLEX] += 2
-
- # Tool-call scoring: a two-band scheme so policy_tier can register
- # multi-step orchestration without needing the escalation override
- # to do all the heavy lifting. Previously only the >=4 threshold
- # bumped, leaving 2-3 tool turns scored at MEDIUM with no signal.
- if self.tool_call_count_estimate >= 4:
- scores[RouteTier.COMPLEX] += 1
- elif self.tool_call_count_estimate >= 2:
- scores[RouteTier.MEDIUM] += 1
-
- # Codebase context is signal that the turn needs to read across
- # the repo, but it's not on its own a hard-task marker once the
- # weak tier has a long context window (V4 Pro / K2.6 both
- # 200K+; V4 Pro is 1M). Earlier sizing as +1 COMPLEX assumed a
- # short-context weak tier where codebase reach was a stress
- # signal — now it's just orientation. Keep the +1 nudge but
- # land it on MEDIUM so it doesn't single-handedly tip a turn
- # into COMPLEX.
- if self.requires_codebase_context:
- scores[RouteTier.MEDIUM] += 1
-
- return _argmax_tier(scores)
-
- def requires_tool_planning(self) -> bool:
- """Coding-agent turns escalate only on real multi-step orchestration.
-
- Concordance rule: two independent signals must agree that the
- turn would defeat the weak tier before we override a weak-tier
- verdict. Three configurations qualify:
-
- * ``turn_type == PLANNING`` — the classifier explicitly labeled
- the turn as planning; unconditional escalation (the one
- unambiguous signal).
- * ``tool_call_count_estimate >= 3`` **and** scope reaches
- FUNCTION or deeper — a tool-heavy turn that also touches
- code. The AND-coupling kills the "ls + cat + grep" pure-probe
- case where many tools fire but nothing is being changed.
-
- 2-tool turns (the typical "read file + edit" cycle) stay on
- the weak tier — modern weak models (V4 Pro, K2.6) handle that
- loop without help. The ≥3 threshold matches the boundary where
- turn shape starts to look like genuine multi-step planning
- rather than the agent's normal read→write rhythm.
- """
- scope_is_modifying = self.code_modification_scope in (
- CodeModificationScope.FUNCTION,
- CodeModificationScope.FILE,
- CodeModificationScope.MULTI_FILE,
- CodeModificationScope.CROSS_MODULE,
- )
- return (
- self.turn_type == CodingAgentTurnType.PLANNING
- or (self.tool_call_count_estimate >= 3 and scope_is_modifying)
- )
-
-
-class OpenClawRouteDecision(RouteDecision):
- """Decision schema for OpenClaw personal-assistant channels.
-
- OpenClaw agents run across messaging platforms (Telegram, Discord, Slack,
- Signal, iMessage, WhatsApp) and read workspace files for personality
- (``SOUL.md``), user profile (``USER.md``), and long-term memory
- (``MEMORY.md``). Turns are short, conversational, often latency-sensitive,
- and tool calls are common but usually narrow.
-
- The routing-meaningful signals are:
-
- * ``turn_type`` — chitchat vs lookup vs tool orchestration etc.
- * ``tool_call_count_estimate`` — orchestration cost.
- * ``memory_dependency`` — does the turn need to read across MEMORY.md?
- * ``external_action_required`` — is the agent about to send a message,
- post somewhere, or hit an external API? Combined with
- ``precision_requirement=high`` this is the strongest "use the strong
- tier" signal (irreversible actions are unforgiving).
- * ``ambiguity`` — chat is "yes / ok / sure"-heavy and ambiguous turns
- benefit disproportionately from a stronger model that can ask a good
- clarifying question.
- * ``channel_kind`` — per the OpenClaw docs, casual channels bias
- toward a fast/cheap model; deliberate channels toward a stronger one.
- """
-
- turn_type: OpenClawTurnType = OpenClawTurnType.CHITCHAT
- tool_call_count_estimate: int = Field(default=0, ge=0)
- memory_dependency: MemoryDependency = MemoryDependency.NONE
- external_action_required: bool = False
- precision_requirement: PrecisionRequirement = PrecisionRequirement.LOW
- ambiguity: RiskLevel = RiskLevel.LOW
- channel_kind: ChannelKind = ChannelKind.CASUAL
-
- def policy_tier(self) -> RouteTier:
- scores: dict[RouteTier, int] = dict.fromkeys(_TIER_ORDER, 0)
-
- if self.turn_type in (
- OpenClawTurnType.CHITCHAT,
- OpenClawTurnType.LOOKUP,
- OpenClawTurnType.MEMORY_RECALL,
- OpenClawTurnType.CLARIFICATION,
- ):
- scores[RouteTier.SIMPLE] += 2
- elif self.turn_type in (
- OpenClawTurnType.PLANNING,
- OpenClawTurnType.EXPLANATION,
- ):
- scores[RouteTier.MEDIUM] += 2
- elif self.turn_type in (
- OpenClawTurnType.TOOL_ORCHESTRATION,
- OpenClawTurnType.ACTION,
- ):
- scores[RouteTier.COMPLEX] += 2
-
- if self.tool_call_count_estimate >= 4:
- scores[RouteTier.COMPLEX] += 1
- elif self.tool_call_count_estimate >= 1:
- scores[RouteTier.MEDIUM] += 1
-
- if self.memory_dependency is MemoryDependency.HEAVY:
- scores[RouteTier.COMPLEX] += 2
- elif self.memory_dependency is MemoryDependency.LIGHT:
- scores[RouteTier.MEDIUM] += 1
-
- if self.external_action_required and (
- self.precision_requirement is PrecisionRequirement.HIGH
- ):
- scores[RouteTier.COMPLEX] += 2
- elif self.external_action_required:
- scores[RouteTier.MEDIUM] += 1
-
- if self.ambiguity is RiskLevel.HIGH:
- scores[RouteTier.COMPLEX] += 1
- elif self.ambiguity is RiskLevel.MEDIUM:
- scores[RouteTier.MEDIUM] += 1
-
- if self.channel_kind is ChannelKind.CASUAL:
- scores[RouteTier.SIMPLE] += 1
- else:
- scores[RouteTier.COMPLEX] += 1
-
- return _argmax_tier(scores)
-
- def requires_tool_planning(self) -> bool:
- """OpenClaw turns escalate on explicit tool orchestration or 2+ tool calls.
-
- ``TOOL_ORCHESTRATION`` already maps to COMPLEX via ``policy_tier``
- so this rule mostly catches multi-step ``PLANNING`` /
- ``EXPLANATION`` turns whose ``tool_call_count_estimate``
- signals real orchestration intent.
- """
- return (
- self.tool_call_count_estimate >= 2
- or self.turn_type == OpenClawTurnType.TOOL_ORCHESTRATION
- )
-
-
-__all__ = [
- "CTX_DETERMINISTIC_ROUTE_SIGNALS",
- "ChannelKind",
- "CodeModificationScope",
- "CodingAgentRouteDecision",
- "CodingAgentTurnType",
- "Complexity",
- "ContextDependency",
- "MemoryDependency",
- "OpenClawRouteDecision",
- "OpenClawTurnType",
- "PrecisionRequirement",
- "ReasonCode",
- "ReasoningDepth",
- "RiskLevel",
- "RouteDecision",
- "RouteSignals",
- "RouteTier",
- "TaskType",
-]
diff --git a/switchyard/lib/processors/llm_classifier/tier_selector_request_processor.py b/switchyard/lib/processors/llm_classifier/tier_selector_request_processor.py
deleted file mode 100644
index f05ea8c10..000000000
--- a/switchyard/lib/processors/llm_classifier/tier_selector_request_processor.py
+++ /dev/null
@@ -1,362 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Request processor that maps LLM classifier signals to a backend tier."""
-
-from __future__ import annotations
-
-import logging
-from collections.abc import Mapping
-from typing import Literal
-
-from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- CTX_DETERMINISTIC_ROUTING_TIER,
-)
-from switchyard.lib.processors.llm_classifier.signals import (
- CTX_DETERMINISTIC_ROUTE_SIGNALS,
- RouteDecision,
- RouteSignals,
- RouteTier,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.session_affinity import SessionAffinity
-from switchyard_rust.core import ChatRequest
-
-log = logging.getLogger(__name__)
-
-#: ``ProxyContext.metadata`` key for the deterministic tier selector decision.
-CTX_DETERMINISTIC_TIER_DECISION = "_deterministic_tier_decision"
-
-DecisionSource = Literal[
- "policy_tier",
- "low_confidence",
- "abstain",
- "missing_signals",
- "unknown_policy_tier",
- "tool_planning_escalation",
- "llm_alignment_bump",
- "sticky",
-]
-
-#: Confident classifier verdicts — the only decisions pinned for affinity.
-#: An allowlist, so a newly added (possibly fallback) source is non-sticky by
-#: default and can't silently lock a task to the default tier.
-_STICKY_SOURCES: frozenset[str] = frozenset(
- {"policy_tier", "tool_planning_escalation", "llm_alignment_bump"}
-)
-
-
-class TierSelectionDecision(BaseModel):
- """Audit record for one signal-to-tier decision.
-
- Captures both the LLM's own vote (``llm_recommended_tier``) and the
- Python policy's computed verdict (``policy_tier``). They will frequently
- agree; when they disagree it's a signal worth investigating, so both
- are persisted on every decision.
- """
-
- model_config = ConfigDict(frozen=True)
-
- tier: str
- source: DecisionSource
- reason: str
- policy_tier: RouteTier | None = None
- llm_recommended_tier: RouteTier | None = None
- confidence: float | None = Field(default=None, ge=0.0, le=1.0)
-
-
-class SignalTierSelectorConfig(BaseModel):
- """Policy for mapping :class:`RouteDecision` to backend tier labels.
-
- ``tier_mapping`` translates abstract classifier tiers such as
- ``RouteTier.SIMPLE`` into concrete backend labels configured on
- :class:`DeterministicRoutingLLMBackend`. The mapping is applied to the
- *policy* tier (``signals.policy_tier()``), not to the LLM's
- ``recommended_tier`` field.
- """
-
- model_config = ConfigDict(frozen=True)
-
- tier_mapping: Mapping[RouteTier, str] = Field(
- default_factory=lambda: {
- RouteTier.SIMPLE: "simple",
- RouteTier.MEDIUM: "medium",
- RouteTier.COMPLEX: "complex",
- RouteTier.REASONING: "reasoning",
- },
- )
- default_tier: str = Field(default="medium", min_length=1)
- """Fallback target on abstain / missing signals / low confidence /
- unknown policy_tier. Kept distinct from ``escalate_target_tier``
- so the "safer-on-uncertainty" path (typically strong) and the
- explicit escalation target can diverge once a third middle tier
- exists; in a 2-tier setup they coincide by default."""
-
- min_confidence: float = Field(default=0.0, ge=0.0, le=1.0)
- escalate_on_tool_planning: bool = Field(default=False)
- """When ``True``, any policy-decided tier other than the escalation
- target is overridden when ``signals.requires_tool_planning()`` fires.
- One-way gate — never demotes. Used to bias coding-agent / OpenClaw
- traffic toward the strong tier on multi-step tool-driven turns
- where weak models tend to fumble; off by default for general
- traffic (``RouteSignals`` already factors ``tool_planning_required``
- into its ``policy_tier`` scoring)."""
-
- escalate_target_tier: str | None = Field(default=None)
- """Tier label the escalation override should target. ``None`` falls
- back to ``default_tier`` (the 2-tier convention). Set explicitly
- when a 3+-tier setup wants escalation to land on the *middle*
- tier (e.g. SIMPLE+tool_planning → Sonnet) rather than always
- jumping to the safe-on-uncertainty fallback (which is typically
- the *strongest* tier). Kept structurally separate from
- ``default_tier`` so the two concerns — "what fires on uncertainty"
- and "what fires on confident-but-flagged" — stay decoupled."""
-
- align_with_llm_recommendation: bool = Field(default=False)
- """When ``True``, treat a high-confidence LLM ``recommended_tier``
- vote of COMPLEX/REASONING as a one-way bump if the deterministic
- ``policy_tier()`` landed on SIMPLE/MEDIUM. Surfaces signal the
- feature-based scoring under-weights — when the classifier
- confidently says "this is hard" but the discrete features didn't
- quite cross a threshold, trust the LLM's gestalt read. Gated on
- ``alignment_min_confidence`` and never demotes. Off by default;
- enable in agent-shaped profiles (coding_agent, openclaw) where
- the LLM has session-level awareness the schema features lack."""
-
- alignment_min_confidence: float = Field(default=0.7, ge=0.0, le=1.0)
- """Minimum ``signals.confidence`` for ``align_with_llm_recommendation``
- to fire. Below this floor the bump is suppressed even if the LLM's
- recommended tier is harder — protects against low-confidence
- over-routing."""
-
- @field_validator("tier_mapping")
- @classmethod
- def _tier_mapping_values_non_empty(
- cls,
- value: Mapping[RouteTier, str],
- ) -> Mapping[RouteTier, str]:
- if not value:
- raise ValueError("tier_mapping must not be empty")
- empty = [tier.value for tier, label in value.items() if not label]
- if empty:
- raise ValueError(f"tier_mapping has empty labels for {empty}")
- return value
-
- @model_validator(mode="after")
- def _default_tier_known(self) -> SignalTierSelectorConfig:
- labels = set(self.tier_mapping.values())
- if self.default_tier not in labels:
- raise ValueError(
- f"default_tier {self.default_tier!r} must be one of "
- f"tier_mapping values {sorted(labels)}",
- )
- if (
- self.escalate_target_tier is not None
- and self.escalate_target_tier not in labels
- ):
- raise ValueError(
- f"escalate_target_tier {self.escalate_target_tier!r} must be "
- f"one of tier_mapping values {sorted(labels)}",
- )
- return self
-
- @property
- def effective_escalate_target_tier(self) -> str:
- """Concrete escalation target — explicit field if set, else default_tier."""
- return (
- self.escalate_target_tier
- if self.escalate_target_tier is not None
- else self.default_tier
- )
-
-
-class SignalTierSelectorRequestProcessor:
- """Convert a stamped :class:`RouteDecision` into a deterministic backend tier.
-
- This processor performs no LLM calls and does not mutate the request body.
- It routes on ``signals.policy_tier()`` — a deterministic function of the
- classifier's extracted features — and only writes
- ``CTX_DETERMINISTIC_ROUTING_TIER`` for
- :class:`DeterministicRoutingLLMBackend`; the backend remains responsible
- for rewriting ``request.body["model"]`` to the selected model.
-
- The LLM's own ``recommended_tier`` is still captured in
- :class:`TierSelectionDecision` for audit, so disagreements between the
- LLM and the policy are observable.
- """
-
- def __init__(
- self,
- config: SignalTierSelectorConfig | None = None,
- *,
- affinity: SessionAffinity | None = None,
- ) -> None:
- self._config = config or SignalTierSelectorConfig()
- # Shared per-conversation tier pin. When set and enabled, the first
- # turn's tier is recorded and every later turn reuses it; the same
- # coordinator gates the classifier's LLM call (classify once per task).
- self._affinity = affinity
-
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- signals = _read_signals(ctx)
- decision = await self._decide(ctx, request, signals)
-
- ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = decision.tier
- ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION] = decision
-
- log.debug(
- "SignalTierSelectorRequestProcessor: tier=%s source=%s "
- "policy=%s llm=%s reason=%s",
- decision.tier,
- decision.source,
- decision.policy_tier.value if decision.policy_tier else None,
- decision.llm_recommended_tier.value if decision.llm_recommended_tier else None,
- decision.reason,
- )
- return request
-
- async def _decide(
- self, ctx: ProxyContext, request: ChatRequest, signals: RouteDecision | None
- ) -> TierSelectionDecision:
- """Select a tier, reusing a per-conversation pin when affinity is on.
-
- On a reused pin the classifier was skipped upstream, so ``signals`` may
- be absent — the pin alone decides the tier."""
- pinned = await self._affinity.pinned(ctx, request) if self._affinity else None
- if pinned is not None:
- return TierSelectionDecision(
- tier=pinned,
- source="sticky",
- reason="reusing pinned tier for this conversation",
- )
- decision = self._select(signals)
- # Pin only a confident verdict — pinning a fail-open fallback would lock
- # the task to the default tier and the classifier would never re-run.
- if self._affinity is not None and decision.source in _STICKY_SOURCES:
- await self._affinity.pin(ctx, request, decision.tier)
- return decision
-
- def _select(self, signals: RouteDecision | None) -> TierSelectionDecision:
- if signals is None:
- return self._default_decision(
- source="missing_signals",
- reason="no RouteDecision found on ProxyContext metadata",
- )
-
- if signals.abstain:
- return self._default_decision(
- source="abstain",
- reason="classifier abstained",
- signals=signals,
- )
-
- if signals.confidence < self._config.min_confidence:
- return self._default_decision(
- source="low_confidence",
- reason=(
- f"classifier confidence {signals.confidence:.3f} "
- f"< min_confidence {self._config.min_confidence:.3f}"
- ),
- signals=signals,
- )
-
- policy_tier = signals.policy_tier()
-
- # LLM alignment bump: when the classifier confidently says
- # "harder than feature scoring suggests", trust the LLM's vote.
- # One-way (never demotes). Lets the LLM surface session-level
- # signal that the discrete-feature scoring under-weights.
- bumped_from: RouteTier | None = None
- if (
- self._config.align_with_llm_recommendation
- and signals.confidence >= self._config.alignment_min_confidence
- and policy_tier in (RouteTier.SIMPLE, RouteTier.MEDIUM)
- and signals.recommended_tier in (RouteTier.COMPLEX, RouteTier.REASONING)
- ):
- bumped_from = policy_tier
- policy_tier = signals.recommended_tier
-
- tier = self._config.tier_mapping.get(policy_tier)
- if tier is None:
- return self._default_decision(
- source="unknown_policy_tier",
- reason=f"no backend tier mapped for policy_tier {policy_tier.value!r}",
- signals=signals,
- )
-
- if bumped_from is not None:
- return TierSelectionDecision(
- tier=tier,
- source="llm_alignment_bump",
- reason=(
- f"LLM alignment bump: policy_tier={bumped_from.value!r} "
- f"-> {policy_tier.value!r} (LLM recommended {policy_tier.value!r} "
- f"@ confidence={signals.confidence:.3f} >= "
- f"{self._config.alignment_min_confidence:.3f}); mapped to {tier!r}"
- ),
- policy_tier=policy_tier,
- llm_recommended_tier=signals.recommended_tier,
- confidence=signals.confidence,
- )
-
- escalate_target = self._config.effective_escalate_target_tier
- if (
- self._config.escalate_on_tool_planning
- and tier != escalate_target
- and signals.requires_tool_planning()
- ):
- return TierSelectionDecision(
- tier=escalate_target,
- source="tool_planning_escalation",
- reason=(
- f"tool_planning escalation: policy_tier={policy_tier.value!r} "
- f"mapped to {tier!r}; escalated to {escalate_target!r}"
- ),
- policy_tier=policy_tier,
- llm_recommended_tier=signals.recommended_tier,
- confidence=signals.confidence,
- )
-
- return TierSelectionDecision(
- tier=tier,
- source="policy_tier",
- reason=f"policy_tier() returned {policy_tier.value!r}",
- policy_tier=policy_tier,
- llm_recommended_tier=signals.recommended_tier,
- confidence=signals.confidence,
- )
-
- def _default_decision(
- self,
- *,
- source: DecisionSource,
- reason: str,
- signals: RouteDecision | None = None,
- ) -> TierSelectionDecision:
- return TierSelectionDecision(
- tier=self._config.default_tier,
- source=source,
- reason=reason,
- policy_tier=signals.policy_tier() if signals else None,
- llm_recommended_tier=signals.recommended_tier if signals else None,
- confidence=signals.confidence if signals else None,
- )
-
-
-def _read_signals(ctx: ProxyContext) -> RouteDecision | None:
- raw = ctx.metadata.get(CTX_DETERMINISTIC_ROUTE_SIGNALS)
- if isinstance(raw, RouteDecision):
- return raw
- if isinstance(raw, Mapping):
- return RouteSignals.model_validate(dict(raw))
- return None
-
-
-__all__ = [
- "CTX_DETERMINISTIC_TIER_DECISION",
- "SignalTierSelectorConfig",
- "SignalTierSelectorRequestProcessor",
- "TierSelectionDecision",
-]
diff --git a/switchyard/lib/processors/message_condensing.py b/switchyard/lib/processors/message_condensing.py
deleted file mode 100644
index 676ac3d55..000000000
--- a/switchyard/lib/processors/message_condensing.py
+++ /dev/null
@@ -1,185 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Shared helpers for condensing chat requests into routing-side LLM prompts.
-
-The LLM classifier and escalation judge show an inner LLM a compact view of
-the live request: anchors (system + first user message) plus a recent window,
-with bulk trimmed. These helpers are the common plumbing — message trimming,
-content flattening, truncation, and output-fence stripping — shared so the
-routing processors condense requests consistently instead of each keeping a
-private copy.
-"""
-
-from __future__ import annotations
-
-import json
-from typing import Any
-
-
-def trim_messages(messages: list[Any], *, recent_turn_window: int = 0) -> list[Any]:
- """Keep system + first-user anchor + a trailing window of messages.
-
- Anchors retained unconditionally:
-
- * system / developer messages — global framing the inner LLM
- always needs.
- * the **first** user message — agent frameworks like terminus-2
- bundle task framing into ``role="user"`` rather than ``system``;
- losing it leaves the inner LLM blind to what the agent is
- working on.
-
- Trailing window controlled by ``recent_turn_window``:
-
- * ``0`` (default) — keep only the last user message. Smallest
- prompt, but blind to recent assistant tool calls and
- tool results, so signal estimation (``tool_call_count_estimate``,
- DEBUG-vs-EXPLORATION turn type) must guess from a terse
- "Continue" echo. Tends to over-escalate on pessimistic
- classifiers.
- * ``N >= 1`` — keep the last ``N`` non-anchor messages
- (assistant / tool / non-first user) in original order. Gives
- the inner LLM visibility into recent agent activity. Each new
- turn appends to a stable prefix the upstream can prompt-cache,
- so the extra tokens are nearly free on cache-discounted backends
- (DeepSeek V4 Flash ~98% cache discount).
- """
- system_msgs: list[Any] = []
- first_user: Any = None
- first_user_idx: int | None = None
- for idx, m in enumerate(messages):
- if not isinstance(m, dict):
- continue
- role = m.get("role")
- if role in ("system", "developer"):
- system_msgs.append(m)
- elif role == "user" and first_user is None:
- first_user = m
- first_user_idx = idx
-
- if first_user is None:
- return system_msgs
-
- # Candidate tail = everything after the first user message that
- # isn't a system/developer anchor (those are already included).
- tail_candidates = [
- m
- for idx, m in enumerate(messages)
- if idx > (first_user_idx or 0)
- and isinstance(m, dict)
- and m.get("role") not in ("system", "developer")
- ]
-
- if recent_turn_window <= 0:
- # Historical behavior: keep only the last user message.
- last_user: Any = None
- for m in tail_candidates:
- if m.get("role") == "user":
- last_user = m
- if last_user is None:
- return [*system_msgs, first_user]
- if last_user is first_user:
- return [*system_msgs, first_user]
- return [*system_msgs, first_user, last_user]
-
- window = tail_candidates[-recent_turn_window:]
- # Filter out the first user (already pinned) to avoid duplicating
- # it if the window reaches back that far on short conversations.
- window = [m for m in window if m is not first_user]
- return [*system_msgs, first_user, *window]
-
-
-def message_text(message: dict[str, Any]) -> str:
- """Flatten a chat message (or Responses item) to plain text, tool calls included."""
- parts: list[str] = []
- content = message.get("content")
- if isinstance(content, str | list):
- text = content_text(content)
- if text:
- parts.append(text)
- # OpenAI chat tool calls: the command shapes an inner LLM needs for loop detection.
- tool_calls = message.get("tool_calls")
- if isinstance(tool_calls, list):
- for call in tool_calls:
- if not isinstance(call, dict):
- continue
- fn = call.get("function")
- if isinstance(fn, dict):
- parts.append(f"tool_call {fn.get('name')}({fn.get('arguments', '')})")
- # OpenAI Responses items carry their payloads in type-specific fields.
- if message.get("type") == "function_call":
- parts.append(f"tool_call {message.get('name')}({message.get('arguments', '')})")
- output = message.get("output")
- if isinstance(output, str | list):
- text = content_text(output)
- if text:
- parts.append(text)
- return " ".join(parts)
-
-
-def content_text(content: str | list[Any]) -> str:
- """Flatten string-or-content-block message content to text.
-
- Handles plain strings, OpenAI/Anthropic text blocks, nested
- ``tool_result`` content, and Anthropic ``tool_use`` blocks — the last
- carry their payload in ``name``/``input`` rather than ``text``/``content``
- and would otherwise flatten to nothing, erasing exactly the
- repeated-command signal a trajectory judge relies on.
- """
- if isinstance(content, str):
- return content
- parts: list[str] = []
- for block in content:
- if isinstance(block, str):
- parts.append(block)
- continue
- if not isinstance(block, dict):
- continue
- if block.get("type") == "tool_use":
- arguments = json.dumps(
- block.get("input", {}), ensure_ascii=False, sort_keys=True, default=str
- )
- parts.append(f"tool_call {block.get('name')}({arguments})")
- continue
- text = block.get("text")
- if isinstance(text, str):
- parts.append(text)
- continue
- inner = block.get("content")
- if isinstance(inner, str | list):
- parts.append(content_text(inner))
- return " ".join(p for p in parts if p)
-
-
-def truncate_middle(text: str, limit: int) -> str:
- """Keep the head and tail of ``text`` within ``limit`` chars."""
- if len(text) <= limit:
- return text
- marker = " ...[trimmed] "
- keep = max(limit - len(marker), 20)
- head = (keep * 2) // 3
- tail = keep - head
- return text[:head] + marker + text[-tail:]
-
-
-def strip_markdown_fence(raw: str) -> str:
- """Drop a wrapping ``` fence from an inner LLM's JSON output, if present."""
- stripped = raw.strip()
- if not stripped.startswith("```"):
- return stripped
-
- lines = stripped.splitlines()
- if lines and lines[0].startswith("```"):
- lines = lines[1:]
- if lines and lines[-1].startswith("```"):
- lines = lines[:-1]
- return "\n".join(lines).strip()
-
-
-__all__ = [
- "content_text",
- "message_text",
- "strip_markdown_fence",
- "trim_messages",
- "truncate_middle",
-]
diff --git a/switchyard/lib/processors/prompts/escalation_judge.md b/switchyard/lib/processors/prompts/escalation_judge.md
deleted file mode 100644
index 6d57369bb..000000000
--- a/switchyard/lib/processors/prompts/escalation_judge.md
+++ /dev/null
@@ -1,178 +0,0 @@
-You are an escalation judge inside an agentic coding router. The session
-started on the EFFICIENT tier (a cheap but top-class 2026 model). Your
-job is to detect when the run is genuinely in trouble so the router can
-escalate the rest of the task to the STRONG tier (frontier, expensive).
-
-You see a condensed view of one session: the task framing (system prompt
-+ first user message) and the most recent turns of activity (assistant
-messages and tool results). Judge the *trajectory* — is the agent making
-real progress toward the stated task — not the difficulty of the task
-itself. Return exactly one JSON object:
-
-{"escalate": boolean, "reason": "one short sentence naming the pattern"}
-
-Escalation is one-way for the rest of the task and expensive. Escalate
-only on a clear PATTERN of trouble, never on a single failed command.
-When the evidence is thin or ambiguous, return {"escalate": false}.
-
-The bar is not "is there friction" — agentic coding is full of friction
-the efficient tier works through on its own. The bar is "is this run
-likely DOOMED without intervention": the agent is stuck in place and
-its recent behavior shows no mechanism by which the next few turns
-would look different.
-
-# Is the stuck point beyond the efficient tier?
-
-Escalation pays only when the trouble is the KIND the strong tier is
-better at. The efficient tier handles routine coding, file
-exploration, single-file edits, normal debugging, dependency and
-environment setup, and most refactors on its own — being stuck on
-those is usually temporary. Weigh the kind of stuck point:
-
-Escalate sooner — the stuck point exceeds the efficient tier's
-capability, and a stronger model would likely break the loop:
-- Cross-module or cross-codebase synthesis: the fix requires learning
- a convention, contract, or invariant from elsewhere in the codebase
- and applying it consistently — even when the edit itself is
- single-file.
-- Subtle invariants: plausible-looking fixes keep failing the same
- test because the root cause hinges on a behavior contract none of
- the attempts have touched.
-- Root causes genuinely spanning modules, or multi-step algorithmic /
- formal reasoning the agent keeps getting almost-right.
-
-Hold weak — the efficient tier resolves these with iteration:
-- Procedural or mechanical friction: tool availability, installs,
- service startup, recipe-following scaffolding, localized one-file
- test fixes.
-
-Hold weak — no model can fix these, so escalation is pure waste:
-- The blocker is external: required data or files that simply do not
- exist in the environment, a permanently broken or missing service,
- or requirements the environment contradicts. A stronger model
- changes nothing about a missing resource. One boundary to respect:
- when producing, recovering, or decoding that very artifact IS the
- stated task, its absence is the work itself, not a blocker — judge
- the trajectory on it like any other work.
-
-# Trouble patterns — escalate when you see these
-
-Repetition and loops (the most common way agent runs die):
-- The same command or edit failing 2+ times with materially the same
- error, especially with unrelated changes in between.
-- Near-identical tool calls repeated, or the same files re-read, without
- new information gained — including longer cycles (A -> B -> C -> A).
-- Fighting the environment: repeatedly invoking a missing executable,
- retrying installs that fail the same way, or trying variations of a
- command the environment has already rejected, instead of adapting.
-
-False progress (looks like progress, is not):
-- Declaring success or moving on while the latest visible evidence
- (test output, exit code, error text) shows failure.
-- Finishing without running the verification the task specifies, when
- the task states how success is checked (e.g. "make the provided
- tests pass") and running it was possible.
-- A reproduction or test the agent wrote that passes trivially without
- exercising the actual issue, then building on that false signal.
-- The agent's stated reading of a tool result contradicts what the
- result actually says (treating an error or empty output as success).
-
-Drift and dead ends:
-- Recent activity no longer serves the task in the first user message
- (e.g. polishing style while the required feature is unstarted).
- A debugging detour that plausibly unblocks the task — fixing the
- environment, starting a required service, investigating an error in
- a dependency — is NOT drift; call drift only when the detour has
- produced nothing useful for many turns AND the task's real
- verification remains untouched.
-- Violating an explicit task constraint (modifying files the task says
- not to touch, changing the tests instead of the code under test).
-- Editing or reasoning about code without ever having opened the files
- the errors point to — acting on guessed file contents.
-- Contradicting or re-deriving something already established earlier in
- the session (forgetting its own findings).
-- Many turns elapsed with nothing durable produced (no successful
- writes, no passing checks) and no visible narrowing of the problem —
- the run is on pace to exhaust its turn budget.
-
-Desperation:
-- Giving up: declaring the task impossible, asking to stop, or drifting
- into restating the problem instead of acting on it.
-- Destructive flailing: rm -rf, wholesale reinstalls, chmod -R, or
- reverting everything as a reaction to being stuck rather than a
- reasoned step.
-
-# Expected friction — do NOT escalate on these
-
-Agentic coding is full of failures that are part of healthy work:
-- A test written to fail first (TDD) or a bug being reproduced on
- purpose.
-- A compile, lint, or test error fixed or meaningfully acted on in the
- immediately following turn.
-- Exploration dead-ends early in a session (grep with no matches,
- reading a file that turns out to be irrelevant) while the agent is
- still orienting.
-- A missing tool handled adaptively (tries `rg`, falls back to `grep`).
-- Sequential alternatives: trying a DIFFERENT library, tool, or
- approach after one fails is adaptation, not a loop — even when
- several alternatives fail in a row. The loop pattern requires the
- SAME approach retried without material change.
-- A service that is unreachable or not yet running (server not
- started, port closed, connection refused) while the agent is still
- actively working to start, configure, or replace it.
-- Planning activity: todo-list and plan updates (TodoWrite,
- update_plan) are routine agent workflow — planning is neither drift
- nor struggle, and harness-injected skill/instruction reading early
- in a session is orientation, not off-task work.
-- Zero-count summaries: "0 failed", "0 errors", "0 warnings" are
- CLEAN results. Read failure keywords together with their counts —
- only a nonzero count is a failure.
-- A long-running command (build, install, test suite) that simply has
- not finished, or the agent waiting on information it asked for.
-
-The distinguishing question: is each failure producing new information
-that changes the next action? Failing forward is fine; failing in place
-is trouble. Also weigh the session's own recovery record: if this same
-session already shows friction the agent subsequently cleared (a failure
-followed by a verified fix or passing check), lean toward holding — a
-session that has recovered before will usually recover again.
-
-# Worked examples (none drawn from any benchmark task set)
-
-* Turn 3; the agent ran the test suite, 4 tests fail, and it is now
- reading the first failing test. -> {"escalate": false} — reproducing
- failures is the job.
-* The agent has run `pytest tests/test_api.py` 4 times with the same
- ImportError, editing an unrelated config file between attempts. ->
- {"escalate": true, "reason": "same ImportError 4 times while editing
- unrelated files"}
-* `conda` is not installed; the agent has tried `conda install` five
- ways instead of using the `pip` that earlier output showed present.
- -> {"escalate": true, "reason": "fighting missing executable instead
- of adapting"}
-* Task: "make the provided integration tests pass." Recent turns:
- renaming variables and reformatting docstrings; tests not run in 8
- turns. -> {"escalate": true, "reason": "drifted to cosmetic edits,
- verification abandoned"}
-* The agent says "All tests pass, task complete" but the last visible
- test output shows "2 failed, 11 passed". -> {"escalate": true,
- "reason": "claims success contradicted by latest test output"}
-* The agent wrote a reproduction script that exits 0 without invoking
- the code path the issue describes, concluded "bug not reproducible",
- and is wrapping up. -> {"escalate": true, "reason": "reproduction
- never exercised the reported code path"}
-* Two turns of edits, one failed build, then a fixed build and a
- passing test. -> {"escalate": false}
-* `npm install` has been running for one turn with no output yet. ->
- {"escalate": false} — slow command, not a stall.
-* Four different serialization libraries failed to import; the agent
- is now writing the converter with a fifth approach it has not tried
- before. -> {"escalate": false} — sequential alternatives are
- adaptation, even when none has succeeded yet.
-* Task: tune a slow batch pipeline. The agent is investigating why
- the message broker fails to start, since the pipeline cannot be
- measured without it. -> {"escalate": false} — unblocking
- verification serves the task.
-
-Do not emit markdown, commentary, or chain-of-thought — only the JSON
-object.
diff --git a/switchyard/lib/processors/random_routing_request_processor.py b/switchyard/lib/processors/random_routing_request_processor.py
deleted file mode 100644
index e9320ef84..000000000
--- a/switchyard/lib/processors/random_routing_request_processor.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Python compatibility random-routing request processor."""
-
-from __future__ import annotations
-
-import random
-from typing import Any
-
-from switchyard_rust.components import RandomRoutingProcessorConfig
-
-
-class RandomRoutingRequestProcessor:
- """Rewrite requests to the randomly selected strong or weak target.
-
- This is a compatibility component for the current Python profile chain.
- """
-
- def __init__(self, config: RandomRoutingProcessorConfig) -> None:
- """Create a processor from validated random-routing config."""
- self.config = config
- self._rng = random.Random(config.rng_seed)
-
- async def startup(self) -> None:
- """Start the processor; random routing has no owned resources."""
-
- async def shutdown(self) -> None:
- """Stop the processor; random routing has no owned resources."""
-
- def select(self, original_model: str | None = None) -> dict[str, Any]:
- """Select a target and return the routing decision dictionary."""
- draw = self._rng.random()
- if draw < self.config.strong_probability:
- tier = "strong"
- target = self.config.strong
- else:
- tier = "weak"
- target = self.config.weak
- return {
- "tier": tier,
- "selected_target": target.id,
- "selected_model": target.model,
- "original_model": original_model,
- "strong_probability": self.config.strong_probability,
- "draw": draw,
- }
-
- async def process(self, ctx: Any, request: Any) -> Any:
- """Rewrite the request model and stamp the selected target on context."""
- decision = self.select(request.model)
- request.set_model(decision["selected_model"])
- ctx.selected_target = decision["selected_target"]
- return request
-
-__all__ = ["RandomRoutingRequestProcessor"]
diff --git a/switchyard/lib/processors/reasoning_effort_normalizer.py b/switchyard/lib/processors/reasoning_effort_normalizer.py
deleted file mode 100644
index 796ff4150..000000000
--- a/switchyard/lib/processors/reasoning_effort_normalizer.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Normalize non-standard ``reasoning_effort`` values before backend dispatch.
-
-Claude Code's ``/effort`` picker offers an ``xhigh`` level (and the
-Codex launcher's model catalog mirrors it). Both upstream paths reject
-that value: NVIDIA Inference Hub's LiteLLM passthrough returns HTTP
-500 with ``Invalid effort value: xhigh. Must be one of: 'high',
-'medium', 'low', 'max'.`` for Azure Anthropic, and
-``Unmapped reasoning effort: xhigh`` for Bedrock.
-
-This processor runs early in the chain and normalizes the request body
-so the upstream never sees an unsupported value. Unknown values are
-mapped to ``high`` rather than stripped because the user's intent
-(``xhigh`` means "as much reasoning as possible") is closer to ``high``
-than to absent.
-"""
-
-from __future__ import annotations
-
-import logging
-
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.core import ChatRequest
-
-log = logging.getLogger(__name__)
-
-#: OpenAI-compatible values the upstream accepts. ``"max"`` is non-standard
-#: but supported by NVIDIA Hub's LiteLLM for reasoning-budget overrides.
-_VALID_REASONING_EFFORT = frozenset({"low", "medium", "high", "max"})
-
-#: Aliases the upstream rejects → the nearest valid value.
-_REASONING_EFFORT_ALIASES = {
- "xhigh": "high",
-}
-
-
-class ReasoningEffortNormalizer:
- """Normalize ``request.body["reasoning_effort"]`` to an upstream-valid value.
-
- No-op when the field is absent or already a valid value. Maps known
- aliases (``xhigh`` → ``high``); for unrecognized values, replaces
- with ``high`` and emits a warning so operators can spot
- misconfigured client-side enums.
- """
-
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest: # noqa: ARG002
- body = request.body
- if not isinstance(body, dict):
- return request
- effort = body.get("reasoning_effort")
- if not isinstance(effort, str) or effort in _VALID_REASONING_EFFORT:
- return request
-
- mapped = _REASONING_EFFORT_ALIASES.get(effort, "high")
- log.warning(
- "ReasoningEffortNormalizer: unsupported reasoning_effort=%r; "
- "normalizing to %r before dispatch",
- effort,
- mapped,
- )
- body["reasoning_effort"] = mapped
- request.replace_body(body)
- return request
-
-
-__all__ = ["ReasoningEffortNormalizer"]
diff --git a/switchyard/lib/processors/reasoning_hint.py b/switchyard/lib/processors/reasoning_hint.py
deleted file mode 100644
index 66c358820..000000000
--- a/switchyard/lib/processors/reasoning_hint.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Model-id compatibility check for the vLLM ``enable_thinking`` hint."""
-
-from __future__ import annotations
-
-# Anthropic-served (Claude) ids reject the vLLM ``chat_template_kwargs`` field;
-# vLLM-served reasoning models that need the hint never carry these tokens.
-_NO_REASONING_HINT_TAGS = ("anthropic", "bedrock", "claude")
-
-
-def model_accepts_reasoning_hint(model: str) -> bool:
- """Whether ``model`` tolerates the vLLM ``enable_thinking=False`` hint.
-
- ``False`` for Anthropic/Bedrock/Claude ids (they 400 on it), else ``True``
- — usable directly as a classifier ``disable_reasoning`` default.
- """
- lowered = model.lower()
- return not any(tag in lowered for tag in _NO_REASONING_HINT_TAGS)
-
-
-__all__ = ["model_accepts_reasoning_hint"]
diff --git a/switchyard/lib/processors/routing_log_response_processor.py b/switchyard/lib/processors/routing_log_response_processor.py
index 7046c851f..cc87c0183 100644
--- a/switchyard/lib/processors/routing_log_response_processor.py
+++ b/switchyard/lib/processors/routing_log_response_processor.py
@@ -18,7 +18,7 @@ from switchyard.lib.chat_response.streaming_response_accumulator import (
attach_final_response_callback,
)
from switchyard.lib.proxy_context import CTX_PROXY_ACTUAL_MODEL, ProxyContext
-from switchyard.lib.request_metadata import CTX_PROFILE_REQUEST_HEADERS, CTX_REQUEST_METADATA
+from switchyard.lib.request_metadata import CTX_REQUEST_HEADERS, CTX_REQUEST_METADATA
from switchyard_rust.core import ChatResponse
if TYPE_CHECKING:
@@ -60,7 +60,7 @@ class RoutingLogResponseProcessor:
def _write_record(self, ctx: ProxyContext, served_model: str, response: ChatResponse) -> None:
metadata = ctx.metadata.get(CTX_REQUEST_METADATA)
- headers = ctx.metadata.get(CTX_PROFILE_REQUEST_HEADERS) or {}
+ headers = ctx.metadata.get(CTX_REQUEST_HEADERS) or {}
# Prefer the model id the backend actually served so buckets reconcile
# with the global routing_stats schema, which keys on the served id.
actual_model = _field(response.body, "model")
@@ -70,7 +70,7 @@ class RoutingLogResponseProcessor:
"trial_id": headers.get("x-switchyard-trial-id"),
"session_id": getattr(metadata, "session_id", None),
"model": actual_model if isinstance(actual_model, str) and actual_model else served_model,
- "tier": ctx.metadata.get("_random_routing_tier", "") or (ctx.selected_target or ""),
+ "tier": ctx.selected_target or "",
**_usage_tokens(response.body),
}
try:
diff --git a/switchyard/lib/processors/stage_router/__init__.py b/switchyard/lib/processors/stage_router/__init__.py
deleted file mode 100644
index 2eadd2fcc..000000000
--- a/switchyard/lib/processors/stage_router/__init__.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""StageRouter — Rust two-axis scorer + picker, with a selective LLM classifier.
-
-The routing logic lives in Rust (``switchyard_components::stage_router``); this
-package is the Python shell: the picker orchestration, the async classifier,
-handoff notes, and decision logging."""
-
-from switchyard.lib.processors.stage_router.classifier import (
- CAPABLE_TIER,
- EFFICIENT_TIER,
- TierClassifier,
-)
-from switchyard.lib.processors.stage_router.decision_log import (
- CONTEXT_KEY,
- DecisionSource,
- StageRouterDecisionLog,
-)
-from switchyard.lib.processors.stage_router.handoff_notes import (
- DEFAULT_ESCALATION_NOTE,
- HandoffNoteInjector,
-)
-from switchyard.lib.processors.stage_router.picker import (
- CAPABLE,
- EFFICIENT,
- pick_capable_first,
- pick_efficient_first,
-)
-
-__all__ = [
- "CONTEXT_KEY",
- "DEFAULT_ESCALATION_NOTE",
- "CAPABLE",
- "CAPABLE_TIER",
- "HandoffNoteInjector",
- "StageRouterDecisionLog",
- "DecisionSource",
- "TierClassifier",
- "EFFICIENT",
- "EFFICIENT_TIER",
- "pick_capable_first",
- "pick_efficient_first",
-]
diff --git a/switchyard/lib/processors/stage_router/classifier.py b/switchyard/lib/processors/stage_router/classifier.py
deleted file mode 100644
index 86c4057fb..000000000
--- a/switchyard/lib/processors/stage_router/classifier.py
+++ /dev/null
@@ -1,259 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""LLM tier classifier — invoked on low-confidence turns, fails open to ``None``."""
-
-import json
-import logging
-import time
-from importlib.resources import files
-from typing import TYPE_CHECKING, Any, Protocol, cast
-
-from switchyard.lib.llm_client import OpenAILLMClient
-
-if TYPE_CHECKING:
- from switchyard.lib.proxy_context import ProxyContext
- from switchyard_rust.components import StatsAccumulator, ToolResultSignal
-
-log = logging.getLogger(__name__)
-
-#: Tier values returned by :meth:`TierClassifier.classify`. Mirrors picker constants.
-CAPABLE_TIER: str = "capable"
-EFFICIENT_TIER: str = "efficient"
-
-#: ``ctx.metadata`` key for inbound conversation messages — read by the classifier.
-RECENT_MESSAGES_KEY: str = "stage_router_recent_messages"
-
-#: Per-message char cap when rendering recent turns into the classifier prompt.
-_MAX_MESSAGE_CHARS: int = 400
-
-
-def _load_system_prompt() -> str:
- """Read the classifier system prompt from the prompts/ package-data file."""
- return (
- files("switchyard.lib.processors.stage_router.prompts")
- .joinpath("tier_classifier.md")
- .read_text(encoding="utf-8")
- .strip()
- )
-
-
-_SYSTEM_PROMPT: str = _load_system_prompt()
-
-
-def _summarise(
- signal: "ToolResultSignal",
- *,
- recent_messages: list[Any] | None = None,
-) -> str:
- """Compact snapshot of agent state; appends trailing messages when present."""
- parts = [
- f"turn_depth={signal.turn_depth}",
- f"severity={signal.severity:.1f}",
- f"writes={signal.write_count}",
- f"edits={signal.edit_count}",
- f"reads={signal.read_count}",
- f"todowrites={signal.todowrite_count}",
- f"recent_writes={signal.recent_write_count}",
- f"recent_edits={signal.recent_edit_count}",
- f"recent_reads={signal.recent_read_count}",
- f"pure_bash_streak={signal.pure_bash_streak}",
- f"no_error_streak={signal.no_error_streak}",
- f"tests_passed={signal.tests_passed}",
- ]
- state_line = "State: " + ", ".join(parts)
- if not recent_messages:
- return "Decide CAPABLE or EFFICIENT for the next call. " + state_line
-
- transcript_lines = ["Recent turns (most recent last):"]
- for msg in recent_messages:
- transcript_lines.append(_format_message(msg))
- return (
- "Decide CAPABLE or EFFICIENT for the next call.\n"
- + state_line
- + "\n"
- + "\n".join(transcript_lines)
- )
-
-
-def _format_message(msg: Any) -> str:
- """Render one message as ``[role] <one-line summary>``; truncated."""
- if not isinstance(msg, dict):
- return f"[?] {str(msg)[:_MAX_MESSAGE_CHARS]}"
- role = msg.get("role", "?")
- content = msg.get("content")
- if isinstance(content, str):
- body = content
- elif isinstance(content, list):
- snippets: list[str] = []
- for block in content:
- if not isinstance(block, dict):
- continue
- block_type = block.get("type")
- if block_type == "tool_use":
- snippets.append(f"<tool_use:{block.get('name', '?')}>")
- elif block_type == "tool_result":
- inner = block.get("content")
- if isinstance(inner, list) and inner and isinstance(inner[0], dict):
- snippets.append(f"<tool_result:{str(inner[0].get('text', ''))[:120]}>")
- elif isinstance(inner, str):
- snippets.append(f"<tool_result:{inner[:120]}>")
- else:
- snippets.append("<tool_result>")
- elif block_type == "text":
- snippets.append(str(block.get("text", ""))[:_MAX_MESSAGE_CHARS])
- body = " ".join(snippets) if snippets else "(empty)"
- else:
- body = str(content)
- if len(body) > _MAX_MESSAGE_CHARS:
- body = body[:_MAX_MESSAGE_CHARS] + "..."
- return f"[{role}] {body}"
-
-
-def _parse_tier(response: object) -> str | None:
- """Extract ``"capable"`` / ``"efficient"`` from a chat-completion response, or ``None``.
-
- Mirrors ``_completion_content`` in
- :mod:`switchyard.lib.processors.llm_classifier.request_processor` — only reads
- ``message.content``. Empty content is treated as a failure (fall-open);
- the ``enable_thinking=False`` extra_body in :meth:`TierClassifier.classify`
- is what keeps reasoning-model output in ``content``.
- """
- try:
- content = response.choices[0].message.content # type: ignore[attr-defined]
- except (AttributeError, IndexError):
- log.warning("classifier response missing choices/message/content")
- return None
- if not isinstance(content, str) or not content.strip():
- log.warning("classifier response had empty content (reasoning model misroute?)")
- return None
- try:
- payload = json.loads(content)
- except json.JSONDecodeError:
- log.warning("classifier response not valid JSON: %r", content[:120])
- return None
- tier = payload.get("tier") if isinstance(payload, dict) else None
- if tier == CAPABLE_TIER or tier == EFFICIENT_TIER:
- return tier
- log.warning("classifier returned unexpected tier %r", tier)
- return None
-
-
-class _LLMClient(Protocol):
- async def acompletion(self, **kwargs: object) -> object: ...
-
-
-class TierClassifier:
- """Async LLM classifier — one call per invocation; fails open to ``None``."""
-
- def __init__(
- self,
- *,
- model: str,
- api_key: str,
- base_url: str | None = None,
- timeout_secs: float = 30.0,
- recent_turn_window: int = 3,
- disable_reasoning: bool = True,
- client: _LLMClient | None = None,
- stats_accumulator: "StatsAccumulator | None" = None,
- ) -> None:
- if recent_turn_window < 0:
- raise ValueError(f"recent_turn_window must be >= 0, got {recent_turn_window}")
- self._model = model
- self._api_key = api_key
- self._recent_turn_window = recent_turn_window
- self._stats = stats_accumulator
- # Reasoning models on the NVIDIA Inference Hub (DeepSeek V4 Flash /
- # V4 Pro, R1-style) misroute strict-`response_format` JSON into
- # `reasoning_content` while leaving `content` empty. Passing
- # `enable_thinking=False` via vLLM's `chat_template_kwargs` extra_body
- # restores `content`. Mirrors `LLMClassifierConfig.disable_reasoning`
- # from `switchyard/lib/processors/llm_classifier/`. Non-reasoning models
- # ignore the hint as a no-op, so the default is safe.
- self._disable_reasoning = disable_reasoning
- if client is None:
- client = cast(_LLMClient, OpenAILLMClient(
- api_key=api_key,
- base_url=base_url,
- timeout=timeout_secs,
- max_retries=0,
- ))
- self._client = client
-
- def attach_stats_accumulator(self, stats_accumulator: "StatsAccumulator") -> None:
- """Attach the serving-level accumulator used for classifier overhead."""
- self._stats = stats_accumulator
-
- async def classify(self, ctx: "ProxyContext", signal: "ToolResultSignal") -> str | None:
- """Return ``"capable"``, ``"efficient"``, or ``None`` (fall-open)."""
- recent_messages: list[Any] = []
- if self._recent_turn_window > 0:
- stashed = ctx.metadata.get(RECENT_MESSAGES_KEY) if hasattr(ctx, "metadata") else None
- if isinstance(stashed, list):
- recent_messages = stashed[-self._recent_turn_window:]
- user_prompt = _summarise(signal, recent_messages=recent_messages)
- extra_body: dict[str, Any] | None = None
- if self._disable_reasoning:
- # Mirror `OpenAIChatLLMClassifierClient` in
- # `switchyard/lib/processors/llm_classifier/`: vLLM-side hint that
- # forces reasoning models (DeepSeek V4 Flash / V4 Pro,
- # R1-style) to emit the JSON tier in `content` rather than
- # `reasoning_content`. Without this, `content` stays None on
- # the NVIDIA Inference Hub and every call falls open.
- extra_body = {"chat_template_kwargs": {"enable_thinking": False}}
- # 4096 matches the canonical LLM-classifier default — wide
- # enough to fit reasoning_content + JSON when the
- # `enable_thinking=False` hint is ignored upstream, but never
- # binds on non-reasoning models that emit ~10 tokens of JSON.
- started_at = time.perf_counter()
- try:
- response = await self._client.acompletion(
- model=self._model,
- messages=[
- {"role": "system", "content": _SYSTEM_PROMPT},
- {"role": "user", "content": user_prompt},
- ],
- temperature=0,
- response_format={"type": "json_object"},
- max_tokens=4096,
- extra_body=extra_body,
- )
- except Exception:
- log.warning("classifier call failed; falling open", exc_info=True)
- if self._stats is not None:
- try:
- await self._stats.record_classifier_error(self._model)
- except Exception:
- pass
- return None
- latency_ms = (time.perf_counter() - started_at) * 1000.0
- prompt_tokens, completion_tokens, cached_tokens = _tier_token_counts(
- getattr(response, "usage", None),
- )
- if self._stats is not None:
- try:
- await self._stats.record_classifier_usage(
- self._model,
- prompt_tokens=prompt_tokens,
- completion_tokens=completion_tokens,
- cached_tokens=cached_tokens,
- latency_ms=latency_ms,
- )
- except Exception:
- pass
- return _parse_tier(response)
-
-
-def _tier_token_counts(usage: Any) -> tuple[int, int, int]:
- """Return ``(prompt, completion, cached)`` tokens from an SDK usage object."""
- if usage is None:
- return 0, 0, 0
- prompt = getattr(usage, "prompt_tokens", 0) or 0
- completion = getattr(usage, "completion_tokens", 0) or 0
- details = getattr(usage, "prompt_tokens_details", None)
- cached = (getattr(details, "cached_tokens", 0) or 0) if details is not None else 0
- return prompt, completion, cached
-
-
-__all__ = ["CAPABLE_TIER", "TierClassifier", "EFFICIENT_TIER"]
diff --git a/switchyard/lib/processors/stage_router/decision_log.py b/switchyard/lib/processors/stage_router/decision_log.py
deleted file mode 100644
index 97077b31d..000000000
--- a/switchyard/lib/processors/stage_router/decision_log.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Thread-safe per-source decision counter for the stage_router picker."""
-
-from dataclasses import dataclass, field
-from threading import Lock
-from typing import Literal
-
-DecisionSource = Literal[
- "override", # _apply_overrides short-circuited (severity ≥ 1.0, CRITICAL)
- "tests_passed", # settled run (recent test-pass + recent write, no error) → EFFICIENT
- "dimensions", # scorer confidence ≥ confidence_threshold → routed by sign
- "llm-classifier", # classifier consulted and returned a tier
- "fall_open", # classifier configured but returned None, OR not configured at all
- "no_signal", # ToolResultSignal not present (first turn)
-]
-
-#: ``ctx.metadata`` key the picker writes its decision source to.
-CONTEXT_KEY: str = "stage_router_decision_source"
-
-
-@dataclass
-class StageRouterDecisionLog:
- """Thread-safe counter for stage_router decision sources."""
-
- counts: dict[str, int] = field(default_factory=dict)
- _lock: Lock = field(default_factory=Lock, repr=False, compare=False)
-
- def record(self, source: DecisionSource) -> None:
- with self._lock:
- self.counts[source] = self.counts.get(source, 0) + 1
-
- def snapshot(self) -> dict[str, int]:
- """Return a copy of the current counts; safe to publish."""
- with self._lock:
- return dict(self.counts)
-
- def total(self) -> int:
- with self._lock:
- return sum(self.counts.values())
-
-
-__all__ = ["CONTEXT_KEY", "StageRouterDecisionLog", "DecisionSource"]
diff --git a/switchyard/lib/processors/stage_router/handoff_notes.py b/switchyard/lib/processors/stage_router/handoff_notes.py
deleted file mode 100644
index 375bc997c..000000000
--- a/switchyard/lib/processors/stage_router/handoff_notes.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Optional signal-driven handoff notes for the capable (strong) tier.
-
-When the stage_router picks the capable tier because a real signal fired, this
-injector appends a short, ephemeral guidance note to the outgoing request so the
-model knows *why* it was handed the task.
-
-**Stateless.** No per-session state is tracked. The note fires on every turn
-where the capable tier is picked *and* the decision source was a real signal
-(``dimensions`` or ``override``). This avoids the stale-state and hash-collision
-problems that came with tracking previous tier per session across a shared proxy.
-
-**Ephemeral by construction.** The note is appended to the request the proxy
-forwards for this turn only; it is never written back into the agent's
-conversation store. On the next turn the agent re-sends its own history (which
-never contained the note) and the injector decides afresh.
-
-**Truthful escalation gate.** With ``only_on_wrong_signal_escalation`` (default),
-the note fires only when the escalation was driven by a real signal
-(``override`` — critical severity or compaction — or ``dimensions`` — the scorer
-crossing threshold on wrong signals), not by an ambiguous default
-(``fall_open`` / ``llm-classifier``). This keeps "the previous model was
-stalling" from being asserted when no such signal was seen.
-"""
-
-from __future__ import annotations
-
-import logging
-from typing import TYPE_CHECKING
-
-from switchyard.lib.processors.stage_router.picker import CAPABLE, EFFICIENT
-
-if TYPE_CHECKING:
- from switchyard_rust.core import ChatRequest
-
-log = logging.getLogger(__name__)
-
-#: Default weak→strong note. Kept to two sentences: state the reason, then the
-#: corrective (don't blindly repeat the weak model's last approach).
-DEFAULT_ESCALATION_NOTE: str = (
- "[router-guidance] A weaker model was handling this task and showed signs of "
- "stalling, looping, or repeated errors on the preceding steps, so control was "
- "escalated to you, a stronger model. Re-examine the current state directly and "
- "do not simply repeat the previous approach."
-)
-
-#: Default strong→weak note (only used when a deescalation note is configured;
-#: the direction is off unless the operator sets one).
-DEFAULT_DEESCALATION_NOTE: str = (
- "[router-guidance] A stronger model just completed the hard reasoning for this "
- "task; the remaining work is expected to be mechanical. Execute the established "
- "plan and avoid re-architecting what is already in place."
-)
-
-#: Decision sources that represent a *real* signal-driven escalation (vs. an
-#: ambiguous default). Used by the escalation gate.
-_WRONG_ESCALATION_SOURCES: frozenset[str] = frozenset({"override", "dimensions"})
-
-class HandoffNoteInjector:
- """Appends a guidance note to a tier-picked request driven by real signals.
-
- Stateless — no per-session tracking. On the capable tier the note fires on
- every turn where the decision source was a real wrong signal; on the
- efficient tier a de-escalation note fires only when one is configured
- (``deescalation_note``), which is off by default.
- """
-
- def __init__(
- self,
- *,
- escalation_note: str = DEFAULT_ESCALATION_NOTE,
- deescalation_note: str | None = None,
- only_on_wrong_signal_escalation: bool = True,
- ) -> None:
- self._escalation_note = escalation_note
- self._deescalation_note = deescalation_note
- self._only_on_wrong_signal = only_on_wrong_signal_escalation
-
- def maybe_inject(
- self,
- request: ChatRequest,
- *,
- tier: int,
- source: str | None,
- ) -> bool:
- """Inject the guidance note appropriate to the picked tier.
-
- Capable tier: the escalation note, gated to real wrong signals when
- ``only_on_wrong_signal_escalation``. Efficient tier: the de-escalation
- note, only when one is configured. Returns ``True`` iff a note was
- appended. Never raises — any failure degrades to "no note" so it can't
- block routing.
- """
- try:
- if tier == CAPABLE:
- if self._only_on_wrong_signal and source not in _WRONG_ESCALATION_SOURCES:
- return False
- return _append_note(request, self._escalation_note)
- if tier == EFFICIENT and self._deescalation_note is not None:
- return _append_note(request, self._deescalation_note)
- return False
- except Exception:
- log.debug("handoff-note injection failed; continuing without note", exc_info=True)
- return False
-
-
-
-def _append_note(request: ChatRequest, note: str) -> bool:
- """Append ``note`` to the request's final user turn.
-
- Anthropic: add a ``text`` block after the trailing user turn's content
- (``tool_result`` blocks must stay first, so the note goes last). OpenAI Chat:
- append a fresh trailing ``user`` message (the trailing turn is often a
- ``tool`` result, and OpenAI permits a following user message). Both preserve
- the cached prefix — only the suffix of the request changes.
- """
- body = getattr(request, "body", None)
- if not isinstance(body, dict):
- return False
- messages = body.get("messages")
- if not isinstance(messages, list) or not messages:
- return False
-
- fmt = getattr(getattr(request, "request_type", None), "value", None)
- block = {"type": "text", "text": note}
-
- if fmt == "anthropic":
- last = messages[-1]
- if not isinstance(last, dict) or last.get("role") != "user":
- return False
- content = last.get("content")
- if isinstance(content, list):
- content.append(block)
- elif isinstance(content, str):
- last["content"] = [{"type": "text", "text": content}, block]
- else:
- return False
- else:
- # OpenAI Chat / Responses: a trailing user message is the portable append.
- messages.append({"role": "user", "content": note})
-
- request.replace_body(body)
- return True
-
-
-__all__ = [
- "DEFAULT_DEESCALATION_NOTE",
- "DEFAULT_ESCALATION_NOTE",
- "HandoffNoteInjector",
-]
diff --git a/switchyard/lib/processors/stage_router/picker.py b/switchyard/lib/processors/stage_router/picker.py
deleted file mode 100644
index c144cf3e6..000000000
--- a/switchyard/lib/processors/stage_router/picker.py
+++ /dev/null
@@ -1,108 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Thin Python shell over the Rust stage_router picker.
-
-The routing logic — the two-axis scorer, the escalate/de-escalate shortcuts, and
-tier selection — lives in Rust (``switchyard_components::stage_router::pick_tier``,
-exposed as ``stage_pick_tier``). This module only adds the Python-side pieces the
-Rust core deliberately leaves to the caller: the ``no_signal`` case, the async LLM
-classifier, and decision-source recording."""
-
-import logging
-from typing import TYPE_CHECKING, cast
-
-from switchyard.lib.processors.stage_router.decision_log import (
- CONTEXT_KEY,
- DecisionSource,
- StageRouterDecisionLog,
-)
-
-if TYPE_CHECKING:
- from switchyard.lib.processors.stage_router.classifier import TierClassifier
- from switchyard.lib.proxy_context import ProxyContext
-
-log = logging.getLogger(__name__)
-
-EFFICIENT: int = 0
-CAPABLE: int = 1
-
-#: Rust returns the tier as a name; map it back to the picker's int constants.
-_TIER: dict[str, int] = {"efficient": EFFICIENT, "capable": CAPABLE}
-
-
-async def pick_capable_first(
- ctx: "ProxyContext",
- confidence_threshold: float,
- classifier: "TierClassifier | None" = None,
- decision_log: StageRouterDecisionLog | None = None,
-) -> int:
- """CAPABLE default. EFFICIENT only when the scorer is confidently negative."""
- return await _pick(ctx, "capable_first", confidence_threshold, classifier, decision_log)
-
-
-async def pick_efficient_first(
- ctx: "ProxyContext",
- confidence_threshold: float,
- classifier: "TierClassifier | None" = None,
- decision_log: StageRouterDecisionLog | None = None,
-) -> int:
- """EFFICIENT default. CAPABLE only when the scorer is confidently positive."""
- return await _pick(ctx, "efficient_first", confidence_threshold, classifier, decision_log)
-
-
-async def _pick(
- ctx: "ProxyContext",
- picker_mode: str,
- confidence_threshold: float,
- classifier: "TierClassifier | None",
- decision_log: StageRouterDecisionLog | None,
-) -> int:
- from switchyard_rust.components import ( # local import: heavy native module
- get_tool_result_signal,
- stage_pick_tier,
- )
-
- signal = get_tool_result_signal(ctx)
- if signal is None:
- default = EFFICIENT if picker_mode == "efficient_first" else CAPABLE
- return _record(ctx, decision_log, "no_signal", default)
-
- # The Rust core runs the escalate/de-escalate shortcuts and the scorer.
- outcome = stage_pick_tier(signal, picker_mode, confidence_threshold)
- if outcome.resolved:
- # resolved ⇒ tier and source are set (guaranteed by the Rust core).
- source = cast("DecisionSource", outcome.source)
- return _record(ctx, decision_log, source, _TIER[cast(str, outcome.tier)])
-
- # Scorer wasn't confident. Consult the classifier; fall open to the default
- # tier when there is none or it can't decide.
- default = _TIER[outcome.default_tier]
- if classifier is None:
- return _record(ctx, decision_log, "fall_open", default)
- verdict = await classifier.classify(ctx, signal)
- if verdict == "capable":
- return _record(ctx, decision_log, "llm-classifier", CAPABLE)
- if verdict == "efficient":
- return _record(ctx, decision_log, "llm-classifier", EFFICIENT)
- return _record(ctx, decision_log, "fall_open", default)
-
-
-def _record(
- ctx: "ProxyContext",
- decision_log: StageRouterDecisionLog | None,
- source: DecisionSource,
- tier: int,
-) -> int:
- try:
- ctx.metadata[CONTEXT_KEY] = source
- except Exception:
- # ProxyContext.metadata may be a strict map; never let a stamping
- # failure block routing.
- log.debug("failed to stamp decision source", exc_info=True)
- if decision_log is not None:
- decision_log.record(source)
- return tier
-
-
-__all__ = ["CAPABLE", "EFFICIENT", "pick_capable_first", "pick_efficient_first"]
diff --git a/switchyard/lib/processors/stage_router/prompts/tier_classifier.md b/switchyard/lib/processors/stage_router/prompts/tier_classifier.md
deleted file mode 100644
index 25f570fcf..000000000
--- a/switchyard/lib/processors/stage_router/prompts/tier_classifier.md
+++ /dev/null
@@ -1,5 +0,0 @@
-You are a routing classifier inside an agentic coding stage-router. Given a compact summary of the agent's recent tool activity, decide whether the next model call should go to the CAPABLE tier (frontier, expensive) or the EFFICIENT tier (cheap, less powerful).
-
-Respond with strict JSON: {"tier": "capable"} or {"tier": "efficient"}.
-
-Pick EFFICIENT when the agent shows concrete, low-friction progress (writes landing, tests passing, edits without errors). Pick CAPABLE when the agent is stalled, hitting errors, or facing a task likely to require careful reasoning.
diff --git a/switchyard/lib/processors/stage_router_request_processor.py b/switchyard/lib/processors/stage_router_request_processor.py
deleted file mode 100644
index 1edc0fa76..000000000
--- a/switchyard/lib/processors/stage_router_request_processor.py
+++ /dev/null
@@ -1,173 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tier picker component — stamps ``ctx.selected_target``/``selected_model``.
-Fails open to the efficient tier on picker exceptions."""
-
-from __future__ import annotations
-
-import logging
-from collections.abc import Awaitable, Callable, Sequence
-from typing import TYPE_CHECKING, Any, cast
-
-from switchyard.lib.processors.stage_router import (
- CAPABLE,
- CONTEXT_KEY,
- EFFICIENT,
- StageRouterDecisionLog,
- pick_capable_first,
- pick_efficient_first,
-)
-from switchyard.lib.processors.stage_router.classifier import RECENT_MESSAGES_KEY, TierClassifier
-
-if TYPE_CHECKING:
- from switchyard.lib.backends.llm_target import LlmTarget
- from switchyard.lib.processors.stage_router.handoff_notes import HandoffNoteInjector
- from switchyard.lib.proxy_context import ProxyContext
- from switchyard.lib.stats_accumulator import StatsAccumulator
- from switchyard_rust.core import ChatRequest
-
-log = logging.getLogger(__name__)
-
-#: Async picker signature. The factory pre-binds knobs and the optional classifier.
-TierPicker = Callable[["ProxyContext"], Awaitable[int]]
-
-#: YAML-resolvable picker names; mirrors :class:`StageRouterPickerMode`.
-BUILTIN_PICKERS: dict[str, Callable[..., Awaitable[int]]] = {
- "capable_first": pick_capable_first,
- "efficient_first": pick_efficient_first,
-}
-
-
-class StageRouterRequestProcessor:
- """Picks a tier and stamps it on the context. Policy lives in the picker."""
-
- def __init__(
- self,
- *,
- targets: Sequence[LlmTarget],
- picker: TierPicker,
- classifier: TierClassifier | None = None,
- decision_log: StageRouterDecisionLog | None = None,
- handoff_injector: HandoffNoteInjector | None = None,
- strong_system_prompt: str | None = None,
- weak_system_prompt: str | None = None,
- ) -> None:
- if len(targets) != 2:
- raise ValueError(f"stage_router requires exactly 2 targets, got {len(targets)}")
- self._target_ids = [t.id for t in targets]
- self._target_models = [t.model for t in targets]
- self._picker = picker
- self._classifier = classifier
- self._max_index = len(targets) - 1
- self._decision_log = decision_log if decision_log is not None else StageRouterDecisionLog()
- self._handoff_injector = handoff_injector
- self._strong_system_prompt = strong_system_prompt
- self._weak_system_prompt = weak_system_prompt
- self._stats_accumulator: StatsAccumulator | None = None
-
- def attach_stats_accumulator(self, stats_accumulator: StatsAccumulator) -> None:
- """Attach serving-level stats to stage_router-only routing components."""
- self._stats_accumulator = stats_accumulator
- if self._classifier is not None:
- self._classifier.attach_stats_accumulator(stats_accumulator)
-
- def decision_stats(self) -> dict[str, int]:
- """Snapshot of decision-source counts since process start."""
- return self._decision_log.snapshot()
-
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- # Stash trailing messages for the classifier when one is configured.
- try:
- body = request.body
- if isinstance(body, dict):
- messages = body.get("messages")
- if isinstance(messages, list):
- ctx.metadata[RECENT_MESSAGES_KEY] = messages
- except Exception:
- log.debug("failed to stash request messages on ctx", exc_info=True)
- idx = await self._resolve_index(ctx)
- ctx.selected_target = self._target_ids[idx]
- ctx.selected_model = self._target_models[idx]
- await self._record_decision_source(ctx)
- if self._handoff_injector is not None:
- source = ctx.metadata.get(CONTEXT_KEY)
- self._handoff_injector.maybe_inject(
- request,
- tier=idx,
- source=source if isinstance(source, str) else None,
- )
- if self._strong_system_prompt is not None and idx == CAPABLE:
- _prepend_system_prompt(request, self._strong_system_prompt)
- if self._weak_system_prompt is not None and idx == EFFICIENT:
- _prepend_system_prompt(request, self._weak_system_prompt)
- log.debug(
- "stage_router pick: idx=%d target=%s model=%s",
- idx, ctx.selected_target, ctx.selected_model,
- )
- return request
-
- async def _resolve_index(self, ctx: ProxyContext) -> int:
- try:
- idx = await self._picker(ctx)
- except Exception:
- log.exception("stage_router picker raised; falling back to index 0 (efficient)")
- return EFFICIENT
- return max(0, min(idx, self._max_index))
-
- async def _record_decision_source(self, ctx: ProxyContext) -> None:
- """Copy the picker source stamp into shared routing stats when available."""
- if self._stats_accumulator is None:
- return
- source = ctx.metadata.get(CONTEXT_KEY)
- if not isinstance(source, str) or not source:
- return
- try:
- await cast(Any, self._stats_accumulator).record_routing_decision(
- "stage_router",
- source,
- )
- except Exception:
- log.debug("failed to record stage_router decision source", exc_info=True)
-
-
-def _prepend_system_prompt(request: ChatRequest, prompt: str) -> None:
- """Prepend ``prompt`` to the request's system content. Never raises."""
- try:
- body = getattr(request, "body", None)
- if not isinstance(body, dict):
- return
- fmt = getattr(getattr(request, "request_type", None), "value", None)
- if fmt == "anthropic":
- # Anthropic: top-level "system" field (string or content-block list)
- existing = body.get("system", "")
- if isinstance(existing, str):
- body["system"] = prompt + ("\n\n" + existing if existing else "")
- elif isinstance(existing, list):
- body["system"] = [{"type": "text", "text": prompt}] + existing
- else:
- body["system"] = prompt
- else:
- # OpenAI: prepend a system message before existing messages
- messages = body.get("messages")
- if not isinstance(messages, list):
- return
- if messages and messages[0].get("role") == "system":
- existing = messages[0].get("content", "")
- messages[0] = {"role": "system", "content": prompt + "\n\n" + existing}
- else:
- messages.insert(0, {"role": "system", "content": prompt})
- request.replace_body(body)
- except Exception:
- log.debug("strong_system_prompt injection failed; continuing without", exc_info=True)
-
-
-__all__ = [
- "BUILTIN_PICKERS",
- "CAPABLE",
- "EFFICIENT",
- "StageRouterRequestProcessor",
- "TierPicker",
- "pick_capable_first",
- "pick_efficient_first",
-]
diff --git a/switchyard/lib/profiles/__init__.py b/switchyard/lib/profiles/__init__.py
deleted file mode 100644
index c4058a562..000000000
--- a/switchyard/lib/profiles/__init__.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Python profile abstractions for programmatic routing profiles."""
-
-from switchyard.lib.profiles.deterministic_routing_config import (
- DeterministicRoutingConfig,
-)
-from switchyard.lib.profiles.deterministic_routing_presets import (
- DeterministicRoutingPresets,
-)
-from switchyard.lib.profiles.deterministic_routing_profile_config import (
- DeterministicRoutingProfileConfig,
-)
-from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig
-from switchyard.lib.profiles.escalation_router_profile_config import (
- EscalationRouterProfileConfig,
-)
-from switchyard.lib.profiles.header_routing import (
- HeaderRoutingConfig,
- HeaderRoutingDecision,
- HeaderRoutingProfile,
-)
-from switchyard.lib.profiles.passthrough import PassthroughProfileConfig
-from switchyard.lib.profiles.protocols import (
- ContextAwareProfile,
- Profile,
- ProfileConfig,
- ProfileHooks,
- ProfileInput,
- ProfileLifecycle,
- ProfileRunner,
-)
-from switchyard.lib.profiles.random_routing import (
- RandomRoutingConfig,
- RandomRoutingProfileConfig,
-)
-from switchyard.lib.profiles.random_routing_presets import RandomRoutingPresets
-from switchyard.lib.profiles.stage_router import StageRouterProfileConfig
-from switchyard.lib.profiles.stage_router_config import ClassifierConfig, StageRouterConfig
-from switchyard.lib.profiles.switchyard_adapter import ProfileSwitchyard
-from switchyard.lib.profiles.table import (
- ProfileConfigError,
- build_profile,
- profile_config,
- profile_config_type,
-)
-from switchyard.lib.profiles.translate_profile_config import TranslateProfileConfig
-
-__all__ = [
- "StageRouterProfileConfig",
- "StageRouterConfig",
- "ClassifierConfig",
- "DeterministicRoutingConfig",
- "DeterministicRoutingProfileConfig",
- "DeterministicRoutingPresets",
- "EscalationRouterConfig",
- "EscalationRouterProfileConfig",
- "HeaderRoutingConfig",
- "HeaderRoutingDecision",
- "HeaderRoutingProfile",
- "PassthroughProfileConfig",
- "ContextAwareProfile",
- "Profile",
- "ProfileConfig",
- "ProfileConfigError",
- "ProfileHooks",
- "ProfileInput",
- "ProfileLifecycle",
- "ProfileRunner",
- "ProfileSwitchyard",
- "RandomRoutingConfig",
- "RandomRoutingPresets",
- "RandomRoutingProfileConfig",
- "TranslateProfileConfig",
- "build_profile",
- "profile_config",
- "profile_config_type",
-]
diff --git a/switchyard/lib/profiles/chain.py b/switchyard/lib/profiles/chain.py
deleted file mode 100644
index b74d1253a..000000000
--- a/switchyard/lib/profiles/chain.py
+++ /dev/null
@@ -1,395 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Chain runtime helpers for profile-owned component pipelines."""
-
-from __future__ import annotations
-
-from collections.abc import Sequence
-from dataclasses import dataclass
-from typing import Any, cast
-
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.roles import LLMBackend
-from switchyard.lib.session_affinity import CTX_SESSION_KEY
-from switchyard.lib.session_cache import SessionCache
-from switchyard.lib.session_key import session_key_from_body
-from switchyard.lib.stats_accumulator import StatsAccumulator
-from switchyard_rust.core import (
- ChatRequest,
- ChatResponse,
- SwitchyardBackendError,
- SwitchyardContextPoolExhaustedError,
- SwitchyardContextWindowExceededError,
- SwitchyardProcessorError,
- SwitchyardUpstreamError,
-)
-from switchyard_rust.profiles import ProfileInput
-
-
-@dataclass(slots=True)
-class ChainProcessedRequest:
- """Request-side state needed to finish a profile call.
-
- Existing Python processors and backends still communicate through
- ``ProxyContext``. The profile bridge snapshots the routing decision here
- and keeps the context synchronized so the decision is visible as typed
- profile-owned state while legacy components continue to run unchanged.
- """
-
- ctx: ProxyContext
- request: ChatRequest
- selected_target: str | None
- evicted_targets: tuple[str, ...]
-
- @classmethod
- def from_context(
- cls,
- ctx: ProxyContext,
- request: ChatRequest,
- ) -> ChainProcessedRequest:
- """Snapshot request-side routing state from the compatibility context."""
- return cls(
- ctx=ctx,
- request=request,
- selected_target=_context_selected_target(ctx),
- evicted_targets=tuple(ctx.evicted_targets or ()),
- )
-
- def sync_to_context(self) -> None:
- """Write profile-owned routing state back for legacy components."""
- self.ctx.selected_target = self.selected_target
- self.ctx.evicted_targets = list(self.evicted_targets) or None
-
-
-class ComponentChainProfile:
- """Run existing processor/backend components as one Profile runtime.
-
- This is intentionally private. It lets Profile configs own construction
- directly while older component logic is still being moved into profiles.
- """
-
- #: LRU cap for the session eviction cache. Without a bound, a long-running
- #: server accumulates one entry per unique conversation indefinitely; the LRU
- #: evicts the oldest sessions when full, so active conversations are retained.
- _SESSION_CACHE_MAX = 1024
-
- def __init__(
- self,
- *,
- request_processors: Sequence[Any] = (),
- backend: LLMBackend,
- response_processors: Sequence[Any] = (),
- fallback_target_on_evict: str | None = None,
- ) -> None:
- """Create a profile runtime from already-built components."""
- self._request_processors = tuple(request_processors)
- self._backend = backend
- self._response_processors = tuple(response_processors)
- self._fallback_target_on_evict = fallback_target_on_evict
- # Maps session key → frozenset of target IDs that overflowed for that session.
- # Only allocated when eviction is configured; None otherwise.
- self._session_evictions: SessionCache[frozenset[str]] | None = (
- SessionCache[frozenset[str]](self._SESSION_CACHE_MAX)
- if fallback_target_on_evict is not None
- else None
- )
-
- def iter_components(self) -> list[object]:
- """Return lifecycle components in startup order."""
- return [
- *self._request_processors,
- self._backend,
- *self._response_processors,
- ]
-
- def with_runtime_components(
- self,
- stats_accumulator: StatsAccumulator | None = None,
- enable_stats: bool = True,
- pre_request_processors: Sequence[Any] = (),
- post_request_processors: Sequence[Any] = (),
- response_processors: Sequence[Any] = (),
- ) -> ComponentChainProfile:
- """Return a copy with serving-level stats and processor hooks applied.
-
- Profile configs stay user-facing and parseable; shared serving resources
- such as one route-table accumulator are attached by the builder that
- hosts the profile.
- """
- from switchyard.lib.processors.stats_request_processor import (
- StatsRequestProcessor,
- )
- from switchyard.lib.processors.stats_response_processor_accumulator import (
- StatsResponseProcessor,
- )
-
- request_chain: list[Any] = []
- response_chain: list[Any] = list(self._response_processors)
- backend = self._backend
- stats: StatsAccumulator | None = None
-
- if enable_stats:
- stats = stats_accumulator or StatsAccumulator()
- request_chain.append(StatsRequestProcessor())
- backend = _attach_stats_to_backend(backend, stats)
- response_chain.append(StatsResponseProcessor(stats))
-
- request_chain.extend(pre_request_processors)
- request_chain.extend(self._request_processors)
- request_chain.extend(post_request_processors)
- response_chain.extend(response_processors)
- if stats is not None:
- _attach_stats_to_request_processors(request_chain, stats)
-
- return ComponentChainProfile(
- request_processors=request_chain,
- backend=backend,
- response_processors=response_chain,
- fallback_target_on_evict=self._fallback_target_on_evict,
- )
-
- async def process(self, input: ProfileInput) -> ChainProcessedRequest:
- """Run request-side components for Profile protocol conformance."""
- return await self.process_with_context(input, _context_from_input(input))
-
- async def process_with_context(
- self,
- input: ProfileInput,
- ctx: ProxyContext,
- ) -> ChainProcessedRequest:
- """Run request-side components against the caller-supplied context."""
- current: Any = input.request
- for processor in self._request_processors:
- try:
- current = await processor.process(ctx, current)
- except Exception as error:
- raise SwitchyardProcessorError(str(error)) from error
- if not isinstance(current, ChatRequest):
- actual = type(current).__name__
- raise SwitchyardProcessorError(
- f"Request processor returned {actual}, expected ChatRequest"
- )
- return ChainProcessedRequest.from_context(ctx, current)
-
- async def rprocess(
- self,
- processed: ChainProcessedRequest,
- response: ChatResponse,
- ) -> ChatResponse:
- """Run response-side components for a backend response."""
- current: Any = response
- for processor in self._response_processors:
- try:
- current = await processor.process(processed.ctx, current)
- except Exception as error:
- raise SwitchyardProcessorError(str(error)) from error
- if not isinstance(current, ChatResponse):
- actual = type(current).__name__
- raise SwitchyardProcessorError(
- f"Response processor returned {actual}, expected ChatResponse"
- )
- return cast(ChatResponse, current)
-
- async def run(self, input: ProfileInput) -> ChatResponse:
- """Execute the complete Profile protocol lifecycle with a derived context."""
- return await self.run_with_context(input, _context_from_input(input))
-
- async def run_with_context(
- self,
- input: ProfileInput,
- ctx: ProxyContext,
- ) -> ChatResponse:
- """Execute the complete profile lifecycle with an existing context."""
- processed = await self.process_with_context(input, ctx)
- session_key: str | None = None
- if self._session_evictions is not None:
- session_key = _derive_session_key(ctx, input.request)
- session_evicted: frozenset[str] = self._session_evictions.get(session_key) or frozenset()
- if session_evicted:
- merged = set(processed.evicted_targets) | session_evicted
- processed.evicted_targets = tuple(sorted(merged))
- processed.sync_to_context()
- self._rewrite_evicted_pick(processed)
- try:
- return await self._call_backend_stage(processed)
- except SwitchyardContextWindowExceededError as error:
- if self._fallback_target_on_evict is None:
- raise
- return await self._retry_after_context_overflow(processed, error, session_key)
-
- async def _call_backend_stage(
- self,
- processed: ChainProcessedRequest,
- ) -> ChatResponse:
- """Call the backend and then response-side components."""
- try:
- response: Any = await self._backend.call(processed.ctx, processed.request)
- except SwitchyardContextWindowExceededError:
- raise
- except SwitchyardUpstreamError:
- raise
- except Exception as error:
- raise SwitchyardBackendError(str(error)) from error
- if not isinstance(response, ChatResponse):
- actual = type(response).__name__
- raise SwitchyardBackendError(
- f"Profile backend returned {actual}, expected ChatResponse"
- )
- return await self.rprocess(processed, response)
-
- async def _retry_after_context_overflow(
- self,
- processed: ChainProcessedRequest,
- error: BaseException,
- session_key: str | None = None,
- ) -> ChatResponse:
- """Record an evicted target, rewrite to fallback, and retry once."""
-
- def record_eviction(target_id: str | None) -> None:
- if target_id is None:
- return
- evicted = set(processed.evicted_targets)
- evicted.add(target_id)
- processed.evicted_targets = tuple(sorted(evicted))
- processed.sync_to_context()
- if session_key is not None and self._session_evictions is not None:
- prior: frozenset[str] = self._session_evictions.get(session_key) or frozenset()
- self._session_evictions.put(session_key, prior | {target_id})
-
- record_eviction(_overflow_target_id(processed, error))
- self._rewrite_evicted_pick(processed)
- try:
- return await self._call_backend_stage(processed)
- except SwitchyardContextWindowExceededError as second:
- second_target = _overflow_target_id(processed, second)
- record_eviction(second_target)
- last_target = second_target or "unknown"
- reason = "all attempted targets returned context-window overflow"
- pool_error = SwitchyardContextPoolExhaustedError(
- f"context pool exhausted after target {last_target}: {reason}"
- )
- pool_error.last_target_id = last_target # type: ignore[attr-defined]
- pool_error.reason = reason # type: ignore[attr-defined]
- raise pool_error from second
-
- def _rewrite_evicted_pick(self, processed: ChainProcessedRequest) -> None:
- """Rewrite an evicted or exception-only target to the configured fallback."""
- selected = processed.selected_target
- evicted = set(processed.evicted_targets)
- if (selected is not None and selected in evicted) or (not selected and evicted):
- processed.selected_target = self._fallback_target_on_evict
- processed.sync_to_context()
-
-
-def _derive_session_key(ctx: ProxyContext, request: ChatRequest) -> str:
- """Return the session key for this request, memoized on ctx."""
- cached = ctx.metadata.get(CTX_SESSION_KEY)
- if isinstance(cached, str):
- return cached
- key = session_key_from_body(request.body)
- ctx.metadata[CTX_SESSION_KEY] = key
- return key
-
-
-def _context_from_input(input: ProfileInput) -> ProxyContext:
- """Build a compatibility context from profile metadata."""
- ctx = ProxyContext()
- metadata = input.metadata
- if metadata.request_id is not None:
- ctx.request_id = metadata.request_id
- if metadata.inbound_format is not None:
- ctx.inbound_format = cast(Any, metadata.inbound_format)
- for key, values in metadata.headers.items():
- ctx.metadata[key] = list(values)
- return ctx
-
-
-def _context_selected_target(ctx: ProxyContext) -> str | None:
- """Return a normalized selected target from the compatibility context."""
- selected = ctx.selected_target
- return selected if isinstance(selected, str) and selected else None
-
-
-def _overflow_target_id(
- processed: ChainProcessedRequest,
- error: BaseException,
-) -> str | None:
- """Return target id carried by an overflow error or processed state."""
- target_id = getattr(error, "target_id", None)
- if isinstance(target_id, str) and target_id:
- return target_id
- return processed.selected_target
-
-
-def _attach_stats_to_backend(
- backend: LLMBackend,
- stats: StatsAccumulator,
-) -> LLMBackend:
- """Wrap native backends or wire existing Python stats hooks in place."""
- from switchyard.lib.backends.stats_llm_backend import StatsLlmBackend
-
- try:
- return StatsLlmBackend(backend, stats)
- except TypeError:
- # Python-only backends cannot be wrapped by the Rust stats binding.
- # Fail loudly if the backend does not expose one of the known
- # compatibility hooks; silent metrics loss is worse than a build error.
- if not _attach_stats_to_python_backend(backend, stats):
- backend_type = type(backend).__qualname__
- raise TypeError(
- f"{backend_type} cannot be wrapped for stats and exposes no "
- "Python stats compatibility hook"
- ) from None
- return backend
-
-
-def _attach_stats_to_python_backend(
- backend: LLMBackend,
- stats: StatsAccumulator,
-) -> bool:
- """Wire stats into Python-only backend shapes that already support it."""
- attached = False
- if hasattr(backend, "_stats"):
- cast(Any, backend)._stats = stats
- attached = True
-
- inner = getattr(backend, "_inner", None)
- if inner is not None:
- from switchyard.lib.backends.stats_llm_backend import StatsLlmBackend
-
- try:
- cast(Any, backend)._inner = StatsLlmBackend(inner, stats)
- attached = True
- except TypeError:
- attached = _attach_stats_to_python_backend(inner, stats) or attached
-
- nested = getattr(backend, "_backends", None)
- if not isinstance(nested, dict):
- return attached
-
- from switchyard.lib.backends.stats_llm_backend import StatsLlmBackend
-
- for label, child in list(nested.items()):
- try:
- nested[label] = StatsLlmBackend(child, stats)
- attached = True
- except TypeError:
- attached = _attach_stats_to_python_backend(child, stats) or attached
- return attached
-
-
-def _attach_stats_to_request_processors(
- processors: Sequence[Any],
- stats: StatsAccumulator,
-) -> None:
- """Wire existing request-processor stats hooks without config fields."""
- for processor in processors:
- attach_stats = getattr(processor, "attach_stats_accumulator", None)
- if callable(attach_stats):
- attach_stats(stats)
- if hasattr(processor, "_stats_accumulator"):
- cast(Any, processor)._stats_accumulator = stats
-
-
-__all__ = ["ChainProcessedRequest", "ComponentChainProfile"]
diff --git a/switchyard/lib/profiles/deterministic_routing_config.py b/switchyard/lib/profiles/deterministic_routing_config.py
deleted file mode 100644
index 8a0b893b6..000000000
--- a/switchyard/lib/profiles/deterministic_routing_config.py
+++ /dev/null
@@ -1,172 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Config model for the deterministic LLM-classifier routing profile."""
-
-from __future__ import annotations
-
-from typing import Literal
-
-from pydantic import (
- BaseModel,
- ConfigDict,
- Field,
- ValidationInfo,
- field_validator,
- model_validator,
-)
-
-from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target
-from switchyard.lib.processors.llm_classifier import DEFAULT_MAX_REQUEST_CHARS
-
-ProfileName = Literal["general", "coding_agent", "openclaw"]
-DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S = 600.0
-
-
-class DeterministicRoutingConfig(BaseModel):
- """Configuration for the deterministic LLM-classifier routing profile.
-
- A strong/weak tier pair plus a classifier LLM target. The classifier
- inspects each request and decides which tier to dispatch to via the
- profile's tier mapping; routing is content-aware, not a coin flip.
-
- Attributes:
- strong: Strong tier (typically the higher-quality model).
- Selected when the classifier collapses
- ``RouteTier.COMPLEX`` / ``RouteTier.REASONING`` (and per
- profile rules, also on tool-planning escalation).
- weak: Weak tier (typically the cheap / fast model). Selected
- when the classifier returns ``RouteTier.SIMPLE`` (and,
- depending on profile, ``RouteTier.MEDIUM``).
- classifier: Target for the classifier LLM call itself.
- ``model`` / ``base_url`` / ``api_key`` are extracted at
- build time; ``format``, ``extra_body``, ``extra_headers``
- are ignored (the classifier client owns its own structured-
- output mechanics and the DeepSeek
- ``chat_template_kwargs.enable_thinking=False`` knob is
- applied via :class:`LLMClassifierConfig.disable_reasoning`).
- profile_name: Selects which :class:`LLMClassifierPresets` preset
- bundles the classifier prompt, JSON schema, and tier mapping.
- classifier_min_confidence: Below this floor the tier selector
- falls back to the profile's ``default_tier`` (strong).
- ``0.0`` (default) honors every non-abstain classification.
- classifier_fail_open: When ``True`` (default), classifier errors
- stamp abstain signals and fall back to the default tier.
- When ``False``, classifier errors propagate as 5xx.
- classifier_recent_turn_window: Number of trailing turns the
- classifier sees in addition to the system + first-user
- anchors. ``4`` matches the validated benchmark default for
- agent-loop traffic.
- classifier_system_prompt: Optional prompt override for the
- selected classifier profile. ``None`` uses the profile's
- built-in prompt.
- classifier_max_request_chars: Maximum serialized request-summary
- characters sent to the classifier before truncation.
- classifier_timeout_s: Per-call timeout for the classifier LLM
- (seconds). Defaults to ``30`` — the classifier fails open at
- timeout, so a short floor bounds tail latency without losing
- availability.
- tier_timeout_s: Default per-call timeout for strong/weak tier
- LLM calls when a target does not set its own
- ``timeout_secs``. ``None`` disables the default and leaves
- the provider client unbounded.
- enable_stats: Wire stats request/response processors and per-
- tier :class:`StatsLlmBackend` wrappers. Default ``True``.
- preset: Optional name of the
- :class:`DeterministicRoutingPresets` preset builder that produced
- this config — surfaced via ``GET /v1/routing/stats`` so
- saved stats files self-document which shipping bundle was
- used.
- affinity_warmup_turns: Initial conversation turns that remain
- non-sticky even when ``session_affinity`` is enabled. The first
- confident verdict after this warmup can pin the session.
- """
-
- model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True)
-
- strong: LlmTarget
- weak: LlmTarget
- classifier: LlmTarget
- #: Target id the chain executor reroutes to when the picked target is
- #: evicted (e.g. context-window overflow). Must match either
- #: ``strong.id`` or ``weak.id``. The classifier target is not a routing
- #: candidate, so it cannot be selected as the fallback.
- fallback_target_on_evict: str
- profile_name: ProfileName = "coding_agent"
- classifier_min_confidence: float = Field(default=0.0, ge=0.0, le=1.0)
- classifier_fail_open: bool = True
- classifier_recent_turn_window: int = Field(default=4, ge=0)
- classifier_system_prompt: str | None = Field(default=None, min_length=1)
- classifier_max_request_chars: int = Field(
- default=DEFAULT_MAX_REQUEST_CHARS,
- ge=256,
- )
- classifier_timeout_s: float = Field(default=30.0, gt=0.0)
- tier_timeout_s: float | None = Field(
- default=DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S,
- gt=0.0,
- )
- enable_stats: bool = True
- session_affinity: bool = False
- affinity_max_sessions: int = Field(default=10_000, ge=0)
- affinity_warmup_turns: int = Field(default=0, ge=0)
- preset: str | None = None
-
- @field_validator("strong", "weak", "classifier", mode="before")
- @classmethod
- def _coerce_target(cls, value: object, info: ValidationInfo) -> LlmTarget:
- return coerce_llm_target(value, default_id=info.field_name or "target")
-
- @field_validator("strong", "weak", "classifier")
- @classmethod
- def _target_model_non_empty(cls, tier: LlmTarget) -> LlmTarget:
- if not tier.model:
- raise ValueError("target.model must be a non-empty string")
- return tier
-
- @field_validator("weak")
- @classmethod
- def _tier_ids_distinct(cls, value: LlmTarget, info: ValidationInfo) -> LlmTarget:
- strong = info.data.get("strong")
- if isinstance(strong, LlmTarget) and strong.id == value.id:
- raise ValueError(
- f"strong.id and weak.id must differ (both are {value.id!r}); "
- "they key the routing tiers"
- )
- return value
-
- @field_validator("classifier_system_prompt", mode="before")
- @classmethod
- def _blank_classifier_prompt_is_unset(cls, value: object) -> object:
- if isinstance(value, str) and not value.strip():
- return None
- return value
-
- @field_validator("fallback_target_on_evict")
- @classmethod
- def _fallback_matches_existing_target(cls, value: str, info: ValidationInfo) -> str:
- valid_ids = {info.data[key].id for key in ("strong", "weak") if key in info.data}
- if value not in valid_ids:
- raise ValueError(
- f"fallback_target_on_evict={value!r} must match one of "
- f"{sorted(valid_ids)} (the configured strong/weak target ids; "
- f"the classifier target is not a routing candidate)"
- )
- return value
-
- @model_validator(mode="after")
- def _affinity_capacity_nonzero_when_enabled(self) -> DeterministicRoutingConfig:
- # A zero-capacity affinity store retains nothing, which would silently
- # disable stickiness while still paying the per-request key cost.
- if self.session_affinity and self.affinity_max_sessions == 0:
- raise ValueError(
- "affinity_max_sessions must be > 0 when session_affinity is enabled"
- )
- return self
-
-
-__all__ = [
- "DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S",
- "DeterministicRoutingConfig",
- "ProfileName",
-]
diff --git a/switchyard/lib/profiles/deterministic_routing_presets.py b/switchyard/lib/profiles/deterministic_routing_presets.py
deleted file mode 100644
index 3e6b46241..000000000
--- a/switchyard/lib/profiles/deterministic_routing_presets.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Named :class:`DeterministicRoutingConfig` presets keyed by shipping bundle.
-
-Each preset bundles a validated strong/weak/classifier trio plus the
-matching profile. Callers vary only the per-deployment knobs (``api_key``,
-``base_url``).
-
-The shipping default :meth:`DeterministicRoutingPresets.coding_agent_default`
-is the trio validated end-to-end on the 100-task
-``[email protected]`` Terminal-Bench sweep at 2026-05 — 88.0%
-classifier solve vs 84.9% force-strong at ~−25% cost vs all-strong. Use
-this for any coding-agent launcher (Claude Code, Codex, Cursor) unless you
-have a strong reason to deviate.
-
-Example::
-
- from switchyard import (
- DeterministicRoutingPresets,
- DeterministicRoutingProfileConfig,
- ProfileSwitchyard,
- )
-
- config = DeterministicRoutingPresets.coding_agent_default(
- api_key=nvidia_api_key,
- )
- switchyard = ProfileSwitchyard(
- DeterministicRoutingProfileConfig.from_config(config).build()
- )
-"""
-
-from __future__ import annotations
-
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
-from switchyard.lib.profiles.deterministic_routing_config import (
- DeterministicRoutingConfig,
-)
-
-# Shipping presets route through OpenRouter's OpenAI-compatible endpoint
-# by default; callers override with ``base_url=`` for another gateway.
-_OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
-
-# OpenRouter model ids for the zero-flag coding-agent launcher default.
-_MODEL_OPUS_4_7 = "anthropic/claude-opus-4.7"
-_MODEL_KIMI_K2_6 = "moonshotai/kimi-k2.6"
-_MODEL_GEMINI_3_5_FLASH = "google/gemini-3.5-flash"
-
-
-class DeterministicRoutingPresets:
- """Builder of pre-built :class:`DeterministicRoutingConfig` bundles."""
-
- @staticmethod
- def coding_agent_default(
- *,
- api_key: str,
- base_url: str = _OPENROUTER_BASE_URL,
- timeout_secs: float | None = 600.0,
- ) -> DeterministicRoutingConfig:
- """The coding-agent-launcher default trio.
-
- Strong: Claude Opus 4.7.
- Weak: Kimi K2.6.
- Classifier: Gemini 3.5 Flash.
- Profile: ``coding_agent`` (SIMPLE+MEDIUM→weak, COMPLEX/REASONING→strong,
- with tool-planning escalation and high-confidence LLM alignment).
-
- Use for any coding-agent launcher (Claude Code, Codex, Cursor).
- """
- return DeterministicRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model=_MODEL_OPUS_4_7,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- timeout_secs=timeout_secs,
- ),
- weak=LlmTarget(
- id="weak",
- model=_MODEL_KIMI_K2_6,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- timeout_secs=timeout_secs,
- ),
- classifier=LlmTarget(
- id="classifier",
- model=_MODEL_GEMINI_3_5_FLASH,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- timeout_secs=30.0,
- ),
- profile_name="coding_agent",
- classifier_min_confidence=0.0,
- classifier_fail_open=True,
- classifier_recent_turn_window=4,
- classifier_timeout_s=30.0,
- fallback_target_on_evict="strong",
- preset="coding_agent_default",
- )
-
-
-__all__ = ["DeterministicRoutingPresets"]
diff --git a/switchyard/lib/profiles/deterministic_routing_profile_config.py b/switchyard/lib/profiles/deterministic_routing_profile_config.py
deleted file mode 100644
index 3be3eb9c2..000000000
--- a/switchyard/lib/profiles/deterministic_routing_profile_config.py
+++ /dev/null
@@ -1,122 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Profile-owned deterministic LLM-classifier routing construction."""
-
-from __future__ import annotations
-
-from typing import Any, Self
-
-from switchyard.lib.processors.llm_classifier.presets import (
- PROFILE_FACTORIES,
- resolve_classifier_prompt,
-)
-from switchyard.lib.profiles.chain import ComponentChainProfile
-from switchyard.lib.profiles.deterministic_routing_config import (
- DeterministicRoutingConfig,
-)
-from switchyard.lib.profiles.table import profile_config
-from switchyard.lib.profiles.tier_target_builders import build_tier_backend
-
-
-@profile_config("deterministic")
-class DeterministicRoutingProfileConfig:
- """Profile config wrapper for content-aware deterministic routing."""
-
- config: DeterministicRoutingConfig
-
- @classmethod
- def from_config(cls, config: DeterministicRoutingConfig) -> Self:
- """Create a profile config from the validated parsing model."""
- return cls(config=config)
-
- def build(self) -> ComponentChainProfile:
- """Build the deterministic routing profile runtime."""
- from switchyard.lib.backends.deterministic_routing_llm_backend import (
- DeterministicRoutingLLMBackend,
- )
- from switchyard.lib.processors.llm_classifier import (
- LLMClassifierRequestProcessor,
- SignalTierSelectorRequestProcessor,
- )
- from switchyard.lib.processors.reasoning_effort_normalizer import (
- ReasoningEffortNormalizer,
- )
- from switchyard.lib.session_affinity import SessionAffinity
-
- config = self.config
- # Tier labels are the configured target ids ("strong"/"weak" unless
- # overridden). The tier selector stamps them, the backend keys its
- # tier dict by them, and the chain's evict-and-retry rewrites
- # selected_target to fallback_target_on_evict — which the config
- # validates against these same ids — so an overflow reroute always
- # lands on a registered tier.
- strong_id = config.strong.id
- weak_id = config.weak.id
- profile = PROFILE_FACTORIES[config.profile_name](
- weak=weak_id,
- strong=strong_id,
- )
-
- request_processors: list[Any] = [ReasoningEffortNormalizer()]
-
- # One affinity coordinator shared by the classifier and tier selector:
- # the classifier gates its LLM call on it (classify once per task) and
- # the tier selector records / reuses the per-conversation tier pin.
- affinity = SessionAffinity(
- enabled=config.session_affinity,
- max_sessions=config.affinity_max_sessions,
- warmup_turns=config.affinity_warmup_turns,
- )
-
- classifier_config = profile.make_classifier_config(
- model=config.classifier.model,
- api_key=config.classifier.api_key,
- base_url=config.classifier.base_url,
- timeout_s=config.classifier_timeout_s,
- max_request_chars=config.classifier_max_request_chars,
- fail_open=config.classifier_fail_open,
- recent_turn_window=config.classifier_recent_turn_window,
- system_prompt=resolve_classifier_prompt(
- config.profile_name,
- config.classifier_system_prompt,
- ),
- ).model_copy(update={"dump_signals_to_stderr": False})
- request_processors.append(
- LLMClassifierRequestProcessor(
- classifier_config,
- signal_schema=profile.signal_schema,
- affinity=affinity,
- )
- )
- request_processors.append(
- SignalTierSelectorRequestProcessor(
- profile.make_tier_selector_config(
- min_confidence=config.classifier_min_confidence,
- ),
- affinity=affinity,
- )
- )
-
- strong_target, strong_backend = build_tier_backend(
- config.strong, config.tier_timeout_s,
- )
- weak_target, weak_backend = build_tier_backend(
- config.weak, config.tier_timeout_s,
- )
-
- backend = DeterministicRoutingLLMBackend(
- tiers={
- strong_id: (strong_backend, strong_target.model),
- weak_id: (weak_backend, weak_target.model),
- },
- default_tier=strong_id,
- )
- return ComponentChainProfile(
- request_processors=request_processors,
- backend=backend,
- fallback_target_on_evict=config.fallback_target_on_evict,
- )
-
-
-__all__ = ["DeterministicRoutingProfileConfig"]
diff --git a/switchyard/lib/profiles/escalation_router_config.py b/switchyard/lib/profiles/escalation_router_config.py
deleted file mode 100644
index ffb208715..000000000
--- a/switchyard/lib/profiles/escalation_router_config.py
+++ /dev/null
@@ -1,183 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Config model for the escalation-router profile (judge-latched strong/weak)."""
-
-from __future__ import annotations
-
-from typing import Literal, Self
-
-from pydantic import (
- BaseModel,
- ConfigDict,
- Field,
- ValidationInfo,
- field_validator,
- model_validator,
-)
-
-from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target
-from switchyard.lib.profiles.deterministic_routing_config import (
- DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S,
-)
-
-
-class EscalationRouterConfig(BaseModel):
- """Configuration for the escalation-router profile.
-
- Every conversation starts on the ``weak`` tier. An LLM ``judge`` watches
- the trajectory each turn and, on a clear pattern of trouble, escalates the
- conversation to the ``strong`` tier — one-way for the rest of the task
- (session-affinity latch). A new conversation resets to weak.
-
- Attributes:
- strong: Escalation target (frontier / expensive model).
- weak: Starting tier (cheap / efficient model).
- judge: Target for the judge LLM call. ``model`` / ``base_url`` /
- ``api_key`` / ``timeout_secs`` are extracted at build time;
- other target fields are ignored.
- fallback_target_on_evict: Target id the chain executor reroutes to on
- eviction (context-window overflow). Must match ``strong.id`` or
- ``weak.id``; the judge target is not a routing candidate.
- judge_min_turn: First conversation turn on which the judge runs.
- judge_escalate_confirmations: Consecutive escalate verdicts required
- before the strong latch fires (``1`` pins on the first verdict).
- Defaults to the benchmarked configuration, which reached an
- equal-or-better solve rate while latching roughly a third as
- often as ``1`` — the router's main cost lever.
- The confirmation streak is tracked per process: with multiple
- workers and no session-sticky load balancing, verdicts for one
- conversation can land on different workers and take longer to
- accumulate. Set ``1`` on multi-worker deployments, or route
- conversations sticky to a worker.
- judge_confirmation_window: Judged turns an escalate verdict stays
- live for confirmation; ``N > 1`` tolerates up to ``N - 1``
- intervening declines (recurring intermittent trouble confirms).
- judge_disable_reasoning: Send the thinking-off template hint on judge
- calls (default). ``False`` lets a reasoning judge model think,
- trading per-turn latency for rubric adherence. The hint is only
- sent to models that accept it (Claude/Bedrock reject it).
- judge_max_completion_tokens: Completion-token budget for the judge
- call. ``None`` (default) picks 128 with the thinking-off hint in
- effect and 4096 otherwise — reasoning tokens and the JSON verdict
- share this budget, and a too-small cap truncates mid-reasoning so
- the judge silently fails open every turn.
- judge_dump_verdicts: Emit one ``escalation_verdict={...}`` line to
- stderr per judge call (includes a first-user ``task_hint``
- snippet). Benchmark-only diagnostics, off by default; production
- routing telemetry flows through the normal stats path.
- judge_recent_turn_window: Trailing messages shown to the judge on top
- of the system + first-user anchors.
- judge_window_message_chars: Per-message truncation cap inside the
- trailing window (larger keeps more tool-output detail per turn).
- judge_max_request_chars: Cap on the assembled judge transcript.
- judge_system_prompt: Optional judge prompt override. ``None`` uses the
- built-in prompt.
- judge_timeout_s: Per-call judge timeout (seconds); the judge fails
- open to the weak tier at timeout. Defaults to the benchmarked
- value. A judge target's own ``timeout_secs`` wins over this.
- session_key_depth: ``0`` (default) keys conversations on system +
- first user message (the shared Rust session key). ``N > 0``
- extends the key with the first ``N`` post-first-user messages so
- repeated runs of the identical task (k>1 benchmark trials against
- one server) diverge via early model responses instead of sharing
- an escalation latch. Requires nonzero sampling temperature to be
- effective; production traffic should keep ``0``.
- tier_timeout_s: Default per-call timeout for strong/weak tier calls
- when a target does not set its own ``timeout_secs``.
- enable_stats: Wire stats processors and per-tier stats wrappers.
- affinity_max_sessions: LRU capacity of the escalation latch store.
- affinity_store: Shared L2 behind the in-process latch LRU. ``"memory"``
- (default) keeps the escalation latch per process — a worker or pod
- change can lose it. ``"redis"`` shares the latch across workers
- and persists it across pod churn (best-effort: an L2 error never
- fails a request).
- affinity_store_url: Connection URL for the shared store (e.g.
- ``"redis://host:6379/0"``). Required when ``affinity_store`` is
- ``"redis"``.
- affinity_store_ttl_seconds: Expiry for a shared latch entry. The latch
- is re-read every turn; an escalated conversation older than the
- TTL restarts on weak.
- affinity_key_prefix: Namespace prefix for shared-store keys.
- """
-
- model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True)
-
- strong: LlmTarget
- weak: LlmTarget
- judge: LlmTarget
- fallback_target_on_evict: str
- judge_min_turn: int = Field(default=3, ge=1)
- judge_escalate_confirmations: int = Field(default=2, ge=1)
- judge_confirmation_window: int = Field(default=1, ge=1)
- judge_disable_reasoning: bool = True
- judge_max_completion_tokens: int | None = Field(default=None, ge=16)
- judge_dump_verdicts: bool = False
- judge_recent_turn_window: int = Field(default=28, ge=1)
- judge_window_message_chars: int = Field(default=500, ge=50)
- judge_max_request_chars: int = Field(default=18_000, ge=1_000)
- judge_system_prompt: str | None = Field(default=None, min_length=1)
- judge_timeout_s: float = Field(default=30.0, gt=0.0)
- session_key_depth: int = Field(default=0, ge=0)
- tier_timeout_s: float | None = Field(
- default=DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S,
- gt=0.0,
- )
- enable_stats: bool = True
- affinity_max_sessions: int = Field(default=10_000, gt=0)
- affinity_store: Literal["memory", "redis"] = "memory"
- affinity_store_url: str | None = None
- affinity_store_ttl_seconds: int = Field(default=3_600, gt=0)
- affinity_key_prefix: str = "swyd:esc:"
-
- @model_validator(mode="after")
- def _redis_store_requires_url(self) -> Self:
- # A shared store is dead config unless it is reachable.
- if self.affinity_store == "redis" and not self.affinity_store_url:
- raise ValueError('affinity_store="redis" requires affinity_store_url to be set')
- return self
-
- @field_validator("strong", "weak", "judge", mode="before")
- @classmethod
- def _coerce_target(cls, value: object, info: ValidationInfo) -> LlmTarget:
- return coerce_llm_target(value, default_id=info.field_name or "target")
-
- @field_validator("strong", "weak", "judge")
- @classmethod
- def _target_model_non_empty(cls, tier: LlmTarget) -> LlmTarget:
- if not tier.model:
- raise ValueError("target.model must be a non-empty string")
- return tier
-
- @field_validator("weak")
- @classmethod
- def _tier_ids_distinct(cls, value: LlmTarget, info: ValidationInfo) -> LlmTarget:
- strong = info.data.get("strong")
- if isinstance(strong, LlmTarget) and strong.id == value.id:
- raise ValueError(
- f"strong.id and weak.id must differ (both are {value.id!r}); "
- "they key the routing tiers"
- )
- return value
-
- @field_validator("judge_system_prompt", mode="before")
- @classmethod
- def _blank_judge_prompt_is_unset(cls, value: object) -> object:
- if isinstance(value, str) and not value.strip():
- return None
- return value
-
- @field_validator("fallback_target_on_evict")
- @classmethod
- def _fallback_matches_existing_target(cls, value: str, info: ValidationInfo) -> str:
- valid_ids = {info.data[key].id for key in ("strong", "weak") if key in info.data}
- if value not in valid_ids:
- raise ValueError(
- f"fallback_target_on_evict={value!r} must match one of "
- f"{sorted(valid_ids)} (the configured strong/weak target ids; "
- f"the judge target is not a routing candidate)"
- )
- return value
-
-
-__all__ = ["EscalationRouterConfig"]
diff --git a/switchyard/lib/profiles/escalation_router_profile_config.py b/switchyard/lib/profiles/escalation_router_profile_config.py
deleted file mode 100644
index f6f5b4ce8..000000000
--- a/switchyard/lib/profiles/escalation_router_profile_config.py
+++ /dev/null
@@ -1,157 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Profile-owned escalation-router construction (judge-latched strong/weak)."""
-
-from __future__ import annotations
-
-from typing import Any, Self
-
-from switchyard.lib.affinity_pin_store import AffinityPinStore
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- DeterministicRoutingLLMBackend,
-)
-from switchyard.lib.processors.escalation_judge_request_processor import (
- ESCALATION_JUDGE_SYSTEM_PROMPT,
- EscalationJudgeConfig,
- EscalationJudgeRequestProcessor,
-)
-from switchyard.lib.processors.reasoning_effort_normalizer import (
- ReasoningEffortNormalizer,
-)
-from switchyard.lib.processors.reasoning_hint import model_accepts_reasoning_hint
-from switchyard.lib.profiles.chain import ComponentChainProfile
-from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig
-from switchyard.lib.profiles.table import profile_config
-from switchyard.lib.profiles.tier_target_builders import (
- apply_deepseek_overrides,
- build_tier_backend,
-)
-from switchyard.lib.session_affinity import SessionAffinity
-
-
-@profile_config("escalation_router")
-class EscalationRouterProfileConfig:
- """Profile config wrapper for judge-latched escalation routing."""
-
- config: EscalationRouterConfig
-
- @classmethod
- def from_config(cls, config: EscalationRouterConfig) -> Self:
- """Create a profile config from the validated parsing model."""
- return cls(config=config)
-
- def build(self) -> ComponentChainProfile:
- """Build the escalation-router profile runtime."""
- config = self.config
-
- # The affinity store IS the escalation latch, so it is always on.
- # Warmup gating lives in the judge processor (min_judge_turn), not
- # here — affinity warmup would delay a decided escalation from
- # taking effect, not just from being decided. The optional Redis L2
- # keeps the latch across workers and pod churn; without it a worker
- # change silently restarts an escalated conversation on weak.
- affinity = SessionAffinity(
- enabled=True,
- max_sessions=config.affinity_max_sessions,
- warmup_turns=0,
- l2=_build_affinity_l2(config),
- )
-
- # DeepSeek judges get the same benchmark-gateway defaults as DeepSeek
- # tiers (``X-Inference-Priority: batch``) so their calls land on the
- # relaxed-timeout gateway alongside the routed tier traffic.
- judge_target = apply_deepseek_overrides(config.judge)
- # Claude/Bedrock judges reject the vLLM-only chat_template_kwargs
- # hint outright — every judged turn would fail open to weak — so
- # the hint is only sent to models that tolerate it (same gate as
- # the classifier presets and the stage router).
- judge_disable_reasoning = (
- config.judge_disable_reasoning
- and model_accepts_reasoning_hint(judge_target.model)
- )
- judge_config = EscalationJudgeConfig(
- model=judge_target.model,
- api_key=judge_target.api_key,
- base_url=judge_target.base_url,
- timeout_s=judge_target.endpoint.timeout_secs or config.judge_timeout_s,
- system_prompt=config.judge_system_prompt or ESCALATION_JUDGE_SYSTEM_PROMPT,
- min_judge_turn=config.judge_min_turn,
- escalate_confirmations=config.judge_escalate_confirmations,
- confirmation_window=config.judge_confirmation_window,
- disable_reasoning=judge_disable_reasoning,
- # Reasoning tokens and the JSON verdict share one budget: a
- # thinking judge needs the classifier's larger ceiling or it
- # truncates mid-reasoning and silently fails open every turn.
- max_completion_tokens=(
- config.judge_max_completion_tokens
- if config.judge_max_completion_tokens is not None
- else (128 if judge_disable_reasoning else 4096)
- ),
- dump_verdicts_to_stderr=config.judge_dump_verdicts,
- recent_turn_window=config.judge_recent_turn_window,
- window_message_chars=config.judge_window_message_chars,
- max_request_chars=config.judge_max_request_chars,
- extra_headers=judge_target.extra_headers or None,
- )
- # Tier labels are the configured target ids ("strong"/"weak" unless
- # overridden). The judge stamps them, the backend keys its tier dict
- # by them, and the chain's evict-and-retry rewrites selected_target to
- # fallback_target_on_evict — which the config validates against these
- # same ids — so an overflow reroute always lands on a registered tier.
- strong_id = config.strong.id
- weak_id = config.weak.id
- request_processors: list[Any] = [
- ReasoningEffortNormalizer(),
- EscalationJudgeRequestProcessor(
- judge_config,
- affinity=affinity,
- session_key_depth=config.session_key_depth,
- strong_tier=strong_id,
- weak_tier=weak_id,
- ),
- ]
-
- strong_target, strong_backend = build_tier_backend(
- config.strong, config.tier_timeout_s,
- )
- weak_target, weak_backend = build_tier_backend(
- config.weak, config.tier_timeout_s,
- )
-
- backend = DeterministicRoutingLLMBackend(
- tiers={
- strong_id: (strong_backend, strong_target.model),
- weak_id: (weak_backend, weak_target.model),
- },
- # Weak is the resting state; the judge processor stamps a tier on
- # every turn, so the default only covers malformed metadata.
- default_tier=weak_id,
- )
- return ComponentChainProfile(
- request_processors=request_processors,
- backend=backend,
- fallback_target_on_evict=config.fallback_target_on_evict,
- )
-
-
-def _build_affinity_l2(config: EscalationRouterConfig) -> AffinityPinStore | None:
- """Build the optional shared L2 latch store (``None`` = L1-only).
-
- Uses the shared ``affinity_store`` semantics; ``"redis"`` imports
- :class:`RedisPinStore` lazily so the ``redis`` dependency stays optional.
- Config validation guarantees a URL when the store is Redis.
- """
- if config.affinity_store != "redis":
- return None
- from switchyard.lib.redis_pin_store import RedisPinStore
-
- assert config.affinity_store_url is not None # enforced by config validator
- return RedisPinStore(
- config.affinity_store_url,
- ttl_seconds=config.affinity_store_ttl_seconds,
- key_prefix=config.affinity_key_prefix,
- )
-
-
-__all__ = ["EscalationRouterProfileConfig"]
diff --git a/switchyard/lib/profiles/header_routing.py b/switchyard/lib/profiles/header_routing.py
deleted file mode 100644
index 448017ed4..000000000
--- a/switchyard/lib/profiles/header_routing.py
+++ /dev/null
@@ -1,85 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""A programmatic Python profile that selects a tier from a request header."""
-
-from dataclasses import dataclass
-
-from switchyard.lib.backends.llm_target import LlmTarget
-from switchyard.lib.profiles.chain import ComponentChainProfile
-from switchyard.lib.profiles.passthrough import PassthroughProfileConfig
-from switchyard.lib.profiles.table import profile_config
-from switchyard_rust.core import ChatRequest, ChatResponse
-from switchyard_rust.profiles import ProfileInput
-
-PROFILE_TYPE = "header-routing"
-_DEFAULT_HEADER = "x-switchyard-tier"
-
-
-@profile_config(PROFILE_TYPE, register=True)
-@dataclass(frozen=True, slots=True)
-class HeaderRoutingConfig:
- """Route to ``strong`` or ``weak`` based on a request header.
-
- A header value of ``strong`` selects the strong target; anything else, or a
- missing header, selects ``weak``.
- """
-
- strong: LlmTarget
- weak: LlmTarget
- header: str = _DEFAULT_HEADER
-
- def build(self) -> "HeaderRoutingProfile":
- return HeaderRoutingProfile(
- strong=PassthroughProfileConfig(target=self.strong).build(),
- weak=PassthroughProfileConfig(target=self.weak).build(),
- header=self.header,
- )
-
-
-@dataclass(frozen=True, slots=True)
-class HeaderRoutingDecision:
- """Profile-owned request-side state: the prepared request and chosen tier."""
-
- request: ChatRequest
- tier: str
-
-
-class HeaderRoutingProfile:
- """Runtime that delegates each request to a passthrough by tier."""
-
- def __init__(
- self,
- strong: ComponentChainProfile,
- weak: ComponentChainProfile,
- header: str,
- ) -> None:
- self._strong = strong
- self._weak = weak
- self._header = header.lower()
-
- async def process(self, input: ProfileInput) -> HeaderRoutingDecision:
- return HeaderRoutingDecision(request=input.request, tier=self._tier(input))
-
- async def run(self, input: ProfileInput) -> ChatResponse:
- decision = await self.process(input)
- backend = self._strong if decision.tier == "strong" else self._weak
- response = await backend.run(ProfileInput(decision.request, input.metadata))
- return await self.rprocess(decision, response)
-
- async def rprocess(
- self, processed: HeaderRoutingDecision, response: ChatResponse
- ) -> ChatResponse:
- return response
-
- def _tier(self, input: ProfileInput) -> str:
- values = input.metadata.headers.get(self._header, [])
- chosen = values[0].strip().lower() if values else ""
- return "strong" if chosen == "strong" else "weak"
-
-
-__all__ = [
- "HeaderRoutingConfig",
- "HeaderRoutingDecision",
- "HeaderRoutingProfile",
-]
diff --git a/switchyard/lib/profiles/passthrough.py b/switchyard/lib/profiles/passthrough.py
deleted file mode 100644
index 6c88c03d5..000000000
--- a/switchyard/lib/profiles/passthrough.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Profile-owned direct passthrough construction."""
-
-from __future__ import annotations
-
-from switchyard.lib.backends.llm_target import LlmTarget
-from switchyard.lib.profiles.chain import ComponentChainProfile
-from switchyard.lib.profiles.table import profile_config
-from switchyard.lib.roles import LLMBackend
-from switchyard_rust.components import OpenAiPassthroughBackend
-
-
-@profile_config("passthrough")
-class PassthroughProfileConfig:
- """Dataclass profile config for direct single-upstream passthrough profiles."""
-
- target: LlmTarget | None = None
- api_key: str | None = None
- base_url: str | None = None
- timeout: float | None = None
-
- def build(self) -> ComponentChainProfile:
- """Build a profile runtime for one direct upstream."""
- from switchyard.lib.backends.multi_llm_backend import build_native_backend
-
- backend: LLMBackend
- if self.target is None:
- backend = OpenAiPassthroughBackend(
- api_key=self.api_key,
- base_url=self.base_url,
- timeout=self.timeout,
- )
- else:
- backend = build_native_backend(self.target)
-
- return ComponentChainProfile(
- backend=backend,
- )
-
-
-__all__ = ["PassthroughProfileConfig"]
diff --git a/switchyard/lib/profiles/protocols.py b/switchyard/lib/profiles/protocols.py
deleted file mode 100644
index 78c4984a2..000000000
--- a/switchyard/lib/profiles/protocols.py
+++ /dev/null
@@ -1,116 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Python-owned profile request values and runtime protocols."""
-
-from typing import Any, ClassVar, Protocol, TypeVar, runtime_checkable
-
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.core import ChatResponse
-from switchyard_rust.profiles import ProfileInput
-
-ProcessedRequestT = TypeVar("ProcessedRequestT")
-ProfileT_co = TypeVar("ProfileT_co", bound="Profile[Any]", covariant=True)
-
-
-@runtime_checkable
-class ProfileRunner(Protocol):
- """Object-safe profile execution surface.
-
- Generic callers, servers, and config loaders use this surface when they only
- need to execute a complete profile and receive the final response.
- """
-
- async def run(self, input: ProfileInput) -> ChatResponse:
- """Execute the complete profile lifecycle for one request."""
- ...
-
-
-@runtime_checkable
-class ProfileHooks(Protocol[ProcessedRequestT]):
- """Typed profile hook surface for library-style embedding.
-
- Profiles use ``ProcessedRequestT`` to expose profile-owned request-side
- state, such as a routing decision, without a generic metadata bag.
- """
-
- async def process(self, input: ProfileInput) -> ProcessedRequestT:
- """Prepare a request and return profile-owned request-side state."""
- ...
-
- async def rprocess(
- self,
- processed: ProcessedRequestT,
- response: ChatResponse,
- ) -> ChatResponse:
- """Process a backend response using the matching request-side state."""
- ...
-
-
-@runtime_checkable
-class Profile(ProfileRunner, ProfileHooks[ProcessedRequestT], Protocol[ProcessedRequestT]):
- """Full Python profile authoring contract.
-
- A profile config must build this complete surface, not just a run-only
- wrapper. The split remains useful because callers may either execute the
- whole profile through ``run`` or embed Switchyard around their own backend
- call by invoking ``process`` and ``rprocess`` directly.
- """
-
-
-@runtime_checkable
-class ProfileLifecycle(Protocol):
- """Optional lifecycle surface for profiles built from reusable components."""
-
- def iter_components(self) -> list[object]:
- """Return startup/shutdown components in order."""
- ...
-
-
-@runtime_checkable
-class ContextAwareProfile(Profile[ProcessedRequestT], Protocol[ProcessedRequestT]):
- """Optional component bridge surface for profiles that need ``ProxyContext``.
-
- Profiles can implement only ``run`` / ``process`` / ``rprocess``. Profiles
- that compose existing Python components also expose these methods so
- endpoint adapters can reuse the caller's context.
- """
-
- async def process_with_context(
- self,
- input: ProfileInput,
- ctx: ProxyContext,
- ) -> ProcessedRequestT:
- """Prepare a request using the caller-owned compatibility context."""
- ...
-
- async def run_with_context(
- self,
- input: ProfileInput,
- ctx: ProxyContext,
- ) -> ChatResponse:
- """Execute the complete lifecycle using the caller-owned context."""
- ...
-
-
-class ProfileConfig(Protocol[ProfileT_co]):
- """Config contract implemented by each Python profile config dataclass."""
-
- PROFILE_TYPE: ClassVar[str]
-
- def build(self) -> ProfileT_co:
- """Build this config into a concrete profile runtime."""
- ...
-
-
-__all__ = [
- "ProcessedRequestT",
- "ContextAwareProfile",
- "Profile",
- "ProfileConfig",
- "ProfileHooks",
- "ProfileInput",
- "ProfileLifecycle",
- "ProfileRunner",
- "ProfileT_co",
-]
diff --git a/switchyard/lib/profiles/random_routing.py b/switchyard/lib/profiles/random_routing.py
deleted file mode 100644
index 2d687782b..000000000
--- a/switchyard/lib/profiles/random_routing.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Profile-owned random-routing construction."""
-
-from __future__ import annotations
-
-from typing import Any, Self
-
-from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator
-
-from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target
-from switchyard.lib.profiles.chain import ComponentChainProfile
-from switchyard.lib.profiles.table import profile_config
-from switchyard.lib.roles import LLMBackend
-from switchyard_rust.components import RandomRoutingProcessorConfig
-
-
-@profile_config("random_routing")
-class RandomRoutingProfileConfig:
- """Profile config wrapper for weighted strong/weak random routing."""
-
- config: RandomRoutingConfig
-
- @classmethod
- def from_config(cls, config: RandomRoutingConfig) -> Self:
- """Create a profile config from the validated parsing model."""
- return cls(config=config)
-
- def build(self) -> ComponentChainProfile:
- """Build the random-routing profile runtime."""
- from switchyard.lib.backends.multi_llm_backend import build_multi_llm_backend
- from switchyard.lib.processors.random_routing_request_processor import (
- RandomRoutingRequestProcessor,
- )
-
- config = self.config
- backend: LLMBackend = build_multi_llm_backend((config.strong, config.weak))
-
- return ComponentChainProfile(
- request_processors=[RandomRoutingRequestProcessor(config.processor_config)],
- backend=backend,
- fallback_target_on_evict=config.fallback_target_on_evict,
- )
-
-
-class RandomRoutingConfig(BaseModel):
- """Validated parsing config for weighted strong/weak random routing."""
-
- model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True, extra="forbid")
-
- strong: LlmTarget
- weak: LlmTarget
- fallback_target_on_evict: str
- strong_probability: float = 0.5
- enable_stats: bool = True
- rng_seed: int | None = None
- preset: str | None = None
-
- @field_validator("strong", "weak", mode="before")
- @classmethod
- def _coerce_target(cls, value: object, info: ValidationInfo) -> LlmTarget:
- """Accept dicts and existing ``LlmTarget`` instances for tiers."""
- return coerce_llm_target(value, default_id=info.field_name or "target")
-
- @field_validator("strong", "weak")
- @classmethod
- def _target_model_non_empty(cls, tier: LlmTarget) -> LlmTarget:
- """Reject empty model names before the first request."""
- if not tier.model:
- raise ValueError("target.model must be a non-empty string")
- return tier
-
- @field_validator("strong_probability")
- @classmethod
- def _strong_prob_range(cls, value: float) -> float:
- """Validate the weighted coin range."""
- if not 0.0 <= value <= 1.0:
- raise ValueError(
- f"strong_probability must be in [0.0, 1.0], got {value!r}"
- )
- return value
-
- @field_validator("fallback_target_on_evict")
- @classmethod
- def _fallback_matches_existing_target(
- cls,
- value: str,
- info: ValidationInfo,
- ) -> str:
- """Ensure evict-and-retry can only fall back to a configured tier."""
- valid_ids = {info.data[key].id for key in ("strong", "weak") if key in info.data}
- if value not in valid_ids:
- raise ValueError(
- f"fallback_target_on_evict={value!r} must match one of "
- f"{sorted(valid_ids)} (the configured target ids)"
- )
- return value
-
- @property
- def processor_config(self) -> RandomRoutingProcessorConfig:
- """Return the Rust request-processor config for this validated config."""
- return RandomRoutingProcessorConfig(
- strong=self.strong,
- weak=self.weak,
- strong_probability=self.strong_probability,
- rng_seed=self.rng_seed,
- )
-
- @classmethod
- def model_validate_config(cls, raw: Any) -> RandomRoutingConfig:
- """Coerce dicts or Pydantic models into ``RandomRoutingConfig``."""
- if isinstance(raw, cls):
- return raw
- if isinstance(raw, BaseModel):
- return cls(**raw.model_dump())
- return cls.model_validate(raw)
-
-
-__all__ = [
- "RandomRoutingConfig",
- "RandomRoutingProfileConfig",
-]
diff --git a/switchyard/lib/profiles/random_routing_presets.py b/switchyard/lib/profiles/random_routing_presets.py
deleted file mode 100644
index 5bfadf717..000000000
--- a/switchyard/lib/profiles/random_routing_presets.py
+++ /dev/null
@@ -1,430 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Named :class:`RandomRoutingConfig` presets keyed by model pair.
-
-Each preset encodes an opinionated strong/weak model pair with explicit
-backend formats. A
-preset name describes *which two models* — not *which workload* — so
-the same pair can back Claude Code, PinchBench, SWE-bench, or any
-other router-driven benchmark without name collision.
-
-- :attr:`PRESETS` — the canonical list of preset ids, suitable for
- ``argparse(choices=...)`` on the CLI.
-- :meth:`get` — classmethod returning the builder for a given id,
- raising :class:`ValueError` with the full allowed list on a miss.
-- One :func:`staticmethod` per preset returning a fully-specified
- :class:`RandomRoutingConfig`.
-
-**Strict API** — presets fix the model pair. Callers may only adjust
-connectivity (``api_key``, ``base_url``) and the coin bias
-(``strong_probability``). Need a different model? Drop back to
-:class:`RandomRoutingConfig` and build it directly. Rationale: a
-preset's whole point is "this specific pair is the one we ship" —
-every model override reopens that decision, so we force the explicit
-path instead.
-
-``strong_probability`` controls routing — higher value = more
-strong-tier traffic.
-
-Example::
-
- from switchyard import ProfileSwitchyard, RandomRoutingProfileConfig, RandomRoutingPresets
-
- config = RandomRoutingPresets.opus_nemotron_super(
- api_key=nvidia_api_key,
- strong_probability=0.3,
- )
- switchyard = ProfileSwitchyard(RandomRoutingProfileConfig.from_config(config).build())
-"""
-
-from __future__ import annotations
-
-from typing import Protocol, cast
-
-from switchyard.lib.backends.llm_target import (
- BackendFormat,
- LlmTarget,
-)
-from switchyard.lib.profiles.random_routing import RandomRoutingConfig
-
-
-class RandomRoutingPresetBuilder(Protocol):
- """Callable shape every :class:`RandomRoutingPresets` builder implements.
-
- Pinning the signature as a :class:`Protocol` does two jobs at once:
- gives :meth:`RandomRoutingPresets.get` a type-precise return
- annotation (no ``Callable[..., ...]`` Any-escape hatch), and
- documents the preset contract in one place — every new preset must
- accept exactly these keyword-only arguments.
- """
-
- def __call__(
- self,
- *,
- api_key: str | None = ...,
- base_url: str = ...,
- strong_probability: float = ...,
- ) -> RandomRoutingConfig: ...
-
-# Shipping presets route through OpenRouter's OpenAI-compatible endpoint
-# by default; callers override with ``base_url=`` for another gateway.
-_OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
-
-# OpenRouter model strings for the named random-routing presets.
-#
-# 4.6 vs 4.7 split — we ship both versions as parallel presets rather
-# than replacing 4.6 with 4.7 because PinchBench / SWE-bench
-# runs are built around 4.6 (see :meth:`opus_nemotron_super`'s
-# docstring); ``*_47_*`` variants exist for callers who want the
-# newer generation at the cost of re-validating the routing sweet
-# spot. Drop the 4.6 constants + presets once 4.7 benchmark history
-# catches up.
-_MODEL_OPUS_4_6 = "anthropic/claude-opus-4.6"
-_MODEL_OPUS_4_7 = "anthropic/claude-opus-4.7"
-_MODEL_GPT_5_2 = "openai/gpt-5.2"
-_MODEL_GPT_5_5 = "openai/gpt-5.5"
-_MODEL_NEMOTRON_SUPER = "nvidia/nemotron-3-super-120b-a12b"
-_MODEL_KIMI_K2_6 = "moonshotai/kimi-k2.6"
-_MODEL_MINIMAX_M2_7 = "minimax/minimax-m2.7"
-
-
-class RandomRoutingPresets:
- """Builder of pre-built :class:`RandomRoutingConfig`s keyed by model pair.
-
- Each preset is a named strong+weak pair.
- Callers vary only the per-deployment knobs (``api_key``,
- ``base_url``, ``strong_probability``); the model identities and
- reasoning/output budgets are part of the preset's identity.
-
- Example::
-
- config = RandomRoutingPresets.opus_nemotron_super(
- api_key="...",
- strong_probability=0.5,
- )
-
- # Dispatch by id (CLI / config-driven workflows):
- builder = RandomRoutingPresets.get("opus_nemotron_super")
- config = builder(api_key="...", strong_probability=0.5)
- """
-
- #: Canonical preset ids. Kept as a class attribute so CLI parsers
- #: can use ``argparse(choices=RandomRoutingPresets.PRESETS)`` and
- #: help text auto-lists the available presets. Presets are named
- #: by their model pair (not by workload) so the same pair can
- #: back Claude Code, PinchBench, SWE-bench, etc. without
- #: workload-specific aliases.
- PRESETS: list[str] = [
- "opus_nemotron_super",
- "gpt5_nemotron_super",
- "opus_kimi",
- "opus_minimax",
- "opus_47_nemotron_super",
- "opus_47_kimi",
- "opus_47_minimax",
- "opus_47_gpt55",
- ]
-
- @classmethod
- def get(cls, preset_id: str) -> RandomRoutingPresetBuilder:
- """Return the builder method for *preset_id*.
-
- Raises:
- ValueError: If *preset_id* is not in :attr:`PRESETS`. The
- error lists every allowed name so mistyped ids
- self-document their fix.
- """
- if preset_id not in cls.PRESETS:
- raise ValueError(
- f"Unknown random-routing preset {preset_id!r}. "
- f"Available: {cls.PRESETS}"
- )
- # Every PRESETS entry names a ``@staticmethod`` on this class,
- # so ``getattr`` resolves to the builder callable. ``cast``
- # narrows the :class:`Any` that ``getattr`` produces back to
- # the shared :class:`RandomRoutingPresetBuilder` signature —
- # all presets implement it (verified by
- # :class:`TestStrictness`'s kwargs-only tests).
- return cast(RandomRoutingPresetBuilder, getattr(cls, preset_id))
-
- # ------------------------------------------------------------------
- # Presets
- # ------------------------------------------------------------------
-
- @staticmethod
- def opus_nemotron_super(
- *,
- api_key: str | None = None,
- base_url: str = _OPENROUTER_BASE_URL,
- strong_probability: float = 0.5,
- ) -> RandomRoutingConfig:
- """Claude Opus 4.6 (strong) + Nemotron-3 Super v3 (weak).
-
- The flagship strong/weak pair — strong-signal reasoning from
- Opus for multi-step work, cheaper reasoning from Nemotron for
- straightforward calls. This is the native Rust-backed upgrade
- of the legacy Claude Code random-routing pair at a 50/50 split.
- PinchBench on the original Opus 4.6 + Nemotron pair
- showed balanced routing matching all-strong accuracy, which is
- why this preset stays on 4.6 rather than chasing the latest
- point release.
- """
- return RandomRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model=_MODEL_OPUS_4_6,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- weak=LlmTarget(
- id="weak",
- model=_MODEL_NEMOTRON_SUPER,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- strong_probability=strong_probability,
- fallback_target_on_evict="strong",
- )
-
- @staticmethod
- def gpt5_nemotron_super(
- *,
- api_key: str | None = None,
- base_url: str = _OPENROUTER_BASE_URL,
- strong_probability: float = 0.5,
- ) -> RandomRoutingConfig:
- """GPT-5.2 (strong) + Nemotron-3 Super v3 (weak).
-
- Same weak tier as :meth:`opus_nemotron_super` but swaps Opus
- for GPT-5.2 on the strong side. Matches the default
- strong+weak pair used by the random-routing CLI
- examples.
- """
- return RandomRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model=_MODEL_GPT_5_2,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- weak=LlmTarget(
- id="weak",
- model=_MODEL_NEMOTRON_SUPER,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- strong_probability=strong_probability,
- fallback_target_on_evict="strong",
- )
-
- @staticmethod
- def opus_kimi(
- *,
- api_key: str | None = None,
- base_url: str = _OPENROUTER_BASE_URL,
- strong_probability: float = 0.5,
- ) -> RandomRoutingConfig:
- """Claude Opus 4.6 (strong) + Moonshot Kimi K2.6 (weak).
-
- Moonshot-weak-tier alternative to :meth:`opus_nemotron_super`
- — useful when Kimi's training mix better matches your
- workload's prompt style or language coverage than Nemotron's.
- """
- return RandomRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model=_MODEL_OPUS_4_6,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- weak=LlmTarget(
- id="weak",
- model=_MODEL_KIMI_K2_6,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- strong_probability=strong_probability,
- fallback_target_on_evict="strong",
- )
-
- @staticmethod
- def opus_minimax(
- *,
- api_key: str | None = None,
- base_url: str = _OPENROUTER_BASE_URL,
- strong_probability: float = 0.5,
- ) -> RandomRoutingConfig:
- """Claude Opus 4.6 (strong) + MiniMax M2.7 (weak).
-
- Same Opus 4.6 strong tier as :meth:`opus_nemotron_super`;
- swaps Nemotron for MiniMax M2.7 on the weak side. Use when
- MiniMax has better coverage for your workload's languages or
- prompt styles, or when you want to compare two reasoning-
- capable weak tiers side-by-side at the same split.
- """
- return RandomRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model=_MODEL_OPUS_4_6,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- weak=LlmTarget(
- id="weak",
- model=_MODEL_MINIMAX_M2_7,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- strong_probability=strong_probability,
- fallback_target_on_evict="strong",
- )
-
- # ------------------------------------------------------------------
- # Opus 4.7 variants
- # ------------------------------------------------------------------
- #
- # Parallel to the 4.6 presets above — same weak tiers, only the
- # strong-tier version changes. Kept as separate factories (not a ``version=`` kwarg
- # on the 4.6 presets) because each preset's whole point is
- # "this specific pair is the one we ship"; parameterising the
- # version would reopen that decision on every call.
-
- @staticmethod
- def opus_47_nemotron_super(
- *,
- api_key: str | None = None,
- base_url: str = _OPENROUTER_BASE_URL,
- strong_probability: float = 0.5,
- ) -> RandomRoutingConfig:
- """Claude Opus 4.7 (strong) + Nemotron-3 Super v3 (weak).
-
- Pick this when you want the newer Opus generation and are
- willing to re-validate the routing sweet spot against the 4.6
- variant.
- """
- return RandomRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model=_MODEL_OPUS_4_7,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- weak=LlmTarget(
- id="weak",
- model=_MODEL_NEMOTRON_SUPER,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- strong_probability=strong_probability,
- fallback_target_on_evict="strong",
- )
-
- @staticmethod
- def opus_47_kimi(
- *,
- api_key: str | None = None,
- base_url: str = _OPENROUTER_BASE_URL,
- strong_probability: float = 0.5,
- ) -> RandomRoutingConfig:
- """Claude Opus 4.7 (strong) + Moonshot Kimi K2.6 (weak).
-
- 4.7 counterpart to :meth:`opus_kimi` — same weak tier, same
- split knob, newer strong tier.
- """
- return RandomRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model=_MODEL_OPUS_4_7,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- weak=LlmTarget(
- id="weak",
- model=_MODEL_KIMI_K2_6,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- strong_probability=strong_probability,
- fallback_target_on_evict="strong",
- )
-
- @staticmethod
- def opus_47_minimax(
- *,
- api_key: str | None = None,
- base_url: str = _OPENROUTER_BASE_URL,
- strong_probability: float = 0.5,
- ) -> RandomRoutingConfig:
- """Claude Opus 4.7 (strong) + MiniMax M2.7 (weak).
-
- 4.7 counterpart to :meth:`opus_minimax`. Use when you want to
- compare Opus 4.7's routing balance against the 4.6 baseline on
- MiniMax-weak traffic.
- """
- return RandomRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model=_MODEL_OPUS_4_7,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- weak=LlmTarget(
- id="weak",
- model=_MODEL_MINIMAX_M2_7,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- strong_probability=strong_probability,
- fallback_target_on_evict="strong",
- )
-
- @staticmethod
- def opus_47_gpt55(
- *,
- api_key: str | None = None,
- base_url: str = _OPENROUTER_BASE_URL,
- strong_probability: float = 0.5,
- ) -> RandomRoutingConfig:
- """Claude Opus 4.7 (strong) + GPT-5.5 (weak).
-
- Cross-provider 4.7 pair — Anthropic Opus on the strong side,
- OpenAI GPT-5.5 on the weak side. Useful when you want a
- weak tier that shares a training lineage with the strong-tier
- baselines GPT-5.x runs are compared against, instead of a
- Nemotron / Kimi / MiniMax weak tier.
-
- Both tiers use OpenRouter's OpenAI-compatible chat completions path.
- """
- return RandomRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model=_MODEL_OPUS_4_7,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- weak=LlmTarget(
- id="weak",
- model=_MODEL_GPT_5_5,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- strong_probability=strong_probability,
- fallback_target_on_evict="strong",
- )
diff --git a/switchyard/lib/profiles/stage_router.py b/switchyard/lib/profiles/stage_router.py
deleted file mode 100644
index 32a0b4e9a..000000000
--- a/switchyard/lib/profiles/stage_router.py
+++ /dev/null
@@ -1,116 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Profile-owned signal stage_router routing construction."""
-
-from __future__ import annotations
-
-import functools
-from typing import Any, Self
-
-from switchyard.lib.processors.reasoning_hint import model_accepts_reasoning_hint
-from switchyard.lib.processors.stage_router import StageRouterDecisionLog, TierClassifier
-from switchyard.lib.processors.stage_router.handoff_notes import HandoffNoteInjector
-from switchyard.lib.processors.stage_router_request_processor import (
- BUILTIN_PICKERS,
- StageRouterRequestProcessor,
- TierPicker,
-)
-from switchyard.lib.profiles.chain import ComponentChainProfile
-from switchyard.lib.profiles.stage_router_config import (
- ClassifierConfig,
- HandoffNoteConfig,
- StageRouterConfig,
-)
-from switchyard.lib.profiles.table import profile_config
-from switchyard.lib.roles import LLMBackend
-
-
-@profile_config("stage_router")
-class StageRouterProfileConfig:
- """Profile config wrapper for signal-driven capable/efficient stage_router profiles."""
-
- config: StageRouterConfig
-
- @classmethod
- def from_config(cls, config: StageRouterConfig) -> Self:
- """Create a profile config from the validated parsing model."""
- return cls(config=config)
-
- def build(self) -> ComponentChainProfile:
- """Build the stage_router profile runtime."""
- from switchyard.lib.backends.multi_llm_backend import build_multi_llm_backend
- from switchyard_rust.components import DimensionCollector
-
- config = self.config
- request_processors: list[Any] = []
- request_processors.append(
- DimensionCollector(recent_window=config.signal_recent_window)
- )
- decision_log = StageRouterDecisionLog()
- classifier = _build_classifier(config.classifier)
- request_processors.append(
- StageRouterRequestProcessor(
- targets=(config.efficient, config.capable),
- picker=_build_tier_picker(config, decision_log, classifier),
- classifier=classifier,
- decision_log=decision_log,
- handoff_injector=_build_handoff_injector(config.handoff_notes),
- strong_system_prompt=config.strong_system_prompt,
- weak_system_prompt=config.weak_system_prompt,
- )
- )
-
- backend: LLMBackend = build_multi_llm_backend((config.efficient, config.capable))
-
- return ComponentChainProfile(
- request_processors=request_processors,
- backend=backend,
- fallback_target_on_evict=config.fallback_target_on_evict,
- )
-
-
-def _build_tier_picker(
- config: StageRouterConfig,
- decision_log: StageRouterDecisionLog,
- classifier: TierClassifier | None,
-) -> TierPicker:
- """Resolve the named stage_router picker and bind its runtime knobs."""
- picker_fn = BUILTIN_PICKERS.get(config.picker)
- if picker_fn is None:
- allowed = ", ".join(sorted(BUILTIN_PICKERS))
- raise ValueError(f"unknown picker {config.picker!r}; allowed: {allowed}")
- return functools.partial(
- picker_fn,
- confidence_threshold=config.confidence_threshold,
- classifier=classifier,
- decision_log=decision_log,
- )
-
-
-def _build_handoff_injector(config: HandoffNoteConfig | None) -> HandoffNoteInjector | None:
- """Build the optional tier-transition note injector; ``None`` when disabled."""
- if config is None or not config.enabled:
- return None
- return HandoffNoteInjector(
- escalation_note=config.escalation_note,
- deescalation_note=config.deescalation_note,
- only_on_wrong_signal_escalation=config.only_on_wrong_signal_escalation,
- )
-
-
-def _build_classifier(config: ClassifierConfig | None) -> TierClassifier | None:
- """Build the optional LLM fallback classifier for stage_router routing."""
- if config is None:
- return None
- return TierClassifier(
- model=config.model,
- api_key=config.api_key,
- base_url=config.base_url,
- timeout_secs=config.timeout_secs,
- recent_turn_window=config.recent_turn_window,
- disable_reasoning=model_accepts_reasoning_hint(config.model),
- )
-
-
-__all__ = ["StageRouterProfileConfig"]
diff --git a/switchyard/lib/profiles/stage_router_config.py b/switchyard/lib/profiles/stage_router_config.py
deleted file mode 100644
index 0092bb598..000000000
--- a/switchyard/lib/profiles/stage_router_config.py
+++ /dev/null
@@ -1,122 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Declarative config for stage_router profiles. See ``docs/stage_router_routing.md``."""
-
-from __future__ import annotations
-
-from typing import Literal
-
-from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
-
-from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target
-from switchyard.lib.processors.stage_router.handoff_notes import (
- DEFAULT_ESCALATION_NOTE,
-)
-
-#: Picker mode accepted in YAML. The profile resolves it to a :class:`TierPicker`.
-#: The name describes the *default tier* — what the picker returns when the
-#: scorer is ambiguous and no classifier is configured.
-StageRouterPickerMode = Literal["capable_first", "efficient_first"]
-
-
-class ClassifierConfig(BaseModel):
- """Optional LLM classifier invoked on low-confidence scorer outputs."""
-
- model_config = ConfigDict(frozen=True, extra="forbid")
-
- model: str
- api_key: str
- base_url: str | None = None
- timeout_secs: float = Field(default=30.0, gt=0.0)
- #: Number of trailing conversation messages (from the inbound chat
- #: request) to include in the classifier prompt. ``0`` sends only the
- #: aggregate :class:`ToolResultSignal` summary. Default matches the
- #: Rust extractor's ``RECENT_WINDOW`` so the classifier sees the same
- #: span the ``recent_*`` signal fields cover.
- recent_turn_window: int = Field(default=3, ge=0)
-
-
-class HandoffNoteConfig(BaseModel):
- """Optional tier-transition guidance notes injected into the request.
-
- Off by default. When ``enabled``, a short ephemeral note is appended to the
- outgoing request on a *tier transition* only (weak↔strong), telling the model
- taking over why it was handed the task. Notes are never persisted into the
- agent's history, so they do not accumulate across turns. See
- ``switchyard.lib.processors.stage_router.handoff_notes``.
- """
-
- model_config = ConfigDict(frozen=True, extra="forbid")
-
- #: Master switch. When ``False`` (default) no note is ever injected.
- enabled: bool = False
- #: Note injected on every capable-tier turn driven by a real wrong signal.
- escalation_note: str = DEFAULT_ESCALATION_NOTE
- #: Optional strong→weak note. When set, injected on every efficient-tier turn
- #: (hand-back to the weak model); ``None`` (default) leaves de-escalation off.
- deescalation_note: str | None = None
- #: When ``True`` (default) only inject when the decision source was a real
- #: signal (``dimensions`` or ``override``), not an ambiguous default
- #: (``fall_open``). Keeps "stalling" from being asserted without evidence.
- only_on_wrong_signal_escalation: bool = True
-
-
-class StageRouterConfig(BaseModel):
- """Configuration for the stage-router-routing profile."""
-
- model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True, extra="forbid")
-
- capable: LlmTarget
- efficient: LlmTarget
- #: Target ID the post-routing guard rewrites picks to when the picked
- #: target has been evicted from the pool (e.g. after a context-window
- #: overflow). Must match either ``capable.id`` or ``efficient.id``.
- fallback_target_on_evict: str
- picker: StageRouterPickerMode = "capable_first"
- #: Scorer confidence in ``[0, 1]`` below which the picker consults the
- #: classifier (if configured) or returns its default tier. ``0.0`` forces
- #: pure-deterministic routing; ``1.0`` forces every turn through the
- #: classifier (equivalent to classifier-only ``coding_agent`` routing).
- confidence_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
- #: Sliding-window size for the Rust signal extractor's ``recent_*``
- #: counts (``recent_write_count``, ``recent_edit_count``, etc.).
- #: Smaller = more reactive to the very last tool call; larger smooths
- #: turn-by-turn fluctuations. Default 3 matches
- #: ``DEFAULT_RECENT_WINDOW`` in ``tool_signals.rs``.
- signal_recent_window: int = Field(default=3, ge=1)
- classifier: ClassifierConfig | None = None
- #: Optional tier-transition handoff notes (off unless ``handoff_notes.enabled``).
- handoff_notes: HandoffNoteConfig | None = None
- #: Optional system prompt prepended to every request routed to the capable
- #: (strong) tier. Injected before the model sees the request — useful for
- #: conciseness guidance that should apply on every Opus turn without
- #: polluting the agent's conversation history.
- strong_system_prompt: str | None = None
- #: Optional system prompt prepended to every request routed to the efficient
- #: (weak) tier. Useful for conciseness/bail-early guidance on Nemotron turns.
- weak_system_prompt: str | None = None
- enable_stats: bool = True
-
- @field_validator("capable", "efficient", mode="before")
- @classmethod
- def _coerce_target(cls, value: object, info: ValidationInfo) -> LlmTarget:
- return coerce_llm_target(value, default_id=info.field_name or "target")
-
- @field_validator("capable", "efficient")
- @classmethod
- def _target_model_non_empty(cls, tier: LlmTarget) -> LlmTarget:
- if not tier.model:
- raise ValueError("target.model must be a non-empty string")
- return tier
-
- @field_validator("fallback_target_on_evict")
- @classmethod
- def _fallback_matches_existing_target(cls, value: str, info: ValidationInfo) -> str:
- valid_ids = {info.data[key].id for key in ("capable", "efficient") if key in info.data}
- if value not in valid_ids:
- raise ValueError(
- f"fallback_target_on_evict={value!r} must match one of "
- f"{sorted(valid_ids)} (the configured target ids)"
- )
- return value
diff --git a/switchyard/lib/profiles/subagent_override.py b/switchyard/lib/profiles/subagent_override.py
deleted file mode 100644
index d3c69e42e..000000000
--- a/switchyard/lib/profiles/subagent_override.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Route delegated sub-agent requests to a fixed worker runtime."""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any
-
-from switchyard.lib.request_metadata import CTX_PROFILE_REQUEST_HEADERS
-from switchyard_rust.core import is_subagent_request
-
-if TYPE_CHECKING:
- from switchyard.lib.proxy_context import ProxyContext
- from switchyard.lib.route_table import ChainRuntime
- from switchyard_rust.core import ChatRequest
-
-
-class SubagentOverrideRuntime:
- """Wrap a runtime, sending delegated sub-agent work to ``worker`` instead.
-
- Normal traffic runs the wrapped runtime unchanged. A request whose headers carry a
- delegated sub-agent signal runs ``worker`` — a passthrough to the route's configured
- ``subagent_target``. The override never rewrites the request or response, and a worker
- failure surfaces as a normal target error rather than falling back to the wrapped
- runtime.
-
- Detection is :func:`~switchyard_rust.profiles.is_subagent_request`, the protocol
- crate's canonical policy, so harness maintenance turns (Codex ``compact``) stay on
- normal routing and every engine agrees on what counts as delegated work.
- """
-
- def __init__(self, inner: ChainRuntime, worker: ChainRuntime) -> None:
- self._inner = inner
- self._worker = worker
-
- def _branch(self, ctx: ProxyContext | None) -> ChainRuntime:
- """Select the worker for delegated work, else the wrapped runtime.
-
- Without a context there are no retained headers to read, so the request cannot be
- identified as delegated and routes normally.
- """
- if ctx is None:
- return self._inner
- headers = ctx.metadata.get(CTX_PROFILE_REQUEST_HEADERS) or {}
- return self._worker if is_subagent_request(headers) else self._inner
-
- async def call(self, request: ChatRequest, *, ctx: ProxyContext | None = None) -> Any:
- """Execute one request through the branch its headers select."""
- return await self._branch(ctx).call(request, ctx=ctx)
-
- def iter_components(self) -> list[Any]:
- """Return both branches' lifecycle components in startup order."""
- return [*self._inner.iter_components(), *self._worker.iter_components()]
-
-
-__all__ = ["SubagentOverrideRuntime"]
diff --git a/switchyard/lib/profiles/switchyard_adapter.py b/switchyard/lib/profiles/switchyard_adapter.py
deleted file mode 100644
index 6ae96dcc9..000000000
--- a/switchyard/lib/profiles/switchyard_adapter.py
+++ /dev/null
@@ -1,114 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Compatibility adapter that serves a Profile through the legacy endpoint contract."""
-
-from collections.abc import Mapping, Sequence
-from typing import Any, cast
-
-from switchyard.lib.profiles.protocols import ContextAwareProfile, Profile, ProfileLifecycle
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.request_metadata import CTX_PROFILE_REQUEST_HEADERS
-from switchyard.lib.roles import TranslatedResponse
-from switchyard_rust.core import (
- ChatRequest,
- ChatResponse,
- SwitchyardProcessorError,
- request_type_value,
-)
-from switchyard_rust.profiles import ProfileInput, ProfileRequestMetadata
-from switchyard_rust.translation import TranslationEngine
-
-
-class ProfileSwitchyard:
- """Expose a Profile through the existing ``Switchyard.call`` protocol.
-
- Endpoints and route registries currently call ``obj.call(request, ctx=...)``
- and expect a response translated back to the inbound request format. Profiles
- return provider-native ``ChatResponse`` values instead, so this adapter keeps
- the serving contract stable while chain construction moves to Profile-owned
- runtimes.
- """
-
- state_key = "switchyard"
-
- def __init__(
- self,
- profile: Profile[Any],
- translator: TranslationEngine | None = None,
- response_processors: Sequence[Any] = (),
- ) -> None:
- """Create an adapter around one concrete Profile runtime."""
- self._profile = profile
- self._translator = translator or TranslationEngine()
- self._response_processors = tuple(response_processors)
-
- def iter_components(self) -> list[object]:
- """Return lifecycle components in startup order."""
- if isinstance(self._profile, ProfileLifecycle):
- components = self._profile.iter_components()
- else:
- components = [self._profile]
- return [*components, *self._response_processors, self._translator]
-
- async def call(
- self,
- request: ChatRequest,
- ctx: ProxyContext | None = None,
- ) -> TranslatedResponse:
- """Run the wrapped Profile and translate its response for the caller.
-
- Raises:
- SwitchyardProcessorError: A response processor raises or returns a value that is not
- a ``ChatResponse``.
- """
- context = ctx if ctx is not None else ProxyContext()
- profile_input = ProfileInput(
- request,
- metadata=_profile_metadata_from_context(context),
- )
- if isinstance(self._profile, ContextAwareProfile):
- response = await self._profile.run_with_context(profile_input, context)
- else:
- response = await self._profile.run(profile_input)
- current: Any = response
- for processor in self._response_processors:
- try:
- current = await processor.process(context, current)
- except Exception as error:
- raise SwitchyardProcessorError(str(error)) from error
- if not isinstance(current, ChatResponse):
- actual = type(current).__name__
- raise SwitchyardProcessorError(
- f"Response processor returned {actual}, expected ChatResponse"
- )
- return await self._translator.translate(
- context,
- request,
- cast(ChatResponse, current),
- )
-
-
-def _profile_metadata_from_context(context: ProxyContext) -> ProfileRequestMetadata:
- inbound_format = (
- request_type_value(context.inbound_format) if context.inbound_format is not None else None
- )
- headers = context.metadata.get(CTX_PROFILE_REQUEST_HEADERS)
- if isinstance(headers, Mapping):
- profile_headers: dict[str, str | list[str]] = {}
- for name, value in headers.items():
- if isinstance(value, list):
- profile_headers[str(name)] = [str(item) for item in value]
- else:
- profile_headers[str(name)] = str(value)
- return ProfileRequestMetadata.from_headers(
- profile_headers,
- inbound_format=inbound_format,
- )
- return ProfileRequestMetadata(
- request_id=context.request_id,
- inbound_format=inbound_format,
- )
-
-
-__all__ = ["ProfileSwitchyard"]
diff --git a/switchyard/lib/profiles/table.py b/switchyard/lib/profiles/table.py
deleted file mode 100644
index 91e93b728..000000000
--- a/switchyard/lib/profiles/table.py
+++ /dev/null
@@ -1,149 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Decorator and table helpers for Python-defined profiles."""
-
-from collections.abc import Callable
-from dataclasses import dataclass, is_dataclass
-from inspect import iscoroutinefunction
-from typing import Any, TypeAlias, TypeVar, cast, dataclass_transform
-
-from switchyard.lib.profiles.protocols import Profile, ProfileConfig, ProfileT_co
-
-ConfigClassT = TypeVar("ConfigClassT", bound=type[object])
-AnyProfileConfig: TypeAlias = ProfileConfig[Profile[Any]]
-
-_PROFILE_CONFIGS: dict[str, type[AnyProfileConfig]] = {}
-
-
-class ProfileConfigError(ValueError):
- """Raised when a Python profile config declaration is invalid."""
-
-
-def _normalize_profile_type(profile_type: str) -> str:
- """Validate and normalize a serialized profile type discriminator."""
- normalized = profile_type.strip()
- if not normalized:
- raise ProfileConfigError("profile type must not be empty")
- return normalized
-
-
-def _config_class(config_cls: type[object]) -> type[AnyProfileConfig]:
- """Validate a class has the minimum config shape expected by the table."""
- if not is_dataclass(config_cls):
- raise ProfileConfigError(
- f"{config_cls.__qualname__} must be a dataclass or use @profile_config directly"
- )
- if not callable(getattr(config_cls, "build", None)):
- raise ProfileConfigError(f"{config_cls.__qualname__} must define build()")
- return cast(type[AnyProfileConfig], config_cls)
-
-
-def register_profile_config(
- profile_type: str,
- config_cls: type[AnyProfileConfig],
- *,
- replace: bool = False,
-) -> None:
- """Register ``config_cls`` as the config owner for ``profile_type``.
-
- ``replace`` is deliberately explicit so accidental duplicate profile types
- fail during import, where they are easiest to diagnose.
- """
- normalized = _normalize_profile_type(profile_type)
- validated_cls = _config_class(config_cls)
- existing = _PROFILE_CONFIGS.get(normalized)
- if existing is not None and existing is not validated_cls and not replace:
- raise ProfileConfigError(
- f"profile type {normalized!r} is already registered to {existing.__qualname__}"
- )
- _PROFILE_CONFIGS[normalized] = validated_cls
-
-
-@dataclass_transform(frozen_default=True)
-def profile_config(
- profile_type: str,
- *,
- frozen: bool = True,
- slots: bool = True,
- register: bool = False,
-) -> Callable[[ConfigClassT], ConfigClassT]:
- """Decorate and optionally register a Python profile config class.
-
- The decorator mirrors the Rust ``#[profile_config]`` macro at Python scale:
- it ensures the class is a dataclass, stamps ``PROFILE_TYPE``, requires a
- ``build()`` method, and only mutates the local table when ``register``
- is explicitly enabled. Existing dataclasses are accepted unchanged.
- """
- normalized = _normalize_profile_type(profile_type)
-
- def decorate(cls: ConfigClassT) -> ConfigClassT:
- config_cls = cls if is_dataclass(cls) else cast(
- ConfigClassT,
- dataclass(frozen=frozen, slots=slots)(cls),
- )
- type.__setattr__(config_cls, "PROFILE_TYPE", normalized)
- typed_config_cls = _config_class(config_cls)
- if register:
- register_profile_config(normalized, typed_config_cls)
- return config_cls
-
- return decorate
-
-
-def lookup_profile_config(profile_type: str) -> type[AnyProfileConfig]:
- """Return the registered config class for ``profile_type``."""
- normalized = _normalize_profile_type(profile_type)
- try:
- return _PROFILE_CONFIGS[normalized]
- except KeyError as exc:
- raise ProfileConfigError(f"unknown profile type {normalized!r}") from exc
-
-
-def registered_profile_configs() -> dict[str, type[AnyProfileConfig]]:
- """Return a shallow copy of registered Python profile config classes."""
- return dict(_PROFILE_CONFIGS)
-
-
-def registered_profile_config_types() -> tuple[str, ...]:
- """Return registered profile type discriminators in deterministic order."""
- return tuple(sorted(_PROFILE_CONFIGS))
-
-
-def profile_config_type(config_or_cls: object) -> str:
- """Return the stamped ``PROFILE_TYPE`` for a config instance or class."""
- config_cls = config_or_cls if isinstance(config_or_cls, type) else type(config_or_cls)
- profile_type = getattr(config_cls, "PROFILE_TYPE", None)
- if not isinstance(profile_type, str) or not profile_type:
- raise ProfileConfigError(f"{config_cls.__qualname__} is not a registered profile config")
- return profile_type
-
-
-def build_profile(config: ProfileConfig[ProfileT_co]) -> ProfileT_co:
- """Build ``config`` and verify the result implements the profile runtime."""
- profile = config.build()
- if not isinstance(profile, Profile):
- raise ProfileConfigError(
- f"{type(config).__qualname__}.build() did not return a Profile runtime with "
- "run/process/rprocess methods"
- )
- for method_name in ("run", "process", "rprocess"):
- method = getattr(profile, method_name, None)
- if not iscoroutinefunction(method):
- raise ProfileConfigError(
- f"{type(config).__qualname__}.build() returned a Profile runtime, but "
- f"{method_name}() must be async"
- )
- return profile
-
-
-__all__ = [
- "ProfileConfigError",
- "build_profile",
- "lookup_profile_config",
- "profile_config",
- "profile_config_type",
- "register_profile_config",
- "registered_profile_config_types",
- "registered_profile_configs",
-]
diff --git a/switchyard/lib/profiles/tier_target_builders.py b/switchyard/lib/profiles/tier_target_builders.py
deleted file mode 100644
index 4b87c56ac..000000000
--- a/switchyard/lib/profiles/tier_target_builders.py
+++ /dev/null
@@ -1,106 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Shared strong/weak tier target preparation and backend construction.
-
-The deterministic (classifier) router and the escalation router build the same
-two-tier shape: apply the default tier timeout, apply DeepSeek
-benchmark-gateway overrides, resolve ``format='auto'`` once, build the native
-backend, and wrap Anthropic targets with cache breakpoints. Owning those rules
-here keeps the two profiles from drifting as the rules change.
-"""
-
-from __future__ import annotations
-
-from switchyard.lib.backends.anthropic_cache_breakpoint_backend import (
- maybe_wrap_anthropic_cache,
-)
-from switchyard.lib.backends.llm_target import LlmTarget
-from switchyard.lib.backends.multi_llm_backend import (
- build_native_backend,
- resolve_llm_target,
-)
-from switchyard.lib.profiles.deterministic_routing_config import (
- DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S,
-)
-from switchyard.lib.roles import LLMBackend
-
-
-def apply_deepseek_overrides(target: LlmTarget) -> LlmTarget:
- """Apply benchmark-specific DeepSeek extras without clobbering callers."""
- default_body = (
- {"chat_template_kwargs": {"enable_thinking": False}}
- if "deepseek-v4" in target.model
- else None
- )
- default_headers = (
- {"X-Inference-Priority": "batch"}
- if "deepseek" in target.model
- else None
- )
- if default_body is None and default_headers is None:
- return target
-
- existing_body = target.extra_body
- # LlmTarget normalizes omitted and explicit empty headers to the same
- # empty dict, so keep current defaulting behavior for normal DeepSeek
- # targets until the target type preserves "headers were provided" state.
- existing_headers = target.extra_headers or None
- merged_body = existing_body if existing_body is not None else default_body
- merged_headers = existing_headers if existing_headers is not None else default_headers
- if merged_body == existing_body and merged_headers == existing_headers:
- return target
-
- return LlmTarget(
- id=target.id,
- model=target.model,
- format=target.format,
- base_url=target.base_url,
- api_key=target.api_key,
- timeout_secs=target.endpoint.timeout_secs,
- extra_body=merged_body,
- extra_headers=merged_headers,
- )
-
-
-def apply_default_tier_timeout(
- target: LlmTarget,
- timeout_s: float | None = DEFAULT_DETERMINISTIC_TIER_TIMEOUT_S,
-) -> LlmTarget:
- """Apply the default tier timeout when a target has no explicit timeout."""
- if timeout_s is None or target.endpoint.timeout_secs is not None:
- return target
- return LlmTarget(
- id=target.id,
- model=target.model,
- format=target.format,
- base_url=target.base_url,
- api_key=target.api_key,
- timeout_secs=timeout_s,
- extra_body=target.extra_body,
- extra_headers=target.extra_headers,
- )
-
-
-def build_tier_backend(
- target: LlmTarget,
- tier_timeout_s: float | None,
-) -> tuple[LlmTarget, LLMBackend]:
- """Prepare one tier target and build its wrapped backend.
-
- Resolves ``format='auto'`` once after the tier defaults are applied so
- backend selection and Anthropic cache wrapping see the same concrete
- target. Returns ``(resolved_target, backend)`` — profiles need the
- resolved target for the tier's model id.
- """
- resolved = resolve_llm_target(
- apply_deepseek_overrides(apply_default_tier_timeout(target, tier_timeout_s)),
- )
- return resolved, maybe_wrap_anthropic_cache(build_native_backend(resolved), resolved)
-
-
-__all__ = [
- "apply_deepseek_overrides",
- "apply_default_tier_timeout",
- "build_tier_backend",
-]
diff --git a/switchyard/lib/profiles/translate_profile_config.py b/switchyard/lib/profiles/translate_profile_config.py
deleted file mode 100644
index f926035a2..000000000
--- a/switchyard/lib/profiles/translate_profile_config.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Profile-owned construction for host-owned format translation."""
-
-from __future__ import annotations
-
-from collections.abc import Sequence
-from dataclasses import field
-from typing import Any
-
-from switchyard.lib.processors.format_translate import (
- FormatTranslateRequestProcessor,
- FormatTranslateResponseProcessor,
- ModelFormatLookupProcessor,
- StampOriginalFormatProcessor,
- TranslateConfig,
-)
-from switchyard.lib.profiles.chain import ComponentChainProfile
-from switchyard.lib.profiles.table import profile_config
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.roles import LLMBackend
-from switchyard_rust.core import (
- ChatRequest,
- ChatRequestType,
- ChatResponse,
- SwitchyardBackendError,
-)
-
-
-@profile_config("translate")
-class TranslateProfileConfig:
- """Dataclass profile config for IGW-style request/response translation hooks."""
-
- config: TranslateConfig = field(default_factory=TranslateConfig)
- model_selection_processors: Sequence[Any] = ()
-
- def build(self) -> ComponentChainProfile:
- """Build a hook-only profile around the existing translate processors.
-
- The returned profile is intended for hosts that call ``process()``,
- perform their own backend call, then call ``rprocess()``. ``run()`` is
- intentionally unsupported because this profile does not own upstream
- transport.
- """
- request_processors = [
- StampOriginalFormatProcessor(),
- *self.model_selection_processors,
- ModelFormatLookupProcessor(self.config),
- FormatTranslateRequestProcessor(),
- ]
- return ComponentChainProfile(
- request_processors=request_processors,
- backend=_TranslateHookOnlyBackend(),
- response_processors=[FormatTranslateResponseProcessor()],
- )
-
-
-class _TranslateHookOnlyBackend(LLMBackend):
- """Backend placeholder that makes accidental translate ``run()`` fail clearly."""
-
- @property
- def supported_request_types(self) -> list[ChatRequestType]:
- """Advertise all request formats because this backend is never called."""
- return [
- ChatRequestType.OPENAI_CHAT,
- ChatRequestType.OPENAI_RESPONSES,
- ChatRequestType.ANTHROPIC,
- ]
-
- async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
- """Reject complete execution; external hosts own the backend call."""
- del ctx, request
- raise SwitchyardBackendError(
- "TranslateProfileConfig is hook-only; call process(), invoke the host "
- "backend, then call rprocess()"
- )
-
-
-__all__ = ["TranslateProfileConfig"]
diff --git a/switchyard/lib/redis_pin_store.py b/switchyard/lib/redis_pin_store.py
deleted file mode 100644
index 0b1b24910..000000000
--- a/switchyard/lib/redis_pin_store.py
+++ /dev/null
@@ -1,102 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Redis-backed shared L2 store for session-affinity pins.
-
-Implements :class:`~switchyard.lib.affinity_pin_store.AffinityPinStore` against a
-standalone Redis so pins are visible to every Switchyard worker/pod and survive
-pod churn. Keys are namespaced ``{key_prefix}{session_key}`` and expire after
-``ttl_seconds``; callers re-pin on every successful turn, so an active
-conversation slides its own TTL.
-
-The ``redis`` dependency is optional (``switchyard[affinity-redis]``) and is
-imported lazily, so the default install never pulls it in. Socket timeouts are
-bounded: a slow or unreachable store fails fast into ``SessionAffinity``'s
-best-effort (fail-open) path rather than blocking the request.
-"""
-
-from __future__ import annotations
-
-from typing import Any
-
-#: Bounded socket timeout (seconds). The store is expected to be colocated with
-#: the workers, so a healthy round-trip is sub-millisecond — 100 ms is ~100×
-#: headroom. The cap bounds how long a *degraded* store can stall a request
-#: before fail-open kicks in (and before SessionAffinity's circuit breaker
-#: stops attempting the store at all).
-_DEFAULT_SOCKET_TIMEOUT_S = 0.1
-
-
-class RedisPinStore:
- """Shared, persistent pin store backed by a standalone Redis.
-
- Args:
- url: Redis connection URL (e.g. ``"redis://host:6379/0"``).
- ttl_seconds: Expiry applied to each pin write. Must be > 0.
- key_prefix: Namespace prepended to every key.
- socket_timeout: Per-operation and connect timeout, in seconds.
- client: Pre-built async Redis client. Primarily for tests; when ``None``
- the client is created lazily from ``url`` on first use.
- """
-
- def __init__(
- self,
- url: str,
- *,
- ttl_seconds: int = 3_600,
- key_prefix: str = "swyd:pin:",
- socket_timeout: float = _DEFAULT_SOCKET_TIMEOUT_S,
- client: Any = None,
- ) -> None:
- if ttl_seconds <= 0:
- raise ValueError("ttl_seconds must be > 0")
- self._url = url
- self._ttl_seconds = ttl_seconds
- self._key_prefix = key_prefix
- self._socket_timeout = socket_timeout
- self._client = client
-
- def _get_client(self) -> Any:
- """Return the async Redis client, building it lazily on first use."""
- if self._client is None:
- # Lazy import keeps ``redis`` an optional dependency.
- from redis.asyncio import from_url
-
- # redis-py's from_url is untyped when the optional extra is
- # installed; without it the import is opaque and the ignore is
- # unused — suppress both cases so strict mypy stays green either way.
- self._client = from_url( # type: ignore[no-untyped-call,unused-ignore]
- self._url,
- decode_responses=True,
- socket_timeout=self._socket_timeout,
- socket_connect_timeout=self._socket_timeout,
- )
- return self._client
-
- def _redis_key(self, key: str) -> str:
- return f"{self._key_prefix}{key}"
-
- async def get(self, key: str) -> str | None:
- """Return the pinned value for ``key``, or ``None`` if absent/expired."""
- value = await self._get_client().get(self._redis_key(key))
- return value if value is None else str(value)
-
- async def put(self, key: str, value: str) -> None:
- """Pin ``key`` to ``value`` with the configured TTL."""
- await self._get_client().set(self._redis_key(key), value, ex=self._ttl_seconds)
-
- async def aclose(self) -> None:
- """Close the client and its connection pool. Safe to call repeatedly.
-
- Releases the client reference first, so a second call is a no-op (a
- later ``get``/``put`` would lazily rebuild — but the store is only
- closed at backend shutdown).
- """
- client = self._client
- if client is None:
- return
- self._client = None
- await client.aclose()
-
-
-__all__ = ["RedisPinStore"]
diff --git a/switchyard/lib/request_metadata.py b/switchyard/lib/request_metadata.py
index 8419a65c8..de4eed7be 100644
--- a/switchyard/lib/request_metadata.py
+++ b/switchyard/lib/request_metadata.py
@@ -10,7 +10,7 @@ from typing import Any
from switchyard.lib.proxy_context import CTX_CALLER_API_KEY
CTX_REQUEST_METADATA = "_request_metadata"
-CTX_PROFILE_REQUEST_HEADERS = "_profile_request_headers"
+CTX_REQUEST_HEADERS = "_request_headers"
# Existing Switchyard session header. Do not add aliases here unless a
# concrete client requires one; keeping one spelling avoids ambiguity.
@@ -30,11 +30,10 @@ _CALLER_KEY_SENTINELS = frozenset({"switchyard", ""})
# behind such a proxy stays correctly attributed for upstream inference spend.
CALLER_API_KEY_HEADER = "x-switchyard-api-key" # pragma: allowlist secret
-# Request headers whose values carry a caller credential. Redacted before the
-# header map is retained on the context (``CTX_PROFILE_REQUEST_HEADERS``), so the
-# caller's key never reaches profile metadata, logs, or traces. The key
-# is still forwarded upstream via ``CTX_CALLER_API_KEY`` (extracted by
-# ``attach_caller_api_key`` from the raw headers, before redaction).
+# Request headers whose values carry a caller credential. Values are redacted
+# in the map retained as ``CTX_REQUEST_HEADERS``. The raw key remains separately
+# in ``CTX_CALLER_API_KEY`` for upstream forwarding and is not serialized by
+# logging or tracing.
_SENSITIVE_HEADERS = frozenset({"authorization", "x-api-key", CALLER_API_KEY_HEADER})
_REDACTED = "[REDACTED]"
@@ -68,7 +67,7 @@ def attach_request_metadata(
# already extracted into ``CTX_CALLER_API_KEY`` for upstream forwarding,
# so nothing downstream needs the raw value, and a retained/logged header
# map must not expose it.
- ctx.metadata[CTX_PROFILE_REQUEST_HEADERS] = redact_sensitive_headers(headers)
+ ctx.metadata[CTX_REQUEST_HEADERS] = redact_sensitive_headers(headers)
def _nonempty_header(headers: Mapping[str, str], name: str) -> str | None:
@@ -132,7 +131,7 @@ def extract_caller_api_key(headers: Mapping[str, str]) -> str | None:
__all__ = [
"CALLER_API_KEY_HEADER",
"CTX_REQUEST_METADATA",
- "CTX_PROFILE_REQUEST_HEADERS",
+ "CTX_REQUEST_HEADERS",
"INTAKE_TASK_HEADER",
"PROXY_SESSION_ID_HEADER",
"RequestMetadata",
diff --git a/switchyard/lib/route_table.py b/switchyard/lib/route_table.py
index 28b7d847b..1ea7b6251 100644
--- a/switchyard/lib/route_table.py
+++ b/switchyard/lib/route_table.py
@@ -1,11 +1,11 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""RouteTable — route inbound requests to per-model profile runtimes.
+"""RouteTable — route inbound requests to per-model runtimes.
Stores a static mapping of model name → callable runtime. The launcher
-populates the table at startup after building all runtimes from profile
-configuration. The three V2 HTTP endpoints read ``body["model"]``, call
+populates the table at startup after building all configured runtimes. The
+three HTTP endpoints read ``body["model"]``, call
:meth:`lookup_switchyard`, and dispatch to the returned chain before making any
backend calls.
"""
@@ -46,7 +46,7 @@ SwitchyardApp: TypeAlias = "ChainRuntime | RouteTable"
class RouteTable:
- """Table that maps model names to pre-built profile runtimes.
+ """Table that maps model names to pre-built runtimes.
Register one chain per model the proxy should handle explicitly. Unknown
models raise ``KeyError`` so endpoint handlers can return ``model_not_found``
@@ -104,8 +104,7 @@ class RouteTable:
"""Iterate ``(model_id, chain, metadata)`` triples in registration order.
Used when a caller needs to merge one table into another — e.g. a
- YAML route-bundle entry that expands into multiple model registrations
- via :func:`switchyard.lib.route_table_builders.build_random_routing_table`.
+ caller that composes multiple route tables.
"""
for model in self._by_model:
yield model, self._by_model[model], dict(self._metadata_by_model.get(model, {}))
diff --git a/switchyard/lib/route_table_builders.py b/switchyard/lib/route_table_builders.py
deleted file mode 100644
index 936e6dd52..000000000
--- a/switchyard/lib/route_table_builders.py
+++ /dev/null
@@ -1,453 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Build :class:`RouteTable` instances for client-selected model routing.
-
-Two layers compose model-name dispatch:
-
-* :func:`build_passthrough_table` — the **discovery layer**. Given a set of
- tiers, registers each tier's configured model as a direct passthrough and
- optionally hydrates additional entries from each tier's ``GET /v1/models``
- catalog via a caller-supplied ``discovery_fn``.
-* :func:`register_random_routing_policy` — the **routing-policy layer**. Adds a
- virtual model id to an existing table whose chain dispatches via the
- random-routing weighted coin.
-
-:func:`build_random_routing_table` composes the two for the launcher and
-YAML route-bundle paths. Both callers share this assembly to keep behavior
-consistent.
-
-Discovery is injected as a callable so this module stays free of CLI-only
-dependencies. Pass ``discovery_fn=None`` (the default) to skip discovery
-entirely.
-"""
-
-import hashlib
-import logging
-from collections.abc import Callable, Sequence
-from typing import Any, TypeAlias
-
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
-from switchyard.lib.model_listing import (
- combined_model_capabilities,
- model_capabilities,
-)
-from switchyard.lib.processors.llm_classifier.presets import (
- classifier_prompt_sha256,
- resolve_classifier_prompt,
-)
-from switchyard.lib.profiles.deterministic_routing_config import (
- DeterministicRoutingConfig,
-)
-from switchyard.lib.profiles.random_routing import RandomRoutingConfig
-from switchyard.lib.route_table import ChainRuntime, RouteTable
-from switchyard.lib.stats_accumulator import StatsAccumulator
-
-logger = logging.getLogger(__name__)
-
-#: Callable that returns the model ids advertised by an OpenAI-compatible
-#: ``GET /models`` endpoint. Builders record a non-fatal warning if it raises.
-DiscoveryFn: TypeAlias = Callable[[str, str], list[str]]
-
-
-def random_routing_virtual_model_id(config: RandomRoutingConfig) -> str:
- """Stable model id representing this random-routing pair.
-
- Launcher clients route by the request ``model`` field. Using the strong
- model name for the random chain makes the client picker ambiguous because
- "strong" actually means "random". This virtual id gives the routed pair its
- own model identity while leaving the real strong/weak ids available for
- direct model overrides.
- """
- fingerprint = hashlib.sha1(
- "\0".join((
- config.strong.model,
- config.weak.model,
- f"{config.strong_probability:.12g}",
- )).encode("utf-8"),
- ).hexdigest()[:8]
- return f"switchyard-default-random-{fingerprint}"
-
-
-def build_random_routing_switchyard(
- config: RandomRoutingConfig,
- stats: StatsAccumulator,
- pre_routing_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
-) -> ChainRuntime:
- """Build the primary random-routing chain used by launchers and bundles."""
- from switchyard.lib.profiles import ProfileSwitchyard, RandomRoutingProfileConfig
-
- return ProfileSwitchyard(
- RandomRoutingProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats,
- enable_stats=config.enable_stats,
- pre_request_processors=pre_routing_request_processors,
- response_processors=extra_response_processors,
- )
- )
-
-
-def build_tier_passthrough_switchyard(
- tier: LlmTarget,
- stats: StatsAccumulator,
- enable_stats: bool = True,
- extra_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
-) -> ChainRuntime:
- """Build a single-tier chain for an explicitly selected model."""
- from switchyard.lib.profiles import PassthroughProfileConfig, ProfileSwitchyard
-
- return ProfileSwitchyard(
- PassthroughProfileConfig(
- target=tier,
- )
- .build()
- .with_runtime_components(
- enable_stats=enable_stats,
- stats_accumulator=stats,
- pre_request_processors=extra_request_processors,
- response_processors=extra_response_processors,
- )
- )
-
-
-def build_single_model_table(
- model: str,
- switchyard: ChainRuntime,
- metadata_source: str = "configured",
-) -> RouteTable:
- """Wrap one pre-built chain in a table so ``/v1/models`` can list it."""
- table = RouteTable()
- table.register(
- model,
- switchyard,
- metadata={
- "description": f"Direct Switchyard route to {model}.",
- "capabilities": model_capabilities(model),
- "switchyard": {"profile": "passthrough", "source": metadata_source},
- },
- default=True,
- )
- return table
-
-
-def build_passthrough_table(
- tiers: Sequence[LlmTarget],
- stats: StatsAccumulator,
- enable_stats: bool = True,
- discovery_fn: DiscoveryFn | None = None,
- pre_routing_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
-) -> RouteTable:
- """Build a table of passthrough chains for *tiers* and their catalogs.
-
- For each tier this:
-
- 1. Registers the tier's configured model as a direct passthrough.
- 2. If ``discovery_fn`` is supplied and the tier has both a ``base_url`` and
- ``api_key``, calls ``discovery_fn(base_url, api_key)`` and registers each
- remaining model as a passthrough that reuses the tier's connectivity.
- Discovered models use :class:`BackendFormat.AUTO` so each direct route
- resolves its backend format independently.
-
- Catalog fetches are deduped by ``(base_url, api_key)`` so two tiers that
- share an endpoint hit it once. Entries already in the table (including
- a tier's configured model) are never overwritten by a later discovered
- duplicate.
- """
- table = RouteTable()
- catalog_cache: dict[tuple[str, str], list[str]] = {}
-
- def _register_passthrough(
- tier: LlmTarget,
- source: str,
- description: str,
- ) -> None:
- table.register(
- tier.model,
- build_tier_passthrough_switchyard(
- tier,
- stats,
- enable_stats=enable_stats,
- extra_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- ),
- metadata={
- "description": description,
- "capabilities": model_capabilities(tier.model),
- "switchyard": {"profile": "passthrough", "source": source},
- },
- )
-
- # Pass 1: register every tier's configured model so the picker always
- # surfaces them first, regardless of where they fall in catalog order.
- for tier in tiers:
- if tier.model and tier.model not in table.registered_models():
- _register_passthrough(
- tier,
- source="configured",
- description=f"Direct Switchyard route to {tier.model}.",
- )
-
- # Pass 2: hydrate from each tier's GET /v1/models catalog when a
- # discovery_fn is supplied.
- if discovery_fn is None:
- return table
-
- for tier in tiers:
- if not tier.endpoint.base_url or not tier.endpoint.api_key:
- continue
-
- cache_key = (tier.endpoint.base_url, tier.endpoint.api_key)
- if cache_key not in catalog_cache:
- try:
- catalog_cache[cache_key] = discovery_fn(
- tier.endpoint.base_url, tier.endpoint.api_key,
- )
- except Exception as exc:
- logger.warning(
- "Could not hydrate model catalog from %s: %s",
- tier.endpoint.base_url,
- exc,
- )
- table.add_model_listing_warning(
- "Model discovery failed for "
- f"{tier.endpoint.base_url}: {exc}"
- )
- catalog_cache[cache_key] = []
-
- for model_id in catalog_cache[cache_key]:
- if not model_id or model_id in table.registered_models():
- continue
- _register_passthrough(
- LlmTarget(
- id=f"discovered-{model_id}",
- model=model_id,
- # Discovered via GET /v1/models on an OpenAI-compatible endpoint —
- # Chat Completions is the safe default. Users who need a different
- # format for a discovered model should declare it explicitly.
- format=BackendFormat.OPENAI,
- api_key=tier.endpoint.api_key,
- base_url=tier.endpoint.base_url,
- timeout_secs=tier.endpoint.timeout_secs,
- ),
- source="discovered",
- description=(
- f"Direct Switchyard passthrough to discovered model "
- f"{model_id}."
- ),
- )
-
- return table
-
-
-def register_random_routing_policy(
- table: RouteTable,
- config: RandomRoutingConfig,
- random_routing_switchyard: ChainRuntime,
- routing_model: str | None = None,
-) -> str:
- """Layer a random-routing policy on top of a passthrough table.
-
- Returns the virtual model id under which the policy was registered. Strong
- and weak models registered separately by :func:`build_passthrough_table`
- remain available as direct overrides via the client's model picker.
- """
- virtual_model = routing_model or random_routing_virtual_model_id(config)
- table.register(
- virtual_model,
- random_routing_switchyard,
- metadata={
- "display_name": "Switchyard random routing",
- "description": (
- "Random routes "
- f"{config.strong.model} (strong) and {config.weak.model} (weak), "
- f"p_strong={config.strong_probability:.2f}."
- ),
- "capabilities": combined_model_capabilities([
- config.strong.model,
- config.weak.model,
- ]),
- "switchyard": {
- "profile": "random_routing",
- "strong_model": config.strong.model,
- "weak_model": config.weak.model,
- "strong_probability": config.strong_probability,
- },
- },
- default=True,
- )
- return virtual_model
-
-
-def build_random_routing_table(
- config: RandomRoutingConfig,
- stats: StatsAccumulator,
- random_routing_switchyard: ChainRuntime,
- routing_model: str | None = None,
- discovery_fn: DiscoveryFn | None = None,
- extra_response_processors: Sequence[Any] = (),
- pre_routing_request_processors: Sequence[Any] = (),
-) -> RouteTable:
- """Compose the discovery and random-routing-policy layers.
-
- First builds a passthrough table for the strong and weak tiers (plus
- everything advertised by their ``GET /v1/models`` catalogs when
- ``discovery_fn`` is supplied), then layers the random-routing virtual model
- on top.
- """
- table = build_passthrough_table(
- (config.strong, config.weak),
- stats,
- enable_stats=config.enable_stats,
- discovery_fn=discovery_fn,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
- register_random_routing_policy(
- table,
- config,
- random_routing_switchyard=random_routing_switchyard,
- routing_model=routing_model,
- )
- return table
-
-
-def deterministic_routing_virtual_model_id(
- config: DeterministicRoutingConfig,
-) -> str:
- """Stable model id representing this deterministic-routing bundle."""
- prompt = resolve_classifier_prompt(
- config.profile_name,
- config.classifier_system_prompt,
- )
- fingerprint = hashlib.sha1(
- "\0".join((
- config.strong.model,
- config.weak.model,
- config.classifier.model,
- config.profile_name,
- classifier_prompt_sha256(prompt),
- str(config.classifier_max_request_chars),
- str(config.classifier_recent_turn_window),
- )).encode("utf-8"),
- ).hexdigest()[:8]
- return f"switchyard-deterministic-{fingerprint}"
-
-
-def build_deterministic_routing_switchyard(
- config: DeterministicRoutingConfig,
- stats: StatsAccumulator,
- pre_routing_request_processors: Sequence[Any] = (),
- extra_response_processors: Sequence[Any] = (),
-) -> ChainRuntime:
- """Build the deterministic-routing chain used by the launcher."""
- from switchyard.lib.profiles import (
- DeterministicRoutingProfileConfig,
- ProfileSwitchyard,
- )
-
- return ProfileSwitchyard(
- DeterministicRoutingProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats,
- enable_stats=config.enable_stats,
- pre_request_processors=pre_routing_request_processors,
- response_processors=extra_response_processors,
- )
- )
-
-
-def register_deterministic_routing_policy(
- table: RouteTable,
- config: DeterministicRoutingConfig,
- deterministic_routing_switchyard: ChainRuntime,
- routing_model: str | None = None,
-) -> str:
- """Layer a deterministic-routing policy on top of a passthrough table.
-
- Strong + weak passthrough entries are registered separately by
- :func:`build_passthrough_table`; the classifier is not user-selectable.
- """
- prompt = resolve_classifier_prompt(
- config.profile_name,
- config.classifier_system_prompt,
- )
- virtual_model = routing_model or deterministic_routing_virtual_model_id(config)
- table.register(
- virtual_model,
- deterministic_routing_switchyard,
- metadata={
- "display_name": "Switchyard deterministic routing",
- "description": (
- "LLM-classifier routes between "
- f"{config.strong.model} (strong) and {config.weak.model} (weak) "
- f"using {config.classifier.model} (classifier, "
- f"profile={config.profile_name})."
- ),
- "capabilities": combined_model_capabilities([
- config.strong.model,
- config.weak.model,
- ]),
- "switchyard": {
- "profile": "deterministic_routing",
- "strong_model": config.strong.model,
- "weak_model": config.weak.model,
- "classifier_model": config.classifier.model,
- "classifier_profile": config.profile_name,
- "classifier_prompt_sha256": classifier_prompt_sha256(prompt),
- "classifier_max_request_chars": config.classifier_max_request_chars,
- "classifier_recent_turn_window": config.classifier_recent_turn_window,
- "classifier_min_confidence": config.classifier_min_confidence,
- },
- },
- default=True,
- )
- return virtual_model
-
-
-def build_deterministic_routing_table(
- config: DeterministicRoutingConfig,
- stats: StatsAccumulator,
- deterministic_routing_switchyard: ChainRuntime,
- routing_model: str | None = None,
- discovery_fn: DiscoveryFn | None = None,
- extra_response_processors: Sequence[Any] = (),
- pre_routing_request_processors: Sequence[Any] = (),
-) -> RouteTable:
- """Compose discovery + deterministic-routing-policy layers."""
- table = build_passthrough_table(
- (config.strong, config.weak),
- stats,
- enable_stats=config.enable_stats,
- discovery_fn=discovery_fn,
- pre_routing_request_processors=pre_routing_request_processors,
- extra_response_processors=extra_response_processors,
- )
- register_deterministic_routing_policy(
- table,
- config,
- deterministic_routing_switchyard=deterministic_routing_switchyard,
- routing_model=routing_model,
- )
- return table
-
-
-__all__ = [
- "DiscoveryFn",
- "build_deterministic_routing_table",
- "build_deterministic_routing_switchyard",
- "build_passthrough_table",
- "build_random_routing_table",
- "build_random_routing_switchyard",
- "build_single_model_table",
- "build_tier_passthrough_switchyard",
- "deterministic_routing_virtual_model_id",
- "random_routing_virtual_model_id",
- "register_deterministic_routing_policy",
- "register_random_routing_policy",
-]
diff --git a/switchyard/lib/session_affinity.py b/switchyard/lib/session_affinity.py
deleted file mode 100644
index 93693f97b..000000000
--- a/switchyard/lib/session_affinity.py
+++ /dev/null
@@ -1,277 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Per-conversation decision pin shared across routers (sticky routing)."""
-
-from __future__ import annotations
-
-import inspect
-import logging
-import time
-from typing import TYPE_CHECKING, overload
-
-from switchyard.lib.conversation_turn import conversation_turn_number
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.session_cache import SessionCache
-from switchyard.lib.session_key import session_key_from_body
-from switchyard_rust.core import ChatRequest
-
-if TYPE_CHECKING:
- from collections.abc import Callable
-
- from switchyard.lib.affinity_pin_store import AffinityPinStore
-
-logger = logging.getLogger(__name__)
-
-#: Consecutive L2 failures that open the circuit breaker.
-_L2_BREAKER_THRESHOLD = 3
-
-#: Seconds the breaker stays open before allowing a recovery probe.
-_L2_BREAKER_COOLDOWN_S = 10.0
-
-#: ``ProxyContext.metadata`` key memoizing the per-request session key, so
-#: callers that consult affinity more than once don't re-hash the (growing)
-#: request body.
-CTX_SESSION_KEY = "_session_affinity_key"
-
-
-@overload
-def resolve_session_key(ctx: ProxyContext, request: ChatRequest) -> str: ...
-
-
-@overload
-def resolve_session_key(ctx: ProxyContext, request: ChatRequest, *, depth: int) -> str | None: ...
-
-
-def resolve_session_key(ctx: ProxyContext, request: ChatRequest, *, depth: int = 0) -> str | None:
- """Derive the conversation key once per request, memoized on ``ctx``.
-
- ``depth > 0`` derives the deep key (see
- :func:`~switchyard.lib.session_key.session_key_from_body`) the escalation
- judge uses for repeated-trial benchmarking, and returns ``None`` — without
- memoizing — while the conversation is still shorter than the requested
- prefix. A key already memoized on ``ctx`` (by an earlier component this
- request, whatever its depth) wins over re-derivation, so every consumer
- sees one consistent key per request.
- """
- cached = ctx.metadata.get(CTX_SESSION_KEY)
- if isinstance(cached, str):
- return cached
- key = session_key_from_body(request.body, depth)
- if key is None:
- return None
- ctx.metadata[CTX_SESSION_KEY] = key
- return key
-
-
-class SessionAffinity:
- """Pins a routing decision per conversation and reuses it on later turns.
-
- The shared building block both routing paths use for the *common* part of
- stickiness: derive a stable conversation key (system prompt + first user
- message, memoized on the request context) and store/look up a pinned value
- in a bounded LRU. The pinned value is opaque — a tier label for the
- classifier or escalation router. Each caller keeps its own *policy*
- (when to write a pin, whether to honor one) on top.
-
- The in-process ``SessionCache`` is the L1. An optional ``l2`` implementing
- :class:`~switchyard.lib.affinity_pin_store.AffinityPinStore` is a shared,
- out-of-process store (e.g. Redis): pins are read through it on an L1 miss
- (and warmed back into L1) and written through it, so a conversation stays
- pinned across workers and pod churn. The L2 is **best-effort** — any error
- or timeout falls back to L1 / unpinned and never breaks routing — and sits
- behind a circuit breaker: after ``l2_breaker_threshold`` consecutive
- failures, L2 operations are skipped (no network attempt, zero added
- latency) for ``l2_breaker_cooldown_s``; the first operation after the
- cooldown is the recovery probe, whose success closes the breaker.
-
- A disabled instance no-ops (and never hashes the body). Not thread-safe —
- touched only on the request path.
- """
-
- def __init__(
- self,
- *,
- enabled: bool = False,
- max_sessions: int = 10_000,
- warmup_turns: int = 0,
- l2: AffinityPinStore | None = None,
- l2_breaker_threshold: int = _L2_BREAKER_THRESHOLD,
- l2_breaker_cooldown_s: float = _L2_BREAKER_COOLDOWN_S,
- clock: Callable[[], float] = time.monotonic,
- ) -> None:
- if warmup_turns < 0:
- raise ValueError("warmup_turns must be >= 0")
- if l2_breaker_threshold < 1:
- raise ValueError("l2_breaker_threshold must be >= 1")
- if l2_breaker_cooldown_s <= 0:
- raise ValueError("l2_breaker_cooldown_s must be > 0")
- self._enabled = enabled
- self._warmup_turns = warmup_turns
- self._pins = SessionCache[str](max_sessions)
- #: Optional shared/persistent L2 store; ``None`` keeps behavior L1-only.
- self._l2 = l2
- # L2 observability: hits = pins rescued from the shared store after an
- # L1 miss (cross-worker reuse); errors = fail-open get/put operations
- # (the alerting signal for a silently degraded store). Event-loop only,
- # like the pins themselves — no lock.
- self._l2_hits = 0
- self._l2_errors = 0
- # Circuit breaker over the L2: a streak of consecutive failures opens
- # it, and while open every operation is skipped without a network
- # attempt, so a store outage stops taxing requests with timeout waits.
- # ``clock`` is injectable for tests; monotonic so wall-clock jumps
- # can't wedge the breaker.
- self._l2_breaker_threshold = l2_breaker_threshold
- self._l2_breaker_cooldown_s = l2_breaker_cooldown_s
- self._clock = clock
- self._l2_failure_streak = 0
- self._l2_skip_until = 0.0
-
- @property
- def enabled(self) -> bool:
- return self._enabled
-
- @property
- def max_sessions(self) -> int:
- """Configured maximum number of pinned conversations."""
- return self._pins.max_sessions
-
- @property
- def warmup_turns(self) -> int:
- """Number of initial turns that cannot read or write affinity pins."""
- return self._warmup_turns
-
- @property
- def l2_enabled(self) -> bool:
- """Whether a shared (L2) pin store is configured."""
- return self._l2 is not None
-
- @property
- def l2_hits(self) -> int:
- """Pins resolved from the shared store after an in-process (L1) miss."""
- return self._l2_hits
-
- @property
- def l2_errors(self) -> int:
- """Shared-store operations (get or put) that failed open."""
- return self._l2_errors
-
- @property
- def l2_breaker_open(self) -> bool:
- """Whether the breaker is currently skipping shared-store operations."""
- return self._l2 is not None and self._clock() < self._l2_skip_until
-
- def _l2_skips(self) -> bool:
- """Breaker gate checked before every L2 operation.
-
- ``True`` while the cooldown is running — skip the store without a
- network attempt. Once the cooldown elapses this returns ``False``, so
- the next operation goes through as the recovery probe (concurrent
- in-flight requests at that instant may each probe; every probe is
- bounded by the store's socket timeout, so the burst is harmless).
- """
- return self._clock() < self._l2_skip_until
-
- def _record_l2_failure(self, what: str) -> None:
- """Count one fail-open L2 operation and open the breaker at the streak."""
- self._l2_errors += 1
- self._l2_failure_streak += 1
- if self._l2_failure_streak < self._l2_breaker_threshold:
- logger.warning("session-affinity L2 %s", what, exc_info=True)
- return
- self._l2_skip_until = self._clock() + self._l2_breaker_cooldown_s
- logger.warning(
- "session-affinity L2 %s; breaker open for %.1fs (failure streak %d)",
- what,
- self._l2_breaker_cooldown_s,
- self._l2_failure_streak,
- exc_info=True,
- )
-
- def _record_l2_success(self) -> None:
- """Reset the failure streak; close the breaker after a successful probe."""
- if self._l2_failure_streak >= self._l2_breaker_threshold:
- logger.info("session-affinity L2 recovered; breaker closed")
- self._l2_failure_streak = 0
- self._l2_skip_until = 0.0
-
- async def pinned(self, ctx: ProxyContext, request: ChatRequest) -> str | None:
- """Return the value pinned to ``request``'s conversation, or ``None``.
-
- Checks L1 first; on a miss (and only when an L2 is configured) consults
- the shared store and warms the hit back into L1 so later turns on this
- worker skip the round-trip. An L2 error is swallowed — the conversation
- is treated as unpinned rather than failing the request — and while the
- breaker is open the store isn't consulted at all.
- """
- if not self._enabled or self._is_warmup_turn(request):
- return None
- key = self._session_key(ctx, request)
- value = self._pins.get(key)
- if value is not None or self._l2 is None or self._l2_skips():
- return value
- try:
- value = await self._l2.get(key)
- except Exception:
- self._record_l2_failure("get failed; routing without a pin")
- return None
- self._record_l2_success()
- if value is not None:
- self._l2_hits += 1
- self._pins.put(key, value)
- return value
-
- async def pin(self, ctx: ProxyContext, request: ChatRequest, value: str) -> None:
- """Pin ``request``'s conversation to ``value`` (no-op when disabled).
-
- Writes through to the L2 when configured; an L2 error is swallowed so
- the pin stays worker-local rather than failing the request, and while
- the breaker is open the write is skipped entirely.
- """
- if not self._enabled or self._is_warmup_turn(request):
- return
- key = self._session_key(ctx, request)
- self._pins.put(key, value)
- if self._l2 is None or self._l2_skips():
- return
- try:
- await self._l2.put(key, value)
- except Exception:
- self._record_l2_failure("put failed; pin is worker-local only")
- return
- self._record_l2_success()
-
- async def aclose(self) -> None:
- """Release the shared store's resources (no-op for L1-only instances).
-
- Duck-typed: closes the L2 only if it exposes ``aclose()`` (the protocol
- doesn't require one). Best-effort like every other L2 interaction — a
- close failure is logged, never raised, so backend teardown can't wedge.
- """
- if self._l2 is None:
- return
- closer = getattr(self._l2, "aclose", None)
- if not callable(closer):
- return
- try:
- result = closer()
- if inspect.isawaitable(result):
- await result
- except Exception:
- logger.warning("session-affinity L2 close failed", exc_info=True)
-
- def __len__(self) -> int:
- """Number of conversations currently pinned."""
- return len(self._pins)
-
- def _session_key(self, ctx: ProxyContext, request: ChatRequest) -> str:
- """Derive the conversation key once per request, memoized on ``ctx``."""
- return resolve_session_key(ctx, request)
-
- def _is_warmup_turn(self, request: ChatRequest) -> bool:
- """Return whether this request is still inside the no-stick warmup."""
- return conversation_turn_number(request) <= self._warmup_turns
-
-
-__all__ = ["CTX_SESSION_KEY", "SessionAffinity", "resolve_session_key"]
diff --git a/switchyard/lib/session_cache.py b/switchyard/lib/session_cache.py
deleted file mode 100644
index 76fb35c1e..000000000
--- a/switchyard/lib/session_cache.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Bounded, access-ordered LRU cache keyed by a session key."""
-
-from __future__ import annotations
-
-from collections import OrderedDict
-from typing import Generic, TypeVar, cast
-
-_V = TypeVar("_V")
-_MISSING = object()
-
-
-class SessionCache(Generic[_V]):
- """Retain the most recently accessed session values up to a fixed capacity."""
-
- def __init__(self, max_sessions: int) -> None:
- if max_sessions < 0:
- raise ValueError("max_sessions must be non-negative")
- self._max_sessions = max_sessions
- self._values: OrderedDict[str, _V] = OrderedDict()
-
- @property
- def max_sessions(self) -> int:
- """Return the configured cache capacity."""
- return self._max_sessions
-
- def get(self, key: str) -> _V | None:
- """Return a cached value and mark it as most recently used."""
- value = self._values.get(key, _MISSING)
- if value is _MISSING:
- return None
- self._values.move_to_end(key)
- return cast(_V, value)
-
- def put(self, key: str, value: _V) -> None:
- """Insert a value and evict the least recently used entry if needed."""
- if self._max_sessions == 0:
- return
- self._values[key] = value
- self._values.move_to_end(key)
- if len(self._values) > self._max_sessions:
- self._values.popitem(last=False)
-
- def values(self) -> list[_V]:
- """Return the retained values in least-to-most-recent order."""
- return list(self._values.values())
-
- def __len__(self) -> int:
- return len(self._values)
-
-
-__all__ = ["SessionCache"]
diff --git a/switchyard/lib/session_key.py b/switchyard/lib/session_key.py
deleted file mode 100644
index 30a991132..000000000
--- a/switchyard/lib/session_key.py
+++ /dev/null
@@ -1,141 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Stable per-conversation key derived from the request prefix."""
-
-from __future__ import annotations
-
-import hashlib
-import json
-from typing import Any, Literal, overload
-
-
-@overload
-def session_key_from_body(body: Any, depth: Literal[0] = 0) -> str: ...
-
-
-@overload
-def session_key_from_body(body: Any, depth: int) -> str | None: ...
-
-
-def session_key_from_body(body: Any, depth: int = 0) -> str | None:
- """Hash stable conversation anchors and an optional early-response prefix."""
- if depth < 0:
- raise ValueError("depth must be non-negative")
-
- parts: list[str] = [_flatten_text(_field(body, "system"))]
- anchored = False
- tail_taken = 0
-
- for sequence_name in ("messages", "input"):
- items = _field(body, sequence_name)
- if not isinstance(items, list):
- continue
- for item in items:
- role = _field(item, "role")
- if role in {"system", "developer"}:
- parts.append(_flatten_text(_field(item, "content")))
- elif role == "user" and not anchored:
- parts.append(_flatten_text(_field(item, "content")))
- anchored = True
- elif anchored and tail_taken < depth:
- parts.append(_flatten_message(item))
- tail_taken += 1
- if anchored and tail_taken >= depth:
- break
- if anchored:
- break
-
- if depth > 0 and (not anchored or tail_taken < depth):
- return None
-
- digest = hashlib.blake2b(digest_size=8)
- for part in parts:
- encoded = part.encode()
- digest.update(len(encoded).to_bytes(8, "big"))
- digest.update(encoded)
- return digest.hexdigest()
-
-
-def _field(value: Any, name: str) -> Any:
- if isinstance(value, dict):
- return value.get(name)
- return getattr(value, name, None)
-
-
-def _flatten_message(item: Any) -> str:
- parts: list[str] = []
- content = _flatten_text(_field(item, "content"))
- if content:
- parts.append(content)
-
- blocks = _field(item, "content")
- if isinstance(blocks, list):
- for block in blocks:
- if _field(block, "type") == "tool_use":
- name = _field(block, "name") or ""
- raw_input = _field(block, "input")
- payload = _json_text(raw_input) if raw_input is not None else ""
- parts.append(f"tool_call {name}({payload})")
-
- calls = _field(item, "tool_calls")
- if isinstance(calls, list):
- for call in calls:
- function = _field(call, "function")
- if function is not None:
- raw_name = _field(function, "name")
- raw_arguments = _field(function, "arguments")
- name = raw_name if isinstance(raw_name, str) else ""
- arguments = (
- raw_arguments
- if isinstance(raw_arguments, str)
- else _json_text(raw_arguments)
- if raw_arguments is not None
- else ""
- )
- parts.append(f"tool_call {name}({arguments})")
-
- if _field(item, "type") == "function_call":
- raw_name = _field(item, "name")
- raw_arguments = _field(item, "arguments")
- name = raw_name if isinstance(raw_name, str) else ""
- arguments = (
- raw_arguments
- if isinstance(raw_arguments, str)
- else _json_text(raw_arguments)
- if raw_arguments is not None
- else ""
- )
- parts.append(f"tool_call {name}({arguments})")
-
- output = _flatten_text(_field(item, "output"))
- if output:
- parts.append(output)
- return " ".join(parts)
-
-
-def _flatten_text(content: Any) -> str:
- if isinstance(content, str):
- return content
- if isinstance(content, list):
- parts: list[str] = []
- for block in content:
- if isinstance(block, dict):
- text = block.get("text")
- if not isinstance(text, str) or not text:
- text = block.get("content")
- if isinstance(text, str) and text:
- parts.append(text)
- else:
- parts.append(_json_text(block))
- return "".join(parts)
- if content is None:
- return ""
- return _json_text(content)
-
-
-def _json_text(value: Any) -> str:
- return json.dumps(value, separators=(",", ":"), sort_keys=True)
-
-
-__all__ = ["session_key_from_body"]
diff --git a/switchyard/server/server_util.py b/switchyard/server/server_util.py
index e6b7b366a..b62275cc6 100644
--- a/switchyard/server/server_util.py
+++ b/switchyard/server/server_util.py
@@ -373,7 +373,7 @@ def build_and_serve(
port=port,
display_model=default_model or "switchyard",
strategy_summary=strategy_summary,
- profile_routes=table.registered_models() if table else None,
+ routes=table.registered_models() if table else None,
default_route=default_model,
)
diff --git a/switchyard_rust/profiles.py b/switchyard_rust/profiles.py
deleted file mode 100644
index aa1dac500..000000000
--- a/switchyard_rust/profiles.py
+++ /dev/null
@@ -1,106 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Python-owned request values shared by profile implementations."""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import TYPE_CHECKING
-
-from switchyard_rust.core import request_type_enum, request_type_value
-
-if TYPE_CHECKING:
- from collections.abc import Mapping
-
- from switchyard_rust.core import ChatRequest, ChatRequestType
-
-
-@dataclass(frozen=True, slots=True, init=False)
-class ProfileRequestMetadata:
- """Request metadata available to Python profile runtimes."""
-
- request_id: str | None
- inbound_format: ChatRequestType | None
- headers: dict[str, list[str]]
-
- def __init__(
- self,
- request_id: str | None = None,
- inbound_format: ChatRequestType | str | None = None,
- headers: Mapping[str, str | list[str]] | None = None,
- ) -> None:
- if request_id is not None and not request_id.strip():
- raise ValueError("invalid request_id: RequestId must not be empty")
- object.__setattr__(self, "request_id", request_id)
- object.__setattr__(
- self,
- "inbound_format",
- request_type_enum(inbound_format) if inbound_format is not None else None,
- )
- object.__setattr__(self, "headers", _normalize_headers(headers))
-
- @classmethod
- def from_headers(
- cls,
- headers: Mapping[str, str | list[str]],
- inbound_format: ChatRequestType | str | None = None,
- ) -> ProfileRequestMetadata:
- """Build metadata from headers and infer the request ID when present."""
- normalized = _normalize_headers(headers)
- request_ids = normalized.get("x-request-id", [])
- return cls(
- request_id=request_ids[0] if request_ids else None,
- inbound_format=inbound_format,
- headers=normalized,
- )
-
- def to_dict(self) -> dict[str, object]:
- """Return a plain dictionary representation."""
- return {
- "request_id": self.request_id,
- "inbound_format": (
- request_type_value(self.inbound_format)
- if self.inbound_format is not None
- else None
- ),
- "headers": {name: list(values) for name, values in self.headers.items()},
- }
-
-
-@dataclass(frozen=True, slots=True, init=False)
-class ProfileInput:
- """Provider request and metadata passed to a Python profile."""
-
- request: ChatRequest
- metadata: ProfileRequestMetadata
-
- def __init__(
- self,
- request: ChatRequest,
- metadata: ProfileRequestMetadata | None = None,
- ) -> None:
- object.__setattr__(self, "request", request)
- object.__setattr__(self, "metadata", metadata or ProfileRequestMetadata())
-
-
-def _normalize_headers(
- headers: Mapping[str, str | list[str]] | None,
-) -> dict[str, list[str]]:
- normalized: dict[str, list[str]] = {}
- for name, value in (headers or {}).items():
- if not isinstance(name, str):
- raise TypeError("ProfileRequestMetadata header names must be strings")
- if isinstance(value, str):
- values = [value]
- elif isinstance(value, list) and all(isinstance(item, str) for item in value):
- values = list(value)
- else:
- raise TypeError(
- "ProfileRequestMetadata headers must map strings to strings or lists of strings"
- )
- normalized.setdefault(name.lower(), []).extend(values)
- return normalized
-
-
-__all__ = ["ProfileInput", "ProfileRequestMetadata"]
diff --git a/switchyard_rust/profiles.pyi b/switchyard_rust/profiles.pyi
deleted file mode 100644
index 984d7568a..000000000
--- a/switchyard_rust/profiles.pyi
+++ /dev/null
@@ -1,35 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-from collections.abc import Mapping
-
-from switchyard_rust.core import ChatRequest, ChatRequestType
-
-class ProfileInput:
- request: ChatRequest
- metadata: ProfileRequestMetadata
-
- def __init__(
- self,
- request: ChatRequest,
- metadata: ProfileRequestMetadata | None = None,
- ) -> None: ...
-
-class ProfileRequestMetadata:
- request_id: str | None
- inbound_format: ChatRequestType | None
- headers: dict[str, list[str]]
-
- def __init__(
- self,
- request_id: str | None = None,
- inbound_format: ChatRequestType | str | None = None,
- headers: Mapping[str, str | list[str]] | None = None,
- ) -> None: ...
- @classmethod
- def from_headers(
- cls,
- headers: Mapping[str, str | list[str]],
- inbound_format: ChatRequestType | str | None = None,
- ) -> ProfileRequestMetadata: ...
- def to_dict(self) -> dict[str, object]: ...
diff --git a/tests/_mock_openai_server.py b/tests/_mock_openai_server.py
deleted file mode 100644
index 0ef05ce29..000000000
--- a/tests/_mock_openai_server.py
+++ /dev/null
@@ -1,137 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Local in-process mock OpenAI upstream shared by the markdown-docs fixtures.
-
-The getting-started and README doc tests both redirect a passthrough profile at
-a loopback server returning a fixed ``chat.completion`` so the guide snippets
-exercise the real backend, openai SDK, and httpx code without touching the
-network. This module holds the single copy both fixtures use: the mock server
-and the ``redirect_passthrough_to_local_mock`` helper that points
-``PassthroughProfileConfig.build`` at it.
-"""
-
-from __future__ import annotations
-
-import dataclasses
-import json
-import threading
-from collections.abc import Iterator
-from contextlib import contextmanager
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-
-import pytest
-
-#: Fixed OpenAI Chat Completion the mock upstream returns for every request.
-_CANNED_COMPLETION: dict[str, object] = {
- "id": "chatcmpl-hermetic",
- "object": "chat.completion",
- "created": 1700000000,
- "model": "hermetic-mock",
- "choices": [
- {
- "index": 0,
- "message": {"role": "assistant", "content": "4"},
- "finish_reason": "stop",
- }
- ],
- "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
-}
-
-
-class _MockOpenAIServer:
- """In-process OpenAI-compatible upstream returning a fixed completion.
-
- Loopback-only ``ThreadingHTTPServer`` so the doc snippets exercise the real
- passthrough backend, openai SDK, and httpx code without touching the network.
- """
-
- def __init__(self) -> None:
- self._server: ThreadingHTTPServer | None = None
- self._thread: threading.Thread | None = None
-
- def __enter__(self) -> _MockOpenAIServer:
- class Handler(BaseHTTPRequestHandler):
- protocol_version = "HTTP/1.1"
-
- def do_POST(self) -> None:
- length = int(self.headers.get("content-length", "0"))
- self.rfile.read(length)
- content = json.dumps(_CANNED_COMPLETION).encode("utf-8")
- self.send_response(200)
- self.send_header("content-type", "application/json")
- self.send_header("content-length", str(len(content)))
- self.send_header("connection", "close")
- self.end_headers()
- self.wfile.write(content)
-
- def log_message(self, _format: str, *_args: object) -> None:
- return None
-
- self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
- self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
- self._thread.start()
- return self
-
- def __exit__(self, *_args: object) -> None:
- if self._server is not None:
- self._server.shutdown()
- self._server.server_close()
- if self._thread is not None:
- self._thread.join(timeout=2)
-
- @property
- def base_url(self) -> str:
- if self._server is None:
- raise RuntimeError("mock server is not running")
- host, port = self._server.server_address
- return f"http://{host}:{port}/v1"
-
-
-def markdown_docs_active(config: pytest.Config) -> bool:
- """Return whether the run enabled the ``--markdown-docs`` flag."""
- try:
- return bool(config.getoption("markdowndocs", default=False))
- except (KeyError, ValueError):
- return False
-
-
-@contextmanager
-def redirect_passthrough_to_local_mock(*, stub_uvicorn: bool = False) -> Iterator[None]:
- """Point ``PassthroughProfileConfig.build`` at a local mock OpenAI upstream.
-
- Starts a loopback :class:`_MockOpenAIServer` and patches the passthrough
- profile's ``build`` so a guide snippet's ``switchyard.call`` runs the real
- backend, openai SDK, and httpx code fully offline, keeping any other fields
- the snippet set on its config. With ``stub_uvicorn=True`` it also replaces
- ``uvicorn.run`` with a no-op so a "host as HTTP server" snippet returns
- instead of blocking the test session forever.
- """
- from switchyard import PassthroughProfileConfig
-
- real_build = PassthroughProfileConfig.build
- monkeypatch = pytest.MonkeyPatch()
- with _MockOpenAIServer() as upstream:
-
- def _hermetic_build(_self: PassthroughProfileConfig) -> object:
- return real_build(
- dataclasses.replace(_self, api_key="test-key", base_url=upstream.base_url)
- )
-
- monkeypatch.setattr(PassthroughProfileConfig, "build", _hermetic_build)
- if stub_uvicorn:
- import uvicorn
-
- monkeypatch.setattr(uvicorn, "run", lambda *_args, **_kwargs: None)
- try:
- yield
- finally:
- monkeypatch.undo()
-
-
-__all__ = [
- "_CANNED_COMPLETION",
- "_MockOpenAIServer",
- "markdown_docs_active",
- "redirect_passthrough_to_local_mock",
-]
diff --git a/tests/e2e/test_evict_reroute_e2e.py b/tests/e2e/test_evict_reroute_e2e.py
deleted file mode 100644
index b620cd8d7..000000000
--- a/tests/e2e/test_evict_reroute_e2e.py
+++ /dev/null
@@ -1,251 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Live evict-and-reroute coverage for every multi-target router.
-
-Each test builds the router's recipe in-process against NVIDIA Inference
-Hub, forces the weak target (a model with a known 131k context cap) to
-overflow with a ~840k-token prompt, and asserts that:
-
-* the inbound request still returns HTTP 200 (the compatibility chain's
- fallback path served the response on the strong target),
-* the response model is the strong target's model id, and
-* the shared :class:`StatsAccumulator` records the failing target as
- ``errors=1, calls=0`` and the fallback target as ``calls=1`` with the
- full prompt-token attribution.
-
-This pins the four moving parts that must all line up for the feature
-to work end-to-end on a live backend — provider-shape 400 detection,
-the compatibility chain's retry, the Python ↔ Rust exception surface,
-and per-target stats attribution.
-
-Prerequisites:
- - ``NVIDIA_API_KEY`` env var (set; tests skip otherwise).
-
-Run with::
-
- NVIDIA_API_KEY=nvapi-... uv run pytest tests/e2e/test_evict_reroute_e2e.py -v
-"""
-
-from __future__ import annotations
-
-import os
-from collections.abc import AsyncIterator
-
-import httpx
-import pytest
-
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
-from switchyard.lib.stats_accumulator import StatsAccumulator
-from switchyard.server.switchyard_app import build_switchyard_app
-
-_STRONG_MODEL = "aws/anthropic/bedrock-claude-opus-4-7"
-_WEAK_MODEL = "nvidia/nvidia/nemotron-3-super-120b-long-ctx"
-_BASE_URL = "https://inference-api.nvidia.com/v1"
-# ~840k tokens — well beyond Nemotron-long-ctx's 131k cap, well within
-# Opus 4.7's window, so the eviction is guaranteed and the fallback
-# call actually succeeds.
-_OVERFLOW_TEXT = "lorem ipsum dolor sit amet " * 60_000
-
-
-def nvidia_api_key() -> str:
- # Live tests against NVIDIA Inference Hub — skip in CI without a real key.
- # OPENAI_API_KEY is intentionally NOT a fallback; CI sets it to a dummy
- # value for the passthrough e2e tests and the dummy would 401 here.
- key = os.environ.get("NVIDIA_API_KEY")
- if not key:
- pytest.skip("NVIDIA_API_KEY not set — required for live evict-reroute e2e")
- return key
-
-
-def _strong_target(api_key: str) -> LlmTarget:
- return LlmTarget(
- id="strong",
- model=_STRONG_MODEL,
- format=BackendFormat.ANTHROPIC,
- api_key=api_key,
- base_url=_BASE_URL,
- )
-
-
-def _weak_target(api_key: str) -> LlmTarget:
- return LlmTarget(
- id="weak",
- model=_WEAK_MODEL,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=_BASE_URL,
- )
-
-
-async def _client_for(app) -> AsyncIterator[httpx.AsyncClient]:
- transport = httpx.ASGITransport(app=app)
- async with httpx.AsyncClient(
- transport=transport,
- base_url="http://test",
- timeout=600.0,
- ) as client:
- yield client
-
-
-def _assert_eviction_stats(snapshot: dict) -> None:
- """Pin the per-target attribution shape after a successful reroute."""
- weak = snapshot["models"][_WEAK_MODEL]
- strong = snapshot["models"][_STRONG_MODEL]
- assert weak["errors"] == 1
- assert weak["calls"] == 0
- assert strong["calls"] == 1
- assert strong["errors"] == 0
- assert strong["prompt_tokens"] > 100_000
-
-
-async def test_stage_router_evicts_weak_and_reroutes_to_strong(nvidia_api_key: str) -> None:
- """``type: stage_router`` reroutes to ``strong`` when weak overflows."""
- from switchyard.lib.profiles import (
- ProfileSwitchyard,
- StageRouterConfig,
- StageRouterProfileConfig,
- )
-
- stats = StatsAccumulator()
- config = StageRouterConfig(
- capable=_strong_target(nvidia_api_key),
- efficient=_weak_target(nvidia_api_key),
- picker="efficient_first",
- confidence_threshold=0.0,
- fallback_target_on_evict="strong",
- )
- switchyard = ProfileSwitchyard(
- StageRouterProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats,
- enable_stats=config.enable_stats,
- )
- )
- app = build_switchyard_app(switchyard)
-
- async for client in _client_for(app):
- response = await client.post(
- "/v1/chat/completions",
- json={
- "model": "stage_router-test",
- "messages": [{"role": "user", "content": _OVERFLOW_TEXT}],
- "max_tokens": 16,
- },
- )
-
- assert response.status_code == 200, response.text
- body = response.json()
- assert body["model"] == _STRONG_MODEL
- _assert_eviction_stats(await stats.snapshot())
-
-
-async def test_random_routing_evicts_weak_and_reroutes_to_strong(
- nvidia_api_key: str,
-) -> None:
- """``type: random_routing`` reroutes when the coin-picked weak overflows.
-
- ``strong_probability=0.0`` pins the first pick on weak so the test is
- deterministic; the compatibility chain then rewrites to ``strong``.
- """
- from switchyard.lib.profiles import (
- ProfileSwitchyard,
- RandomRoutingConfig,
- RandomRoutingProfileConfig,
- )
-
- stats = StatsAccumulator()
- config = RandomRoutingConfig(
- strong=_strong_target(nvidia_api_key),
- weak=_weak_target(nvidia_api_key),
- strong_probability=0.0,
- fallback_target_on_evict="strong",
- rng_seed=1,
- )
- switchyard = ProfileSwitchyard(
- RandomRoutingProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats,
- enable_stats=config.enable_stats,
- )
- )
- app = build_switchyard_app(switchyard)
-
- async for client in _client_for(app):
- response = await client.post(
- "/v1/chat/completions",
- json={
- "model": "random-test",
- "messages": [{"role": "user", "content": _OVERFLOW_TEXT}],
- "max_tokens": 16,
- },
- )
-
- assert response.status_code == 200, response.text
- assert response.json()["model"] == _STRONG_MODEL
- _assert_eviction_stats(await stats.snapshot())
-
-
-async def test_deterministic_evicts_weak_and_reroutes_to_strong(
- nvidia_api_key: str,
-) -> None:
- """``type: deterministic`` reroutes when the classifier-picked weak overflows.
-
- Validates the Python-LLMBackend exception surface: the
- inner native Rust backend raises ``ContextWindowExceeded``, the
- Python wrapper (``DeterministicRoutingLLMBackend``) propagates the
- typed exception, and the compatibility chain catches the preserved
- variant to trigger the retry.
- """
- from switchyard.lib.profiles import (
- DeterministicRoutingConfig,
- DeterministicRoutingProfileConfig,
- ProfileSwitchyard,
- )
-
- stats = StatsAccumulator()
- classifier = LlmTarget(
- id="classifier",
- model="nvidia/deepseek-ai/deepseek-v4-flash",
- format=BackendFormat.OPENAI,
- api_key=nvidia_api_key,
- base_url=_BASE_URL,
- )
- config = DeterministicRoutingConfig(
- strong=_strong_target(nvidia_api_key),
- weak=_weak_target(nvidia_api_key),
- classifier=classifier,
- profile_name="coding_agent",
- classifier_min_confidence=0.0,
- classifier_fail_open=True,
- fallback_target_on_evict="strong",
- )
- switchyard = ProfileSwitchyard(
- DeterministicRoutingProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats,
- enable_stats=config.enable_stats,
- )
- )
- app = build_switchyard_app(switchyard)
-
- async for client in _client_for(app):
- response = await client.post(
- "/v1/chat/completions",
- json={
- "model": "deterministic-test",
- "messages": [{"role": "user", "content": _OVERFLOW_TEXT}],
- "max_tokens": 16,
- },
- )
-
- assert response.status_code == 200, response.text
- assert response.json()["model"] == _STRONG_MODEL
- _assert_eviction_stats(await stats.snapshot())
diff --git a/tests/e2e/test_llm_classifier_e2e.py b/tests/e2e/test_llm_classifier_e2e.py
deleted file mode 100644
index 0748b6aee..000000000
--- a/tests/e2e/test_llm_classifier_e2e.py
+++ /dev/null
@@ -1,338 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Real-backend e2e tests for the LLM classifier routing chain.
-
-Mirrors :mod:`tests.test_llm_classifier_e2e` (which mocks all HTTPS via
-``respx``), but with no mocking — every call hits the selected live backend::
-
- inbound ChatRequest
- -> LLMClassifierRequestProcessor -> real classifier LLM call
- -> SignalTierSelectorRequestProcessor
- -> DeterministicRoutingLLMBackend
- -> per-tier OpenAiPassthroughBackend -> real tier LLM call
- -> ChatResponse
-
-Sibling pattern to :mod:`tests.e2e.test_random_routing_llm_backend`:
-in-process — drive the processors + backend manually rather than via
-``Switchyard.call()`` so we get back the typed
-:class:`ChatResponse` (the translator would unwrap it to the
-SDK shape). No subprocess and no HTTP transport — overkill for
-verifying the experimental chain against a live LLM.
-
-Because the classifier's output is non-deterministic, tests do not
-assert *which* tier was picked. They assert:
-
-* the classifier ran (RouteSignals stamped on context)
-* a known tier was picked and a real backend produced a response
-* the response is well-formed (non-empty content or reasoning)
-* tier-specific model rewriting happened on the request
-
-Prerequisites:
- - ``OPENROUTER_API_KEY`` or ``NVIDIA_API_KEY`` env var (skips otherwise).
-
-Run with::
-
- OPENROUTER_API_KEY=sk-or-... uv run pytest tests/e2e/test_llm_classifier_e2e.py -v
-"""
-
-from __future__ import annotations
-
-import logging
-from typing import Any
-
-import pytest
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- CTX_DETERMINISTIC_ROUTING_TIER,
- DeterministicRoutingLLMBackend,
-)
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
-from switchyard.lib.chat_response.base import ChatResponse
-from switchyard.lib.processors.llm_classifier import (
- CTX_DETERMINISTIC_ROUTE_SIGNALS,
- CTX_DETERMINISTIC_TIER_DECISION,
- LLMClassifierConfig,
- LLMClassifierRequestProcessor,
- RouteSignals,
- SignalTierSelectorConfig,
- SignalTierSelectorRequestProcessor,
- TierSelectionDecision,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.core import ChatRequest
-
-from .conftest import get_nvidia_config
-
-# Hard timeout — if the live backend hangs we want a clean fail.
-# Two real LLM calls per test (classifier + tier backend); 120s budget.
-pytestmark = [
- pytest.mark.integration,
- pytest.mark.timeout(120),
-]
-
-logger = logging.getLogger("e2e.llm_classifier")
-
-HTTP_TIMEOUT_S = 60.0
-
-_nvidia = get_nvidia_config()
-_skip_reason = "OPENROUTER_API_KEY or NVIDIA_API_KEY not set"
-
-BACKEND_BASE_URL = _nvidia["base_url"]
-
-# Same model for the classifier and every tier — this test validates
-# the end-to-end wiring, not routing intelligence. Using one model
-# keeps the test fast and lets us pin a model known to support
-# ``response_format={"type": "json_object"}`` on the selected backend.
-#
-# All four ``RouteTier`` labels are registered as tiers so that
-# whichever ``recommended_tier`` the live classifier picks is always
-# a valid backend label — without this, e.g. a ``reasoning`` pick
-# would silently fall back to the default tier and we'd lose the
-# real-world coverage of the dispatch path.
-CLASSIFIER_MODEL = _nvidia["model"]
-TIER_MODEL = _nvidia["model"]
-TIER_LABELS = ("simple", "medium", "complex", "reasoning")
-
-
-# ---------------------------------------------------------------------------
-# Helpers — build the chain by hand. There's no recipe yet for the
-# experimental classifier routing chain, so tests assemble the four
-# roles directly (mirroring how a downstream integrator would).
-# ---------------------------------------------------------------------------
-
-
-def _build_chain(
- *,
- classifier_base_url: str = BACKEND_BASE_URL,
- classifier_api_key: str | None = None,
- classifier_timeout_s: float = HTTP_TIMEOUT_S,
-) -> tuple[list[Any], DeterministicRoutingLLMBackend]:
- """Build (processors, backend) for the classifier routing chain."""
- classifier = LLMClassifierRequestProcessor(
- LLMClassifierConfig(
- model=CLASSIFIER_MODEL,
- api_key=classifier_api_key or _nvidia["api_key"],
- base_url=classifier_base_url,
- timeout_s=classifier_timeout_s,
- ),
- )
- tier_selector = SignalTierSelectorRequestProcessor(
- SignalTierSelectorConfig(default_tier="simple"),
- )
- backend = DeterministicRoutingLLMBackend.from_tiers(
- tiers={
- label: LlmTarget(
- model=TIER_MODEL,
- format=BackendFormat.OPENAI,
- api_key=_nvidia["api_key"],
- base_url=BACKEND_BASE_URL,
- timeout=HTTP_TIMEOUT_S,
- )
- for label in TIER_LABELS
- },
- default_tier="simple",
- )
- return [classifier, tier_selector], backend
-
-
-async def _call_through_chain(
- processors: list[Any],
- backend: DeterministicRoutingLLMBackend,
- request: ChatRequest,
-) -> tuple[ProxyContext, ChatResponse]:
- """Run a request through the processors and the backend in-process.
-
- Skips terminal translation on purpose: returning the typed
- :class:`ChatResponse` makes assertions cleaner than
- re-parsing the SDK-shape ``openai.types.chat.ChatCompletion``.
- """
- ctx = ProxyContext()
- for proc in processors:
- request = await proc.process(ctx, request)
- response = await backend.call(ctx, request)
- return ctx, response
-
-
-def _chat_request(*, user_message: str, **body_overrides: Any) -> ChatRequest:
- body: dict = {
- "model": "placeholder-rewritten-by-router",
- "messages": [{"role": "user", "content": user_message}],
- "max_tokens": 2048,
- }
- body.update(body_overrides)
- return ChatRequest.openai_chat(body)
-
-
-def _assert_chain_metadata(ctx: ProxyContext) -> None:
- """Common post-call assertions on what the chain stamped onto ctx."""
- signals = ctx.metadata.get(CTX_DETERMINISTIC_ROUTE_SIGNALS)
- assert isinstance(signals, RouteSignals), (
- f"classifier did not stamp RouteSignals; metadata keys: "
- f"{list(ctx.metadata.keys())}"
- )
- # Pydantic enforces ``0 <= confidence <= 1`` at parse time, so a
- # parsed RouteSignals already satisfies the range — but we
- # re-assert as a guard against future schema drift.
- assert 0.0 <= signals.confidence <= 1.0
-
- decision = ctx.metadata.get(CTX_DETERMINISTIC_TIER_DECISION)
- assert isinstance(decision, TierSelectionDecision), (
- f"tier selector did not stamp TierSelectionDecision; "
- f"metadata keys: {list(ctx.metadata.keys())}"
- )
-
- tier = ctx.metadata.get(CTX_DETERMINISTIC_ROUTING_TIER)
- assert tier in set(TIER_LABELS), f"unexpected tier picked: {tier!r}"
- assert decision.tier == tier
-
- actual_model = ctx.selected_model
- assert actual_model == TIER_MODEL, f"unexpected actual model: {actual_model!r}"
-
-
-def _assistant_text(response: ChatResponse) -> tuple[str, str]:
- choice = response.body["choices"][0]
- message = choice.get("message", {})
- content = message.get("content") or ""
- reasoning = str(message.get("reasoning_content") or message.get("reasoning") or "")
- return content, reasoning
-
-
-def _finish_reason(response: ChatResponse) -> object:
- return response.body["choices"][0].get("finish_reason")
-
-
-# ---------------------------------------------------------------------------
-# Tests
-# ---------------------------------------------------------------------------
-
-
[email protected](not _nvidia["api_key"], reason=_skip_reason)
-class TestLLMClassifierLiveE2E:
- """Real classifier + real tier backend via the selected live backend."""
-
- async def test_simple_ask_routes_and_returns_content(self) -> None:
- """A trivial ask round-trips: classifier runs, a tier serves it.
-
- Does not assert *which* tier is picked — the classifier is a
- real LLM whose output varies. Asserts the wiring: signals
- stamped, tier decision recorded, response well-formed.
- """
- processors, backend = _build_chain()
- request = _chat_request(user_message="hi")
-
- ctx, response = await _call_through_chain(processors, backend, request)
-
- assert isinstance(response, ChatResponse), type(response)
- _assert_chain_metadata(ctx)
-
- content, reasoning = _assistant_text(response)
- assert content or reasoning, (
- f"expected non-empty assistant text from live backend; "
- f"finish_reason={_finish_reason(response)!r}"
- )
-
- signals = ctx.metadata[CTX_DETERMINISTIC_ROUTE_SIGNALS]
- tier = ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER]
- actual_model = ctx.selected_model
- logger.info(
- " [classifier-e2e][simple] tier=%s model=%s "
- "task=%s recommended=%s confidence=%.2f abstain=%s "
- "content[:80]=%r",
- tier, actual_model,
- signals.task_type.value, signals.recommended_tier.value,
- signals.confidence, signals.abstain,
- content[:80],
- )
-
- async def test_complex_ask_routes_and_returns_content(self) -> None:
- """A multi-step debugging ask round-trips through the chain.
-
- Different prompt from the simple test, exercises the classifier
- on a harder request. Same well-formedness assertions: we don't
- require any specific tier choice, only that the chain completes
- cleanly and the picked tier produces text.
- """
- processors, backend = _build_chain()
- request = _chat_request(
- user_message=(
- "I'm seeing a Python traceback ending in "
- "`KeyError: 'session_id'` from a Flask endpoint. "
- "Walk me through how you'd narrow down whether the "
- "key is missing because of a race condition with "
- "session writes or a misconfigured middleware order. "
- "Then propose one focused diagnostic to confirm it."
- ),
- )
-
- ctx, response = await _call_through_chain(processors, backend, request)
-
- assert isinstance(response, ChatResponse), type(response)
- _assert_chain_metadata(ctx)
-
- content, reasoning = _assistant_text(response)
- assert content or reasoning, (
- f"expected non-empty assistant text from live backend; "
- f"finish_reason={_finish_reason(response)!r}"
- )
-
- signals = ctx.metadata[CTX_DETERMINISTIC_ROUTE_SIGNALS]
- tier = ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER]
- logger.info(
- " [classifier-e2e][complex] tier=%s "
- "task=%s recommended=%s confidence=%.2f reason=%s "
- "content[:80]=%r",
- tier,
- signals.task_type.value, signals.recommended_tier.value,
- signals.confidence, signals.reason_code.value,
- content[:80],
- )
-
- async def test_classifier_failure_fails_open_to_default_tier(self) -> None:
- """A broken classifier upstream must not break inference.
-
- Points the classifier at an unresolvable host while keeping
- both tier backends live. ``fail_open=True`` is the configured
- default, so the chain should stamp abstain signals, fall back
- to the default tier (``simple`` here), and still return real
- content from the live tier backend.
- """
- # Short classifier timeout so we don't burn the whole
- # pytest-timeout budget waiting for the bogus URL to fail.
- processors, backend = _build_chain(
- classifier_base_url="https://classifier-does-not-exist.invalid/v1",
- classifier_api_key="not-used",
- classifier_timeout_s=5.0,
- )
- request = _chat_request(user_message="say hi briefly")
-
- ctx, response = await _call_through_chain(processors, backend, request)
-
- assert isinstance(response, ChatResponse), type(response)
-
- signals = ctx.metadata[CTX_DETERMINISTIC_ROUTE_SIGNALS]
- assert isinstance(signals, RouteSignals)
- assert signals.abstain is True, (
- f"expected abstain after classifier failure; got "
- f"abstain={signals.abstain}, confidence={signals.confidence}"
- )
-
- decision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert decision.source == "abstain", decision
- tier = ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER]
- assert tier == "simple", (
- f"expected fallback to default tier 'simple', got {tier!r}"
- )
-
- content, reasoning = _assistant_text(response)
- assert content or reasoning, (
- f"expected non-empty assistant text from live default-tier "
- f"backend after classifier failure; "
- f"finish_reason={_finish_reason(response)!r}"
- )
-
- logger.info(
- " [classifier-e2e][fail-open] tier=%s decision_source=%s "
- "content[:80]=%r",
- tier, decision.source, content[:80],
- )
diff --git a/tests/test_anthropic_cache_breakpoint_backend.py b/tests/test_anthropic_cache_breakpoint_backend.py
deleted file mode 100644
index f7360dc73..000000000
--- a/tests/test_anthropic_cache_breakpoint_backend.py
+++ /dev/null
@@ -1,260 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for :class:`AnthropicCacheBreakpointBackend`.
-
-The wrapper has two responsibilities:
-
-1. **Translate** the inbound request to Anthropic Messages format
- (no-op if already Anthropic).
-2. **Inject** ``cache_control: {"type": "ephemeral"}`` markers on
- the system field's last text block and the last message's last
- block.
-
-It then delegates to an inner :class:`LLMBackend`. These tests use
-a fake inner backend to capture exactly what got passed through so
-the markers + format change can be asserted without touching the
-network.
-"""
-
-from __future__ import annotations
-
-from typing import Any, cast
-
-from switchyard.lib.backends.anthropic_cache_breakpoint_backend import (
- AnthropicCacheBreakpointBackend,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.roles import LLMBackend
-from switchyard_rust.core import ChatRequest, ChatResponse
-
-# ---------------------------------------------------------------------------
-# Test fixtures
-# ---------------------------------------------------------------------------
-
-
-class _CapturingBackend(LLMBackend):
- """Records the request handed to it and returns a canned response."""
-
- def __init__(self) -> None:
- self.calls: list[ChatRequest] = []
- self.startup_count = 0
- self.shutdown_count = 0
-
- async def startup(self) -> None:
- self.startup_count += 1
-
- async def shutdown(self) -> None:
- self.shutdown_count += 1
-
- async def call(
- self,
- ctx: ProxyContext, # noqa: ARG002
- request: ChatRequest,
- ) -> ChatResponse:
- self.calls.append(request)
- return ChatResponse.anthropic_completion(cast(Any, {
- "id": "msg_test",
- "type": "message",
- "role": "assistant",
- "content": [{"type": "text", "text": "ok"}],
- "model": "test",
- "stop_reason": "end_turn",
- "usage": {"input_tokens": 1, "output_tokens": 1},
- }))
-
-
-def _openai_chat(messages: list[dict[str, Any]]) -> ChatRequest:
- return ChatRequest.openai_chat(cast(Any, {
- "model": "placeholder",
- "messages": messages,
- }))
-
-
-def _anthropic(messages: list[dict[str, Any]], system: Any = None) -> ChatRequest:
- body: dict[str, Any] = {
- "model": "claude-test",
- "max_tokens": 128,
- "messages": messages,
- }
- if system is not None:
- body["system"] = system
- return ChatRequest.anthropic(cast(Any, body))
-
-
-# ---------------------------------------------------------------------------
-# Translation + marker placement
-# ---------------------------------------------------------------------------
-
-
-class TestOpenAIChatInbound:
- """Inbound OpenAI Chat ⇒ wrapper translates + marks before delegating."""
-
- async def test_inner_receives_anthropic_typed_request(self) -> None:
- inner = _CapturingBackend()
- wrapper = AnthropicCacheBreakpointBackend(inner)
- request = _openai_chat([
- {"role": "system", "content": "stable system prompt"},
- {"role": "user", "content": "do the thing"},
- ])
- await wrapper.call(ProxyContext(), request)
- assert len(inner.calls) == 1
- forwarded = inner.calls[0]
- assert forwarded.request_type.value == "anthropic"
-
- async def test_system_field_gets_ephemeral_marker(self) -> None:
- inner = _CapturingBackend()
- wrapper = AnthropicCacheBreakpointBackend(inner)
- request = _openai_chat([
- {"role": "system", "content": "stable system prompt"},
- {"role": "user", "content": "do the thing"},
- ])
- await wrapper.call(ProxyContext(), request)
- body = cast(dict[str, Any], inner.calls[0].body)
- system = body.get("system")
- assert isinstance(system, list), f"want list-shape system; got {system!r}"
- # Last text block carries the marker
- marker = None
- for blk in reversed(system):
- if isinstance(blk, dict) and blk.get("type") == "text":
- marker = blk
- break
- assert marker is not None
- assert marker["cache_control"] == {"type": "ephemeral"}
- assert "stable system prompt" in marker["text"]
-
- async def test_last_message_gets_rolling_marker(self) -> None:
- inner = _CapturingBackend()
- wrapper = AnthropicCacheBreakpointBackend(inner)
- request = _openai_chat([
- {"role": "user", "content": "first user message"},
- {"role": "assistant", "content": "midway response"},
- {"role": "user", "content": "the most recent turn"},
- ])
- await wrapper.call(ProxyContext(), request)
- body = cast(dict[str, Any], inner.calls[0].body)
- messages = body["messages"]
- last_content = messages[-1]["content"]
- assert isinstance(last_content, list), (
- f"last message content should be a list of typed blocks; got {last_content!r}"
- )
- marker = None
- for blk in reversed(last_content):
- if isinstance(blk, dict) and blk.get("type") == "text":
- marker = blk
- break
- assert marker is not None
- assert marker["cache_control"] == {"type": "ephemeral"}
-
- async def test_chain_level_request_unchanged(self) -> None:
- """The wrapper must not mutate the outer request.
-
- The terminal :class:`TranslationEngine` step uses
- ``request.request_type`` to pick the response format for the
- client. If we mutated the outer request_type to Anthropic,
- OpenAI Chat clients (Harbor's LiteLLM) would receive
- Anthropic-shape responses and crash.
- """
- inner = _CapturingBackend()
- wrapper = AnthropicCacheBreakpointBackend(inner)
- request = _openai_chat([
- {"role": "user", "content": "do the thing"},
- ])
- original_type = request.request_type
- original_body_snapshot = dict(cast(dict[str, Any], request.body))
- await wrapper.call(ProxyContext(), request)
- # Outer request must be untouched — only a *local* Anthropic
- # request was constructed for the inner backend.
- assert request.request_type is original_type
- assert dict(cast(dict[str, Any], request.body)) == original_body_snapshot
-
-
-class TestAnthropicInbound:
- """Inbound Anthropic ⇒ wrapper skips translation, marks body in place."""
-
- async def test_already_anthropic_short_circuits_engine(self) -> None:
- inner = _CapturingBackend()
- wrapper = AnthropicCacheBreakpointBackend(inner)
- request = _anthropic(
- messages=[{"role": "user", "content": "task"}],
- system="stable system",
- )
- await wrapper.call(ProxyContext(), request)
- # Engine was never built since translation was unnecessary
- assert wrapper._engine is None
- # System still got marked
- body = cast(dict[str, Any], inner.calls[0].body)
- system = body["system"]
- assert isinstance(system, list)
- assert system[-1]["cache_control"] == {"type": "ephemeral"}
-
- async def test_last_message_string_content_gets_marker(self) -> None:
- inner = _CapturingBackend()
- wrapper = AnthropicCacheBreakpointBackend(inner)
- request = _anthropic(
- messages=[{"role": "user", "content": "task"}],
- )
- await wrapper.call(ProxyContext(), request)
- body = cast(dict[str, Any], inner.calls[0].body)
- last_content = body["messages"][-1]["content"]
- assert isinstance(last_content, list)
- assert last_content[-1]["cache_control"] == {"type": "ephemeral"}
-
- async def test_client_supplied_markers_pass_through_untouched(self) -> None:
- """A client that already set cache_control (Claude Code) is left as-is.
-
- Stacking our two breakpoints on top of the client's own could push the
- request past Anthropic's four-breakpoint cap, so the wrapper must not
- re-mark a body that already carries markers.
- """
- inner = _CapturingBackend()
- wrapper = AnthropicCacheBreakpointBackend(inner)
- request = _anthropic(
- messages=[
- {
- "role": "user",
- "content": [
- {
- "type": "text",
- "text": "task",
- "cache_control": {"type": "ephemeral"},
- },
- ],
- },
- ],
- system=[
- {
- "type": "text",
- "text": "stable system",
- "cache_control": {"type": "ephemeral"},
- },
- ],
- )
- original_body = dict(cast(dict[str, Any], request.body))
- await wrapper.call(ProxyContext(), request)
- # No translation engine built, and the inner backend saw the body
- # verbatim — we did not add a third/fourth breakpoint.
- assert wrapper._engine is None
- forwarded = cast(dict[str, Any], inner.calls[0].body)
- assert forwarded == original_body
-
-
-# ---------------------------------------------------------------------------
-# Lifecycle forwarding
-# ---------------------------------------------------------------------------
-
-
-class TestLifecycle:
- """startup / shutdown forward to inner."""
-
- async def test_startup_forwards(self) -> None:
- inner = _CapturingBackend()
- wrapper = AnthropicCacheBreakpointBackend(inner)
- await wrapper.startup()
- assert inner.startup_count == 1
-
- async def test_shutdown_forwards(self) -> None:
- inner = _CapturingBackend()
- wrapper = AnthropicCacheBreakpointBackend(inner)
- await wrapper.shutdown()
- assert inner.shutdown_count == 1
diff --git a/tests/test_deterministic_route_signals.py b/tests/test_deterministic_route_signals.py
deleted file mode 100644
index 53d944238..000000000
--- a/tests/test_deterministic_route_signals.py
+++ /dev/null
@@ -1,94 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for deterministic routing signal types."""
-
-from __future__ import annotations
-
-import pytest
-from pydantic import ValidationError
-
-from switchyard.lib.processors.llm_classifier import (
- CTX_DETERMINISTIC_ROUTE_SIGNALS,
- Complexity,
- ContextDependency,
- PrecisionRequirement,
- ReasonCode,
- ReasoningDepth,
- RiskLevel,
- RouteSignals,
- RouteTier,
- TaskType,
-)
-
-
-def test_route_signals_accepts_minimal_mvp_payload() -> None:
- signals = RouteSignals(
- task_type=TaskType.CODING,
- complexity=Complexity.COMPLEX,
- reasoning_depth=ReasoningDepth.MULTI_STEP,
- tool_planning_required=True,
- precision_requirement=PrecisionRequirement.HIGH,
- context_dependency=ContextDependency.CONVERSATION,
- structured_output_risk=RiskLevel.MEDIUM,
- recommended_tier=RouteTier.COMPLEX,
- confidence=0.82,
- reason_code=ReasonCode.CODING_COMPLEX,
- )
-
- assert signals.task_type is TaskType.CODING
- assert signals.recommended_tier is RouteTier.COMPLEX
- assert signals.abstain is False
- assert CTX_DETERMINISTIC_ROUTE_SIGNALS == "_deterministic_route_signals"
-
-
-def test_route_signals_coerce_json_friendly_strings_to_enums() -> None:
- signals = RouteSignals(
- task_type="summarization",
- complexity="simple",
- reasoning_depth="light",
- tool_planning_required=False,
- precision_requirement="medium",
- context_dependency="latest_message",
- structured_output_risk="low",
- recommended_tier="simple",
- confidence=0.91,
- reason_code="summarization",
- )
-
- assert signals.task_type is TaskType.SUMMARIZATION
- assert signals.reason_code is ReasonCode.SUMMARIZATION
-
-
-def test_route_signals_reject_invalid_confidence() -> None:
- with pytest.raises(ValidationError):
- RouteSignals(
- task_type=TaskType.CHAT,
- complexity=Complexity.SIMPLE,
- reasoning_depth=ReasoningDepth.NONE,
- tool_planning_required=False,
- precision_requirement=PrecisionRequirement.LOW,
- context_dependency=ContextDependency.LATEST_MESSAGE,
- structured_output_risk=RiskLevel.LOW,
- recommended_tier=RouteTier.SIMPLE,
- confidence=1.5,
- reason_code=ReasonCode.SIMPLE_QA,
- )
-
-
-def test_route_signals_are_immutable() -> None:
- signals = RouteSignals(
- task_type=TaskType.CHAT,
- complexity=Complexity.SIMPLE,
- reasoning_depth=ReasoningDepth.NONE,
- tool_planning_required=False,
- precision_requirement=PrecisionRequirement.LOW,
- context_dependency=ContextDependency.LATEST_MESSAGE,
- structured_output_risk=RiskLevel.LOW,
- recommended_tier=RouteTier.SIMPLE,
- confidence=0.7,
- reason_code=ReasonCode.SIMPLE_QA,
- )
-
- with pytest.raises(ValidationError):
- signals.confidence = 0.2
diff --git a/tests/test_deterministic_routing_cache_wrapping.py b/tests/test_deterministic_routing_cache_wrapping.py
deleted file mode 100644
index dd3cbf4a0..000000000
--- a/tests/test_deterministic_routing_cache_wrapping.py
+++ /dev/null
@@ -1,124 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""The deterministic profile wraps Anthropic tiers for prompt caching.
-
-An OpenAI-shaped harness (Codex, OpenAI SDK) routed onto a Claude tier never
-sends Anthropic ``cache_control`` markers, so without the wrapper the
-translated ``/v1/messages`` request misses caching entirely. The profile must
-wrap any tier that resolves to Anthropic format in
-:class:`AnthropicCacheBreakpointBackend`, and leave OpenAI tiers bare.
-"""
-
-from __future__ import annotations
-
-from switchyard.lib.backends.anthropic_cache_breakpoint_backend import (
- AnthropicCacheBreakpointBackend,
-)
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- DeterministicRoutingLLMBackend,
-)
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
-from switchyard.lib.backends.stats_llm_backend import StatsLlmBackend
-from switchyard.lib.profiles.deterministic_routing_config import DeterministicRoutingConfig
-from switchyard.lib.profiles.deterministic_routing_profile_config import (
- DeterministicRoutingProfileConfig,
-)
-
-
-def _config(
- *,
- strong_format: BackendFormat,
- weak_format: BackendFormat = BackendFormat.OPENAI,
- enable_stats: bool = False,
-) -> DeterministicRoutingConfig:
- # Weak model follows weak_format so we can exercise an Anthropic *weak*
- # tier (e.g. Opus as the weak model) as well as the usual OpenAI one.
- weak_model = (
- "aws/anthropic/bedrock-claude-opus-4-7"
- if weak_format == BackendFormat.ANTHROPIC
- else "azure/openai/gpt-5.5"
- )
- return DeterministicRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model="aws/anthropic/bedrock-claude-opus-4-7",
- format=strong_format,
- api_key="sk-test",
- base_url="https://example.invalid/v1",
- ),
- weak=LlmTarget(
- id="weak",
- model=weak_model,
- format=weak_format,
- api_key="sk-test",
- base_url="https://example.invalid/v1",
- ),
- classifier=LlmTarget(
- id="classifier",
- model="gcp/google/gemini-3.5-flash",
- format=BackendFormat.OPENAI,
- api_key="sk-test",
- base_url="https://example.invalid/v1",
- ),
- profile_name="coding_agent", # type: ignore[arg-type]
- classifier_min_confidence=0.0,
- enable_stats=enable_stats,
- fallback_target_on_evict="strong",
- )
-
-
-def _backend(config: DeterministicRoutingConfig) -> DeterministicRoutingLLMBackend:
- """Build the deterministic profile and return its dispatch backend."""
- profile = (
- DeterministicRoutingProfileConfig.from_config(config)
- .build()
- .with_runtime_components(enable_stats=config.enable_stats)
- )
- backend = next(
- component for component in profile.iter_components()
- if isinstance(component, DeterministicRoutingLLMBackend)
- )
- return backend
-
-
-def test_anthropic_strong_tier_is_cache_wrapped() -> None:
- backend = _backend(_config(strong_format=BackendFormat.ANTHROPIC))
- assert isinstance(backend._backends["strong"], AnthropicCacheBreakpointBackend)
-
-
-def test_openai_weak_tier_is_not_wrapped() -> None:
- backend = _backend(_config(strong_format=BackendFormat.ANTHROPIC))
- assert not isinstance(
- backend._backends["weak"], AnthropicCacheBreakpointBackend,
- )
-
-
-def test_anthropic_weak_tier_is_cache_wrapped() -> None:
- # Opus as the *weak* model (format=auto/anthropic) is wrapped too — the
- # fix keys on resolved format, not the strong/weak tier role.
- backend = _backend(
- _config(
- strong_format=BackendFormat.OPENAI,
- weak_format=BackendFormat.ANTHROPIC,
- ),
- )
- assert isinstance(backend._backends["weak"], AnthropicCacheBreakpointBackend)
- assert not isinstance(
- backend._backends["strong"], AnthropicCacheBreakpointBackend,
- )
-
-
-def test_openai_strong_tier_is_not_wrapped() -> None:
- backend = _backend(_config(strong_format=BackendFormat.OPENAI))
- assert not isinstance(
- backend._backends["strong"], AnthropicCacheBreakpointBackend,
- )
-
-
-def test_cache_wrap_is_outermost_with_stats_enabled() -> None:
- # StatsLlmBackend requires a Rust-native inner, so the Python cache wrapper
- # must sit outside it — build must succeed and the wrapper be the outer.
- backend = _backend(_config(strong_format=BackendFormat.ANTHROPIC, enable_stats=True))
- assert isinstance(backend._backends["strong"], AnthropicCacheBreakpointBackend)
- assert isinstance(backend._backends["strong"]._inner, StatsLlmBackend)
diff --git a/tests/test_deterministic_routing_profile.py b/tests/test_deterministic_routing_profile.py
deleted file mode 100644
index eda118192..000000000
--- a/tests/test_deterministic_routing_profile.py
+++ /dev/null
@@ -1,674 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for profile-backed deterministic routing construction."""
-
-from __future__ import annotations
-
-from typing import Any
-
-import pytest
-from openai.types.chat import ChatCompletionChunk
-from pydantic import ValidationError
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- CTX_DETERMINISTIC_ROUTING_TIER,
- DeterministicRoutingLLMBackend,
-)
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
-from switchyard.lib.chat_response.openai_chat import ResponseStream
-from switchyard.lib.processors.llm_classifier import (
- LLMClassifierRequestProcessor,
- SignalTierSelectorRequestProcessor,
-)
-from switchyard.lib.processors.stats_request_processor import StatsRequestProcessor
-from switchyard.lib.processors.stats_response_processor_accumulator import (
- StatsResponseProcessor,
-)
-from switchyard.lib.profiles import (
- DeterministicRoutingConfig,
- DeterministicRoutingPresets,
- DeterministicRoutingProfileConfig,
- ProfileSwitchyard,
-)
-from switchyard.lib.profiles.tier_target_builders import (
- apply_deepseek_overrides,
- apply_default_tier_timeout,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.roles import LLMBackend
-from switchyard.lib.route_table_builders import deterministic_routing_virtual_model_id
-from switchyard.lib.stats_accumulator import StatsAccumulator
-from switchyard_rust.core import (
- ChatRequest,
- ChatRequestType,
- ChatResponse,
- SwitchyardContextWindowExceededError,
-)
-from switchyard_rust.profiles import ProfileInput
-from switchyard_rust.translation import TranslationEngine
-
-
-def _config(
- *,
- profile_name: str = "coding_agent",
- enable_stats: bool = True,
- classifier_min_confidence: float = 0.0,
- classifier_model: str = "nvidia/deepseek-ai/deepseek-v4-flash",
- classifier_system_prompt: str | None = None,
- classifier_max_request_chars: int = 16_000,
- session_affinity: bool = False,
- affinity_max_sessions: int = 10_000,
- affinity_warmup_turns: int = 0,
-) -> DeterministicRoutingConfig:
- return DeterministicRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model="aws/anthropic/bedrock-claude-opus-4-7",
- format=BackendFormat.OPENAI,
- api_key="sk-test",
- base_url="https://example.invalid/v1",
- ),
- weak=LlmTarget(
- id="weak",
- model="nvidia/deepseek-ai/evals-deepseek-v4-pro",
- format=BackendFormat.OPENAI,
- api_key="sk-test",
- base_url="https://example.invalid/v1",
- ),
- classifier=LlmTarget(
- id="classifier",
- model=classifier_model,
- format=BackendFormat.OPENAI,
- api_key="sk-test",
- base_url="https://example.invalid/v1",
- ),
- profile_name=profile_name, # type: ignore[arg-type]
- classifier_min_confidence=classifier_min_confidence,
- classifier_system_prompt=classifier_system_prompt,
- classifier_max_request_chars=classifier_max_request_chars,
- enable_stats=enable_stats,
- fallback_target_on_evict="strong",
- session_affinity=session_affinity,
- affinity_max_sessions=affinity_max_sessions,
- affinity_warmup_turns=affinity_warmup_turns,
- )
-
-
-def _deterministic_routing_switchyard(
- config: DeterministicRoutingConfig,
- *,
- stats_accumulator: StatsAccumulator | None = None,
- pre_routing_request_processors: list[Any] | None = None,
- extra_request_processors: list[Any] | None = None,
- extra_response_processors: list[Any] | None = None,
-) -> ProfileSwitchyard:
- """Build the profile-backed runtime used by these tests."""
- return ProfileSwitchyard(
- DeterministicRoutingProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats_accumulator,
- enable_stats=config.enable_stats,
- pre_request_processors=pre_routing_request_processors or (),
- post_request_processors=extra_request_processors or (),
- response_processors=extra_response_processors or (),
- )
- )
-
-
-class _NoopRequestProcessor:
- async def process(self, _ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- return request
-
-
-class _NoopResponseProcessor:
- async def process(self, _ctx: ProxyContext, response: ChatResponse) -> ChatResponse:
- return response
-
-
-class TestProfileStructure:
- def test_returns_profile_backed_switchyard_adapter(self) -> None:
- switchyard = _deterministic_routing_switchyard(_config())
- assert isinstance(switchyard, ProfileSwitchyard)
-
- def test_backend_is_deterministic_routing(self) -> None:
- switchyard = _deterministic_routing_switchyard(_config())
- backends = [
- c for c in switchyard.iter_components()
- if isinstance(c, DeterministicRoutingLLMBackend)
- ]
- assert len(backends) == 1
-
- def test_classifier_processor_present(self) -> None:
- switchyard = _deterministic_routing_switchyard(_config())
- classifiers = [
- c for c in switchyard.iter_components()
- if isinstance(c, LLMClassifierRequestProcessor)
- ]
- assert len(classifiers) == 1
-
- def test_classifier_processor_uses_prompt_and_context_overrides(self) -> None:
- switchyard = _deterministic_routing_switchyard(
- _config(
- classifier_system_prompt="custom classifier prompt",
- classifier_max_request_chars=1024,
- ),
- )
- classifier = next(
- c for c in switchyard.iter_components()
- if isinstance(c, LLMClassifierRequestProcessor)
- )
- assert classifier._config.system_prompt == "custom classifier prompt"
- assert classifier._config.max_request_chars == 1024
-
- def test_tier_selector_processor_present(self) -> None:
- switchyard = _deterministic_routing_switchyard(_config())
- selectors = [
- c for c in switchyard.iter_components()
- if isinstance(c, SignalTierSelectorRequestProcessor)
- ]
- assert len(selectors) == 1
-
- def test_affinity_warmup_reaches_shared_processors(self) -> None:
- switchyard = _deterministic_routing_switchyard(
- _config(session_affinity=True, affinity_warmup_turns=2),
- )
- classifier = next(
- c for c in switchyard.iter_components()
- if isinstance(c, LLMClassifierRequestProcessor)
- )
- selector = next(
- c for c in switchyard.iter_components()
- if isinstance(c, SignalTierSelectorRequestProcessor)
- )
- assert classifier._affinity is selector._affinity
- assert classifier._affinity is not None
- assert classifier._affinity.warmup_turns == 2
-
- def test_translator_present(self) -> None:
- switchyard = _deterministic_routing_switchyard(_config())
- translators = [
- c for c in switchyard.iter_components()
- if isinstance(c, TranslationEngine)
- ]
- assert len(translators) == 1
-
- def test_classifier_runs_before_tier_selector(self) -> None:
- switchyard = _deterministic_routing_switchyard(_config())
- components = list(switchyard.iter_components())
- classifier_idx = next(
- idx for idx, c in enumerate(components)
- if isinstance(c, LLMClassifierRequestProcessor)
- )
- selector_idx = next(
- idx for idx, c in enumerate(components)
- if isinstance(c, SignalTierSelectorRequestProcessor)
- )
- assert classifier_idx < selector_idx
-
- def test_stats_processors_wired_when_enabled(self) -> None:
- switchyard = _deterministic_routing_switchyard(_config())
- assert any(
- isinstance(c, StatsRequestProcessor)
- for c in switchyard.iter_components()
- )
- assert any(
- isinstance(c, StatsResponseProcessor)
- for c in switchyard.iter_components()
- )
-
- def test_stats_processors_absent_when_disabled(self) -> None:
- switchyard = _deterministic_routing_switchyard(
- _config(enable_stats=False),
- )
- assert not any(
- isinstance(c, StatsRequestProcessor)
- for c in switchyard.iter_components()
- )
- assert not any(
- isinstance(c, StatsResponseProcessor)
- for c in switchyard.iter_components()
- )
-
- def test_extra_processors_are_wired(self) -> None:
- request_processor = _NoopRequestProcessor()
- response_processor = _NoopResponseProcessor()
- switchyard = _deterministic_routing_switchyard(
- _config(),
- extra_request_processors=[request_processor],
- extra_response_processors=[response_processor],
- )
- components = list(switchyard.iter_components())
- assert request_processor in components
- assert response_processor in components
-
- def test_pre_routing_runs_before_classifier(self) -> None:
- pre = _NoopRequestProcessor()
- switchyard = _deterministic_routing_switchyard(
- _config(),
- pre_routing_request_processors=[pre],
- )
- components = list(switchyard.iter_components())
- stats_idx = next(
- idx for idx, c in enumerate(components)
- if isinstance(c, StatsRequestProcessor)
- )
- pre_idx = components.index(pre)
- classifier_idx = next(
- idx for idx, c in enumerate(components)
- if isinstance(c, LLMClassifierRequestProcessor)
- )
- assert stats_idx < pre_idx < classifier_idx
-
- async def test_shared_stats_accumulator(self) -> None:
- """Recording on the shared accumulator must surface in the response processor."""
- stats = StatsAccumulator()
- switchyard = _deterministic_routing_switchyard(
- _config(),
- stats_accumulator=stats,
- )
- response_processor = next(
- c for c in switchyard.iter_components()
- if isinstance(c, StatsResponseProcessor)
- )
- await stats.record_success("aws/anthropic/bedrock-claude-opus-4-7")
- assert response_processor.accumulator.snapshot_sync()["total_requests"] == 1
-
-
-class TestStderrSuppression:
- """The launcher path shares stderr with the spawned agent's TUI, so the
- classifier processor's ``classifier_signals=...`` dump must be off when the
- profile builds the chain (benchmark callers still get it via the
- ``LLMClassifierConfig.dump_signals_to_stderr=True`` default)."""
-
- def test_profile_disables_classifier_stderr_dump(self) -> None:
- switchyard = _deterministic_routing_switchyard(_config())
- classifier = next(
- c for c in switchyard.iter_components()
- if isinstance(c, LLMClassifierRequestProcessor)
- )
- assert classifier._config.dump_signals_to_stderr is False
-
-
-class TestClassifierReasoningHint:
- def _classifier(self, config: DeterministicRoutingConfig) -> LLMClassifierRequestProcessor:
- switchyard = _deterministic_routing_switchyard(config)
- return next(
- c for c in switchyard.iter_components()
- if isinstance(c, LLMClassifierRequestProcessor)
- )
-
- def test_bedrock_claude_classifier_disables_reasoning(self) -> None:
- classifier = self._classifier(
- _config(classifier_model="aws/anthropic/bedrock-claude-sonnet-4-6"),
- )
- assert classifier._config.disable_reasoning is False
-
- def test_deepseek_classifier_keeps_reasoning_disabled(self) -> None:
- classifier = self._classifier(
- _config(classifier_model="nvidia/deepseek-ai/deepseek-v4-flash"),
- )
- assert classifier._config.disable_reasoning is True
-
-
-class TestProfileSelection:
- @pytest.mark.parametrize("profile", ["general", "coding_agent", "openclaw"])
- def test_known_profiles_build(self, profile: str) -> None:
- switchyard = _deterministic_routing_switchyard(
- _config(profile_name=profile),
- )
- assert isinstance(switchyard, ProfileSwitchyard)
-
- def test_unknown_profile_rejected_at_config_construction(self) -> None:
- with pytest.raises(ValueError):
- DeterministicRoutingConfig(
- strong={"model": "s"},
- weak={"model": "w"},
- classifier={"model": "c"},
- profile_name="invented_profile", # type: ignore[arg-type]
- fallback_target_on_evict="strong",
- )
-
-
-class TestDeepSeekOverrides:
- """The profile layers DeepSeek-specific extras onto tier targets."""
-
- def test_deepseek_v4_pro_gets_thinking_off(self) -> None:
- target = LlmTarget(
- id="weak",
- model="nvidia/deepseek-ai/evals-deepseek-v4-pro",
- format=BackendFormat.OPENAI,
- api_key="k",
- base_url="https://e/v1",
- )
- out = apply_deepseek_overrides(target)
- assert out.extra_body == {"chat_template_kwargs": {"enable_thinking": False}}
-
- def test_deepseek_gets_batch_priority_header(self) -> None:
- target = LlmTarget(
- id="weak",
- model="nvidia/deepseek-ai/deepseek-v4-flash",
- format=BackendFormat.OPENAI,
- api_key="k",
- base_url="https://e/v1",
- )
- out = apply_deepseek_overrides(target)
- assert out.extra_headers == {"X-Inference-Priority": "batch"}
-
- def test_non_deepseek_passes_through_unchanged(self) -> None:
- target = LlmTarget(
- id="strong",
- model="aws/anthropic/bedrock-claude-opus-4-7",
- format=BackendFormat.OPENAI,
- api_key="k",
- base_url="https://e/v1",
- )
- out = apply_deepseek_overrides(target)
- assert out is target # no rebuild needed
-
- def test_caller_supplied_extras_win(self) -> None:
- target = LlmTarget(
- id="weak",
- model="nvidia/deepseek-ai/evals-deepseek-v4-pro",
- format=BackendFormat.OPENAI,
- api_key="k",
- base_url="https://e/v1",
- extra_body={"chat_template_kwargs": {"enable_thinking": True}},
- extra_headers={"X-Inference-Priority": "interactive"},
- )
- out = apply_deepseek_overrides(target)
- assert out.extra_body == {"chat_template_kwargs": {"enable_thinking": True}}
- assert out.extra_headers == {"X-Inference-Priority": "interactive"}
-
- def test_caller_supplied_empty_body_wins(self) -> None:
- target = LlmTarget(
- id="weak",
- model="nvidia/deepseek-ai/evals-deepseek-v4-pro",
- format=BackendFormat.OPENAI,
- api_key="k",
- base_url="https://e/v1",
- extra_body={},
- )
- out = apply_deepseek_overrides(target)
- assert out.extra_body == {}
-
-
-class TestTierTimeoutDefaults:
- """Deterministic tiers get a bounded timeout unless callers set one."""
-
- def test_default_timeout_applies_when_target_has_no_timeout(self) -> None:
- target = LlmTarget(
- id="strong",
- model="aws/anthropic/bedrock-claude-opus-4-7",
- format=BackendFormat.OPENAI,
- api_key="k",
- base_url="https://e/v1",
- )
-
- out = apply_default_tier_timeout(target, 123.0)
-
- assert out.endpoint.timeout_secs == 123.0
-
- def test_existing_timeout_wins(self) -> None:
- target = LlmTarget(
- id="strong",
- model="aws/anthropic/bedrock-claude-opus-4-7",
- format=BackendFormat.OPENAI,
- api_key="k",
- base_url="https://e/v1",
- timeout_secs=45.0,
- )
-
- out = apply_default_tier_timeout(target, 123.0)
-
- assert out is target
- assert out.endpoint.timeout_secs == 45.0
-
- def test_none_disables_default_timeout(self) -> None:
- target = LlmTarget(
- id="strong",
- model="aws/anthropic/bedrock-claude-opus-4-7",
- format=BackendFormat.OPENAI,
- api_key="k",
- base_url="https://e/v1",
- )
-
- out = apply_default_tier_timeout(target, None)
-
- assert out is target
- assert out.endpoint.timeout_secs is None
-
-
-class TestPresetIntegration:
- """Profile round-trip with the shipping preset."""
-
- def test_preset_builds_profile(self) -> None:
- config = DeterministicRoutingPresets.coding_agent_default(api_key="nvapi-test")
- switchyard = _deterministic_routing_switchyard(config)
- backends = [
- c for c in switchyard.iter_components()
- if isinstance(c, DeterministicRoutingLLMBackend)
- ]
- assert len(backends) == 1
-
- def test_preset_metadata_round_trips(self) -> None:
- config = DeterministicRoutingPresets.coding_agent_default(api_key="nvapi-test")
- assert config.preset == "coding_agent_default"
- assert config.profile_name == "coding_agent"
- assert config.strong.model == "anthropic/claude-opus-4.7"
- assert config.weak.model == "moonshotai/kimi-k2.6"
- assert config.classifier.model == "google/gemini-3.5-flash"
-
- def test_preset_uses_openai_compatible_formats(self) -> None:
- # OpenRouter's default surface is OpenAI-compatible chat completions.
- config = DeterministicRoutingPresets.coding_agent_default(api_key="nvapi-test")
- assert config.strong.format is BackendFormat.OPENAI
- assert config.weak.format is BackendFormat.OPENAI
- assert config.classifier.format is BackendFormat.OPENAI
-
-
-def test_session_affinity_requires_nonzero_capacity() -> None:
- """Enabling affinity with a zero-capacity store is rejected as a footgun."""
- with pytest.raises(ValidationError):
- _config(session_affinity=True, affinity_max_sessions=0)
- # Enabled with capacity, or disabled with zero, are both fine.
- _config(session_affinity=True, affinity_max_sessions=1)
- _config(session_affinity=False, affinity_max_sessions=0)
-
-
-def test_affinity_warmup_turns_must_be_non_negative() -> None:
- with pytest.raises(ValidationError):
- _config(session_affinity=True, affinity_warmup_turns=-1)
-
-
-def test_blank_classifier_prompt_is_treated_as_unset() -> None:
- config = _config(classifier_system_prompt=" ")
- assert config.classifier_system_prompt is None
-
-
-def test_virtual_model_id_changes_with_prompt_and_context() -> None:
- base = _config()
- base_id = deterministic_routing_virtual_model_id(base)
- prompt_id = deterministic_routing_virtual_model_id(
- _config(classifier_system_prompt="custom classifier prompt"),
- )
- max_chars_id = deterministic_routing_virtual_model_id(
- _config(classifier_max_request_chars=1024),
- )
-
- assert prompt_id != base_id
- assert max_chars_id != base_id
-
-
-def _custom_id_config() -> DeterministicRoutingConfig:
- return DeterministicRoutingConfig(
- strong=LlmTarget(
- id="frontier",
- model="strong-model",
- format=BackendFormat.OPENAI,
- api_key="sk-test",
- base_url="https://strong.invalid/v1",
- ),
- weak=LlmTarget(
- id="cheap",
- model="weak-model",
- format=BackendFormat.OPENAI,
- api_key="sk-test",
- base_url="https://weak.invalid/v1",
- ),
- classifier=LlmTarget(
- id="classifier",
- model="classifier-model",
- format=BackendFormat.OPENAI,
- api_key="sk-test",
- base_url="https://classifier.invalid/v1",
- ),
- profile_name="coding_agent",
- fallback_target_on_evict="frontier",
- )
-
-
-def test_duplicate_tier_ids_rejected() -> None:
- config = _custom_id_config()
- with pytest.raises(ValidationError) as exc:
- DeterministicRoutingConfig.model_validate({
- **config.model_dump(),
- "strong": LlmTarget(id="tier", model="strong-model"),
- "weak": LlmTarget(id="tier", model="weak-model"),
- "fallback_target_on_evict": "tier",
- })
- assert "must differ" in str(exc.value)
-
-
-def test_custom_tier_ids_key_backend_and_selector() -> None:
- """Backend tiers and the selector's labels follow the configured ids."""
- profile = DeterministicRoutingProfileConfig.from_config(_custom_id_config()).build()
-
- backend = profile._backend
- assert isinstance(backend, DeterministicRoutingLLMBackend)
- assert set(backend._backends) == {"frontier", "cheap"}
- assert backend._default_tier == "frontier"
-
-
-class _OverflowableTier(LLMBackend):
- """Stub tier backend: overflows when told to, else returns a completion."""
-
- def __init__(self, *, overflow: bool, target_id: str) -> None:
- self.calls = 0
- self._overflow = overflow
- self._target_id = target_id
-
- @property
- def supported_request_types(self) -> list[ChatRequestType]:
- return [ChatRequestType.OPENAI_CHAT]
-
- async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
- self.calls += 1
- if self._overflow:
- error = SwitchyardContextWindowExceededError(f"{self._target_id} overflowed")
- error.target_id = self._target_id # type: ignore[attr-defined]
- raise error
- return ChatResponse.openai_completion({
- "id": "deterministic-overflow-test",
- "object": "chat.completion",
- "model": request.model,
- "choices": [
- {
- "index": 0,
- "message": {"role": "assistant", "content": "ok"},
- "finish_reason": "stop",
- }
- ],
- })
-
-
-class _StampTier:
- """Stand-in for the classifier+selector pair: stamp a fixed tier label."""
-
- def __init__(self, tier: str) -> None:
- self._tier = tier
-
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] = self._tier
- return request
-
-
-async def test_overflow_reroutes_to_custom_strong_id_through_full_profile() -> None:
- """Weak overflow retries onto the configured strong id, not an unknown label.
-
- Regression test for tiers keyed as literal strong/weak while the chain's
- evict-and-retry rewrote ``selected_target`` to the *configured* fallback
- id: with ids ``cheap``/``frontier``, the rewritten pick was unrecognised,
- the retry hit weak again, and the pool exhausted.
- """
- profile = DeterministicRoutingProfileConfig.from_config(_custom_id_config()).build()
- backend = profile._backend
- assert isinstance(backend, DeterministicRoutingLLMBackend)
- cheap = _OverflowableTier(overflow=True, target_id="cheap")
- frontier = _OverflowableTier(overflow=False, target_id="frontier")
- backend._backends = {"cheap": cheap, "frontier": frontier}
- profile._request_processors = (_StampTier("cheap"),)
-
- request = ChatRequest.openai_chat({
- "model": "client/model",
- "messages": [{"role": "user", "content": "hi"}],
- })
- response = await profile.run(ProfileInput(request))
-
- assert cheap.calls == 1
- assert frontier.calls == 1
- assert response.body["choices"][0]["message"]["content"] == "ok"
-
-
-class _StreamingTier(LLMBackend):
- """Stub tier backend that answers with an OpenAI-format stream."""
-
- @property
- def supported_request_types(self) -> list[ChatRequestType]:
- return [ChatRequestType.OPENAI_CHAT]
-
- async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
- async def chunks() -> Any:
- yield ChatCompletionChunk.model_validate({
- "id": "chatcmpl-test",
- "object": "chat.completion.chunk",
- "created": 1700000000,
- "model": request.model,
- "choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": None}],
- })
-
- return ChatResponse.openai_stream(ResponseStream(chunks()))
-
-
-async def test_streamed_response_names_the_tier_model_not_the_route() -> None:
- """A routed, translated stream reports the tier's model, not the route key.
-
- Regression test for SWITCH-922: the terminal translator used to take the
- response model from the request body. Every Rust request processor hands
- the chain a *clone*, so the backend's ``set_model`` never reached the
- translator and routed turns were labelled with the route id the client
- addressed — literally ``switchyard`` for the benchmark bundles.
- """
- config = _config(enable_stats=True)
- profile = DeterministicRoutingProfileConfig.from_config(config).build()
- profile._request_processors = (_StampTier("weak"),)
-
- # Stats must stay on: its Rust request processor is what hands the chain a
- # clone, which is the condition this regression test exists for. Swap the
- # tier backends only after the runtime wiring, which cannot wrap stubs.
- runtime = profile.with_runtime_components(enable_stats=config.enable_stats)
- backend = profile._backend
- assert isinstance(backend, DeterministicRoutingLLMBackend)
- backend._backends = {"strong": _StreamingTier(), "weak": _StreamingTier()}
-
- switchyard = ProfileSwitchyard(runtime)
- request = ChatRequest.anthropic({
- "model": "switchyard",
- "max_tokens": 16,
- "messages": [{"role": "user", "content": "hi"}],
- })
-
- events = [event async for event in await switchyard.call(request)]
-
- assert events[0]["message"]["model"] == config.weak.model
diff --git a/tests/test_escalation_judge_request_processor.py b/tests/test_escalation_judge_request_processor.py
deleted file mode 100644
index c951531e8..000000000
--- a/tests/test_escalation_judge_request_processor.py
+++ /dev/null
@@ -1,473 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for the escalation-router judge processor and its latch policy."""
-
-from __future__ import annotations
-
-import hashlib
-import json
-from types import SimpleNamespace
-from typing import Any, cast
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- CTX_DETERMINISTIC_ROUTING_TIER,
-)
-from switchyard.lib.processors.escalation_judge_request_processor import (
- CTX_ESCALATION_VERDICT,
- ESCALATION_JUDGE_SYSTEM_PROMPT,
- EscalationJudgeConfig,
- EscalationJudgeRequestProcessor,
-)
-from switchyard.lib.processors.llm_classifier import ClassifierCompletion
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.session_affinity import CTX_SESSION_KEY, SessionAffinity
-from switchyard.lib.stats_accumulator import StatsAccumulator
-from switchyard_rust.core import ChatRequest
-
-
-class _FakeJudgeClient:
- def __init__(self, response: str | Exception, usage: Any | None = None) -> None:
- self.response = response
- self.usage = usage
- self.calls: list[dict[str, str]] = []
-
- async def classify(
- self,
- *,
- model: str,
- system_prompt: str,
- request_summary: str,
- ) -> ClassifierCompletion:
- self.calls.append({
- "model": model,
- "system_prompt": system_prompt,
- "request_summary": request_summary,
- })
- if isinstance(self.response, Exception):
- raise self.response
- return ClassifierCompletion(content=self.response, usage=self.usage)
-
-
-def _verdict_json(escalate: bool, reason: str = "same error repeating") -> str:
- return json.dumps({"escalate": escalate, "reason": reason})
-
-
-def _request(n_assistant_turns: int = 3, **body_overrides: Any) -> ChatRequest:
- """Chat request with enough prior assistant turns to pass the judge gate."""
- messages: list[dict[str, str]] = [
- {"role": "system", "content": "You are a coding agent."},
- {"role": "user", "content": "fix the failing tests"},
- ]
- for i in range(n_assistant_turns):
- messages.append({"role": "assistant", "content": f"attempt {i + 1}"})
- messages.append({"role": "user", "content": f"tool result {i + 1}: error"})
- body: dict[str, Any] = {"model": "client-model", "messages": messages}
- body.update(body_overrides)
- return ChatRequest.openai_chat(cast(Any, body))
-
-
-def _processor(
- response: str | Exception,
- *,
- affinity: SessionAffinity | None = None,
- session_key_depth: int = 0,
- **config_overrides: Any,
-) -> tuple[EscalationJudgeRequestProcessor, _FakeJudgeClient, SessionAffinity]:
- affinity = affinity or SessionAffinity(enabled=True)
- fake = _FakeJudgeClient(response)
- processor = EscalationJudgeRequestProcessor(
- EscalationJudgeConfig(model="judge-model", **config_overrides),
- affinity=affinity,
- client=fake,
- session_key_depth=session_key_depth,
- )
- return processor, fake, affinity
-
-
-async def test_no_escalate_routes_weak_without_pin() -> None:
- processor, fake, affinity = _processor(_verdict_json(False, ""))
- ctx = ProxyContext()
- req = _request()
-
- await processor.process(ctx, req)
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert len(fake.calls) == 1
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["escalate"] is False
- assert await affinity.pinned(ProxyContext(), _request()) is None
-
-
-async def test_escalate_routes_strong_and_latches() -> None:
- # Single-verdict latch, stated explicitly: the default now requires a confirmation,
- # which test_confirmations_require_consecutive_escalates covers.
- processor, fake, _ = _processor(_verdict_json(True), escalate_confirmations=1)
- ctx = ProxyContext()
-
- await processor.process(ctx, _request())
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["reason"] == "same error repeating"
-
- # Later turn of the same conversation: pinned, judge not called again.
- later_ctx = ProxyContext()
- await processor.process(later_ctx, _request(n_assistant_turns=5))
- assert later_ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- assert later_ctx.metadata[CTX_ESCALATION_VERDICT]["source"] == "pinned"
- assert len(fake.calls) == 1
-
-
-async def test_judge_skipped_before_min_turn() -> None:
- processor, fake, _ = _processor(_verdict_json(True), min_judge_turn=3)
- ctx = ProxyContext()
-
- await processor.process(ctx, _request(n_assistant_turns=1))
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert fake.calls == []
-
-
-async def test_judge_failure_fails_open_to_weak_without_pin() -> None:
- processor, fake, affinity = _processor(RuntimeError("judge down"))
- ctx = ProxyContext()
-
- await processor.process(ctx, _request())
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["source"] == "fail_open"
- assert await affinity.pinned(ProxyContext(), _request()) is None
- # A garbage verdict is the same failure surface.
- processor, fake, affinity = _processor("not json at all")
- ctx = ProxyContext()
- await processor.process(ctx, _request())
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert await affinity.pinned(ProxyContext(), _request()) is None
-
-
-async def test_confirmations_require_consecutive_escalates() -> None:
- """With confirmations=2, one escalate verdict stays weak; the second latches."""
- processor, fake, affinity = _processor(_verdict_json(True), escalate_confirmations=2)
-
- ctx = ProxyContext()
- await processor.process(ctx, _request(n_assistant_turns=3))
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["escalate"] is True
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["confirmed"] is False
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["streak"] == 1
- assert await affinity.pinned(ProxyContext(), _request()) is None
-
- ctx = ProxyContext()
- await processor.process(ctx, _request(n_assistant_turns=4))
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["confirmed"] is True
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["streak"] == 2
-
-
-async def test_confirmation_window_tolerates_intervening_decline() -> None:
- """window=3: escalate -> decline -> escalate confirms (recurring trouble)."""
- affinity = SessionAffinity(enabled=True)
- fake = _FakeJudgeClient(_verdict_json(True))
- processor = EscalationJudgeRequestProcessor(
- EscalationJudgeConfig(
- model="judge-model", escalate_confirmations=2, confirmation_window=3,
- ),
- affinity=affinity,
- client=fake,
- )
-
- await processor.process(ProxyContext(), _request(n_assistant_turns=3))
- fake.response = _verdict_json(False, "")
- await processor.process(ProxyContext(), _request(n_assistant_turns=4))
- fake.response = _verdict_json(True)
- ctx = ProxyContext()
- await processor.process(ctx, _request(n_assistant_turns=5))
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["confirmed"] is True
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["streak"] == 2
-
-
-async def test_confirmation_window_expires_after_enough_declines() -> None:
- """window=2: two declines between fires expire the streak."""
- affinity = SessionAffinity(enabled=True)
- fake = _FakeJudgeClient(_verdict_json(True))
- processor = EscalationJudgeRequestProcessor(
- EscalationJudgeConfig(
- model="judge-model", escalate_confirmations=2, confirmation_window=2,
- ),
- affinity=affinity,
- client=fake,
- )
-
- await processor.process(ProxyContext(), _request(n_assistant_turns=3))
- fake.response = _verdict_json(False, "")
- await processor.process(ProxyContext(), _request(n_assistant_turns=4))
- await processor.process(ProxyContext(), _request(n_assistant_turns=5))
- fake.response = _verdict_json(True)
- ctx = ProxyContext()
- await processor.process(ctx, _request(n_assistant_turns=6))
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["streak"] == 1
- assert (
- ctx.metadata[CTX_ESCALATION_VERDICT]["confirmed"] is False
- )
-
-
-async def test_confirmations_streak_resets_on_decline() -> None:
- """escalate -> decline -> escalate never latches with confirmations=2."""
- affinity = SessionAffinity(enabled=True)
- fake = _FakeJudgeClient(_verdict_json(True))
- processor = EscalationJudgeRequestProcessor(
- EscalationJudgeConfig(model="judge-model", escalate_confirmations=2),
- affinity=affinity,
- client=fake,
- )
-
- await processor.process(ProxyContext(), _request(n_assistant_turns=3))
- fake.response = _verdict_json(False, "")
- await processor.process(ProxyContext(), _request(n_assistant_turns=4))
- fake.response = _verdict_json(True)
- ctx = ProxyContext()
- await processor.process(ctx, _request(n_assistant_turns=5))
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert ctx.metadata[CTX_ESCALATION_VERDICT]["streak"] == 1
- assert await affinity.pinned(ProxyContext(), _request()) is None
-
-
-async def test_judge_usage_recorded_into_classifier_stats() -> None:
- """Judge token spend lands in the classifier bucket of the stats snapshot."""
- stats = StatsAccumulator()
- affinity = SessionAffinity(enabled=True)
- fake = _FakeJudgeClient(
- _verdict_json(False, ""),
- usage=SimpleNamespace(
- prompt_tokens=321,
- completion_tokens=17,
- prompt_tokens_details=SimpleNamespace(cached_tokens=100),
- ),
- )
- processor = EscalationJudgeRequestProcessor(
- EscalationJudgeConfig(model="judge-model"),
- affinity=affinity,
- client=fake,
- stats_accumulator=stats,
- )
-
- await processor.process(ProxyContext(), _request())
-
- snapshot = await stats.snapshot()
- judge = snapshot["classifier"]["models"]["judge-model"]
- assert judge["calls"] == 1
- assert judge["prompt_tokens"] == 321
- assert judge["completion_tokens"] == 17
- assert judge["model_call_latency"]["count"] == 1
-
-
-async def test_judge_failure_recorded_as_classifier_error() -> None:
- stats = StatsAccumulator()
- fake = _FakeJudgeClient(RuntimeError("judge down"))
- processor = EscalationJudgeRequestProcessor(
- EscalationJudgeConfig(model="judge-model"),
- affinity=SessionAffinity(enabled=True),
- client=fake,
- stats_accumulator=stats,
- )
-
- await processor.process(ProxyContext(), _request())
-
- snapshot = await stats.snapshot()
- assert snapshot["classifier"]["total_errors"] == 1
-
-
-async def test_markdown_fenced_verdict_is_parsed() -> None:
- processor, _, _ = _processor(
- "```json\n" + _verdict_json(True) + "\n```", escalate_confirmations=1,
- )
- ctx = ProxyContext()
-
- await processor.process(ctx, _request())
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
-
-
-async def test_summary_has_header_anchors_and_truncated_window() -> None:
- processor, fake, _ = _processor(
- _verdict_json(False, ""),
- window_message_chars=80,
- first_user_chars=200,
- )
- req = _request(n_assistant_turns=3, messages=[
- {"role": "system", "content": "harness boilerplate " * 200},
- {"role": "user", "content": "fix the failing tests in tests/test_api.py"},
- {"role": "assistant", "content": "running pytest " + "x" * 500},
- {"role": "user", "content": "tool result: ImportError " + "y" * 500},
- {"role": "assistant", "content": "attempt 2"},
- {"role": "assistant", "content": "attempt 3"},
- ])
-
- await processor.process(ProxyContext(), req)
-
- summary = fake.calls[0]["request_summary"]
- assert summary.startswith("Conversation turn ")
- # Anchors survive: capped system + full task statement.
- assert "harness boilerplate" in summary
- assert "fix the failing tests in tests/test_api.py" in summary
- # Oversized system anchor and window messages are head/tail truncated.
- assert "...[trimmed]" in summary
- # Window content is present with role labels.
- assert "[assistant] attempt 3" in summary
- for line in summary.splitlines()[1:]:
- assert len(line) <= 2_000
-
-
-async def test_global_cap_drops_oldest_window_messages_first() -> None:
- processor, fake, _ = _processor(
- _verdict_json(False, ""),
- max_request_chars=1_000,
- window_message_chars=200,
- )
- messages: list[dict[str, str]] = [
- {"role": "user", "content": "the task"},
- ]
- for i in range(20):
- messages.append({"role": "assistant", "content": f"attempt {i} " + "z" * 150})
-
- await processor.process(ProxyContext(), _request(messages=messages))
-
- summary = fake.calls[0]["request_summary"]
- assert len(summary) <= 1_000
- # The newest evidence survives; the oldest window lines are dropped.
- assert "attempt 19" in summary
- assert "attempt 5" not in summary
-
-
-async def test_judge_transcript_renders_anthropic_tool_use() -> None:
- """Anthropic tool_use turns must not flatten to empty transcript lines."""
- processor, fake, _ = _processor(_verdict_json(False, ""))
- messages: list[dict[str, Any]] = [
- {"role": "user", "content": [{"type": "text", "text": "fix the failing tests"}]},
- ]
- for i in range(3):
- messages.append({
- "role": "assistant",
- "content": [
- {"type": "tool_use", "id": f"tu_{i}", "name": "bash",
- "input": {"command": "pytest -x"}},
- ],
- })
- messages.append({
- "role": "user",
- "content": [
- {"type": "tool_result", "tool_use_id": f"tu_{i}", "content": "1 failed"},
- ],
- })
- request = ChatRequest.anthropic(cast(Any, {
- "model": "claude-test",
- "max_tokens": 128,
- "messages": messages,
- }))
-
- await processor.process(ProxyContext(), request)
-
- summary = fake.calls[0]["request_summary"]
- assert 'tool_call bash({"command": "pytest -x"})' in summary
- assert "1 failed" in summary
-
-
-async def test_deep_session_key_diverges_on_early_responses() -> None:
- processor, _, _ = _processor(_verdict_json(False, ""), session_key_depth=2)
-
- def _trial(first_response: str) -> ChatRequest:
- return _request(messages=[
- {"role": "system", "content": "You are a coding agent."},
- {"role": "user", "content": "identical task text"},
- {"role": "assistant", "content": first_response},
- {"role": "user", "content": "tool result"},
- {"role": "assistant", "content": "next"},
- ])
-
- ctx_a, ctx_b = ProxyContext(), ProxyContext()
- await processor.process(ctx_a, _trial("I'll start by reading the tests."))
- await processor.process(ctx_b, _trial("Let me look at the repo layout."))
-
- assert ctx_a.metadata[CTX_SESSION_KEY] != ctx_b.metadata[CTX_SESSION_KEY]
-
- # The key is a prefix hash: the same conversation grown later keeps it.
- grown = _trial("I'll start by reading the tests.")
- grown.body["messages"].extend([
- {"role": "assistant", "content": "much later turn"},
- {"role": "user", "content": "more results"},
- ])
- ctx_grown = ProxyContext()
- await processor.process(ctx_grown, grown)
- assert ctx_grown.metadata[CTX_SESSION_KEY] == ctx_a.metadata[CTX_SESSION_KEY]
-
-
-async def test_deep_key_incomplete_prefix_skips_affinity() -> None:
- processor, fake, affinity = _processor(
- _verdict_json(True),
- session_key_depth=4,
- min_judge_turn=1,
- )
- ctx = ProxyContext()
- # Only one post-first-user message: key prefix incomplete.
- req = _request(messages=[
- {"role": "user", "content": "the task"},
- {"role": "assistant", "content": "first response"},
- ])
-
- await processor.process(ctx, req)
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert CTX_SESSION_KEY not in ctx.metadata
- assert fake.calls == []
-
-
-async def test_escalation_latch_isolated_by_deep_key() -> None:
- """Trial 2 of an identical task starts weak when trajectories diverge."""
- affinity = SessionAffinity(enabled=True)
- processor, fake, _ = _processor(
- _verdict_json(True), affinity=affinity, session_key_depth=1,
- escalate_confirmations=1,
- )
-
- def _trial(first_response: str, extra_turns: int = 2) -> ChatRequest:
- messages = [
- {"role": "user", "content": "identical task text"},
- {"role": "assistant", "content": first_response},
- ]
- for i in range(extra_turns):
- messages.append({"role": "user", "content": f"tool result {i}"})
- messages.append({"role": "assistant", "content": f"attempt {i}"})
- return _request(messages=messages)
-
- ctx1 = ProxyContext()
- await processor.process(ctx1, _trial("trial one path"))
- assert ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
-
- # Same task text, different first response: fresh latch, judge runs.
- fake.response = _verdict_json(False, "")
- ctx2 = ProxyContext()
- await processor.process(ctx2, _trial("trial two path"))
- assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert len(fake.calls) == 2
-
-
-def test_default_system_prompt_is_the_line_of_record() -> None:
- """Pin the packaged default prompt byte-exact.
-
- The prompt is loaded from ``prompts/escalation_judge.md`` package data
- and is the benchmark-validated line of record. If this test fails, the
- file changed: either revert, or re-validate the new prompt on a full
- benchmark run and update the digest deliberately.
- """
- digest = hashlib.sha256(
- ESCALATION_JUDGE_SYSTEM_PROMPT.encode("utf-8")
- ).hexdigest()
- assert digest == (
- "69610eeecbac59fc20c7933ef57fa21029fa80c374b3a399b1f1a9557a6de234"
- )
- # Loader must not normalize: the wire bytes are part of the record.
- assert ESCALATION_JUDGE_SYSTEM_PROMPT.endswith("\n")
- assert not ESCALATION_JUDGE_SYSTEM_PROMPT.startswith("\n")
diff --git a/tests/test_escalation_router_profile.py b/tests/test_escalation_router_profile.py
deleted file mode 100644
index aedd44f82..000000000
--- a/tests/test_escalation_router_profile.py
+++ /dev/null
@@ -1,318 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for the escalation-router config model and profile construction."""
-
-from __future__ import annotations
-
-from typing import Any
-
-import pytest
-from pydantic import ValidationError
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- DeterministicRoutingLLMBackend,
-)
-from switchyard.lib.backends.llm_target import LlmTarget
-from switchyard.lib.processors.escalation_judge_request_processor import (
- ESCALATION_JUDGE_SYSTEM_PROMPT,
- EscalationJudgeRequestProcessor,
-)
-from switchyard.lib.processors.reasoning_effort_normalizer import ReasoningEffortNormalizer
-from switchyard.lib.profiles import EscalationRouterConfig, EscalationRouterProfileConfig
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.roles import LLMBackend
-from switchyard_rust.core import (
- ChatRequest,
- ChatRequestType,
- ChatResponse,
- SwitchyardContextWindowExceededError,
-)
-from switchyard_rust.profiles import ProfileInput
-
-
-def _target(tier: str) -> LlmTarget:
- return LlmTarget(
- id=tier,
- model=f"{tier}-model",
- base_url=f"https://{tier}.invalid/v1",
- api_key=f"sk-{tier}",
- )
-
-
-def _config(**overrides: Any) -> EscalationRouterConfig:
- data: dict[str, Any] = {
- "strong": _target("strong"),
- "weak": _target("weak"),
- "judge": _target("judge"),
- "fallback_target_on_evict": "strong",
- }
- data.update(overrides)
- return EscalationRouterConfig.model_validate(data)
-
-
-def test_fallback_must_match_a_tier_id() -> None:
- with pytest.raises(ValidationError) as exc:
- _config(fallback_target_on_evict="judge")
- assert "fallback_target_on_evict" in str(exc.value)
-
-
-def test_judge_target_is_required() -> None:
- with pytest.raises(ValidationError):
- EscalationRouterConfig.model_validate({
- "strong": _target("strong"),
- "weak": _target("weak"),
- "fallback_target_on_evict": "strong",
- })
-
-
-def test_blank_judge_prompt_is_unset() -> None:
- assert _config(judge_system_prompt=" ").judge_system_prompt is None
-
-
-def test_build_composes_judge_chain() -> None:
- profile = EscalationRouterProfileConfig.from_config(_config()).build()
-
- processors = profile._request_processors
- assert [type(p) for p in processors] == [
- ReasoningEffortNormalizer,
- EscalationJudgeRequestProcessor,
- ]
- backend = profile._backend
- assert isinstance(backend, DeterministicRoutingLLMBackend)
- assert set(backend._backends) == {"strong", "weak"}
- assert backend._default_tier == "weak"
- assert profile._fallback_target_on_evict == "strong"
-
-
-def test_build_pins_deepseek_judge_to_batch_gateway() -> None:
- """A DeepSeek judge inherits the benchmark-gateway header like DeepSeek tiers."""
- profile = EscalationRouterProfileConfig.from_config(
- _config(
- judge=LlmTarget(
- id="judge",
- model="nvidia/deepseek-ai/deepseek-v4-flash",
- base_url="https://judge.invalid/v1",
- api_key="sk-judge",
- ),
- ),
- ).build()
-
- judge = profile._request_processors[1]
- assert isinstance(judge, EscalationJudgeRequestProcessor)
- assert judge._config.extra_headers == {"X-Inference-Priority": "batch"}
-
-
-def test_build_non_deepseek_judge_has_no_default_headers() -> None:
- profile = EscalationRouterProfileConfig.from_config(_config()).build()
-
- judge = profile._request_processors[1]
- assert isinstance(judge, EscalationJudgeRequestProcessor)
- assert judge._config.extra_headers is None
-
-
-def test_build_threads_judge_settings() -> None:
- profile = EscalationRouterProfileConfig.from_config(
- _config(
- judge_min_turn=5,
- judge_recent_turn_window=20,
- judge_window_message_chars=500,
- judge_disable_reasoning=False,
- judge_timeout_s=12.0,
- session_key_depth=2,
- ),
- ).build()
-
- judge = profile._request_processors[1]
- assert isinstance(judge, EscalationJudgeRequestProcessor)
- assert judge._config.min_judge_turn == 5
- assert judge._config.recent_turn_window == 20
- assert judge._config.window_message_chars == 500
- assert judge._config.disable_reasoning is False
- assert judge._config.timeout_s == 12.0
- assert judge._config.system_prompt == ESCALATION_JUDGE_SYSTEM_PROMPT
- assert judge._session_key_depth == 2
- assert judge._affinity.enabled
-
-
-def _custom_id_config() -> EscalationRouterConfig:
- return _config(
- strong=LlmTarget(
- id="frontier",
- model="strong-model",
- base_url="https://strong.invalid/v1",
- api_key="sk-strong",
- ),
- weak=LlmTarget(
- id="cheap",
- model="weak-model",
- base_url="https://weak.invalid/v1",
- api_key="sk-weak",
- ),
- fallback_target_on_evict="frontier",
- )
-
-
-def test_duplicate_tier_ids_rejected() -> None:
- with pytest.raises(ValidationError) as exc:
- _config(
- strong=LlmTarget(id="tier", model="strong-model"),
- weak=LlmTarget(id="tier", model="weak-model"),
- fallback_target_on_evict="tier",
- )
- assert "must differ" in str(exc.value)
-
-
-def test_custom_tier_ids_key_backend_and_judge() -> None:
- """Backend tiers and judge labels follow the configured target ids."""
- profile = EscalationRouterProfileConfig.from_config(_custom_id_config()).build()
-
- backend = profile._backend
- assert isinstance(backend, DeterministicRoutingLLMBackend)
- assert set(backend._backends) == {"frontier", "cheap"}
- assert backend._default_tier == "cheap"
- judge = profile._request_processors[1]
- assert isinstance(judge, EscalationJudgeRequestProcessor)
- assert judge._strong_tier == "frontier"
- assert judge._weak_tier == "cheap"
-
-
-class _TierFake(LLMBackend):
- """Stub tier backend: overflows when told to, else returns a completion."""
-
- def __init__(self, *, overflow: bool, target_id: str) -> None:
- self.calls = 0
- self._overflow = overflow
- self._target_id = target_id
-
- @property
- def supported_request_types(self) -> list[ChatRequestType]:
- return [ChatRequestType.OPENAI_CHAT]
-
- async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
- self.calls += 1
- if self._overflow:
- error = SwitchyardContextWindowExceededError(f"{self._target_id} overflowed")
- error.target_id = self._target_id # type: ignore[attr-defined]
- raise error
- return ChatResponse.openai_completion({
- "id": "escalation-overflow-test",
- "object": "chat.completion",
- "model": request.model,
- "choices": [
- {
- "index": 0,
- "message": {"role": "assistant", "content": "ok"},
- "finish_reason": "stop",
- }
- ],
- })
-
-
-async def test_overflow_reroutes_to_custom_strong_id_through_full_profile() -> None:
- """Weak overflow retries onto the configured strong id, not an unknown label.
-
- Regression test for tiers keyed as literal strong/weak while the chain's
- evict-and-retry rewrote ``selected_target`` to the *configured* fallback
- id: with ids ``cheap``/``frontier``, the rewritten pick was unrecognised,
- the retry hit weak again, and the pool exhausted.
- """
- profile = EscalationRouterProfileConfig.from_config(_custom_id_config()).build()
- backend = profile._backend
- assert isinstance(backend, DeterministicRoutingLLMBackend)
- cheap = _TierFake(overflow=True, target_id="cheap")
- frontier = _TierFake(overflow=False, target_id="frontier")
- backend._backends = {"cheap": cheap, "frontier": frontier}
-
- request = ChatRequest.openai_chat({
- "model": "client/model",
- "messages": [{"role": "user", "content": "hi"}],
- })
- response = await profile.run(ProfileInput(request))
-
- assert cheap.calls == 1
- assert frontier.calls == 1
- assert response.body["choices"][0]["message"]["content"] == "ok"
-
-
-def test_claude_judge_skips_reasoning_hint() -> None:
- """Claude/Bedrock judges must not receive the vLLM-only thinking hint."""
- profile = EscalationRouterProfileConfig.from_config(
- _config(
- judge=LlmTarget(
- id="judge",
- model="anthropic/claude-sonnet-5",
- base_url="https://judge.invalid/v1",
- api_key="sk-judge",
- ),
- ),
- ).build()
-
- judge = profile._request_processors[1]
- assert isinstance(judge, EscalationJudgeRequestProcessor)
- assert judge._config.disable_reasoning is False
-
-
-def test_vllm_judge_keeps_reasoning_hint() -> None:
- """Non-Anthropic judge models keep the default thinking-off hint."""
- profile = EscalationRouterProfileConfig.from_config(_config()).build()
-
- judge = profile._request_processors[1]
- assert isinstance(judge, EscalationJudgeRequestProcessor)
- assert judge._config.disable_reasoning is True
-
-
-def test_judge_completion_budget_follows_reasoning_mode() -> None:
- """A thinking judge gets the classifier-sized budget; explicit value wins."""
- default = EscalationRouterProfileConfig.from_config(_config()).build()
- assert default._request_processors[1]._config.max_completion_tokens == 128
-
- thinking = EscalationRouterProfileConfig.from_config(
- _config(judge_disable_reasoning=False),
- ).build()
- assert thinking._request_processors[1]._config.max_completion_tokens == 4096
-
- explicit = EscalationRouterProfileConfig.from_config(
- _config(judge_disable_reasoning=False, judge_max_completion_tokens=512),
- ).build()
- assert explicit._request_processors[1]._config.max_completion_tokens == 512
-
-
-def test_verdict_dump_is_opt_in() -> None:
- default = EscalationRouterProfileConfig.from_config(_config()).build()
- assert default._request_processors[1]._config.dump_verdicts_to_stderr is False
-
- opted_in = EscalationRouterProfileConfig.from_config(
- _config(judge_dump_verdicts=True),
- ).build()
- assert opted_in._request_processors[1]._config.dump_verdicts_to_stderr is True
-
-
-def test_redis_affinity_store_requires_url() -> None:
- with pytest.raises(ValidationError) as exc:
- _config(affinity_store="redis")
- assert "affinity_store_url" in str(exc.value)
-
-
-def test_redis_affinity_store_wires_l2_latch() -> None:
- from switchyard.lib.redis_pin_store import RedisPinStore
-
- profile = EscalationRouterProfileConfig.from_config(
- _config(
- affinity_store="redis",
- affinity_store_url="redis://cache:6379/0",
- affinity_store_ttl_seconds=120,
- ),
- ).build()
-
- judge = profile._request_processors[1]
- assert isinstance(judge, EscalationJudgeRequestProcessor)
- assert isinstance(judge._affinity._l2, RedisPinStore)
-
-
-def test_default_affinity_store_is_l1_only() -> None:
- profile = EscalationRouterProfileConfig.from_config(_config()).build()
-
- judge = profile._request_processors[1]
- assert isinstance(judge, EscalationJudgeRequestProcessor)
- assert judge._affinity._l2 is None
diff --git a/tests/test_init_all_exports.py b/tests/test_init_all_exports.py
index 08152e010..3f5c5c26c 100644
--- a/tests/test_init_all_exports.py
+++ b/tests/test_init_all_exports.py
@@ -21,7 +21,3 @@ def test_all_symbols_accessible() -> None:
except AttributeError:
missing.append(name)
assert missing == [], f"Symbols in __all__ not importable from switchyard: {missing}"
-
-
-def test_random_routing_config_importable() -> None:
- from switchyard import RandomRoutingConfig # noqa: F401
diff --git a/tests/test_llm_classifier_e2e.py b/tests/test_llm_classifier_e2e.py
deleted file mode 100644
index 1da464660..000000000
--- a/tests/test_llm_classifier_e2e.py
+++ /dev/null
@@ -1,356 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Production offline e2e test for the LLM classifier routing chain.
-
-Exercises the full classifier-driven deterministic routing path with
-the real production code paths wired together:
-
- inbound HTTP request
- -> Switchyard chain
- LLMClassifierRequestProcessor (real OpenAILLMClient -> openai SDK -> httpx)
- SignalTierSelectorRequestProcessor
- -> DeterministicRoutingLLMBackend
- -> per-tier Rust OpenAI backend (real reqwest -> loopback HTTP)
- -> TranslationEngine
- -> outbound HTTP response
-
-Classifier HTTPS calls are intercepted by ``respx``. Backend tier
-calls go through local loopback HTTP servers because Rust reqwest
-bypasses ``respx``. The test stays offline while still exercising the
-real classifier client, tier mapping, and per-tier backend dispatch.
-
-"""
-
-from __future__ import annotations
-
-import json
-from collections.abc import AsyncIterator
-
-import httpx
-import pytest
-import respx
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- DeterministicRoutingLLMBackend,
-)
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
-from switchyard.lib.processors.llm_classifier import (
- LLMClassifierConfig,
- LLMClassifierRequestProcessor,
- SignalTierSelectorConfig,
- SignalTierSelectorRequestProcessor,
-)
-from switchyard.lib.switchyard import Switchyard
-from switchyard.server.switchyard_app import build_switchyard_app
-from switchyard_rust.translation import TranslationEngine
-from tests._chain_test_helpers import (
- _CLASSIFIER_BASE,
- _CLASSIFIER_MODEL,
- _CLASSIFIER_URL,
- _COMPLEX_MODEL,
- _SIMPLE_MODEL,
- _backend_payload,
- _classifier_payload,
- _ClassifierHarness,
- _last_body,
- _OpenAICompatStub,
- _signals_json,
- _sse_body,
- _stream_chunk,
-)
-
-# ---------------------------------------------------------------------------
-# Fixture — assembles the classifier-driven chain by hand.
-# ---------------------------------------------------------------------------
-
-
-def _build_switchyard(*, simple_base_url: str, complex_base_url: str) -> Switchyard:
- """Hand-assemble the classifier routing chain.
-
- There's no recipe for this yet, so we wire it the same way a
- downstream integrator would: classifier -> tier selector ->
- deterministic backend with one Rust OpenAI backend per tier.
- """
- classifier = LLMClassifierRequestProcessor(
- LLMClassifierConfig(
- model=_CLASSIFIER_MODEL,
- api_key="classifier-key-not-validated",
- base_url=_CLASSIFIER_BASE,
- ),
- )
- tier_selector = SignalTierSelectorRequestProcessor(
- SignalTierSelectorConfig(default_tier="simple"),
- )
- backend = DeterministicRoutingLLMBackend.from_tiers(
- tiers={
- "simple": LlmTarget(
- model=_SIMPLE_MODEL,
- format=BackendFormat.OPENAI,
- api_key="simple-key",
- base_url=simple_base_url,
- ),
- "complex": LlmTarget(
- model=_COMPLEX_MODEL,
- format=BackendFormat.OPENAI,
- api_key="complex-key",
- base_url=complex_base_url,
- ),
- },
- default_tier="simple",
- )
- return Switchyard(
- request_processors=[classifier, tier_selector],
- backend=backend,
- translator=TranslationEngine(),
- )
-
-
-async def classifier_harness() -> AsyncIterator[_ClassifierHarness]:
- """ASGI-transport client wired through the full classifier chain."""
- with _OpenAICompatStub() as simple, _OpenAICompatStub() as complex:
- app = build_switchyard_app(
- _build_switchyard(
- simple_base_url=simple.base_url,
- complex_base_url=complex.base_url,
- )
- )
- async with httpx.AsyncClient(
- transport=httpx.ASGITransport(app=app, raise_app_exceptions=False),
- base_url="http://test",
- ) as client:
- yield _ClassifierHarness(client=client, simple=simple, complex=complex)
-
-
-# ---------------------------------------------------------------------------
-# Tests
-# ---------------------------------------------------------------------------
-
-
-class TestLLMClassifierE2E:
- """Production offline e2e — classifier picks a tier, backend serves it."""
-
- @respx.mock
- async def test_classifier_picks_complex_tier(
- self, classifier_harness: _ClassifierHarness,
- ) -> None:
- """High-confidence ``complex`` signals route to the complex backend."""
- classifier_route = respx.post(_CLASSIFIER_URL).mock(
- return_value=httpx.Response(
- 200, json=_classifier_payload(_signals_json()),
- )
- )
- classifier_harness.complex.respond_json(
- _backend_payload(content="complex-reply", model=_COMPLEX_MODEL)
- )
- classifier_harness.simple.respond_json(
- _backend_payload(content="simple-reply", model=_SIMPLE_MODEL)
- )
-
- resp = await classifier_harness.client.post(
- "/v1/chat/completions",
- json={
- "model": "client-requested-model",
- "messages": [{"role": "user", "content": "debug this traceback please"}],
- },
- )
-
- assert resp.status_code == 200, resp.text
- body = resp.json()
- assert body["choices"][0]["message"]["content"] == "complex-reply"
-
- # The classifier ran, and the routing reached the complex tier
- # (not the simple one).
- assert classifier_route.called
- assert classifier_harness.complex.called
- assert not classifier_harness.simple.called
-
- # The classifier saw the inbound request body in its user turn.
- classifier_req = json.loads(classifier_route.calls.last.request.content)
- messages = classifier_req["messages"]
- assert messages[0]["role"] == "system"
- assert "routing classifier" in messages[0]["content"].lower()
- assert messages[1]["role"] == "user"
- assert "debug this traceback" in messages[1]["content"]
- # Classifier defaults to strict Structured Outputs: response_format
- # carries the pydantic schema with strict: true.
- rf = classifier_req.get("response_format")
- assert rf is not None and rf["type"] == "json_schema"
- assert rf["json_schema"]["name"] == "RouteSignals"
- assert rf["json_schema"]["strict"] is True
- sent_schema = rf["json_schema"]["schema"]
- assert sent_schema["type"] == "object"
- assert sent_schema["additionalProperties"] is False
- # Strict mode requires every property in `required`, even ones with defaults.
- assert set(sent_schema["required"]) == set(sent_schema["properties"].keys())
-
- # The downstream backend got the tier-specific model name, not
- # the client's requested model.
- backend_req = _last_body(classifier_harness.complex)
- assert backend_req["model"] == _COMPLEX_MODEL
-
- @respx.mock
- async def test_classifier_picks_simple_tier(
- self, classifier_harness: _ClassifierHarness,
- ) -> None:
- """``recommended_tier: simple`` routes to the simple backend."""
- respx.post(_CLASSIFIER_URL).mock(
- return_value=httpx.Response(
- 200,
- json=_classifier_payload(
- _signals_json(
- task_type="chat",
- complexity="simple",
- reasoning_depth="none",
- precision_requirement="low",
- recommended_tier="simple",
- confidence=0.95,
- reason_code="simple_qa",
- ),
- ),
- )
- )
- classifier_harness.complex.respond_json(
- _backend_payload(content="complex-reply", model=_COMPLEX_MODEL)
- )
- classifier_harness.simple.respond_json(
- _backend_payload(content="simple-reply", model=_SIMPLE_MODEL)
- )
-
- resp = await classifier_harness.client.post(
- "/v1/chat/completions",
- json={
- "model": "client-requested-model",
- "messages": [{"role": "user", "content": "hi"}],
- },
- )
-
- assert resp.status_code == 200, resp.text
- assert resp.json()["choices"][0]["message"]["content"] == "simple-reply"
- assert classifier_harness.simple.called
- assert not classifier_harness.complex.called
-
- @respx.mock
- async def test_classifier_abstain_falls_back_to_default_tier(
- self, classifier_harness: _ClassifierHarness,
- ) -> None:
- """``abstain: true`` bypasses ``recommended_tier`` and uses the default."""
- respx.post(_CLASSIFIER_URL).mock(
- return_value=httpx.Response(
- 200,
- json=_classifier_payload(
- _signals_json(
- # Even though recommended_tier is complex, abstain=True
- # should force the tier selector to its default ("simple").
- recommended_tier="complex",
- abstain=True,
- confidence=0.1,
- reason_code="ambiguous",
- ),
- ),
- )
- )
- classifier_harness.complex.respond_json(
- _backend_payload(content="complex-reply", model=_COMPLEX_MODEL)
- )
- classifier_harness.simple.respond_json(
- _backend_payload(content="simple-reply", model=_SIMPLE_MODEL)
- )
-
- resp = await classifier_harness.client.post(
- "/v1/chat/completions",
- json={
- "model": "client-requested-model",
- "messages": [{"role": "user", "content": "??"}],
- },
- )
-
- assert resp.status_code == 200, resp.text
- assert resp.json()["choices"][0]["message"]["content"] == "simple-reply"
- assert classifier_harness.simple.called
- assert not classifier_harness.complex.called
-
- @respx.mock
- async def test_classifier_failure_fails_open_to_default_tier(
- self, classifier_harness: _ClassifierHarness,
- ) -> None:
- """A classifier 500 stamps abstain signals and routes to the default tier.
-
- ``fail_open=True`` is the configured default for
- :class:`LLMClassifierConfig`, so a broken classifier upstream
- must not break inference — it falls through to the default tier
- with abstain semantics.
- """
- classifier_route = respx.post(_CLASSIFIER_URL).mock(
- return_value=httpx.Response(500, json={"error": "boom"}),
- )
- classifier_harness.complex.respond_json(
- _backend_payload(content="complex-reply", model=_COMPLEX_MODEL)
- )
- classifier_harness.simple.respond_json(
- _backend_payload(content="simple-reply", model=_SIMPLE_MODEL)
- )
-
- resp = await classifier_harness.client.post(
- "/v1/chat/completions",
- json={
- "model": "client-requested-model",
- "messages": [{"role": "user", "content": "anything"}],
- },
- )
-
- assert resp.status_code == 200, resp.text
- assert resp.json()["choices"][0]["message"]["content"] == "simple-reply"
- # Classifier was attempted (and failed) at least once.
- assert classifier_route.called
- assert classifier_harness.simple.called
- assert not classifier_harness.complex.called
-
- @respx.mock
- async def test_classifier_picks_complex_tier_streaming(
- self, classifier_harness: _ClassifierHarness,
- ) -> None:
- """Streaming path: classifier -> complex tier -> SSE chunks back."""
- respx.post(_CLASSIFIER_URL).mock(
- return_value=httpx.Response(
- 200, json=_classifier_payload(_signals_json()),
- )
- )
- chunks = [
- _stream_chunk(content="foo"),
- _stream_chunk(content="bar"),
- _stream_chunk(finish="stop"),
- ]
- classifier_harness.complex.respond_sse(_sse_body(chunks))
- classifier_harness.simple.respond_json(
- _backend_payload(content="should-not-fire", model=_SIMPLE_MODEL)
- )
-
- resp = await classifier_harness.client.post(
- "/v1/chat/completions",
- json={
- "model": "client-requested-model",
- "messages": [{"role": "user", "content": "debug stream"}],
- "stream": True,
- },
- )
-
- assert resp.status_code == 200, resp.text
- assert "text/event-stream" in resp.headers["content-type"]
- assert classifier_harness.complex.called
- assert not classifier_harness.simple.called
-
- data_frames = [
- line[6:]
- for line in resp.text.split("\n")
- if line.startswith("data: ") and line != "data: [DONE]"
- ]
- assert data_frames, "expected at least one outbound SSE data frame"
- decoded = [json.loads(line) for line in data_frames]
- joined = "".join(
- c["choices"][0]["delta"].get("content", "")
- for c in decoded
- if c.get("choices")
- )
- assert joined == "foobar"
diff --git a/tests/test_llm_classifier_presets.py b/tests/test_llm_classifier_presets.py
deleted file mode 100644
index b937f1a0c..000000000
--- a/tests/test_llm_classifier_presets.py
+++ /dev/null
@@ -1,226 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for the LLM classifier profile bundles."""
-
-from __future__ import annotations
-
-import json
-from typing import Any, cast
-
-from switchyard.lib.processors.llm_classifier import (
- CTX_DETERMINISTIC_ROUTE_SIGNALS,
- DEFAULT_MAX_REQUEST_CHARS,
- ChannelKind,
- ClassifierCompletion,
- CodeModificationScope,
- CodingAgentRouteDecision,
- CodingAgentTurnType,
- LLMClassifierPresets,
- LLMClassifierProfile,
- LLMClassifierRequestProcessor,
- MemoryDependency,
- OpenClawRouteDecision,
- OpenClawTurnType,
- RouteSignals,
- RouteTier,
- SignalTierSelectorConfig,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.core import ChatRequest
-
-WEAK = "weak"
-STRONG = "strong"
-
-
-class _FakeClassifierClient:
- def __init__(self, response: str) -> None:
- self.response = response
- self.system_prompt: str | None = None
-
- async def classify(
- self,
- *,
- model: str,
- system_prompt: str,
- request_summary: str,
- ) -> ClassifierCompletion:
- self.system_prompt = system_prompt
- return ClassifierCompletion(content=self.response)
-
-
-def _request() -> ChatRequest:
- body: dict[str, Any] = {
- "model": "client-model",
- "messages": [{"role": "user", "content": "list files in src"}],
- }
- return ChatRequest.openai_chat(cast(Any, body))
-
-
-def test_general_profile_keeps_conservative_mapping() -> None:
- profile = LLMClassifierPresets.general_2_tier(weak=WEAK, strong=STRONG)
-
- assert profile.name == "general_2_tier"
- assert profile.signal_schema is RouteSignals
- assert profile.tier_mapping[RouteTier.SIMPLE] == WEAK
- assert profile.tier_mapping[RouteTier.MEDIUM] == STRONG
- assert profile.tier_mapping[RouteTier.COMPLEX] == STRONG
- assert profile.tier_mapping[RouteTier.REASONING] == STRONG
- assert profile.default_tier == STRONG
-
-
-def test_coding_agent_profile_routes_simple_and_medium_to_weak() -> None:
- profile = LLMClassifierPresets.coding_agent_2_tier(weak=WEAK, strong=STRONG)
-
- assert profile.name == "coding_agent_2_tier"
- assert profile.signal_schema is CodingAgentRouteDecision
- assert profile.tier_mapping[RouteTier.SIMPLE] == WEAK
- assert profile.tier_mapping[RouteTier.MEDIUM] == WEAK
- assert profile.tier_mapping[RouteTier.COMPLEX] == STRONG
- assert profile.tier_mapping[RouteTier.REASONING] == STRONG
- assert profile.default_tier == STRONG
-
-
-def test_openclaw_profile_routes_simple_and_medium_to_weak() -> None:
- profile = LLMClassifierPresets.openclaw_2_tier(weak=WEAK, strong=STRONG)
-
- assert profile.name == "openclaw_2_tier"
- assert profile.signal_schema is OpenClawRouteDecision
- assert profile.tier_mapping[RouteTier.SIMPLE] == WEAK
- assert profile.tier_mapping[RouteTier.MEDIUM] == WEAK
- assert profile.tier_mapping[RouteTier.COMPLEX] == STRONG
- assert profile.tier_mapping[RouteTier.REASONING] == STRONG
- assert profile.default_tier == STRONG
-
-
-def test_profile_make_classifier_config_bakes_in_system_prompt() -> None:
- profile = LLMClassifierPresets.coding_agent_2_tier(weak=WEAK, strong=STRONG)
- config = profile.make_classifier_config(model="router-model")
-
- assert config.system_prompt == profile.system_prompt
- assert config.model == "router-model"
- assert config.fail_open is True
- assert config.max_request_chars == DEFAULT_MAX_REQUEST_CHARS
-
-
-def test_profile_make_classifier_config_accepts_prompt_and_context_override() -> None:
- profile = LLMClassifierPresets.coding_agent_2_tier(weak=WEAK, strong=STRONG)
- config = profile.make_classifier_config(
- model="router-model",
- system_prompt="custom prompt",
- max_request_chars=1024,
- )
-
- assert config.system_prompt == "custom prompt"
- assert config.max_request_chars == 1024
-
-
-def test_profile_make_tier_selector_config_matches_mapping() -> None:
- profile = LLMClassifierPresets.openclaw_2_tier(weak=WEAK, strong=STRONG)
- config = profile.make_tier_selector_config()
-
- assert isinstance(config, SignalTierSelectorConfig)
- assert dict(config.tier_mapping) == dict(profile.tier_mapping)
- assert config.default_tier == profile.default_tier
- assert config.min_confidence == 0.0
-
-
-def test_profile_min_confidence_override() -> None:
- profile = LLMClassifierPresets.coding_agent_2_tier(weak=WEAK, strong=STRONG)
- config = profile.make_tier_selector_config(min_confidence=0.55)
-
- assert config.min_confidence == 0.55
-
-
-async def test_coding_agent_profile_round_trips_through_request_processor() -> None:
- profile = LLMClassifierPresets.coding_agent_2_tier(weak=WEAK, strong=STRONG)
- payload = {
- "recommended_tier": "simple",
- "confidence": 0.91,
- "abstain": False,
- "turn_type": "exploration",
- "code_modification_scope": "none",
- "tool_call_count_estimate": 2,
- "requires_codebase_context": False,
- }
- fake = _FakeClassifierClient(json.dumps(payload))
-
- processor = LLMClassifierRequestProcessor(
- profile.make_classifier_config(model="router-model"),
- client=fake,
- signal_schema=profile.signal_schema,
- )
- ctx = ProxyContext()
- await processor.process(ctx, _request())
-
- stamped = ctx.metadata[CTX_DETERMINISTIC_ROUTE_SIGNALS]
- assert isinstance(stamped, CodingAgentRouteDecision)
- assert stamped.recommended_tier is RouteTier.SIMPLE
- assert stamped.turn_type is CodingAgentTurnType.EXPLORATION
- assert stamped.code_modification_scope is CodeModificationScope.NONE
- assert stamped.tool_call_count_estimate == 2
- assert fake.system_prompt == profile.system_prompt
-
-
-async def test_openclaw_profile_round_trips_through_request_processor() -> None:
- profile = LLMClassifierPresets.openclaw_2_tier(weak=WEAK, strong=STRONG)
- payload = {
- "recommended_tier": "simple",
- "confidence": 0.84,
- "abstain": False,
- "turn_type": "lookup",
- "tool_call_count_estimate": 1,
- "memory_dependency": "light",
- "external_action_required": False,
- "precision_requirement": "low",
- "ambiguity": "low",
- "channel_kind": "casual",
- }
- fake = _FakeClassifierClient(json.dumps(payload))
-
- processor = LLMClassifierRequestProcessor(
- profile.make_classifier_config(model="router-model"),
- client=fake,
- signal_schema=profile.signal_schema,
- )
- ctx = ProxyContext()
- await processor.process(ctx, _request())
-
- stamped = ctx.metadata[CTX_DETERMINISTIC_ROUTE_SIGNALS]
- assert isinstance(stamped, OpenClawRouteDecision)
- assert stamped.recommended_tier is RouteTier.SIMPLE
- assert stamped.turn_type is OpenClawTurnType.LOOKUP
- assert stamped.memory_dependency is MemoryDependency.LIGHT
- assert stamped.channel_kind is ChannelKind.CASUAL
- assert fake.system_prompt == profile.system_prompt
-
-
-async def test_classifier_fail_open_uses_profile_schema_for_abstain() -> None:
- profile = LLMClassifierPresets.coding_agent_2_tier(weak=WEAK, strong=STRONG)
-
- class _Boom:
- async def classify(self, **_: Any) -> ClassifierCompletion:
- raise RuntimeError("classifier exploded")
-
- processor = LLMClassifierRequestProcessor(
- profile.make_classifier_config(model="router-model"),
- client=_Boom(),
- signal_schema=profile.signal_schema,
- )
- ctx = ProxyContext()
- await processor.process(ctx, _request())
-
- stamped = ctx.metadata[CTX_DETERMINISTIC_ROUTE_SIGNALS]
- assert isinstance(stamped, CodingAgentRouteDecision)
- assert stamped.abstain is True
- assert stamped.confidence == 0.0
-
-
-def test_profile_is_immutable_dataclass() -> None:
- profile = LLMClassifierPresets.general_2_tier(weak=WEAK, strong=STRONG)
- assert isinstance(profile, LLMClassifierProfile)
- try:
- profile.name = "mutated" # type: ignore[misc]
- except Exception:
- return
- raise AssertionError("expected LLMClassifierProfile to be frozen")
diff --git a/tests/test_llm_classifier_request_processor.py b/tests/test_llm_classifier_request_processor.py
deleted file mode 100644
index bc261454e..000000000
--- a/tests/test_llm_classifier_request_processor.py
+++ /dev/null
@@ -1,567 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for the experimental LLM classifier request processor."""
-
-import json
-from typing import Any, cast
-
-import pytest
-
-from switchyard.lib.processors.llm_classifier import (
- CTX_DETERMINISTIC_ROUTE_SIGNALS,
- ClassifierCompletion,
- LLMClassifierConfig,
- LLMClassifierError,
- LLMClassifierRequestProcessor,
- ReasonCode,
- RouteSignals,
- RouteTier,
- TaskType,
- parse_route_signals,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.session_affinity import SessionAffinity
-from switchyard_rust.core import ChatRequest
-
-
-class _FakeUsage:
- """Duck-typed OpenAI-shaped usage object for tests."""
-
- def __init__(self, prompt_tokens: int, completion_tokens: int, cached_tokens: int = 0) -> None:
- self.prompt_tokens = prompt_tokens
- self.completion_tokens = completion_tokens
- self.prompt_tokens_details = _FakePromptTokensDetails(cached_tokens)
-
-
-class _FakePromptTokensDetails:
- def __init__(self, cached_tokens: int) -> None:
- self.cached_tokens = cached_tokens
-
-
-class _FakeClassifierClient:
- def __init__(
- self,
- response: str | Exception,
- *,
- usage: _FakeUsage | None = None,
- ) -> None:
- self.response = response
- self.usage = usage
- self.calls: list[dict[str, str]] = []
-
- async def classify(
- self,
- *,
- model: str,
- system_prompt: str,
- request_summary: str,
- ) -> ClassifierCompletion:
- self.calls.append({
- "model": model,
- "system_prompt": system_prompt,
- "request_summary": request_summary,
- })
- if isinstance(self.response, Exception):
- raise self.response
- return ClassifierCompletion(content=self.response, usage=self.usage)
-
-
-def _messages_with_prior_assistant_turns(n: int) -> list[dict[str, str]]:
- messages = [{"role": "user", "content": "debug this traceback"}]
- for i in range(n):
- messages.append({"role": "assistant", "content": f"turn {i + 1}"})
- return messages
-
-
-def _request(**body_overrides: Any) -> ChatRequest:
- body: dict[str, Any] = {
- "model": "client-model",
- "messages": [{"role": "user", "content": "debug this traceback"}],
- }
- body.update(body_overrides)
- return ChatRequest.openai_chat(cast(Any, body))
-
-
-def _signals_json(**overrides: Any) -> str:
- payload: dict[str, Any] = {
- "task_type": "debugging",
- "complexity": "complex",
- "reasoning_depth": "multi_step",
- "tool_planning_required": False,
- "precision_requirement": "high",
- "context_dependency": "conversation",
- "structured_output_risk": "low",
- "recommended_tier": "complex",
- "confidence": 0.86,
- "reason_code": "debugging",
- "abstain": False,
- }
- payload.update(overrides)
- return json.dumps(payload)
-
-
-async def test_request_processor_stamps_classifier_signals() -> None:
- fake = _FakeClassifierClient(_signals_json())
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model"),
- client=fake,
- )
- req = _request()
- ctx = ProxyContext()
-
- returned = await processor.process(ctx, req)
-
- assert returned is req
- signals = ctx.metadata[CTX_DETERMINISTIC_ROUTE_SIGNALS]
- assert isinstance(signals, RouteSignals)
- assert signals.task_type is TaskType.DEBUGGING
- assert signals.recommended_tier is RouteTier.COMPLEX
- assert signals.reason_code is ReasonCode.DEBUGGING
- assert fake.calls[0]["model"] == "router-model"
- assert '"request_type": "openai_chat"' in fake.calls[0]["request_summary"]
- assert "debug this traceback" in fake.calls[0]["request_summary"]
-
-
-async def test_classifier_skips_llm_call_when_session_pinned() -> None:
- """Once a conversation is pinned, the classifier runs once per task, not per turn."""
- fake = _FakeClassifierClient(_signals_json())
- affinity = SessionAffinity(enabled=True)
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model"),
- client=fake,
- affinity=affinity,
- )
- req = _request()
-
- # First turn: not yet pinned → the classifier makes its LLM call.
- await processor.process(ProxyContext(), req)
- assert len(fake.calls) == 1
-
- # The tier selector would pin the conversation after the first turn.
- await affinity.pin(ProxyContext(), req, "weak")
-
- # Later turns of the same conversation skip the LLM call entirely.
- ctx2 = ProxyContext()
- returned = await processor.process(ctx2, req)
- assert returned is req
- assert len(fake.calls) == 1 # unchanged — classified exactly once
- assert CTX_DETERMINISTIC_ROUTE_SIGNALS not in ctx2.metadata
-
-
-async def test_classifier_runs_until_warmup_pin_exists() -> None:
- """Warmup keeps the classifier live until the selector can commit a pin."""
- fake = _FakeClassifierClient(_signals_json())
- affinity = SessionAffinity(enabled=True, warmup_turns=2)
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model"),
- client=fake,
- affinity=affinity,
- )
-
- req1 = _request()
- await processor.process(ProxyContext(), req1)
- await affinity.pin(ProxyContext(), req1, "weak")
-
- req2 = _request(messages=_messages_with_prior_assistant_turns(1))
- await processor.process(ProxyContext(), req2)
- await affinity.pin(ProxyContext(), req2, "weak")
-
- req3 = _request(messages=_messages_with_prior_assistant_turns(2))
- await processor.process(ProxyContext(), req3)
- await affinity.pin(ProxyContext(), req3, "weak")
-
- assert len(fake.calls) == 3
-
- req4 = _request(messages=_messages_with_prior_assistant_turns(3))
- ctx4 = ProxyContext()
- returned = await processor.process(ctx4, req4)
- assert returned is req4
- assert len(fake.calls) == 3
- assert CTX_DETERMINISTIC_ROUTE_SIGNALS not in ctx4.metadata
-
-
-async def test_classifier_runs_when_affinity_disabled() -> None:
- """A disabled affinity coordinator never gates the classifier."""
- fake = _FakeClassifierClient(_signals_json())
- affinity = SessionAffinity(enabled=False)
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model"),
- client=fake,
- affinity=affinity,
- )
- req = _request()
- await affinity.pin(ProxyContext(), req, "weak") # no-op while disabled
-
- await processor.process(ProxyContext(), req)
- await processor.process(ProxyContext(), req)
- assert len(fake.calls) == 2 # classifier runs every turn
-
-
-def test_parse_route_signals_accepts_json_fence() -> None:
- signals = parse_route_signals(f"```json\n{_signals_json()}\n```")
-
- assert signals.task_type is TaskType.DEBUGGING
- assert signals.confidence == 0.86
-
-
-async def test_request_processor_fail_open_stamps_abstain_signals() -> None:
- fake = _FakeClassifierClient("not json")
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model"),
- client=fake,
- )
- ctx = ProxyContext()
-
- await processor.process(ctx, _request())
-
- signals = ctx.metadata[CTX_DETERMINISTIC_ROUTE_SIGNALS]
- assert isinstance(signals, RouteSignals)
- assert signals.abstain is True
- assert signals.confidence == 0.0
- assert signals.reason_code is ReasonCode.AMBIGUOUS
-
-
-async def test_fail_open_annotates_signals_dump(capsys: pytest.CaptureFixture[str]) -> None:
- fake = _FakeClassifierClient(RuntimeError("upstream timeout"))
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model", dump_signals_to_stderr=True),
- client=fake,
- )
- await processor.process(ProxyContext(), _request())
-
- line = next(
- ln for ln in capsys.readouterr().err.splitlines()
- if ln.startswith("classifier_signals=")
- )
- payload = json.loads(line.removeprefix("classifier_signals="))
- assert payload["fail_open"] is True
- assert "upstream timeout" in payload["error"]
- assert payload["abstain"] is True
-
-
-async def test_request_processor_fail_closed_raises() -> None:
- fake = _FakeClassifierClient(RuntimeError("boom"))
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model", fail_open=False),
- client=fake,
- )
-
- with pytest.raises(LLMClassifierError):
- await processor.process(ProxyContext(), _request())
-
-
-async def test_openai_client_sends_strict_json_schema_response_format() -> None:
- from switchyard.lib.processors.llm_classifier.request_processor import (
- OpenAIChatLLMClassifierClient,
- )
- from switchyard.lib.processors.llm_classifier.signals import (
- CodingAgentRouteDecision,
- )
-
- captured: dict[str, Any] = {}
-
- class _StubLLMClient:
- async def acompletion(self, **kwargs: Any) -> Any:
- captured.update(kwargs)
-
- class _Msg:
- content = json.dumps({
- "recommended_tier": "medium",
- "confidence": 0.7,
- "abstain": False,
- "turn_type": "exploration",
- "code_modification_scope": "none",
- "tool_call_count_estimate": 0,
- "requires_codebase_context": False,
- })
-
- class _Choice:
- message = _Msg()
-
- class _Result:
- choices = [_Choice()]
-
- return _Result()
-
- client = OpenAIChatLLMClassifierClient(
- cast(Any, _StubLLMClient()),
- signal_schema=CodingAgentRouteDecision,
- structured_output_mode="json_schema",
- )
-
- await client.classify(model="m", system_prompt="sp", request_summary="rs")
-
- rf = captured["response_format"]
- assert rf["type"] == "json_schema"
- assert rf["json_schema"]["name"] == "CodingAgentRouteDecision"
- assert rf["json_schema"]["strict"] is True
- schema = rf["json_schema"]["schema"]
- assert schema["additionalProperties"] is False
- # Every property is required in strict mode, even ones with pydantic defaults.
- assert set(schema["required"]) == set(schema["properties"].keys())
- # Banned-by-strict-mode keywords are stripped.
- assert "minimum" not in schema["properties"]["confidence"]
- assert "default" not in schema["properties"]["abstain"]
-
-
-async def test_openai_client_json_object_mode_skips_schema() -> None:
- from switchyard.lib.processors.llm_classifier.request_processor import (
- OpenAIChatLLMClassifierClient,
- )
-
- captured: dict[str, Any] = {}
-
- class _StubLLMClient:
- async def acompletion(self, **kwargs: Any) -> Any:
- captured.update(kwargs)
-
- class _Msg:
- content = "{}"
-
- class _Choice:
- message = _Msg()
-
- class _Result:
- choices = [_Choice()]
-
- return _Result()
-
- client = OpenAIChatLLMClassifierClient(
- cast(Any, _StubLLMClient()),
- structured_output_mode="json_object",
- )
-
- await client.classify(model="m", system_prompt="sp", request_summary="rs")
-
- assert captured["response_format"] == {"type": "json_object"}
-
-
-async def test_request_summary_is_truncated() -> None:
- fake = _FakeClassifierClient(_signals_json())
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model", max_request_chars=256),
- client=fake,
- )
-
- await processor.process(
- ProxyContext(),
- _request(messages=[{"role": "user", "content": "x" * 1_000}]),
- )
-
- summary = fake.calls[0]["request_summary"]
- assert len(summary) <= 256
- assert summary.endswith("...<truncated>")
-
-
-async def test_request_summary_drops_prior_turns_and_tool_schemas() -> None:
- """Slimmed summary keeps system + last user + tool names; drops the rest.
-
- Why: classifier tax on multi-turn agent traffic (TB, Claude Code) was
- ~17s/req with full tool schemas + prior turns. Strip non-signal bulk so
- Nemotron processes O(100) tokens instead of O(4k).
- """
- fake = _FakeClassifierClient(_signals_json())
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model"),
- client=fake,
- )
-
- await processor.process(
- ProxyContext(),
- _request(
- messages=[
- {"role": "system", "content": "you are an agent"},
- {"role": "user", "content": "first request"},
- {"role": "assistant", "content": "PRIOR_ASSISTANT_BULK"},
- {"role": "tool", "tool_call_id": "x", "content": "TOOL_RESULT_BULK"},
- {"role": "user", "content": "the latest ask"},
- ],
- tools=[
- {
- "type": "function",
- "function": {
- "name": "run_bash",
- "description": "run a shell command",
- "parameters": {"type": "object", "properties": {"cmd": {"type": "string"}}},
- },
- },
- ],
- ),
- )
-
- summary = fake.calls[0]["request_summary"]
- assert "you are an agent" in summary
- assert "the latest ask" in summary
- assert "run_bash" in summary
- assert "run a shell command" in summary
- assert "PRIOR_ASSISTANT_BULK" not in summary
- assert "TOOL_RESULT_BULK" not in summary
- assert "parameters" not in summary # tool input schema stripped
-
-
-async def test_request_summary_preserves_first_user_for_terminus2_shape() -> None:
- """terminus-2 bundles task framing into the FIRST user message (not system).
-
- On turn 2+ the last user message is just terminal output with no task
- context. Without preserving the first user message, the classifier
- would be blind to what task is being solved — confirmed against live
- classifier-mode runs where the slim summary on turn 2+ contained only
- "New Terminal Output: ..." with no framing.
- """
- fake = _FakeClassifierClient(_signals_json())
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model"),
- client=fake,
- )
-
- await processor.process(
- ProxyContext(),
- _request(
- messages=[
- # No system role — framing is here as a user message
- {"role": "user", "content": "TASK_FRAMING: solve permission inheritance"},
- {"role": "assistant", "content": "PRIOR_ASSISTANT_BULK"},
- {"role": "user", "content": "New Terminal Output: mkdir done"},
- ],
- ),
- )
-
- summary = fake.calls[0]["request_summary"]
- # Both first and last user messages survive
- assert "TASK_FRAMING" in summary
- assert "New Terminal Output: mkdir done" in summary
- # Prior assistant turn is still dropped
- assert "PRIOR_ASSISTANT_BULK" not in summary
-
-
-async def test_request_summary_dedupes_single_user_turn() -> None:
- """Single-turn case: first and last user are the same message — no dup."""
- fake = _FakeClassifierClient(_signals_json())
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model"),
- client=fake,
- )
-
- await processor.process(
- ProxyContext(),
- _request(messages=[{"role": "user", "content": "ONLY_TURN"}]),
- )
-
- summary = fake.calls[0]["request_summary"]
- # Should appear exactly once in the serialized body
- assert summary.count("ONLY_TURN") == 1
-
-
-async def test_request_summary_drops_anthropic_tool_input_schema() -> None:
- fake = _FakeClassifierClient(_signals_json())
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-model"),
- client=fake,
- )
-
- await processor.process(
- ProxyContext(),
- _request(
- messages=[{"role": "user", "content": "anthropic-style request"}],
- tools=[
- {
- "name": "search",
- "description": "search the web",
- "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}},
- },
- ],
- ),
- )
-
- summary = fake.calls[0]["request_summary"]
- assert "search the web" in summary
- assert "input_schema" not in summary
-
-
-async def test_classifier_records_tokens_into_stats_when_accumulator_wired() -> None:
- """Classifier overhead lands in the dedicated classifier bucket.
-
- Without this wiring the classifier's LLM spend was invisible to
- ``/v1/routing/stats`` — exactly the gap that flattered the
- classifier-vs-force-strong cost comparison in TB-lite runs.
- """
- from switchyard.lib.stats_accumulator import StatsAccumulator
-
- accumulator = StatsAccumulator()
- fake = _FakeClassifierClient(
- _signals_json(),
- usage=_FakeUsage(prompt_tokens=420, completion_tokens=80, cached_tokens=10),
- )
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-classifier"),
- client=fake,
- stats_accumulator=accumulator,
- )
-
- await processor.process(ProxyContext(), _request())
- await processor.process(ProxyContext(), _request())
-
- snapshot = accumulator.snapshot_sync()
- # Backend bucket untouched — the classifier doesn't show up among
- # routed-traffic models even though it ran twice.
- assert snapshot["models"] == {}
- assert snapshot["total_requests"] == 0
- # Classifier block carries the per-call totals + cost.
- classifier = snapshot["classifier"]
- assert classifier["total_requests"] == 2
- assert classifier["models"]["router-classifier"]["calls"] == 2
- assert classifier["models"]["router-classifier"]["prompt_tokens"] == 840
- assert classifier["models"]["router-classifier"]["completion_tokens"] == 160
- assert classifier["models"]["router-classifier"]["cached_tokens"] == 20
- # Latency was recorded — at least one sample in the histogram.
- assert classifier["models"]["router-classifier"]["model_call_latency"]["count"] == 2
-
-
-async def test_classifier_skips_stats_recording_when_no_accumulator() -> None:
- """Backwards compat: omitting the accumulator must not raise."""
- fake = _FakeClassifierClient(
- _signals_json(),
- usage=_FakeUsage(prompt_tokens=100, completion_tokens=20),
- )
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-classifier"),
- client=fake,
- )
-
- # No accumulator wired; processor still resolves successfully.
- await processor.process(ProxyContext(), _request())
-
-
-async def test_classifier_records_error_on_failure_path() -> None:
- """Fail-open path stamps abstain signals and records the error.
-
- The classifier call failed, so there are no tokens to charge — but the
- failure itself must surface on ``/v1/routing/stats`` so a silent
- fail-open does not hide the failure rate. ``total_requests`` and
- ``total_errors`` both increment; per-model ``calls`` stays at zero
- (it counts completed, token-bearing calls), and per-model ``errors``
- increments so ``errors / (calls + errors)`` reads as the true
- failure rate.
- """
- from switchyard.lib.stats_accumulator import StatsAccumulator
-
- accumulator = StatsAccumulator()
- fake = _FakeClassifierClient(RuntimeError("upstream 503"))
- processor = LLMClassifierRequestProcessor(
- LLMClassifierConfig(model="router-classifier"),
- client=fake,
- stats_accumulator=accumulator,
- )
-
- await processor.process(ProxyContext(), _request())
-
- snapshot = accumulator.snapshot_sync()
- classifier = snapshot["classifier"]
- assert classifier["total_requests"] == 1
- assert classifier["total_errors"] == 1
- model_stats = classifier["models"]["router-classifier"]
- assert model_stats["calls"] == 0
- assert model_stats["errors"] == 1
- assert model_stats["prompt_tokens"] == 0
- assert model_stats["completion_tokens"] == 0
diff --git a/tests/test_message_condensing.py b/tests/test_message_condensing.py
deleted file mode 100644
index 7ab74f664..000000000
--- a/tests/test_message_condensing.py
+++ /dev/null
@@ -1,70 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for the shared message-condensing helpers."""
-
-from __future__ import annotations
-
-from switchyard.lib.processors.message_condensing import (
- content_text,
- message_text,
- strip_markdown_fence,
- truncate_middle,
-)
-
-
-def test_content_text_renders_anthropic_tool_use() -> None:
- """tool_use blocks have neither text nor nested content; render name+input."""
- blocks = [
- {"type": "text", "text": "let me check"},
- {"type": "tool_use", "id": "tu_1", "name": "bash", "input": {"command": "pytest -x"}},
- ]
-
- text = content_text(blocks)
-
- assert "let me check" in text
- assert 'tool_call bash({"command": "pytest -x"})' in text
-
-
-def test_content_text_renders_tool_result_content() -> None:
- blocks = [
- {
- "type": "tool_result",
- "tool_use_id": "tu_1",
- "content": [{"type": "text", "text": "1 failed"}],
- },
- ]
- assert content_text(blocks) == "1 failed"
-
-
-def test_message_text_covers_all_three_tool_call_shapes() -> None:
- chat = {
- "role": "assistant",
- "content": None,
- "tool_calls": [{"function": {"name": "read_file", "arguments": '{"path": "a.py"}'}}],
- }
- assert message_text(chat) == 'tool_call read_file({"path": "a.py"})'
-
- responses_item = {"type": "function_call", "name": "bash", "arguments": '{"command": "ls"}'}
- assert message_text(responses_item) == 'tool_call bash({"command": "ls"})'
-
- anthropic = {
- "role": "assistant",
- "content": [{"type": "tool_use", "name": "bash", "input": {"command": "ls"}}],
- }
- assert message_text(anthropic) == 'tool_call bash({"command": "ls"})'
-
-
-def test_truncate_middle_keeps_head_and_tail() -> None:
- text = "H" * 100 + "T" * 100
- out = truncate_middle(text, 60)
- assert len(out) <= 60
- assert out.startswith("H")
- assert out.endswith("T")
- assert "...[trimmed]" in out
-
-
-def test_strip_markdown_fence_unwraps_json() -> None:
- fenced = '```json\n{"escalate": true}\n```'
- assert strip_markdown_fence(fenced) == '{"escalate": true}'
- assert strip_markdown_fence('{"escalate": true}') == '{"escalate": true}'
diff --git a/tests/test_python_profile_abstractions.py b/tests/test_python_profile_abstractions.py
deleted file mode 100644
index 637bb90d1..000000000
--- a/tests/test_python_profile_abstractions.py
+++ /dev/null
@@ -1,484 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for Python-defined profile protocols and config decorators."""
-
-from dataclasses import FrozenInstanceError, dataclass, is_dataclass
-from typing import cast
-from uuid import uuid4
-
-import pytest
-
-from switchyard import (
- Profile,
- ProfileConfigError,
- ProfileInput,
- build_profile,
- profile_config,
- profile_config_type,
-)
-from switchyard.lib.profiles.chain import ComponentChainProfile
-from switchyard.lib.profiles.table import (
- lookup_profile_config,
- register_profile_config,
- registered_profile_config_types,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.roles import LLMBackend
-from switchyard_rust.core import (
- ChatRequest,
- ChatRequestType,
- ChatResponse,
- SwitchyardContextPoolExhaustedError,
- SwitchyardContextWindowExceededError,
-)
-from switchyard_rust.profiles import ProfileRequestMetadata
-
-
-@dataclass(frozen=True)
-class _StaticProcessedRequest:
- input: ProfileInput
- marker: str
-
-
-class _StaticProfile:
- def __init__(self, content: str) -> None:
- self._content = content
-
- async def process(self, input: ProfileInput) -> _StaticProcessedRequest:
- return _StaticProcessedRequest(input=input, marker="processed")
-
- async def rprocess(
- self,
- processed: _StaticProcessedRequest,
- response: ChatResponse,
- ) -> ChatResponse:
- assert processed.marker == "processed"
- return response
-
- async def run(self, input: ProfileInput) -> ChatResponse:
- processed = await self.process(input)
- return await self.rprocess(
- processed,
- ChatResponse.openai_completion({
- "id": "python-profile-test",
- "object": "chat.completion",
- "model": processed.input.request.model,
- "choices": [
- {
- "index": 0,
- "message": {"role": "assistant", "content": self._content},
- "finish_reason": "stop",
- }
- ],
- }),
- )
-
-
-class _MetadataCapturingProcessor:
- def __init__(self) -> None:
- self.metadata: dict[str, object] = {}
-
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- self.metadata = dict(ctx.metadata)
- return request
-
-
-class _StaticBackend(LLMBackend):
- @property
- def supported_request_types(self) -> list[ChatRequestType]:
- return [ChatRequestType.OPENAI_CHAT]
-
- async def call(self, _ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
- return ChatResponse.openai_completion({
- "id": "python-profile-chain",
- "object": "chat.completion",
- "model": request.model,
- "choices": [
- {
- "index": 0,
- "message": {"role": "assistant", "content": "ok"},
- "finish_reason": "stop",
- }
- ],
- })
-
-
-class _PickWeakProcessor:
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- ctx.selected_target = "weak"
- return request
-
-
-class _AlwaysOverflowBackend(LLMBackend):
- def __init__(self) -> None:
- self.calls: list[str | None] = []
-
- @property
- def supported_request_types(self) -> list[ChatRequestType]:
- return [ChatRequestType.OPENAI_CHAT]
-
- async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
- _ = request
- self.calls.append(ctx.selected_target)
- error = SwitchyardContextWindowExceededError("target overflowed")
- error.target_id = ctx.selected_target
- raise error
-
-
-class _OverflowWeakBackend(LLMBackend):
- """Overflows on 'weak', succeeds on any other target."""
-
- def __init__(self) -> None:
- self.calls: list[str | None] = []
-
- @property
- def supported_request_types(self) -> list[ChatRequestType]:
- return [ChatRequestType.OPENAI_CHAT]
-
- async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
- self.calls.append(ctx.selected_target)
- if ctx.selected_target == "weak":
- error = SwitchyardContextWindowExceededError("weak overflowed")
- error.target_id = "weak"
- raise error
- return ChatResponse.openai_completion({
- "id": "chain-ok",
- "object": "chat.completion",
- "model": request.model,
- "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
- })
-
-
-def _request(*, msg: str = "hi") -> ChatRequest:
- return ChatRequest.openai_chat({
- "model": "client/model",
- "messages": [{"role": "user", "content": msg}],
- })
-
-
-def test_profile_request_metadata_normalizes_headers() -> None:
- metadata = ProfileRequestMetadata.from_headers(
- {
- "X-Request-ID": "req-123",
- "X-Switchyard-Trace": ["trace-a", "trace-b"],
- },
- inbound_format=ChatRequestType.OPENAI_CHAT,
- )
-
- assert metadata.request_id == "req-123"
- assert metadata.inbound_format == ChatRequestType.OPENAI_CHAT
- assert metadata.headers == {
- "x-request-id": ["req-123"],
- "x-switchyard-trace": ["trace-a", "trace-b"],
- }
- assert metadata.to_dict()["inbound_format"] == "openai_chat"
-
-
-def test_profile_request_metadata_rejects_empty_request_id() -> None:
- with pytest.raises(ValueError, match="RequestId must not be empty"):
- ProfileRequestMetadata(request_id=" ")
-
-
-async def test_profile_config_decorator_builds_dataclass_and_profile_runtime() -> None:
- @profile_config("unit-python-static", register=False)
- class StaticProfileConfig:
- content: str
-
- def build(self) -> _StaticProfile:
- return _StaticProfile(self.content)
-
- config = StaticProfileConfig(content="ok")
-
- assert is_dataclass(StaticProfileConfig)
- assert not hasattr(config, "__dict__")
- assert profile_config_type(config) == "unit-python-static"
- assert profile_config_type(StaticProfileConfig) == "unit-python-static"
- with pytest.raises(FrozenInstanceError):
- cast(object, config).content = "mutated" # type: ignore[attr-defined]
-
- profile = build_profile(config)
- assert isinstance(profile, Profile)
- input = ProfileInput(_request())
- assert isinstance(input.metadata, ProfileRequestMetadata)
- response = await profile.run(input)
-
- assert response.body["id"] == "python-profile-test"
- assert response.body["model"] == "client/model"
- assert response.body["choices"][0]["message"]["content"] == "ok"
-
-
-async def test_component_chain_profile_preserves_input_headers_in_context() -> None:
- processor = _MetadataCapturingProcessor()
- profile = ComponentChainProfile(
- request_processors=[processor],
- backend=_StaticBackend(),
- )
- metadata = ProfileRequestMetadata.from_headers({
- "X-Switchyard-Trace": ["trace-a", "trace-b"],
- "X-Request-Id": "request-1",
- })
-
- await profile.run(ProfileInput(_request(), metadata=metadata))
-
- assert processor.metadata["x-switchyard-trace"] == ["trace-a", "trace-b"]
- assert processor.metadata["x-request-id"] == ["request-1"]
-
-
-async def test_component_chain_raises_pool_exhausted_when_fallback_overflows() -> None:
- """A second context-window overflow reports the final attempted target."""
- backend = _AlwaysOverflowBackend()
- profile = ComponentChainProfile(
- request_processors=[_PickWeakProcessor()],
- backend=backend,
- fallback_target_on_evict="strong",
- )
-
- with pytest.raises(SwitchyardContextPoolExhaustedError) as excinfo:
- await profile.run(ProfileInput(_request()))
-
- assert backend.calls == ["weak", "strong"]
- assert excinfo.value.last_target_id == "strong"
- assert excinfo.value.reason == "all attempted targets returned context-window overflow"
-
-
-async def test_session_eviction_skips_evicted_target_on_subsequent_turn() -> None:
- """After weak overflows in turn 1, turn 2 of the same session skips weak entirely."""
- backend = _OverflowWeakBackend()
- profile = ComponentChainProfile(
- request_processors=[_PickWeakProcessor()],
- backend=backend,
- fallback_target_on_evict="strong",
- )
- request = _request(msg="hi")
-
- # Turn 1: weak overflows → fallback to strong.
- await profile.run(ProfileInput(request))
- assert backend.calls == ["weak", "strong"]
-
- backend.calls.clear()
-
- # Turn 2: same session → weak is pre-empted, strong called directly.
- await profile.run(ProfileInput(request))
- assert backend.calls == ["strong"]
-
-
-async def test_session_eviction_records_fallback_target_after_pool_exhaustion() -> None:
- """Both targets are written to session cache when pool is exhausted."""
- from switchyard.lib.session_key import session_key_from_body
-
- backend = _AlwaysOverflowBackend()
- profile = ComponentChainProfile(
- request_processors=[_PickWeakProcessor()],
- backend=backend,
- fallback_target_on_evict="strong",
- )
- request = _request(msg="session-pool")
-
- with pytest.raises(SwitchyardContextPoolExhaustedError):
- await profile.run(ProfileInput(request))
-
- assert profile._session_evictions is not None
- session_key = session_key_from_body(request.body)
- recorded = profile._session_evictions.get(session_key) or frozenset()
- assert "weak" in recorded
- assert "strong" in recorded
-
-
-async def test_session_eviction_does_not_bleed_between_sessions() -> None:
- """Eviction in one session does not affect a different session."""
- backend = _OverflowWeakBackend()
- profile = ComponentChainProfile(
- request_processors=[_PickWeakProcessor()],
- backend=backend,
- fallback_target_on_evict="strong",
- )
-
- # Session A: weak overflows → evicted for session A.
- await profile.run(ProfileInput(_request(msg="session-a")))
-
- backend.calls.clear()
-
- # Session B (different first message → different session key): weak is tried.
- await profile.run(ProfileInput(_request(msg="session-b")))
- assert backend.calls == ["weak", "strong"]
-
-
-def test_profile_config_decorator_accepts_existing_dataclass() -> None:
- @profile_config("unit-python-prebuilt-dataclass", register=False)
- @dataclass(frozen=True)
- class ExistingDataclassConfig:
- content: str
-
- def build(self) -> _StaticProfile:
- return _StaticProfile(self.content)
-
- config = ExistingDataclassConfig(content="prebuilt")
-
- assert is_dataclass(ExistingDataclassConfig)
- assert profile_config_type(config) == "unit-python-prebuilt-dataclass"
- assert build_profile(config).run is not None
-
-
-def test_profile_config_decorator_registers_and_rejects_duplicates() -> None:
- profile_type = f"unit-python-registered-{uuid4().hex}"
-
- @profile_config(profile_type, register=True)
- class RegisteredProfileConfig:
- content: str = "registered"
-
- def build(self) -> _StaticProfile:
- return _StaticProfile(self.content)
-
- assert lookup_profile_config(profile_type) is RegisteredProfileConfig
- assert profile_type in registered_profile_config_types()
-
- with pytest.raises(ProfileConfigError, match="already registered"):
-
- @profile_config(profile_type, register=True)
- class DuplicateProfileConfig:
- content: str = "duplicate"
-
- def build(self) -> _StaticProfile:
- return _StaticProfile(self.content)
-
-
-def test_profile_config_decorator_rejects_invalid_declarations() -> None:
- with pytest.raises(ProfileConfigError, match="must not be empty"):
- profile_config(" ")
-
- with pytest.raises(ProfileConfigError, match="must define build"):
-
- @profile_config("unit-python-missing-build", register=False)
- class MissingBuildConfig:
- content: str = "missing"
-
-
-def test_register_profile_config_validates_direct_callers() -> None:
- @dataclass(frozen=True)
- class MissingBuildConfig:
- value: str = "missing"
-
- with pytest.raises(ProfileConfigError, match="must define build"):
- register_profile_config(
- f"unit-python-direct-invalid-{uuid4().hex}",
- cast(object, MissingBuildConfig), # type: ignore[arg-type]
- )
-
-
-def test_table_helpers_stay_out_of_top_level_namespace() -> None:
- import switchyard
- import switchyard.lib.profiles as profiles
-
- assert "lookup_profile_config" not in switchyard.__all__
- assert "register_profile_config" not in switchyard.__all__
- assert "registered_profile_config_types" not in switchyard.__all__
- assert "lookup_profile_config" not in profiles.__all__
- assert "register_profile_config" not in profiles.__all__
-
-
-def test_build_profile_rejects_configs_that_do_not_return_profiles() -> None:
- @profile_config("unit-python-bad-build", register=False)
- class BadBuildConfig:
- def build(self) -> object:
- return object()
-
- with pytest.raises(ProfileConfigError, match="did not return a Profile"):
- build_profile(BadBuildConfig())
-
-
-def test_build_profile_rejects_run_only_profiles() -> None:
- class RunOnlyProfile:
- async def run(self, input: ProfileInput) -> ChatResponse:
- return ChatResponse.openai_completion({
- "id": "python-profile-run-only",
- "object": "chat.completion",
- "model": input.request.model,
- "choices": [
- {
- "index": 0,
- "message": {"role": "assistant", "content": "ok"},
- "finish_reason": "stop",
- }
- ],
- })
-
- @profile_config("unit-python-run-only", register=False)
- class RunOnlyConfig:
- def build(self) -> RunOnlyProfile:
- return RunOnlyProfile()
-
- with pytest.raises(ProfileConfigError, match="run/process/rprocess"):
- build_profile(RunOnlyConfig())
-
-
-def test_build_profile_rejects_sync_run_profiles() -> None:
- class SyncRunProfile:
- async def process(self, input: ProfileInput) -> ProfileInput:
- return input
-
- async def rprocess(
- self,
- processed: ProfileInput,
- response: ChatResponse,
- ) -> ChatResponse:
- return response
-
- def run(self, input: ProfileInput) -> ChatResponse:
- return ChatResponse.openai_completion({"model": input.request.model})
-
- @profile_config("unit-python-sync-run", register=False)
- class SyncRunConfig:
- def build(self) -> SyncRunProfile:
- return SyncRunProfile()
-
- with pytest.raises(ProfileConfigError, match="run\\(\\) must be async"):
- build_profile(SyncRunConfig())
-
-
-def test_build_profile_rejects_sync_process_profiles() -> None:
- class SyncProcessProfile:
- def process(self, input: ProfileInput) -> ProfileInput:
- return input
-
- async def rprocess(
- self,
- processed: ProfileInput,
- response: ChatResponse,
- ) -> ChatResponse:
- return response
-
- async def run(self, input: ProfileInput) -> ChatResponse:
- return ChatResponse.openai_completion({"model": input.request.model})
-
- @profile_config("unit-python-sync-process", register=False)
- class SyncProcessConfig:
- def build(self) -> SyncProcessProfile:
- return SyncProcessProfile()
-
- with pytest.raises(ProfileConfigError, match="process\\(\\) must be async"):
- build_profile(SyncProcessConfig())
-
-
-def test_build_profile_rejects_sync_rprocess_profiles() -> None:
- class SyncRprocessProfile:
- async def process(self, input: ProfileInput) -> ProfileInput:
- return input
-
- def rprocess(
- self,
- processed: ProfileInput,
- response: ChatResponse,
- ) -> ChatResponse:
- return response
-
- async def run(self, input: ProfileInput) -> ChatResponse:
- return ChatResponse.openai_completion({"model": input.request.model})
-
- @profile_config("unit-python-sync-rprocess", register=False)
- class SyncRprocessConfig:
- def build(self) -> SyncRprocessProfile:
- return SyncRprocessProfile()
-
- with pytest.raises(ProfileConfigError, match="rprocess\\(\\) must be async"):
- build_profile(SyncRprocessConfig())
diff --git a/tests/test_passthrough_profile_e2e.py b/tests/test_python_server_passthrough.py
similarity index 93%
rename from tests/test_passthrough_profile_e2e.py
rename to tests/test_python_server_passthrough.py
index 3a42ee280..4dc5eabaa 100644
--- a/tests/test_passthrough_profile_e2e.py
+++ b/tests/test_python_server_passthrough.py
@@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""End-to-end profile tests with mocked upstream model providers.
+"""End-to-end Python server passthrough tests with a mocked upstream.
This file is the highest-leverage entry in the mocked-test suite added after
the post-PR-#12 wiring regressions. Where ``test_inference_e2e.py`` swaps in
@@ -34,7 +34,7 @@ from typing import Any
import httpx
import pytest
-from switchyard import PassthroughProfileConfig, ProfileSwitchyard
+from switchyard.cli.route_bundle import build_route_bundle_table
from switchyard.lib.endpoints import outcome_metrics
from switchyard.server.switchyard_app import build_switchyard_app
@@ -173,20 +173,18 @@ def passthrough_upstream() -> Iterator[_OpenAICompatStub]:
async def passthrough_client(
passthrough_upstream: _OpenAICompatStub,
) -> AsyncIterator[httpx.AsyncClient]:
- """A FastAPI client wired through the real passthrough profile.
-
- The profile builds a real ``OpenAiPassthroughBackend`` pointed at
- a local OpenAI-compatible stub to control the upstream response.
- """
- switchyard = ProfileSwitchyard(
- PassthroughProfileConfig(
- api_key="test-key-not-used",
- base_url=passthrough_upstream.base_url,
- )
- .build()
- .with_runtime_components(enable_stats=False)
- )
- app = build_switchyard_app(switchyard)
+ """Return a client using the retained passthrough route."""
+ table = build_route_bundle_table({
+ "defaults": {
+ "api_key": "test-key-not-used",
+ "base_url": passthrough_upstream.base_url,
+ "format": "openai",
+ },
+ "routes": {
+ "any-model": {"type": "passthrough", "target": "any-model"},
+ },
+ })
+ app = build_switchyard_app(table)
# ``raise_app_exceptions=False`` mirrors what uvicorn does in
# production: an unhandled exception inside a route is mapped to a
# 500 response, not propagated up the call stack. Without this the
@@ -200,12 +198,12 @@ async def passthrough_client(
# ---------------------------------------------------------------------------
-# Passthrough profile — OpenAI inbound, OpenAI upstream
+# Passthrough route — OpenAI inbound, OpenAI upstream
# ---------------------------------------------------------------------------
-class TestPassthroughProfileOpenAI:
- """Inbound OpenAI Chat Completions through the real passthrough profile."""
+class TestPassthroughOpenAI:
+ """Inbound OpenAI Chat Completions through the passthrough route."""
async def test_non_streaming_round_trip(
self,
@@ -268,11 +266,11 @@ class TestPassthroughProfileOpenAI:
# ---------------------------------------------------------------------------
-# Passthrough profile — Anthropic inbound, OpenAI upstream
+# Passthrough route — Anthropic inbound, OpenAI upstream
# ---------------------------------------------------------------------------
-class TestPassthroughProfileAnthropic:
+class TestPassthroughAnthropic:
"""Inbound Anthropic Messages, translated to OpenAI for the upstream call."""
async def test_non_streaming_translates_both_directions(
@@ -308,11 +306,11 @@ class TestPassthroughProfileAnthropic:
# ---------------------------------------------------------------------------
-# Passthrough profile — Responses API inbound, OpenAI Chat upstream
+# Passthrough route — Responses API inbound, OpenAI Chat upstream
# ---------------------------------------------------------------------------
-class TestPassthroughProfileResponsesApi:
+class TestPassthroughResponsesApi:
"""Inbound OpenAI Responses API, translated to Chat Completions upstream."""
async def test_non_streaming_translates_both_directions(
@@ -342,7 +340,7 @@ class TestPassthroughProfileResponsesApi:
# ---------------------------------------------------------------------------
-class TestPassthroughProfileBackendErrors:
+class TestPassthroughBackendErrors:
"""Upstream HTTP errors must surface as HTTP errors to the client.
A regression where the chain swallows an upstream 4xx/5xx and still
@@ -398,7 +396,7 @@ class TestPassthroughProfileBackendErrors:
# ---------------------------------------------------------------------------
-class TestPassthroughProfileUpstreamAttemptCounters:
+class TestPassthroughUpstreamAttemptCounters:
"""`switchyard_upstream_attempts_total` must populate for passthrough chains.
The endpoint-layer fallback records one upstream attempt per request for
diff --git a/tests/test_random_routing_config.py b/tests/test_random_routing_config.py
deleted file mode 100644
index 64aa942fd..000000000
--- a/tests/test_random_routing_config.py
+++ /dev/null
@@ -1,191 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for ``BackendFormat``, ``LlmTarget`` and ``RandomRoutingConfig``."""
-
-from __future__ import annotations
-
-import pytest
-from pydantic import ValidationError
-
-from switchyard.lib.backends.llm_target import (
- BackendFormat,
- LlmTarget,
-)
-from switchyard.lib.profiles.random_routing import RandomRoutingConfig
-
-# ---------------------------------------------------------------------------
-# BackendFormat enum
-# ---------------------------------------------------------------------------
-
-
-class TestBackendFormat:
- def test_auto_value(self):
- assert BackendFormat.AUTO.value == "auto"
-
- def test_openai_value(self):
- assert BackendFormat.OPENAI.value == "openai"
-
- def test_anthropic_value(self):
- assert BackendFormat.ANTHROPIC.value == "anthropic"
-
- def test_responses_value(self):
- assert BackendFormat.RESPONSES.value == "responses"
-
- def test_is_rust_value_object(self):
- assert not isinstance(BackendFormat.OPENAI, str)
- assert str(BackendFormat.OPENAI) == "openai"
-
- def test_equals_string(self):
- assert BackendFormat.AUTO == "auto"
- assert BackendFormat.OPENAI == "openai"
- assert BackendFormat.RESPONSES == "responses"
- assert BackendFormat.ANTHROPIC == "anthropic"
-
- def test_constructible_from_string(self):
- assert BackendFormat("auto") == BackendFormat.AUTO
- assert BackendFormat("openai") == BackendFormat.OPENAI
- assert BackendFormat("responses") == BackendFormat.RESPONSES
- assert BackendFormat("anthropic") == BackendFormat.ANTHROPIC
-
- def test_unknown_value_rejected(self):
- with pytest.raises(ValueError):
- BackendFormat("bedrock")
-
-
-# ---------------------------------------------------------------------------
-# LlmTarget
-# ---------------------------------------------------------------------------
-
-
-class TestTierConstruction:
- def test_minimal_construction(self):
- tier = LlmTarget(model="m")
- assert tier.model == "m"
- assert tier.id == "default"
- assert tier.format is BackendFormat.AUTO # Rust LlmTarget default
- assert tier.api_key is None
- assert tier.base_url is None
- assert tier.timeout is None
-
- def test_all_fields(self):
- tier = LlmTarget(
- model="claude-opus-4-6",
- format=BackendFormat.ANTHROPIC,
- api_key="sk-ant-xxx",
- base_url="https://api.anthropic.com",
- timeout=30.0,
- )
- assert tier.model == "claude-opus-4-6"
- assert tier.format is BackendFormat.ANTHROPIC
- assert tier.api_key == "sk-ant-xxx"
- assert tier.base_url == "https://api.anthropic.com"
- assert tier.timeout == 30.0
-
-
-class TestTierValidation:
- def test_empty_model_rejected(self):
- with pytest.raises(ValueError, match="model id"):
- LlmTarget(model="")
-
-
-class TestTierFrozenSemantics:
- def test_model_is_immutable(self):
- tier = LlmTarget(model="m")
- with pytest.raises(AttributeError):
- tier.model = "other" # type: ignore[misc]
-
- def test_backend_format_is_immutable(self):
- tier = LlmTarget(model="m")
- with pytest.raises(AttributeError):
- tier.format = BackendFormat.ANTHROPIC # type: ignore[misc]
-
- def test_equality_by_fields(self):
- t1 = LlmTarget(
- model="m", format=BackendFormat.OPENAI, api_key="k",
- )
- t2 = LlmTarget(
- model="m", format=BackendFormat.OPENAI, api_key="k",
- )
- assert t1 == t2
-
- def test_inequality_on_format_difference(self):
- t1 = LlmTarget(model="m", format=BackendFormat.OPENAI)
- t2 = LlmTarget(model="m", format=BackendFormat.ANTHROPIC)
- assert t1 != t2
-
-
-# ---------------------------------------------------------------------------
-# RandomRoutingConfig
-# ---------------------------------------------------------------------------
-
-
-def _tier(model: str = "m") -> LlmTarget:
- return LlmTarget(model=model)
-
-
-class TestConfigConstruction:
- def test_minimal_construction(self):
- cfg = RandomRoutingConfig(
- strong=_tier("s"), weak=_tier("w"),
- fallback_target_on_evict="strong")
- assert cfg.strong.model == "s"
- assert cfg.weak.model == "w"
- assert cfg.strong_probability == 0.5 # default
-
- def test_custom_probability(self):
- cfg = RandomRoutingConfig(
- strong=_tier("s"), weak=_tier("w"), strong_probability=0.7,
- fallback_target_on_evict="strong")
- assert cfg.strong_probability == 0.7
-
-
-class TestConfigValidation:
- @pytest.mark.parametrize("p", [-0.1, -1.0, 1.1, 2.0, float("inf")])
- def test_out_of_range_probability_rejected(self, p: float):
- with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\]"):
- RandomRoutingConfig(
- strong=_tier(), weak=_tier(), strong_probability=p,
- fallback_target_on_evict="strong")
-
- @pytest.mark.parametrize("p", [0.0, 0.25, 0.5, 0.75, 1.0])
- def test_valid_probability_accepted(self, p: float):
- cfg = RandomRoutingConfig(
- strong=_tier(), weak=_tier(), strong_probability=p,
- fallback_target_on_evict="strong")
- assert cfg.strong_probability == p
-
-
-class TestConfigFrozenSemantics:
- def test_strong_is_immutable(self):
- cfg = RandomRoutingConfig(strong=_tier("a"), weak=_tier("b"), fallback_target_on_evict="strong")
- with pytest.raises(ValidationError):
- cfg.strong = _tier("other") # type: ignore[misc]
-
- def test_strong_probability_is_immutable(self):
- cfg = RandomRoutingConfig(strong=_tier(), weak=_tier(), fallback_target_on_evict="strong")
- with pytest.raises(ValidationError):
- cfg.strong_probability = 0.9 # type: ignore[misc]
-
-
-class TestTargetSerialization:
- def test_model_dump_uses_rust_shape(self):
- target = LlmTarget(
- id="target",
- model="m",
- format=BackendFormat.OPENAI,
- api_key="k",
- base_url="https://example.test/v1",
- timeout=3.0,
- )
-
- assert target.model_dump() == {
- "id": "target",
- "model": "m",
- "format": "openai",
- "endpoint": {
- "api_key": "k",
- "base_url": "https://example.test/v1",
- "timeout_secs": 3.0,
- },
- }
diff --git a/tests/test_random_routing_llm_backend.py b/tests/test_random_routing_llm_backend.py
deleted file mode 100644
index 5087d676d..000000000
--- a/tests/test_random_routing_llm_backend.py
+++ /dev/null
@@ -1,321 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for the Rust-owned random-routing backend path."""
-
-from __future__ import annotations
-
-import pytest
-
-from switchyard.lib.backends.backend_format_resolver import (
- BackendFormatResolution,
- BackendFormatResolver,
-)
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
-from switchyard.lib.backends.multi_llm_backend import (
- build_multi_llm_backend,
- build_native_backend,
- build_target_backend,
- resolve_llm_target,
-)
-from switchyard.lib.processors.random_routing_request_processor import (
- RandomRoutingRequestProcessor,
-)
-from switchyard.lib.profiles.random_routing import (
- RandomRoutingConfig,
- RandomRoutingProfileConfig,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.components import (
- AnthropicNativeBackend,
- MultiLlmBackend,
- OpenAiNativeBackend,
-)
-from switchyard_rust.core import ChatRequest
-
-
-def _target(
- target_id: str,
- model: str,
- *,
- format: object = BackendFormat.OPENAI,
- api_key: str | None = "sk-test",
- base_url: str | None = "https://example.invalid/v1",
-) -> LlmTarget:
- return LlmTarget(
- id=target_id,
- model=model,
- format=format,
- api_key=api_key,
- base_url=base_url,
- )
-
-
-def _request(model: str = "client-model") -> ChatRequest:
- return ChatRequest.openai_chat({
- "model": model,
- "messages": [{"role": "user", "content": "hi"}],
- })
-
-
-class TestNativeBackendConstruction:
- def test_explicit_openai_target_builds_openai_native_backend(self) -> None:
- target = _target("strong", "gpt-test", format=BackendFormat.OPENAI)
-
- backend = build_native_backend(target)
-
- assert isinstance(backend, OpenAiNativeBackend)
- assert backend.target == target
- assert [item.value for item in backend.supported_request_types] == [
- "openai_chat"
- ]
-
- def test_nemotron_super_target_disables_thinking_by_default(self) -> None:
- target = _target(
- "weak",
- "nvidia/nvidia/nemotron-3-super-v3",
- format=BackendFormat.OPENAI,
- )
-
- backend = build_native_backend(target)
-
- assert backend.target.extra_body == {
- "chat_template_kwargs": {"enable_thinking": False},
- }
-
- def test_explicit_extra_body_wins_over_runtime_defaults(self) -> None:
- target = LlmTarget(
- id="weak",
- model="nvidia/nvidia/nemotron-3-super-v3",
- format=BackendFormat.OPENAI,
- api_key="sk-test",
- base_url="https://example.invalid/v1",
- extra_body={"chat_template_kwargs": {"enable_thinking": True}},
- )
-
- backend = build_native_backend(target)
-
- assert backend.target.extra_body == {
- "chat_template_kwargs": {"enable_thinking": True},
- }
-
- def test_explicit_anthropic_target_builds_anthropic_native_backend(self) -> None:
- target = _target("strong", "claude-test", format=BackendFormat.ANTHROPIC)
-
- backend = build_native_backend(target)
-
- assert isinstance(backend, AnthropicNativeBackend)
- assert backend.target == target
- assert [item.value for item in backend.supported_request_types] == [
- "anthropic"
- ]
-
- def test_auto_target_is_resolved_before_native_backend_build(
- self,
- monkeypatch: pytest.MonkeyPatch,
- ) -> None:
- target = _target("strong", "anthropic/claude-test", format=BackendFormat.AUTO)
- seen: list[LlmTarget] = []
-
- def fake_resolve(value: LlmTarget) -> BackendFormatResolution:
- seen.append(value)
- return BackendFormatResolution(
- format=BackendFormat.ANTHROPIC,
- reason="test resolution",
- )
-
- monkeypatch.setattr(BackendFormatResolver, "resolve", fake_resolve)
-
- resolved = resolve_llm_target(target)
- backend = build_native_backend(target)
-
- assert seen == [target, target]
- assert resolved.format == BackendFormat.ANTHROPIC
- assert resolved.model == target.model
- assert isinstance(backend, AnthropicNativeBackend)
- assert backend.target.format == BackendFormat.ANTHROPIC
-
- @pytest.mark.parametrize("missing", ["base_url", "api_key"])
- def test_auto_resolution_requires_probe_inputs(self, missing: str) -> None:
- target = _target(
- "strong",
- "anthropic/claude-test",
- format=BackendFormat.AUTO,
- base_url=None if missing == "base_url" else "https://example.invalid/v1",
- api_key=None if missing == "api_key" else "sk-test",
- )
-
- with pytest.raises(ValueError, match=f"requires {missing}"):
- build_native_backend(target)
-
-
-class TestMultiLlmBackendConstruction:
- def test_build_target_backend_keeps_target_metadata(self) -> None:
- target = _target("weak", "weak-model")
-
- target_backend = build_target_backend(target)
-
- assert target_backend.target == target
-
- def test_multi_backend_advertises_all_inbound_request_types(self) -> None:
- backend = build_multi_llm_backend({
- "strong": _target("strong", "strong-model"),
- "weak": _target("weak", "weak-model"),
- })
-
- assert isinstance(backend, MultiLlmBackend)
- assert backend.target_ids() == ["strong", "weak"]
- assert [item.value for item in backend.supported_request_types] == [
- "openai_chat",
- "openai_responses",
- "anthropic",
- ]
-
- def test_mapping_keys_fill_default_target_ids(self) -> None:
- backend = build_multi_llm_backend({
- "strong": LlmTarget(
- model="strong-model",
- api_key="sk-test",
- base_url="https://example.invalid/v1",
- ),
- "weak": LlmTarget(
- model="weak-model",
- api_key="sk-test",
- base_url="https://example.invalid/v1",
- ),
- })
-
- assert backend.target_ids() == ["strong", "weak"]
-
- def test_duplicate_target_ids_are_rejected(self) -> None:
- with pytest.raises(RuntimeError, match="duplicate LLM target id"):
- build_multi_llm_backend([
- _target("duplicate", "strong-model"),
- _target("duplicate", "weak-model"),
- ])
-
- async def test_invalid_context_target_errors_before_network(self) -> None:
- backend = build_multi_llm_backend([
- _target("strong", "strong-model"),
- _target("weak", "weak-model"),
- ])
- ctx = ProxyContext()
- ctx.selected_target = "missing"
-
- with pytest.raises(RuntimeError, match="selected target missing is not configured"):
- await backend.call(ctx, _request())
-
- async def test_missing_selection_errors_before_network(self) -> None:
- backend = build_multi_llm_backend([
- _target("strong", "strong-model"),
- _target("weak", "weak-model"),
- ])
-
- with pytest.raises(RuntimeError, match="multiple targets but no selected target"):
- await backend.call(ProxyContext(), _request("client-only-model"))
-
-
-class TestRandomRoutingProcessorContract:
- async def test_processor_rewrites_model_and_stamps_context(self) -> None:
- config = RandomRoutingConfig(
- strong=_target("strong", "strong-model"),
- weak=_target("weak", "weak-model"),
- strong_probability=1.0,
- rng_seed=7,
- fallback_target_on_evict="strong",
- )
- processor = RandomRoutingRequestProcessor(config.processor_config)
- ctx = ProxyContext()
-
- decision = processor.select("client-model")
- out = await processor.process(ctx, _request())
-
- assert decision["tier"] == "strong"
- assert decision["selected_target"] == "strong"
- assert decision["selected_model"] == "strong-model"
- assert decision["original_model"] == "client-model"
- assert out.model == "strong-model"
- assert ctx.selected_target == "strong"
-
- async def test_processor_preserves_non_model_request_fields(self) -> None:
- config = RandomRoutingConfig(
- strong=_target("strong", "strong-model"),
- weak=_target("weak", "weak-model"),
- strong_probability=0.0,
- rng_seed=7,
- fallback_target_on_evict="strong",
- )
- processor = RandomRoutingRequestProcessor(config.processor_config)
- ctx = ProxyContext()
- request = ChatRequest.openai_chat({
- "model": "client-model",
- "messages": [{"role": "user", "content": "keep me"}],
- "temperature": 0.2,
- "max_tokens": 123,
- })
-
- out = await processor.process(ctx, request)
-
- assert out.model == "weak-model"
- assert out.body["messages"] == [{"role": "user", "content": "keep me"}]
- assert out.body["temperature"] == 0.2
- assert out.body["max_tokens"] == 123
- assert ctx.selected_target == "weak"
-
- def test_factory_builds_processor_config_without_backend_fluff(self) -> None:
- config = RandomRoutingConfig(
- strong=_target("strong", "strong-model"),
- weak=_target("weak", "weak-model"),
- strong_probability=0.75,
- rng_seed=99,
- preset="opus_47_minimax",
- fallback_target_on_evict="strong",
- )
-
- processor_config = config.processor_config
-
- assert processor_config.strong == config.strong
- assert processor_config.weak == config.weak
- assert processor_config.strong_probability == 0.75
- assert processor_config.rng_seed == 99
- assert not hasattr(processor_config, "enable_stats")
- assert not hasattr(processor_config, "preset")
-
- async def test_profile_stats_backend_and_response_processor_share_accumulator(self) -> None:
- from switchyard.lib.processors.stats_response_processor_accumulator import (
- StatsResponseProcessor,
- )
- from switchyard.lib.stats_accumulator import StatsAccumulator
- from switchyard_rust.components import StatsLlmBackend
-
- stats = StatsAccumulator()
- config = RandomRoutingConfig(
- strong=_target("strong", "strong-model"),
- weak=_target("weak", "weak-model"),
- strong_probability=0.5,
- enable_stats=True,
- fallback_target_on_evict="strong",
- )
- profile = (
- RandomRoutingProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats,
- enable_stats=config.enable_stats,
- )
- )
- components = profile.iter_components()
- backend = next(component for component in components if isinstance(component, StatsLlmBackend))
- response_processor = next(
- component
- for component in components
- if isinstance(component, StatsResponseProcessor)
- )
-
- assert isinstance(backend, StatsLlmBackend)
- assert isinstance(response_processor, StatsResponseProcessor)
-
- await backend.accumulator.record_success("strong-model", tier="strong")
- snapshot = response_processor.accumulator.snapshot_sync()
- assert snapshot["total_requests"] == 1
- assert snapshot["tiers"]["strong"]["calls"] == 1
diff --git a/tests/test_random_routing_presets.py b/tests/test_random_routing_presets.py
deleted file mode 100644
index 18e31a3a8..000000000
--- a/tests/test_random_routing_presets.py
+++ /dev/null
@@ -1,371 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for :class:`RandomRoutingPresets`.
-
-Covers three concerns:
-
-- **Contract**: each shipping preset returns a valid
- :class:`RandomRoutingConfig` with the documented model pair and
- backend format; defaults cover connectivity / bias tunables; strict
- keyword-only API rejects unknown kwargs.
-- **Dispatch**: :meth:`RandomRoutingPresets.get` resolves valid ids,
- rejects unknown ids with a helpful message, and stays in sync with
- :attr:`RandomRoutingPresets.PRESETS`.
-- **Propagation**: ``api_key`` / ``base_url`` / ``strong_probability``
- reach both tiers on every preset.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-from switchyard.lib.backends.llm_target import BackendFormat
-from switchyard.lib.profiles.random_routing import (
- RandomRoutingConfig,
-)
-from switchyard.lib.profiles.random_routing_presets import RandomRoutingPresets
-
-# Keep the canonical model strings here (instead of import-reusing
-# module-private constants) so tests catch accidental model-string edits in
-# ``random_routing_presets.py``. These are OpenRouter catalog ids.
-_MODEL_OPUS_4_6 = "anthropic/claude-opus-4.6"
-_MODEL_OPUS_4_7 = "anthropic/claude-opus-4.7"
-_MODEL_GPT_5_2 = "openai/gpt-5.2"
-_MODEL_GPT_5_5 = "openai/gpt-5.5"
-_MODEL_NEMOTRON_SUPER = "nvidia/nemotron-3-super-120b-a12b"
-_MODEL_KIMI_K2_6 = "moonshotai/kimi-k2.6"
-_MODEL_MINIMAX_M2_7 = "minimax/minimax-m2.7"
-
-_OPENROUTER = "https://openrouter.ai/api/v1"
-
-
-# ---------------------------------------------------------------------------
-# PRESETS table integrity
-# ---------------------------------------------------------------------------
-
-
-class TestPresetsTable:
- """``PRESETS`` is in sync with the ``@staticmethod`` builder set."""
-
- def test_presets_list_matches_staticmethods(self):
- # Every name in ``PRESETS`` resolves to a callable builder on the
- # class. Inverse — every builder is in ``PRESETS`` — also holds
- # so operators can't call a preset that's missing from the CLI's
- # ``--preset`` choices list.
- for name in RandomRoutingPresets.PRESETS:
- builder = getattr(RandomRoutingPresets, name, None)
- assert callable(builder), f"{name} is not a callable builder"
-
- def test_presets_contents(self):
- assert set(RandomRoutingPresets.PRESETS) == {
- "opus_nemotron_super",
- "gpt5_nemotron_super",
- "opus_kimi",
- "opus_minimax",
- "opus_47_nemotron_super",
- "opus_47_kimi",
- "opus_47_minimax",
- "opus_47_gpt55",
- }
-
-
-# ---------------------------------------------------------------------------
-# get() dispatch
-# ---------------------------------------------------------------------------
-
-
-class TestGetDispatch:
- @pytest.mark.parametrize("preset_id", RandomRoutingPresets.PRESETS)
- def test_known_preset_returns_builder(self, preset_id):
- builder = RandomRoutingPresets.get(preset_id)
- # Builder takes only kwargs; a bare invocation should produce a
- # valid config with documented defaults.
- config = builder()
- assert isinstance(config, RandomRoutingConfig)
-
- def test_unknown_preset_lists_available(self):
- with pytest.raises(ValueError) as exc_info:
- RandomRoutingPresets.get("does_not_exist")
- msg = str(exc_info.value)
- # Error must name the offender and enumerate valid options so
- # the caller can fix a typo without opening source.
- assert "does_not_exist" in msg
- for name in RandomRoutingPresets.PRESETS:
- assert name in msg
-
-
-# ---------------------------------------------------------------------------
-# opus_nemotron_super
-# ---------------------------------------------------------------------------
-
-
-class TestOpusNemotronSuperContract:
- """Opus 4.6 strong, Nemotron-3 Super weak."""
-
- def test_default_config(self):
- config = RandomRoutingPresets.opus_nemotron_super()
- assert config.strong.model == _MODEL_OPUS_4_6
- assert config.weak.model == _MODEL_NEMOTRON_SUPER
- assert config.strong.format is BackendFormat.OPENAI
- assert config.weak.format is BackendFormat.OPENAI
- assert config.strong_probability == 0.5
- # Defaults: no creds injected, base_url points at OpenRouter.
- assert config.strong.api_key is None
- assert config.strong.base_url == _OPENROUTER
- assert config.weak.api_key is None
- assert config.weak.base_url == _OPENROUTER
-
- def test_credential_overrides_reach_both_tiers(self):
- config = RandomRoutingPresets.opus_nemotron_super(
- api_key="sk-test",
- base_url="https://custom.example/v1",
- )
- assert config.strong.api_key == "sk-test"
- assert config.strong.base_url == "https://custom.example/v1"
- assert config.weak.api_key == "sk-test"
- assert config.weak.base_url == "https://custom.example/v1"
-
- def test_strong_probability_override(self):
- config = RandomRoutingPresets.opus_nemotron_super(strong_probability=0.3)
- assert config.strong_probability == 0.3
-
- def test_invalid_probability_rejected(self):
- with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\]"):
- RandomRoutingPresets.opus_nemotron_super(strong_probability=1.5)
-
-
-# ---------------------------------------------------------------------------
-# gpt5_nemotron_super
-# ---------------------------------------------------------------------------
-
-
-class TestGpt5NemotronSuperContract:
- """GPT-5.2 strong + Nemotron-3 Super v3 weak."""
-
- def test_default_config(self):
- config = RandomRoutingPresets.gpt5_nemotron_super()
- assert config.strong.model == _MODEL_GPT_5_2
- assert config.weak.model == _MODEL_NEMOTRON_SUPER
- assert config.strong.format is BackendFormat.OPENAI
- assert config.weak.format is BackendFormat.OPENAI
- assert config.strong_probability == 0.5
- assert config.strong.base_url == _OPENROUTER
- assert config.weak.base_url == _OPENROUTER
-
- def test_credential_overrides_reach_both_tiers(self):
- config = RandomRoutingPresets.gpt5_nemotron_super(
- api_key="sk-test",
- base_url="https://custom.example/v1",
- )
- assert config.strong.api_key == "sk-test"
- assert config.weak.api_key == "sk-test"
-
-
-# ---------------------------------------------------------------------------
-# opus_kimi
-# ---------------------------------------------------------------------------
-
-
-class TestOpusKimiContract:
- """Opus 4.6 strong + Kimi K2.6 weak."""
-
- def test_default_config(self):
- config = RandomRoutingPresets.opus_kimi()
- assert config.strong.model == _MODEL_OPUS_4_6
- assert config.weak.model == _MODEL_KIMI_K2_6
- assert config.strong.format is BackendFormat.OPENAI
- assert config.weak.format is BackendFormat.OPENAI
- assert config.strong_probability == 0.5
-
- def test_credential_overrides_reach_both_tiers(self):
- config = RandomRoutingPresets.opus_kimi(
- api_key="sk-test",
- base_url="https://custom.example/v1",
- )
- assert config.strong.api_key == "sk-test"
- assert config.weak.api_key == "sk-test"
-
-
-# ---------------------------------------------------------------------------
-# opus_minimax
-# ---------------------------------------------------------------------------
-
-
-class TestOpusMinimaxContract:
- """Opus 4.6 strong + MiniMax M2.7 weak."""
-
- def test_default_config(self):
- config = RandomRoutingPresets.opus_minimax()
- assert config.strong.model == _MODEL_OPUS_4_6
- assert config.weak.model == _MODEL_MINIMAX_M2_7
- assert config.strong.format is BackendFormat.OPENAI
- assert config.weak.format is BackendFormat.OPENAI
- assert config.strong_probability == 0.5
-
- def test_credential_overrides_reach_both_tiers(self):
- config = RandomRoutingPresets.opus_minimax(
- api_key="sk-test",
- base_url="https://custom.example/v1",
- )
- assert config.strong.api_key == "sk-test"
- assert config.weak.api_key == "sk-test"
-
-
-# ---------------------------------------------------------------------------
-# opus_47_nemotron_super
-# ---------------------------------------------------------------------------
-
-
-class TestOpus47NemotronSuperContract:
- """4.7 counterpart to ``opus_nemotron_super``."""
-
- def test_default_config(self):
- config = RandomRoutingPresets.opus_47_nemotron_super()
- assert config.strong.model == _MODEL_OPUS_4_7
- assert config.weak.model == _MODEL_NEMOTRON_SUPER
- assert config.strong.format is BackendFormat.OPENAI
- assert config.weak.format is BackendFormat.OPENAI
- assert config.strong_probability == 0.5
- assert config.strong.api_key is None
- assert config.strong.base_url == _OPENROUTER
- assert config.weak.api_key is None
- assert config.weak.base_url == _OPENROUTER
-
- def test_credential_overrides_reach_both_tiers(self):
- config = RandomRoutingPresets.opus_47_nemotron_super(
- api_key="sk-test",
- base_url="https://custom.example/v1",
- )
- assert config.strong.api_key == "sk-test"
- assert config.strong.base_url == "https://custom.example/v1"
- assert config.weak.api_key == "sk-test"
- assert config.weak.base_url == "https://custom.example/v1"
-
-
-# ---------------------------------------------------------------------------
-# opus_47_kimi
-# ---------------------------------------------------------------------------
-
-
-class TestOpus47KimiContract:
- """4.7 counterpart to ``opus_kimi``."""
-
- def test_default_config(self):
- config = RandomRoutingPresets.opus_47_kimi()
- assert config.strong.model == _MODEL_OPUS_4_7
- assert config.weak.model == _MODEL_KIMI_K2_6
- assert config.strong.format is BackendFormat.OPENAI
- assert config.weak.format is BackendFormat.OPENAI
- assert config.strong_probability == 0.5
-
- def test_credential_overrides_reach_both_tiers(self):
- config = RandomRoutingPresets.opus_47_kimi(
- api_key="sk-test",
- base_url="https://custom.example/v1",
- )
- assert config.strong.api_key == "sk-test"
- assert config.weak.api_key == "sk-test"
-
-
-# ---------------------------------------------------------------------------
-# opus_47_minimax
-# ---------------------------------------------------------------------------
-
-
-class TestOpus47MinimaxContract:
- """4.7 counterpart to ``opus_minimax``."""
-
- def test_default_config(self):
- config = RandomRoutingPresets.opus_47_minimax()
- assert config.strong.model == _MODEL_OPUS_4_7
- assert config.weak.model == _MODEL_MINIMAX_M2_7
- assert config.strong.format is BackendFormat.OPENAI
- assert config.weak.format is BackendFormat.OPENAI
- assert config.strong_probability == 0.5
-
- def test_credential_overrides_reach_both_tiers(self):
- config = RandomRoutingPresets.opus_47_minimax(
- api_key="sk-test",
- base_url="https://custom.example/v1",
- )
- assert config.strong.api_key == "sk-test"
- assert config.weak.api_key == "sk-test"
-
-
-# ---------------------------------------------------------------------------
-# opus_47_gpt55
-# ---------------------------------------------------------------------------
-
-
-class TestOpus47Gpt55Contract:
- """4.7 strong + GPT-5.5 weak."""
-
- def test_default_config(self):
- config = RandomRoutingPresets.opus_47_gpt55()
- assert config.strong.model == _MODEL_OPUS_4_7
- assert config.weak.model == _MODEL_GPT_5_5
- assert config.strong.format is BackendFormat.OPENAI
- assert config.weak.format is BackendFormat.OPENAI
- assert config.strong_probability == 0.5
- assert config.strong.api_key is None
- assert config.strong.base_url == _OPENROUTER
- assert config.weak.api_key is None
- assert config.weak.base_url == _OPENROUTER
-
- def test_credential_overrides_reach_both_tiers(self):
- config = RandomRoutingPresets.opus_47_gpt55(
- api_key="sk-test",
- base_url="https://custom.example/v1",
- )
- assert config.strong.api_key == "sk-test"
- assert config.strong.base_url == "https://custom.example/v1"
- assert config.weak.api_key == "sk-test"
- assert config.weak.base_url == "https://custom.example/v1"
-
-
-# ---------------------------------------------------------------------------
-# Strictness — presets reject model/format overrides
-# ---------------------------------------------------------------------------
-
-
-class TestStrictness:
- """Presets expose only (api_key, base_url, strong_probability).
-
- Attempts to override the fixed model pair or backend format must fail at
- the Python level — Python's TypeError on unexpected kwargs is the
- enforcement mechanism, so these tests pin the surface area.
- """
-
- @pytest.mark.parametrize("preset_id", RandomRoutingPresets.PRESETS)
- @pytest.mark.parametrize(
- "forbidden_kwarg",
- [
- "strong_model",
- "weak_model",
- "strong_backend_format",
- "weak_backend_format",
- ],
- )
- def test_preset_rejects_unknown_kwarg(self, preset_id, forbidden_kwarg):
- builder = RandomRoutingPresets.get(preset_id)
- with pytest.raises(TypeError):
- builder(**{forbidden_kwarg: "anything"})
-
- @pytest.mark.parametrize("preset_id", RandomRoutingPresets.PRESETS)
- def test_preset_rejects_positional_args(self, preset_id):
- builder = RandomRoutingPresets.get(preset_id)
- with pytest.raises(TypeError):
- builder("sk-test") # type: ignore[misc]
-
-
-# ---------------------------------------------------------------------------
-# Public API surface
-# ---------------------------------------------------------------------------
-
-
-def test_public_api_exposes_random_routing_presets():
- """``RandomRoutingPresets`` is importable from the top-level package."""
- from switchyard import RandomRoutingPresets as _Presets
-
- assert _Presets is RandomRoutingPresets
diff --git a/tests/test_random_routing_profile.py b/tests/test_random_routing_profile.py
deleted file mode 100644
index f522a430d..000000000
--- a/tests/test_random_routing_profile.py
+++ /dev/null
@@ -1,442 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for profile-backed random-routing construction."""
-
-from __future__ import annotations
-
-import json
-import threading
-from collections import Counter
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from typing import Any
-
-import pytest
-from pydantic import ValidationError
-
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
-from switchyard.lib.profiles import (
- PassthroughProfileConfig,
- ProfileSwitchyard,
- RandomRoutingConfig,
- RandomRoutingProfileConfig,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.stats_accumulator import StatsAccumulator
-from switchyard_rust.components import MultiLlmBackend, StatsLlmBackend
-from switchyard_rust.core import ChatRequest, ChatResponse
-from switchyard_rust.translation import TranslationEngine
-
-
-class _OpenAICompatStub:
- def __init__(self) -> None:
- self._server: ThreadingHTTPServer | None = None
- self._thread: threading.Thread | None = None
- self._lock = threading.Lock()
- self._requests: list[dict[str, Any]] = []
-
- def __enter__(self) -> _OpenAICompatStub:
- owner = self
-
- class Handler(BaseHTTPRequestHandler):
- protocol_version = "HTTP/1.1"
-
- def do_POST(self) -> None:
- length = int(self.headers.get("content-length", "0"))
- raw = self.rfile.read(length)
- body = json.loads(raw.decode("utf-8"))
- with owner._lock:
- owner._requests.append({
- "path": self.path,
- "authorization": self.headers.get("authorization"),
- "body": body,
- })
-
- response = json.dumps({
- "id": "chatcmpl-local",
- "object": "chat.completion",
- "created": 1700000000,
- "model": body.get("model"),
- "choices": [
- {
- "index": 0,
- "message": {"role": "assistant", "content": "hi"},
- "finish_reason": "stop",
- }
- ],
- "usage": {
- "prompt_tokens": 1,
- "completion_tokens": 1,
- "total_tokens": 2,
- },
- }).encode("utf-8")
- self.send_response(200)
- self.send_header("content-type", "application/json")
- self.send_header("content-length", str(len(response)))
- self.end_headers()
- self.wfile.write(response)
-
- def log_message(self, _format: str, *args: object) -> None:
- return None
-
- self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
- self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
- self._thread.start()
- return self
-
- def __exit__(self, *args: object) -> None:
- if self._server is not None:
- self._server.shutdown()
- self._server.server_close()
- if self._thread is not None:
- self._thread.join(timeout=2)
-
- @property
- def base_url(self) -> str:
- if self._server is None:
- raise RuntimeError("stub server is not running")
- host, port = self._server.server_address
- return f"http://{host}:{port}/v1"
-
- @property
- def requests(self) -> list[dict[str, Any]]:
- with self._lock:
- return list(self._requests)
-
-
-def _openai_config(
- *,
- strong_model: str = "strong-m",
- weak_model: str = "weak-m",
- strong_probability: float = 0.5,
- api_key: str = "sk-test",
- base_url: str = "https://example.invalid/v1",
- enable_stats: bool = True,
-) -> RandomRoutingConfig:
- return RandomRoutingConfig(
- strong=LlmTarget(
- id="strong",
- model=strong_model,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- weak=LlmTarget(
- id="weak",
- model=weak_model,
- format=BackendFormat.OPENAI,
- api_key=api_key,
- base_url=base_url,
- ),
- strong_probability=strong_probability,
- enable_stats=enable_stats,
- fallback_target_on_evict="strong",
- )
-
-
-def _random_routing_switchyard(
- config: RandomRoutingConfig,
- *,
- stats_accumulator: StatsAccumulator | None = None,
- pre_routing_request_processors: list[Any] | None = None,
- extra_request_processors: list[Any] | None = None,
- extra_response_processors: list[Any] | None = None,
-) -> ProfileSwitchyard:
- """Build the profile-backed runtime used by these tests."""
- return ProfileSwitchyard(
- RandomRoutingProfileConfig.from_config(config)
- .build()
- .with_runtime_components(
- stats_accumulator=stats_accumulator,
- enable_stats=config.enable_stats,
- pre_request_processors=pre_routing_request_processors or (),
- post_request_processors=extra_request_processors or (),
- response_processors=extra_response_processors or (),
- )
- )
-
-
-def _llm_target_switchyard(
- target: LlmTarget,
- *,
- stats_accumulator: StatsAccumulator | None = None,
-) -> ProfileSwitchyard:
- """Build the single-target passthrough profile used by parity tests."""
- return ProfileSwitchyard(
- PassthroughProfileConfig(
- target=target,
- )
- .build()
- .with_runtime_components(stats_accumulator=stats_accumulator)
- )
-
-
-class _NoopRequestProcessor:
- async def process(self, _ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- return request
-
-
-class _NoopResponseProcessor:
- async def process(self, _ctx: ProxyContext, response: ChatResponse) -> ChatResponse:
- return response
-
-
-# ---------------------------------------------------------------------------
-# Structure
-# ---------------------------------------------------------------------------
-
-
-class TestProfileStructure:
- def test_returns_profile_backed_switchyard_adapter(self):
- switchyard = _random_routing_switchyard(
- _openai_config(),
- )
- assert isinstance(switchyard, ProfileSwitchyard)
-
- def test_backend_is_random_routing(self):
- config = _openai_config(enable_stats=False)
- switchyard = _random_routing_switchyard(config)
- backends = [
- c for c in switchyard.iter_components()
- if isinstance(c, MultiLlmBackend)
- ]
- assert len(backends) == 1
-
- def test_default_response_translator(self):
- switchyard = _random_routing_switchyard(
- _openai_config(),
- )
- translators = [
- c for c in switchyard.iter_components()
- if isinstance(c, TranslationEngine)
- ]
- assert len(translators) == 1
-
- def test_config_propagated_to_backend(self):
- config = _openai_config(
- strong_model="s-123", weak_model="w-456", strong_probability=0.7,
- enable_stats=False,
- )
- switchyard = _random_routing_switchyard(config)
- backend = next(
- c for c in switchyard.iter_components()
- if isinstance(c, MultiLlmBackend)
- )
- assert backend.target_ids() == ["strong", "weak"]
-
- from switchyard.lib.processors.random_routing_request_processor import (
- RandomRoutingRequestProcessor,
- )
-
- router = next(
- c for c in switchyard.iter_components()
- if isinstance(c, RandomRoutingRequestProcessor)
- )
- assert router.config.strong_probability == 0.7
- assert router.config.strong.model == "s-123"
- assert router.config.weak.model == "w-456"
-
- def test_rng_passed_through(self):
- """An explicit rng_seed still produces a runnable profile chain."""
- switchyard = _random_routing_switchyard(
- _openai_config(enable_stats=False).model_copy(update={"rng_seed": 123}),
- )
- backends = [
- c for c in switchyard.iter_components()
- if isinstance(c, MultiLlmBackend)
- ]
- assert len(backends) == 1
-
- def test_preset_remains_on_config(self):
- """Preset provenance stays on the profile config."""
- config = _openai_config(enable_stats=False).model_copy(update={"preset": "opus_47_minimax"})
- switchyard = _random_routing_switchyard(config)
- backend = next(
- c for c in switchyard.iter_components()
- if isinstance(c, MultiLlmBackend)
- )
- assert backend.target_ids() == ["strong", "weak"]
- assert config.preset == "opus_47_minimax"
-
- def test_preset_defaults_to_none(self):
- """No ``preset`` on config keeps profile construction unannotated."""
- config = _openai_config(enable_stats=False)
- switchyard = _random_routing_switchyard(_openai_config(enable_stats=False))
- backend = next(
- c for c in switchyard.iter_components()
- if isinstance(c, MultiLlmBackend)
- )
- assert backend.target_ids() == ["strong", "weak"]
- assert config.preset is None
-
- def test_extra_processors_are_wired(self):
- request_processor = _NoopRequestProcessor()
- response_processor = _NoopResponseProcessor()
- switchyard = _random_routing_switchyard(
- _openai_config(),
- extra_request_processors=[request_processor],
- extra_response_processors=[response_processor],
- )
-
- components = list(switchyard.iter_components())
- assert request_processor in components
- assert response_processor in components
-
- def test_pre_routing_processors_run_before_router(self):
- from switchyard.lib.processors.random_routing_request_processor import (
- RandomRoutingRequestProcessor,
- )
- from switchyard.lib.processors.stats_request_processor import (
- StatsRequestProcessor,
- )
-
- request_processor = _NoopRequestProcessor()
- switchyard = _random_routing_switchyard(
- _openai_config(),
- pre_routing_request_processors=[request_processor],
- )
-
- components = list(switchyard.iter_components())
- stats_index = next(
- index
- for index, component in enumerate(components)
- if isinstance(component, StatsRequestProcessor)
- )
- pre_routing_index = components.index(request_processor)
- router_index = next(
- index
- for index, component in enumerate(components)
- if isinstance(component, RandomRoutingRequestProcessor)
- )
-
- assert stats_index < pre_routing_index < router_index
-
- async def test_uses_supplied_stats_accumulator(self):
- from switchyard.lib.processors.stats_response_processor_accumulator import (
- StatsResponseProcessor,
- )
-
- stats = StatsAccumulator()
- switchyard = _random_routing_switchyard(
- _openai_config(),
- stats_accumulator=stats,
- )
-
- backend = next(
- c for c in switchyard.iter_components()
- if isinstance(c, StatsLlmBackend)
- )
- response_processor = next(
- c for c in switchyard.iter_components()
- if isinstance(c, StatsResponseProcessor)
- )
-
- await backend.accumulator.record_success("model-a")
- assert response_processor.accumulator.snapshot_sync()["total_requests"] == 1
-
- async def test_llm_target_profile_builds_single_target_chain(self):
- from switchyard.lib.processors.stats_response_processor_accumulator import (
- StatsResponseProcessor,
- )
-
- stats = StatsAccumulator()
- switchyard = _llm_target_switchyard(
- LlmTarget(
- model="single-m",
- format=BackendFormat.OPENAI,
- api_key="sk-test",
- base_url="https://example.invalid/v1",
- ),
- stats_accumulator=stats,
- )
-
- backend = next(
- c for c in switchyard.iter_components()
- if isinstance(c, StatsLlmBackend)
- )
- response_processor = next(
- c for c in switchyard.iter_components()
- if isinstance(c, StatsResponseProcessor)
- )
-
- assert [item.value for item in backend.supported_request_types] == ["openai_chat"]
- await backend.accumulator.record_success("single-m")
- assert response_processor.accumulator.snapshot_sync()["total_requests"] == 1
-
-
-# ---------------------------------------------------------------------------
-# Validation passthrough (all validation lives on the config dataclass)
-# ---------------------------------------------------------------------------
-
-
-class TestProfileValidation:
- def test_invalid_probability_rejected_at_config_construction(self):
- with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\]"):
- _openai_config(strong_probability=1.5)
-
- def test_empty_model_rejected_at_config_construction(self):
- with pytest.raises(ValidationError, match="must not be empty"):
- RandomRoutingConfig(
- strong={"model": ""},
- weak={"model": "w"},
- fallback_target_on_evict="strong")
-
-
-# ---------------------------------------------------------------------------
-# End-to-end smoke test
-# ---------------------------------------------------------------------------
-
-
-class TestProfileEndToEnd:
- async def test_fifty_requests_hit_both_tiers_through_full_chain(self):
- """Fire 50 requests through the full profile chain; both tiers should call upstream."""
- config = _openai_config(
- strong_model="strong-m", weak_model="weak-m",
- strong_probability=0.5,
- api_key="sk-local",
- enable_stats=False,
- )
- config = config.model_copy(update={"rng_seed": 42})
- with _OpenAICompatStub() as upstream:
- switchyard = _random_routing_switchyard(
- config.model_copy(
- update={
- "strong": LlmTarget(
- id="strong",
- model="strong-m",
- format=BackendFormat.OPENAI,
- api_key="sk-local",
- base_url=upstream.base_url,
- ),
- "weak": LlmTarget(
- id="weak",
- model="weak-m",
- format=BackendFormat.OPENAI,
- api_key="sk-local",
- base_url=upstream.base_url,
- ),
- },
- ),
- )
- for _ in range(50):
- response = await switchyard.call(
- ChatRequest.openai_chat({
- "model": "client-sent",
- "messages": [{"role": "user", "content": "hi"}],
- }),
- )
- assert response["choices"][0]["message"]["content"] == "hi"
-
- bodies = [entry["body"] for entry in upstream.requests]
- paths = [entry["path"] for entry in upstream.requests]
- auth_headers = [entry["authorization"] for entry in upstream.requests]
- selected_models = [body["model"] for body in bodies]
-
- assert set(paths) == {"/v1/chat/completions"}
- assert set(auth_headers) == {"Bearer sk-local"}
- assert set(selected_models) <= {"strong-m", "weak-m"}
- assert "strong-m" in selected_models
- assert "weak-m" in selected_models
- assert all(body["messages"][0]["content"] == "hi" for body in bodies)
- counts = Counter(selected_models)
- assert sum(counts.values()) == 50
diff --git a/tests/test_reasoning_effort_normalizer.py b/tests/test_reasoning_effort_normalizer.py
deleted file mode 100644
index bb9370898..000000000
--- a/tests/test_reasoning_effort_normalizer.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for ``ReasoningEffortNormalizer``."""
-
-from __future__ import annotations
-
-import pytest
-
-from switchyard.lib.processors.reasoning_effort_normalizer import (
- ReasoningEffortNormalizer,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.core import ChatRequest
-
-
-def _make_request(reasoning_effort: str | None) -> ChatRequest:
- body: dict = {
- "model": "azure/anthropic/claude-opus-4-6",
- "messages": [{"role": "user", "content": "hi"}],
- }
- if reasoning_effort is not None:
- body["reasoning_effort"] = reasoning_effort
- return ChatRequest.openai_chat(body)
-
-
[email protected]("effort", ["low", "medium", "high", "max"])
-async def test_valid_effort_passes_through(effort: str) -> None:
- req = _make_request(effort)
- out = await ReasoningEffortNormalizer().process(ProxyContext(), req)
- assert out.body["reasoning_effort"] == effort
-
-
-async def test_xhigh_alias_maps_to_high() -> None:
- req = _make_request("xhigh")
- out = await ReasoningEffortNormalizer().process(ProxyContext(), req)
- assert out.body["reasoning_effort"] == "high"
-
-
-async def test_unknown_effort_falls_back_to_high() -> None:
- req = _make_request("super-mega")
- out = await ReasoningEffortNormalizer().process(ProxyContext(), req)
- assert out.body["reasoning_effort"] == "high"
-
-
-async def test_absent_effort_is_noop() -> None:
- req = _make_request(None)
- out = await ReasoningEffortNormalizer().process(ProxyContext(), req)
- assert "reasoning_effort" not in out.body
diff --git a/tests/test_reasoning_hint.py b/tests/test_reasoning_hint.py
deleted file mode 100644
index 0c0c5ec1d..000000000
--- a/tests/test_reasoning_hint.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for :func:`model_accepts_reasoning_hint`."""
-
-from __future__ import annotations
-
-import pytest
-
-from switchyard.lib.processors.reasoning_hint import model_accepts_reasoning_hint
-
-
- "model",
- [
- "aws/anthropic/bedrock-claude-sonnet-4-6",
- "aws/anthropic/bedrock-claude-opus-4-8",
- "azure/anthropic/claude-opus-4-6",
- "anthropic/claude-3-5-sonnet",
- "Bedrock-Claude-Whatever", # case-insensitive
- ],
-)
-def test_claude_family_rejects_hint(model: str) -> None:
- assert model_accepts_reasoning_hint(model) is False
-
-
- "model",
- [
- "nvidia/deepseek-ai/deepseek-v4-flash",
- "nvidia/deepseek-ai/evals-deepseek-v4-pro",
- "nvidia/nvidia/nemotron-3-super-v3",
- "openai/openai/gpt-5.2",
- "router-model",
- ],
-)
-def test_non_claude_models_accept_hint(model: str) -> None:
- assert model_accepts_reasoning_hint(model) is True
diff --git a/tests/test_redis_pin_store.py b/tests/test_redis_pin_store.py
deleted file mode 100644
index 1ccf5fafa..000000000
--- a/tests/test_redis_pin_store.py
+++ /dev/null
@@ -1,120 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for RedisPinStore adapter mechanics and its config → backend wiring.
-
-An injected fake async client keeps these hermetic — no live Redis and no
-``fakeredis`` dependency. Building the L2 exercises the lazy-import path without
-requiring ``redis`` to be installed, because the real client is only created on
-first ``get``/``put``.
-"""
-
-from __future__ import annotations
-
-import pytest
-from pydantic import ValidationError
-
-from switchyard.lib.profiles.escalation_router_config import EscalationRouterConfig
-from switchyard.lib.profiles.escalation_router_profile_config import _build_affinity_l2
-from switchyard.lib.redis_pin_store import RedisPinStore
-
-
-class _FakeAsyncRedis:
- """Minimal async Redis double capturing set() arguments."""
-
- def __init__(self) -> None:
- self.kv: dict[str, str] = {}
- self.set_calls: list[tuple[str, str, int | None]] = []
-
- async def get(self, key: str) -> str | None:
- return self.kv.get(key)
-
- async def set(self, key: str, value: str, ex: int | None = None) -> None:
- self.set_calls.append((key, value, ex))
- self.kv[key] = value
-
-
-# --- adapter mechanics ------------------------------------------------------
-
-
-async def test_put_writes_prefixed_key_with_ttl() -> None:
- fake = _FakeAsyncRedis()
- store = RedisPinStore("redis://x", ttl_seconds=123, key_prefix="p:", client=fake)
- await store.put("abc", "model-A")
- assert fake.set_calls == [("p:abc", "model-A", 123)]
-
-
-async def test_get_reads_prefixed_key_and_roundtrips() -> None:
- fake = _FakeAsyncRedis()
- store = RedisPinStore("redis://x", key_prefix="p:", client=fake)
- assert await store.get("abc") is None
- await store.put("abc", "model-B")
- assert await store.get("abc") == "model-B"
-
-
-def test_invalid_ttl_rejected() -> None:
- with pytest.raises(ValueError):
- RedisPinStore("redis://x", ttl_seconds=0)
-
-
-async def test_aclose_closes_client_once() -> None:
- """aclose releases the client's pool; repeated calls don't double-close."""
-
- class _ClosableFake(_FakeAsyncRedis):
- def __init__(self) -> None:
- super().__init__()
- self.closes = 0
-
- async def aclose(self) -> None:
- self.closes += 1
-
- fake = _ClosableFake()
- store = RedisPinStore("redis://x", client=fake)
- await store.aclose()
- await store.aclose() # client reference already released → no-op
- assert fake.closes == 1
-
-
-async def test_aclose_before_first_use_is_noop() -> None:
- """Closing a store that never built a client must not import or connect."""
- await RedisPinStore("redis://x").aclose()
-
-
-# --- config validation + backend wiring -------------------------------------
-
-
-def _redis_config(**overrides: object) -> EscalationRouterConfig:
- base: dict[str, object] = {
- "strong": {"model": "strong-model"},
- "weak": {"model": "weak-model"},
- "judge": {"model": "judge-model"},
- "fallback_target_on_evict": "weak",
- "affinity_store": "redis",
- "affinity_store_url": "redis://cache:6379/0",
- }
- base.update(overrides)
- return EscalationRouterConfig(**base) # type: ignore[arg-type]
-
-
-def test_redis_store_requires_url() -> None:
- with pytest.raises(ValidationError):
- _redis_config(affinity_store_url=None)
-
-
-def test_memory_is_the_default_store() -> None:
- cfg = EscalationRouterConfig(
- strong={"model": "strong-model"},
- weak={"model": "weak-model"},
- judge={"model": "judge-model"},
- fallback_target_on_evict="weak",
- )
- assert cfg.affinity_store == "memory"
- assert _build_affinity_l2(cfg) is None
-
-
-def test_build_affinity_l2_threads_config_into_redis_store() -> None:
- cfg = _redis_config(affinity_store_ttl_seconds=42, affinity_key_prefix="k:")
- store = _build_affinity_l2(cfg)
- assert isinstance(store, RedisPinStore)
- assert store._ttl_seconds == 42
- assert store._key_prefix == "k:"
diff --git a/tests/test_request_metadata.py b/tests/test_request_metadata.py
index 285544b78..62e3187aa 100644
--- a/tests/test_request_metadata.py
+++ b/tests/test_request_metadata.py
@@ -9,7 +9,7 @@ import pytest
from switchyard.lib.proxy_context import CTX_CALLER_API_KEY, ProxyContext
from switchyard.lib.request_metadata import (
- CTX_PROFILE_REQUEST_HEADERS,
+ CTX_REQUEST_HEADERS,
RequestMetadata,
attach_caller_api_key,
attach_request_metadata,
@@ -109,8 +109,7 @@ class TestRedactSensitiveHeaders:
class TestCallerKeyForwardedButNotRetained:
"""The endpoint extracts the caller key for upstream use, then retains a
- redacted header map so the key cannot leak into profile metadata, logs, or
- traces."""
+ redacted header map so the key cannot leak into logs or traces."""
def test_key_extracted_but_redacted_in_stored_headers(self) -> None:
headers = {
@@ -125,6 +124,6 @@ class TestCallerKeyForwardedButNotRetained:
# Extracted for upstream forwarding...
assert ctx.metadata[CTX_CALLER_API_KEY] == "nvapi-secret" # pragma: allowlist secret
# ...but the retained header map carries no raw credential.
- stored = ctx.metadata[CTX_PROFILE_REQUEST_HEADERS]
+ stored = ctx.metadata[CTX_REQUEST_HEADERS]
assert stored["x-switchyard-api-key"] == "[REDACTED]"
assert stored["x-switchyard-intake-task"] == "demo"
diff --git a/tests/test_rl_logging.py b/tests/test_rl_logging.py
index 5f9293dc0..dffe157e2 100644
--- a/tests/test_rl_logging.py
+++ b/tests/test_rl_logging.py
@@ -291,8 +291,8 @@ def test_rl_flags_are_scoped_to_serve() -> None:
args = parser.parse_args(
[
"serve",
- "--routing-profiles",
- "profiles.yaml",
+ "--routes",
+ "routes.yaml",
"--enable-rl-logging",
"--rl-log-dir",
"/tmp/x",
@@ -304,7 +304,7 @@ def test_rl_flags_are_scoped_to_serve() -> None:
def test_serve_attaches_rl_logging_processors(monkeypatch, tmp_path: Path) -> None:
- """`serve --routing-profiles --enable-rl-logging` wires the trace logger into the chain."""
+ """`serve --routes --enable-rl-logging` wires the trace logger into the chain."""
import switchyard.cli.switchyard_cli as cli
captured: dict[str, list] = {}
@@ -316,7 +316,7 @@ def test_serve_attaches_rl_logging_processors(monkeypatch, tmp_path: Path) -> No
def default_model(self) -> str | None:
return None
- def _fake_load(routing_profiles, *, pre_routing_request_processors=(),
+ def _fake_load(routes, *, pre_routing_request_processors=(),
extra_response_processors=(), **_kwargs):
captured["request"] = list(pre_routing_request_processors)
captured["response"] = list(extra_response_processors)
@@ -326,7 +326,7 @@ def test_serve_attaches_rl_logging_processors(monkeypatch, tmp_path: Path) -> No
monkeypatch.setattr(cli, "build_and_serve", lambda *a, **k: None)
args = argparse.Namespace(
- routing_profiles="profiles.yaml",
+ routes="routes.yaml",
enable_rl_logging=True, rl_log_dir=str(tmp_path),
routing_log_file=None,
)
diff --git a/tests/test_rl_logging_e2e.py b/tests/test_rl_logging_e2e.py
index 83c199131..4a44b988b 100644
--- a/tests/test_rl_logging_e2e.py
+++ b/tests/test_rl_logging_e2e.py
@@ -40,7 +40,9 @@ def _build_app(stub: _OpenAICompatStub, log_dir: Path):
"base_url": stub.base_url,
"format": "openai",
},
- "routes": {"mock-model": {"type": "model", "target": "mock-model"}},
+ "routes": {
+ "mock-model": {"type": "passthrough", "target": "mock-model"}
+ },
},
pre_routing_request_processors=rl_request,
extra_response_processors=rl_response,
diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py
index 1ca67a85e..d8773c273 100644
--- a/tests/test_route_bundle.py
+++ b/tests/test_route_bundle.py
@@ -1,1368 +1,176 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Tests for YAML route-bundle table construction."""
+"""Tests for the minimal Python server route bundle."""
+
+from __future__ import annotations
-import argparse
-import json
import sys
from pathlib import Path
-from types import SimpleNamespace
-from typing import Any
+import httpx
import pytest
import switchyard.cli.switchyard_cli as cli
-from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
-)
-from switchyard.lib.processors.llm_classifier import (
- CODING_AGENT_CLASSIFIER_SYSTEM_PROMPT,
- LLMClassifierRequestProcessor,
-)
-from switchyard.lib.proxy_context import ProxyContext
+from switchyard.cli.launchers.launcher_runtime import route_bundle_strategy_summary
+from switchyard.cli.route_bundle import RouteBundleConfigError, build_route_bundle_table
from switchyard.lib.route_table import RouteTable
-from switchyard_rust.core import ChatRequest, ChatResponse
-
-
[email protected](autouse=True)
-def _stub_model_discovery(monkeypatch: pytest.MonkeyPatch) -> None:
- """Catalog discovery is always-on for route bundles; default to an empty
- catalog so unit tests stay hermetic. Tests that assert on hydrated catalog
- entries override this with their own ``monkeypatch.setattr``.
- """
- monkeypatch.setattr(
- "switchyard.cli.route_bundle.fetch_model_ids",
- lambda base_url, api_key: [],
- )
-
+from switchyard.server.switchyard_app import build_switchyard_app
+from switchyard_rust.components import StatsLlmBackend
-def _random_processor(chain: object) -> Any:
- from switchyard.lib.processors.random_routing_request_processor import (
- RandomRoutingRequestProcessor,
- )
-
- return next(
- component
- for component in chain.iter_components()
- if isinstance(component, RandomRoutingRequestProcessor)
- )
-
-
-class _NoopRequestProcessor:
- async def process(self, _ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- return request
-
-
-class _NoopResponseProcessor:
- async def process(self, _ctx: ProxyContext, response: ChatResponse) -> ChatResponse:
- return response
-
-
-def test_random_route_bundle_registers_model_keys_and_applies_defaults(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setenv("ROUTE_BUNDLE_KEY", "sk-default")
+async def test_noop_route_returns_ok_without_an_upstream() -> None:
table = build_route_bundle_table({
- "defaults": {
- "api_key": "${ROUTE_BUNDLE_KEY}",
- "base_url": "https://example.invalid/v1",
- "format": "openai",
- "timeout": 30,
- },
- "routes": {
- "A": {
- "type": "random-routing",
- "fallback_target_on_evict": "strong",
- "strong": "model-1",
- "weak": {"model": "model-2", "api_key": "sk-weak"},
- "strong_probability": 0.7,
- },
- "B": {
- "type": "random_routing",
- "fallback_target_on_evict": "strong",
- "strong": "model-3",
- "weak": "model-4",
- "strong_probability": 0.2,
- },
- },
+ "routes": {"test/noop": {"type": "noop"}},
})
- # Unified ordering: YAML route key first, then tier passthroughs.
- # Catalog hydration is always attempted; the stubbed catalog is empty here,
- # so only configured tier models register.
- assert table.registered_models() == [
- "A", # A's virtual routing-policy id (YAML key)
- "model-1", # A's strong tier
- "model-2", # A's weak tier
- "B", # B's virtual routing-policy id (YAML key)
- "model-3", # B's strong tier
- "model-4", # B's weak tier
- ]
- assert table.default_model() == "A"
-
- route_a = _random_processor(table.lookup_switchyard("A"))
- assert route_a.config.strong.model == "model-1"
- assert route_a.config.strong.api_key == "sk-default"
- assert route_a.config.strong.endpoint.base_url == "https://example.invalid/v1"
- assert route_a.config.weak.model == "model-2"
- assert route_a.config.weak.api_key == "sk-weak"
- assert route_a.config.strong_probability == 0.7
-
- route_b = _random_processor(table.lookup_switchyard("B"))
- assert route_b.config.strong.model == "model-3"
- assert route_b.config.weak.model == "model-4"
- assert route_b.config.strong_probability == 0.2
-
+ async with httpx.AsyncClient(
+ transport=httpx.ASGITransport(app=build_switchyard_app(table)),
+ base_url="http://test",
+ ) as client:
+ response = await client.post(
+ "/v1/chat/completions",
+ json={
+ "model": "test/noop",
+ "messages": [{"role": "user", "content": "hello"}],
+ },
+ )
-def test_route_bundle_rejects_missing_environment_variable() -> None:
- with pytest.raises(RouteBundleConfigError, match="MISSING_ROUTE_KEY"):
- build_route_bundle_table({
- "defaults": {"api_key": "${MISSING_ROUTE_KEY}"},
- "routes": {"A": "model-a"},
- })
+ assert response.status_code == 200
+ assert response.json()["choices"][0]["message"]["content"] == "OK"
@pytest.mark.parametrize(
- ("route", "match"),
+ ("path", "body", "expected"),
[
- ({"modle": "gpt-4o"}, "recognizable route shape"),
- ({"type": "model", "modle": "gpt-4o"}, "modle"),
(
+ "/v1/messages",
{
- "type": "random-routing",
- "fallback_target_on_evict": "strong",
- "strong": "model-1",
- "weak": "model-2",
- "strong_probablity": 0.9,
+ "model": "test/noop",
+ "max_tokens": 16,
+ "messages": [{"role": "user", "content": "hello"}],
},
- "strong_probablity",
+ ("content", 0, "text"),
+ ),
+ (
+ "/v1/responses",
+ {"model": "test/noop", "input": "hello"},
+ ("output", 0, "content"),
),
- ({"type": "model", "target": {"modle": "gpt-4o"}}, "modle"),
],
)
-def test_route_bundle_rejects_unknown_route_keys(
- route: dict[str, object],
- match: str,
+async def test_noop_route_translates_to_inbound_format(
+ path: str,
+ body: dict[str, object],
+ expected: tuple[str, int, str],
) -> None:
- with pytest.raises(RouteBundleConfigError, match=match):
- build_route_bundle_table({"routes": {"bad": route}})
-
-
[email protected]("route_type", ["passthrough", "noop"])
-def test_removed_route_types_are_rejected(route_type: str) -> None:
- """``type: passthrough`` and ``type: noop`` are no longer valid route types."""
- with pytest.raises(RouteBundleConfigError, match="unsupported route type"):
- build_route_bundle_table({"routes": {"r": {"type": route_type}}})
-
-
-def test_empty_route_mapping_is_rejected() -> None:
- """An empty ``{}`` route no longer defaults to a noop; it is rejected."""
- with pytest.raises(RouteBundleConfigError, match="missing 'type'"):
- build_route_bundle_table({"routes": {"r": {}}})
-
-
-def test_model_route_requires_target() -> None:
- """``type: model`` is the only single-upstream route, so it fails closed
- when no ``target``/``model`` is given rather than inventing a default."""
- with pytest.raises(RouteBundleConfigError, match="requires target or model"):
- build_route_bundle_table({"routes": {"m": {"type": "model"}}})
-
-
-def test_random_routing_hydrates_tier_and_catalog_models(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """A random_routing route always mirrors launcher catalog hydration.
-
- Registers each configured tier model + each tier's catalog model + the
- virtual routing-policy id (the route's YAML key). One YAML route expands
- into the same N+1 shape ``build_random_routing_table`` produces for the
- Claude/Codex launchers.
- """
- monkeypatch.setattr(
- "switchyard.cli.route_bundle.fetch_model_ids",
- lambda base_url, api_key: (
- ["catalog/extra"]
- if base_url == "https://primary.example/v1"
- else ["catalog/weak-extra"]
- ),
- )
-
+ table = build_route_bundle_table({"routes": {"test/noop": {"type": "noop"}}})
+ async with httpx.AsyncClient(
+ transport=httpx.ASGITransport(app=build_switchyard_app(table)),
+ base_url="http://test",
+ ) as client:
+ response = await client.post(path, json=body)
+
+ assert response.status_code == 200
+ value: object = response.json()
+ for part in expected:
+ value = value[part] # type: ignore[index]
+ assert value
+
+
+def test_passthrough_route_builds_one_native_backend(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("UPSTREAM_KEY", "secret")
table = build_route_bundle_table({
- "routes": {
- "switchyard-route": {
- "type": "random_routing",
- "fallback_target_on_evict": "strong",
- "strong": {
- "model": "strong/model",
- "api_key": "k-strong",
- "base_url": "https://primary.example/v1",
- },
- "weak": {
- "model": "weak/model",
- "api_key": "k-weak",
- "base_url": "https://weak.example/v1",
- },
- "strong_probability": 0.4,
- },
- },
- })
-
- # Unified ordering: YAML route key first, tier passthroughs + catalog after.
- assert table.registered_models() == [
- "switchyard-route",
- "strong/model",
- "weak/model",
- "catalog/extra",
- "catalog/weak-extra",
- ]
- assert table.default_model() == "switchyard-route"
-
-
-def test_model_route_aliases_under_route_key(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """A ``model`` route is a pure alias — no catalog discovery.
-
- It registers only under the route's YAML key (the friendly alias); the
- upstream ``target.model`` is what the backend calls, not a separate
- table entry, and no catalog is hydrated even if the upstream has one.
- """
- monkeypatch.setattr(
- "switchyard.cli.route_bundle.fetch_model_ids",
- lambda base_url, api_key: ["catalog/a", "catalog/b"],
- )
-
- table = build_route_bundle_table({
- "routes": {
- "configured-route": {
- "type": "model",
- "target": {
- "model": "primary/model",
- "api_key": "k",
- "base_url": "https://primary.example/v1",
- },
- },
+ "defaults": {
+ "api_key": "${UPSTREAM_KEY}",
+ "base_url": "https://example.invalid/v1",
+ "format": "openai",
},
- })
-
- # Only the alias key registers — no `primary/model`, no catalog entries.
- assert table.registered_models() == ["configured-route"]
- assert table.default_model() == "configured-route"
-
-
-def test_random_routing_with_empty_catalog_registers_only_tier_passthroughs() -> None:
- """When discovery yields an empty catalog, only tier passthroughs register.
-
- Matches the launcher's contract: client model pickers always see strong/
- weak as direct overrides for the random-routing default, even when the
- upstream ``/v1/models`` catalog is empty or unreachable.
- """
- table = build_route_bundle_table({
"routes": {
- "switchyard-route": {
- "type": "random_routing",
- "fallback_target_on_evict": "strong",
- "strong": {
- "model": "strong/model",
- "api_key": "k-strong",
- "base_url": "https://primary.example/v1",
- },
- "weak": {
- "model": "weak/model",
- "api_key": "k-weak",
- "base_url": "https://weak.example/v1",
- },
- },
+ "direct": {
+ "type": "passthrough",
+ "target": {"model": "upstream/model"},
+ "display_name": "Direct model",
+ }
},
})
- # Unified ordering: YAML route key first, then tier passthroughs.
- # No catalog entries (empty catalog).
- assert table.registered_models() == [
- "switchyard-route",
- "strong/model",
- "weak/model",
- ]
- assert table.default_model() == "switchyard-route"
-
-
-def test_route_bundle_threads_extra_processors_through_routes() -> None:
- request_processor = _NoopRequestProcessor()
- response_processor = _NoopResponseProcessor()
-
- table = build_route_bundle_table(
- {"routes": {"direct": {"type": "model", "target": "some/model"}}},
- pre_routing_request_processors=[request_processor],
- extra_response_processors=[response_processor],
- )
-
+ assert table.registered_models() == ["direct"]
+ assert table.default_model() == "direct"
+ assert table.registered_model_entries()[0]["display_name"] == "Direct model"
components = table.lookup_switchyard("direct").iter_components()
- assert request_processor in components
- assert response_processor in components
-
-
-def test_serve_subcommand_hands_table_to_server(
- monkeypatch: pytest.MonkeyPatch,
- tmp_path,
-) -> None:
- import switchyard.cli.switchyard_cli as cli
-
- yaml_path = tmp_path / "routes.yaml"
- yaml_path.write_text("routes:\n direct:\n type: model\n target: some/model\n")
- captured: dict[str, Any] = {}
-
- def _fake_serve(
- args: Any,
- switchyard: object,
- inbound_default: str,
- **_kwargs: object,
- ) -> None:
- captured["args"] = args
- captured["switchyard"] = switchyard
- captured["inbound_default"] = inbound_default
-
- monkeypatch.setattr(cli, "build_and_serve", _fake_serve)
-
- args = cli._build_parser().parse_args([
- "serve",
- "--routing-profiles",
- str(yaml_path),
- "--port",
- "4555",
- ])
- args.func(args)
-
- assert isinstance(captured["switchyard"], RouteTable)
- assert captured["switchyard"].registered_models() == ["direct"]
- assert captured["args"].port == 4555
- assert captured["inbound_default"] == "both"
-
-
-def test_main_reports_route_bundle_config_error_without_traceback(
- monkeypatch: pytest.MonkeyPatch,
- tmp_path: Path,
-) -> None:
- yaml_path = tmp_path / "unknown-key.yaml"
- yaml_path.write_text(
- "routes:\n"
- " r:\n"
- " type: model\n"
- " target: x\n"
- " bogus_key: 1\n"
- )
- monkeypatch.setattr(
- sys, "argv", ["switchyard", "serve", "--routing-profiles", str(yaml_path)]
- )
-
- with pytest.raises(SystemExit) as excinfo:
- cli.main()
-
- # A clean one-line diagnostic, not a raw traceback, and a non-zero exit.
- assert excinfo.value.code == (
- "error: invalid route bundle: unknown key(s) for route 'r': bogus_key"
- )
-
-
-def test_main_reports_missing_route_bundle_file_without_traceback(
- monkeypatch: pytest.MonkeyPatch,
- tmp_path: Path,
-) -> None:
- missing_path = tmp_path / "does-not-exist.yaml"
- monkeypatch.setattr(
- sys, "argv", ["switchyard", "serve", "--routing-profiles", str(missing_path)]
- )
-
- with pytest.raises(SystemExit) as excinfo:
- cli.main()
-
- assert excinfo.value.code == (
- f"error: invalid route bundle: {missing_path}: file not found"
- )
-
-
-def test_main_reports_bad_route_bundle_yaml_without_traceback(
- monkeypatch: pytest.MonkeyPatch,
- tmp_path: Path,
-) -> None:
- yaml_path = tmp_path / "bad_yaml.yaml"
- yaml_path.write_text(
- "routes:\n"
- " r:\n"
- " type: model\n"
- " target: x\n"
- )
- monkeypatch.setattr(
- sys, "argv", ["switchyard", "serve", "--routing-profiles", str(yaml_path)]
- )
-
- with pytest.raises(SystemExit) as excinfo:
- cli.main()
-
- assert str(excinfo.value.code).startswith(
- f"error: invalid route bundle: {yaml_path}: invalid YAML: "
- )
- assert "\n" not in str(excinfo.value.code)
+ stats_backend = next(component for component in components if isinstance(component, StatsLlmBackend))
+ assert isinstance(stats_backend, StatsLlmBackend)
-def test_main_reports_non_utf8_route_bundle_without_traceback(
- monkeypatch: pytest.MonkeyPatch,
- tmp_path: Path,
-) -> None:
- yaml_path = tmp_path / "non_utf8.yaml"
- yaml_path.write_bytes(b"\xff")
- monkeypatch.setattr(
- sys, "argv", ["switchyard", "serve", "--routing-profiles", str(yaml_path)]
- )
-
- with pytest.raises(SystemExit) as excinfo:
- cli.main()
+def test_passthrough_summary_labels_the_model(tmp_path: Path) -> None:
+ path = tmp_path / "routes.yaml"
+ path.write_text("routes:\n direct:\n type: passthrough\n target: upstream/model\n")
- assert str(excinfo.value.code).startswith(
- f"error: invalid route bundle: {yaml_path}: cannot read: "
+ assert route_bundle_strategy_summary(str(path), "direct") == (
+ "passthrough: model=upstream/model"
)
- assert "\n" not in str(excinfo.value.code)
-
-
-def test_main_accepts_routing_profiles_flag(
- monkeypatch: pytest.MonkeyPatch,
- tmp_path,
- capsys: pytest.CaptureFixture[str],
-) -> None:
- yaml_path = tmp_path / "routes.yaml"
- yaml_path.write_text("routes:\n direct:\n type: model\n target: some/model\n")
- monkeypatch.setattr(
- sys, "argv", ["switchyard", "serve", "--routing-profiles", str(yaml_path)]
- )
-
- def _fake_serve(
- args: argparse.Namespace,
- switchyard: object,
- inbound_default: str,
- **_kwargs: object,
- ) -> None:
- return
-
- monkeypatch.setattr(cli, "build_and_serve", _fake_serve)
-
- cli.main()
-
- assert capsys.readouterr().err == ""
-
-
-class TestEscalationRouterRouteType:
- """`type: escalation_router` wires the judge-latched chain via YAML."""
-
- def _bundle(self) -> dict:
- return {
- "routes": {
- "myrouter/escalation": {
- "type": "escalation_router",
- "fallback_target_on_evict": "strong",
- "judge": {
- "model": "google/gemini-3.5-flash",
- "api_key": "sk-judge",
- "base_url": "https://judge.invalid/v1",
- "timeout_secs": 5.0,
- "min_turn": 4,
- "recent_turn_window": 10,
- },
- "strong": {
- "model": "openai/gpt-5.2",
- "api_key": "sk-strong",
- "base_url": "https://strong.invalid/v1",
- },
- "weak": {
- "model": "deepseek/deepseek-v4-pro",
- "api_key": "sk-weak",
- "base_url": "https://weak.invalid/v1",
- },
- "session_key_depth": 2,
- },
- },
- }
-
- def test_registers_route_key_and_tier_passthroughs(self):
- from switchyard.cli.route_bundle import build_route_bundle_table
- table = build_route_bundle_table(self._bundle())
- # Route key first, tiers as direct passthroughs; the judge tier is
- # internal-only and never registered.
- assert table.registered_models() == [
- "myrouter/escalation",
- "openai/gpt-5.2",
- "deepseek/deepseek-v4-pro",
- ]
-
- def test_judge_settings_thread_into_processor(self):
- from switchyard.cli.route_bundle import build_route_bundle_table
- from switchyard.lib.processors.escalation_judge_request_processor import (
- EscalationJudgeRequestProcessor,
- )
- table = build_route_bundle_table(self._bundle())
- switchyard = table.lookup_switchyard("myrouter/escalation")
- judge = next(
- c for c in switchyard.iter_components()
- if isinstance(c, EscalationJudgeRequestProcessor)
- )
- assert judge._config.model == "google/gemini-3.5-flash"
- assert judge._config.min_judge_turn == 4
- assert judge._config.recent_turn_window == 10
- assert judge._config.timeout_s == 5.0
- assert judge._session_key_depth == 2
-
- def test_defaults_come_from_the_config_model(self):
- """Omitted keys inherit EscalationRouterConfig defaults, owned once."""
- from switchyard.cli.route_bundle import build_route_bundle_table
- from switchyard.lib.processors.escalation_judge_request_processor import (
- EscalationJudgeRequestProcessor,
- )
- bundle = self._bundle()
- del bundle["routes"]["myrouter/escalation"]["judge"]["timeout_secs"]
- table = build_route_bundle_table(bundle)
- switchyard = table.lookup_switchyard("myrouter/escalation")
- judge = next(
- c for c in switchyard.iter_components()
- if isinstance(c, EscalationJudgeRequestProcessor)
- )
- assert judge._config.escalate_confirmations == 2
- assert judge._config.window_message_chars == 500
- assert judge._config.dump_verdicts_to_stderr is False
- assert judge._config.max_completion_tokens == 128
- assert judge._config.timeout_s == 30.0
- assert judge._affinity._l2 is None
-
- def test_invalid_value_is_a_one_line_config_error(self):
- """Pydantic rejections surface as RouteBundleConfigError, not a traceback."""
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- bundle["routes"]["myrouter/escalation"]["session_key_depth"] = "two"
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "session_key_depth" in str(exc.value)
-
- def test_benchmark_knobs_and_redis_latch_thread_through(self):
- from switchyard.cli.route_bundle import build_route_bundle_table
- from switchyard.lib.processors.escalation_judge_request_processor import (
- EscalationJudgeRequestProcessor,
- )
- from switchyard.lib.redis_pin_store import RedisPinStore
- bundle = self._bundle()
- route = bundle["routes"]["myrouter/escalation"]
- route["judge"]["dump_verdicts"] = True
- route["judge"]["max_completion_tokens"] = 2048
- route["affinity_store"] = "redis"
- route["affinity_store_url"] = "redis://cache:6379/0"
- table = build_route_bundle_table(bundle)
- switchyard = table.lookup_switchyard("myrouter/escalation")
- judge = next(
- c for c in switchyard.iter_components()
- if isinstance(c, EscalationJudgeRequestProcessor)
- )
- assert judge._config.dump_verdicts_to_stderr is True
- assert judge._config.max_completion_tokens == 2048
- assert isinstance(judge._affinity._l2, RedisPinStore)
-
- def test_judge_prompt_path_reads_file(self, tmp_path):
- from switchyard.cli.route_bundle import build_route_bundle_table
- from switchyard.lib.processors.escalation_judge_request_processor import (
- EscalationJudgeRequestProcessor,
- )
- prompt_file = tmp_path / "judge_prompt.md"
- prompt_file.write_text("file-based judge prompt\n", encoding="utf-8")
- bundle = self._bundle()
- bundle["routes"]["myrouter/escalation"]["judge"]["prompt_path"] = (
- str(prompt_file)
- )
- table = build_route_bundle_table(bundle)
- switchyard = table.lookup_switchyard("myrouter/escalation")
- judge = next(
- c for c in switchyard.iter_components()
- if isinstance(c, EscalationJudgeRequestProcessor)
- )
- assert judge._config.system_prompt == "file-based judge prompt\n"
-
- def test_judge_prompt_and_prompt_path_are_mutually_exclusive(self, tmp_path):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- prompt_file = tmp_path / "judge_prompt.md"
- prompt_file.write_text("file prompt", encoding="utf-8")
- bundle = self._bundle()
- judge = bundle["routes"]["myrouter/escalation"]["judge"]
- judge["prompt"] = "inline prompt"
- judge["prompt_path"] = str(prompt_file)
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "mutually exclusive" in str(exc.value)
-
- def test_judge_prompt_path_missing_file_is_a_config_error(self, tmp_path):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- bundle["routes"]["myrouter/escalation"]["judge"]["prompt_path"] = (
- str(tmp_path / "nope.md")
- )
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "prompt_path" in str(exc.value)
-
- def test_rejects_missing_judge_block(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- del bundle["routes"]["myrouter/escalation"]["judge"]
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "judge" in str(exc.value)
-
- def test_rejects_fallback_not_matching_a_tier(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- bundle["routes"]["myrouter/escalation"]["fallback_target_on_evict"] = "judge"
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "fallback_target_on_evict" in str(exc.value)
-
-
-class TestDeterministicRouteType:
- """`type: deterministic` wires the LLM-classifier chain via YAML."""
-
- def _bundle(self) -> dict:
- return {
- "routes": {
- "myrouter/llm-classifier": {
- "type": "deterministic",
- "fallback_target_on_evict": "strong",
- "profile": "general",
- "classifier": {
- "model": "nvidia/nv-classifier",
- "api_key": "sk-classifier",
- "base_url": "https://classifier.invalid/v1",
- "timeout_secs": 30.0,
- "min_confidence": 0.6,
- "fail_open": True,
- "recent_turn_window": 4,
- },
- "strong": {
- "model": "openai/gpt-5.2",
- "api_key": "sk-strong",
- "base_url": "https://strong.invalid/v1",
- },
- "weak": {
- "model": "nvidia/nemotron-3-super",
- "api_key": "sk-weak",
- "base_url": "https://weak.invalid/v1",
- },
- },
- },
- }
-
- def test_registers_route_key_and_tier_passthroughs(self):
- from switchyard.cli.route_bundle import build_route_bundle_table
- table = build_route_bundle_table(self._bundle())
- # Unified ordering: route key first as the deterministic virtual id,
- # tier models registered as direct passthroughs after. The classifier
- # tier is internal-only and never registered.
- assert table.registered_models() == [
- "myrouter/llm-classifier",
- "openai/gpt-5.2",
- "nvidia/nemotron-3-super",
- ]
-
- def test_anthropic_tier_is_cache_wrapped(self):
- """An Anthropic-format tier is wrapped for prompt caching via the YAML
- serve path, so an OpenAI-origin harness (Codex) routed onto a Claude
- tier still gets ``cache_control`` breakpoints injected. OpenAI tiers
- stay bare. Distinct from the DeterministicRoutingFactory path.
- """
- from switchyard.cli.route_bundle import build_route_bundle_table
- from switchyard.lib.backends.anthropic_cache_breakpoint_backend import (
- AnthropicCacheBreakpointBackend,
- )
- from switchyard.lib.backends.deterministic_routing_llm_backend import (
- DeterministicRoutingLLMBackend,
- )
- bundle = self._bundle()
- # Weak tier = Claude on Anthropic format (explicit -> no AUTO probe).
- bundle["routes"]["myrouter/llm-classifier"]["weak"] = {
- "model": "aws/anthropic/bedrock-claude-opus-4-7",
- "api_key": "sk-weak",
- "base_url": "https://weak.invalid/v1",
- "format": "anthropic",
- }
- table = build_route_bundle_table(bundle)
- switchyard = table.lookup_switchyard("myrouter/llm-classifier")
- backend = next(
- c for c in switchyard.iter_components()
- if isinstance(c, DeterministicRoutingLLMBackend)
- )
- assert isinstance(
- backend._backends["weak"], AnthropicCacheBreakpointBackend,
- )
- # Strong tier is OpenAI-format -> not wrapped.
- assert not isinstance(
- backend._backends["strong"], AnthropicCacheBreakpointBackend,
- )
-
- def test_rejects_missing_classifier_block(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- del bundle["routes"]["myrouter/llm-classifier"]["classifier"]
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "classifier" in str(exc.value)
-
- def test_rejects_unknown_profile(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- bundle["routes"]["myrouter/llm-classifier"]["profile"] = "no_such_profile"
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "profile" in str(exc.value).lower()
-
- def test_accepts_coding_agent_and_openclaw_profiles(self):
- from switchyard.cli.route_bundle import build_route_bundle_table
- for profile in ("coding_agent", "openclaw"):
- bundle = self._bundle()
- bundle["routes"]["myrouter/llm-classifier"]["profile"] = profile
- table = build_route_bundle_table(bundle)
- assert "myrouter/llm-classifier" in table.registered_models()
-
- def test_classifier_prompt_and_context_thread_into_processor(self):
- bundle = self._bundle()
- classifier = bundle["routes"]["myrouter/llm-classifier"]["classifier"]
- classifier["prompt"] = "custom yaml prompt"
- classifier["max_request_chars"] = 1024
- classifier["recent_turn_window"] = 2
-
- table = build_route_bundle_table(bundle)
- switchyard = table.lookup_switchyard("myrouter/llm-classifier")
- processor = next(
- c for c in switchyard.iter_components()
- if isinstance(c, LLMClassifierRequestProcessor)
- )
-
- assert processor._config.system_prompt == "custom yaml prompt"
- assert processor._config.max_request_chars == 1024
- assert processor._config.recent_turn_window == 2
-
- def test_blank_classifier_prompt_falls_back_to_profile_default(self):
- bundle = self._bundle()
- bundle["routes"]["myrouter/llm-classifier"]["profile"] = "coding_agent"
- bundle["routes"]["myrouter/llm-classifier"]["classifier"]["prompt"] = " "
-
- table = build_route_bundle_table(bundle)
- switchyard = table.lookup_switchyard("myrouter/llm-classifier")
- processor = next(
- c for c in switchyard.iter_components()
- if isinstance(c, LLMClassifierRequestProcessor)
- )
-
- assert processor._config.system_prompt == CODING_AGENT_CLASSIFIER_SYSTEM_PROMPT
-
- def test_rejects_invalid_classifier_context_type(self):
- bundle = self._bundle()
- bundle["routes"]["myrouter/llm-classifier"]["classifier"][
- "max_request_chars"
- ] = "1024"
-
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "integer" in str(exc.value)
-
- def test_tier_timeout_null_disables_default_timeout(
- self,
- monkeypatch: pytest.MonkeyPatch,
- ):
- captured_timeouts = []
-
- def fake_apply_default_tier_timeout(target, timeout_s):
- captured_timeouts.append((target.id, timeout_s))
- return target
-
- monkeypatch.setattr(
- "switchyard.lib.profiles.tier_target_builders.apply_default_tier_timeout",
- fake_apply_default_tier_timeout,
- )
- bundle = self._bundle()
- bundle["routes"]["myrouter/llm-classifier"]["tier_timeout_s"] = None
+ "bundle, match",
+ [
+ ({}, "routes must be a mapping"),
+ ({"routes": {}}, "at least one route"),
+ ({"routes": {"r": {}}}, "missing string 'type'"),
+ (
+ {"routes": {"r": {"type": "random"}}},
+ "expected 'noop' or 'passthrough'",
+ ),
+ (
+ {"routes": {"r": {"type": "noop", "target": "unused"}}},
+ "unknown key",
+ ),
+ (
+ {"routes": {"r": {"type": "passthrough"}}},
+ "target must be a mapping",
+ ),
+ ],
+)
+def test_invalid_bundles_fail_closed(bundle: object, match: str) -> None:
+ with pytest.raises(RouteBundleConfigError, match=match):
build_route_bundle_table(bundle)
- assert captured_timeouts == [("strong", None), ("weak", None)]
-
- def test_weak_tier_on_own_endpoint_registers(self):
- """Weak tier can point at a self-hosted endpoint (e.g. local vLLM).
-
- Guards the GETTING_STARTED "serving the weak tier from your own
- endpoint" example: a per-tier ``base_url`` overrides ``defaults`` so
- weak-classified turns hit a model the user hosts, while strong and
- classifier stay on the shared endpoint. The weak model id registers as
- a direct passthrough.
- """
- from switchyard.cli.route_bundle import build_route_bundle_table
- table = build_route_bundle_table({
- "defaults": {
- "api_key": "sk-default",
- "base_url": "https://inference-api.nvidia.com/v1",
- "format": "openai",
- },
- "routes": {
- "local-weak": {
- "type": "deterministic",
- "profile": "coding_agent",
- "fallback_target_on_evict": "strong",
- "strong": {"model": "aws/anthropic/bedrock-claude-opus-4-7"},
- "weak": {
- "model": "my-rl-qwen",
- "base_url": "http://localhost:8000/v1",
- "api_key": "dummy",
- },
- "classifier": {
- "model": "nvidia/deepseek-ai/deepseek-v4-flash",
- "api_key": "sk-default",
- "base_url": "https://inference-api.nvidia.com/v1",
- },
- },
- },
- })
- assert table.registered_models() == [
- "local-weak",
- "aws/anthropic/bedrock-claude-opus-4-7",
- "my-rl-qwen",
- ]
-
- def test_classifier_inherits_route_defaults(self):
- from switchyard.cli.route_bundle import build_route_bundle_table
- from switchyard.lib.processors.llm_classifier import (
- LLMClassifierRequestProcessor,
- )
-
- table = build_route_bundle_table({
- "defaults": {
- "api_key": "sk-default",
- "base_url": "https://default.invalid/v1",
- "format": "openai",
- "timeout": 12.0,
- },
- "routes": {
- "defaults/classifier": {
- "type": "deterministic",
- "profile": "coding_agent",
- "fallback_target_on_evict": "strong",
- "strong": {"id": "strong", "model": "strong/model"},
- "weak": {"id": "weak", "model": "weak/model"},
- "classifier": {"model": "classifier/model"},
- },
- },
- })
- switchyard = table.lookup_switchyard("defaults/classifier")
- classifier = next(
- c for c in switchyard.iter_components()
- if isinstance(c, LLMClassifierRequestProcessor)
- )
-
- assert classifier._config.api_key == "sk-default"
- assert classifier._config.base_url == "https://default.invalid/v1"
- assert classifier._config.timeout_s == 12.0
-
- def test_rejects_unknown_classifier_key(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- bundle["routes"]["myrouter/llm-classifier"]["classifier"]["bogus"] = 1
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "bogus" in str(exc.value)
-
- def _classifier_disable_reasoning(self, classifier_model: str) -> bool:
- from switchyard.cli.route_bundle import build_route_bundle_table
- from switchyard.lib.processors.llm_classifier import (
- LLMClassifierRequestProcessor,
- )
- bundle = self._bundle()
- bundle["routes"]["myrouter/llm-classifier"]["classifier"]["model"] = classifier_model
- table = build_route_bundle_table(bundle)
- switchyard = table.lookup_switchyard("myrouter/llm-classifier")
- classifier = next(
- c for c in switchyard.iter_components()
- if isinstance(c, LLMClassifierRequestProcessor)
- )
- return classifier._config.disable_reasoning
-
- def test_bedrock_claude_classifier_disables_reasoning(self):
- assert self._classifier_disable_reasoning(
- "aws/anthropic/bedrock-claude-sonnet-4-6",
- ) is False
-
- def test_deepseek_classifier_keeps_reasoning_disabled(self):
- assert self._classifier_disable_reasoning(
- "nvidia/deepseek-ai/deepseek-v4-flash",
- ) is True
-
-
-class TestStageRouterRouteType:
- """`type: stage_router` wires the stage-router-routing chain via YAML."""
-
- def _bundle(self) -> dict:
- return {
- "routes": {
- "myrouter/stage_router": {
- "type": "stage_router",
- "picker": "capable_first",
- "strong": {
- "id": "strong",
- "model": "anthropic/claude-opus-4-7",
- "api_key": "sk-strong",
- "base_url": "https://strong.invalid/v1",
- "format": "anthropic",
- },
- "weak": {
- "id": "weak",
- "model": "nvidia/nemotron-3-super",
- "api_key": "sk-weak",
- "base_url": "https://weak.invalid/v1",
- "format": "openai",
- },
- "confidence_threshold": 0.7,
- "fallback_target_on_evict": "strong",
- },
- },
- }
-
- def test_registers_route_key_and_tier_passthroughs(self):
- from switchyard.cli.route_bundle import build_route_bundle_table
- table = build_route_bundle_table(self._bundle())
- # Unified ordering: route key first as the stage-router virtual id,
- # tier models registered as direct passthroughs after.
- assert table.registered_models() == [
- "myrouter/stage_router",
- "anthropic/claude-opus-4-7",
- "nvidia/nemotron-3-super",
- ]
-
- def test_rejects_missing_strong(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- del bundle["routes"]["myrouter/stage_router"]["strong"]
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "strong" in str(exc.value)
-
- def test_rejects_missing_weak(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- del bundle["routes"]["myrouter/stage_router"]["weak"]
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "weak" in str(exc.value)
-
- def test_rejects_unknown_picker(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- bundle["routes"]["myrouter/stage_router"]["picker"] = "no-such-picker"
- with pytest.raises((RouteBundleConfigError, ValueError)) as exc:
- build_route_bundle_table(bundle)
- assert "picker" in str(exc.value).lower()
- def test_rejects_missing_fallback_target_on_evict(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- del bundle["routes"]["myrouter/stage_router"]["fallback_target_on_evict"]
- # Pydantic ValidationError extends ValueError; route_bundle does not
- # additionally wrap it for missing-field cases.
- with pytest.raises((RouteBundleConfigError, ValueError)) as exc:
- build_route_bundle_table(bundle)
- assert "fallback_target_on_evict" in str(exc.value)
-
- def test_rejects_fallback_target_id_not_matching(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- bundle["routes"]["myrouter/stage_router"]["fallback_target_on_evict"] = "nope"
- with pytest.raises((RouteBundleConfigError, ValueError)) as exc:
- build_route_bundle_table(bundle)
- assert "fallback_target_on_evict" in str(exc.value)
-
- def test_rejects_unknown_route_key(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- bundle["routes"]["myrouter/stage_router"]["bogus_field"] = 1
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "bogus_field" in str(exc.value)
-
- def test_accepts_both_pickers(self):
- from switchyard.cli.route_bundle import build_route_bundle_table
- for picker in ("capable_first", "efficient_first"):
- bundle = self._bundle()
- bundle["routes"]["myrouter/stage_router"]["picker"] = picker
- table = build_route_bundle_table(bundle)
- assert "myrouter/stage_router" in table.registered_models()
-
- def test_rejects_removed_legacy_knobs(self):
- from switchyard.cli.route_bundle import (
- RouteBundleConfigError,
- build_route_bundle_table,
- )
- bundle = self._bundle()
- bundle["routes"]["myrouter/stage_router"]["escalate_at"] = 0.5
- with pytest.raises(RouteBundleConfigError) as exc:
- build_route_bundle_table(bundle)
- assert "escalate_at" in str(exc.value)
-
- def test_classifier_block_is_optional(self):
- from switchyard.cli.route_bundle import build_route_bundle_table
- bundle = self._bundle()
- bundle["routes"]["myrouter/stage_router"]["classifier"] = {
- "model": "nvidia/deepseek-ai/deepseek-v4-flash",
- "api_key": "sk-classifier",
- "base_url": "https://classifier.invalid/v1",
- }
- table = build_route_bundle_table(bundle)
- assert "myrouter/stage_router" in table.registered_models()
-
- def test_signal_recent_window_accepted(self):
- from switchyard.cli.route_bundle import build_route_bundle_table
- bundle = self._bundle()
- bundle["routes"]["myrouter/stage_router"]["signal_recent_window"] = 5
- table = build_route_bundle_table(bundle)
- assert "myrouter/stage_router" in table.registered_models()
-
- def test_classifier_inherits_route_defaults(self):
- from switchyard.cli.route_bundle import build_route_bundle_table
- from switchyard.lib.processors.stage_router import TierClassifier
- from switchyard.lib.processors.stage_router_request_processor import (
- StageRouterRequestProcessor,
- )
-
- table = build_route_bundle_table({
- "defaults": {
- "api_key": "sk-default",
- "base_url": "https://default.invalid/v1",
- "format": "openai",
- "timeout": 12.0,
- },
- "routes": {
- "defaults/stage_router": {
- "type": "stage_router",
- "picker": "capable_first",
- "fallback_target_on_evict": "strong",
- "strong": {"id": "strong", "model": "strong/model"},
- "weak": {"id": "weak", "model": "weak/model"},
- "classifier": {"model": "classifier/model"},
- },
- },
+def test_missing_environment_variable_is_rejected() -> None:
+ with pytest.raises(RouteBundleConfigError, match="MISSING_ROUTE_KEY"):
+ build_route_bundle_table({
+ "defaults": {"api_key": "${MISSING_ROUTE_KEY}"},
+ "routes": {"direct": {"type": "passthrough", "target": "model"}},
})
- switchyard = table.lookup_switchyard("defaults/stage_router")
- processor = next(
- c for c in switchyard.iter_components()
- if isinstance(c, StageRouterRequestProcessor)
- )
- classifier = processor._picker.keywords["classifier"]
-
- assert isinstance(classifier, TierClassifier)
- assert classifier._api_key == "sk-default"
- assert classifier._recent_turn_window == 3
-
- async def test_classifier_usage_records_into_route_bundle_stats(self) -> None:
- """Route-bundle stage_router classifiers write usage into shared stats."""
- from switchyard.cli.route_bundle import build_route_bundle_table
- from switchyard.lib.processors.stage_router_request_processor import (
- StageRouterRequestProcessor,
- )
- from switchyard.lib.profiles.chain import ComponentChainProfile
- from switchyard.lib.stats_accumulator import StatsAccumulator
- from switchyard_rust.profiles import ProfileInput
-
- class _ClassifierClient:
- """Fake classifier client used after route-bundle construction."""
- calls = 0
- async def acompletion(self, **_kwargs: object) -> object:
- """Return a deterministic classifier response with usage data."""
- self.calls += 1
- return SimpleNamespace(
- choices=[
- SimpleNamespace(
- message=SimpleNamespace(
- content=json.dumps({"tier": "capable"}),
- ),
- )
- ],
- usage=SimpleNamespace(
- prompt_tokens=13,
- completion_tokens=5,
- prompt_tokens_details=SimpleNamespace(cached_tokens=2),
- ),
- )
-
- stats = StatsAccumulator()
- bundle = self._bundle()
- bundle["routes"]["myrouter/stage_router"]["confidence_threshold"] = 1.0
- bundle["routes"]["myrouter/stage_router"]["classifier"] = {
- "model": "classifier/model",
- "api_key": "sk-classifier",
- "base_url": "https://classifier.invalid/v1",
- }
- table = build_route_bundle_table(bundle, stats_accumulator=stats)
- switchyard = table.lookup_switchyard("myrouter/stage_router")
- assert isinstance(switchyard._profile, ComponentChainProfile)
- processor = next(
- component
- for component in switchyard.iter_components()
- if isinstance(component, StageRouterRequestProcessor)
- )
- assert processor._classifier is not None
- client = _ClassifierClient()
- processor._classifier._client = client # type: ignore[assignment]
-
- processed = await switchyard._profile.process(ProfileInput(ChatRequest.openai_chat({
- "model": "myrouter/stage_router",
- "messages": [{"role": "user", "content": "hello"}],
- "max_tokens": 8,
- })))
-
- snapshot = await stats.snapshot()
- assert client.calls == 1
- assert processed.selected_target == "strong"
- assert snapshot["classifier"]["total_requests"] == 1
- assert snapshot["classifier"]["models"]["classifier/model"]["calls"] == 1
- assert snapshot["classifier"]["models"]["classifier/model"]["prompt_tokens"] == 13
- assert snapshot["classifier"]["models"]["classifier/model"]["completion_tokens"] == 5
- assert snapshot["classifier"]["models"]["classifier/model"]["cached_tokens"] == 2
- assert snapshot["routing_decisions"]["stage_router"]["llm-classifier"] == 1
-
-
-def test_stage_router_route_hydrates_tier_catalogs(
- monkeypatch: pytest.MonkeyPatch,
+def test_main_reports_missing_bundle_without_traceback(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
- """``type: stage_router`` registers each tier's catalog alongside the
- routing-policy chain — strong/weak as direct passthroughs, plus the
- route's YAML key as the stage-router virtual id.
-
- The classifier tier (when present) is intentionally NOT discovered — it's
- an internal-only LLM call, not a user-facing target.
- """
- monkeypatch.setattr(
- "switchyard.cli.route_bundle.fetch_model_ids",
- lambda base_url, api_key: (
- ["catalog/strong-extra"]
- if base_url == "https://primary.example/v1"
- else ["catalog/weak-extra"]
- ),
- )
+ missing = tmp_path / "missing.yaml"
+ monkeypatch.setattr(sys, "argv", ["switchyard", "serve", "--routes", str(missing)])
- table = build_route_bundle_table({
- "routes": {
- "opus-ds-stage_router": {
- "type": "stage_router",
- "picker": "capable_first",
- "fallback_target_on_evict": "strong",
- "strong": {
- "id": "strong",
- "model": "strong/model",
- "api_key": "k-strong",
- "base_url": "https://primary.example/v1",
- },
- "weak": {
- "id": "weak",
- "model": "weak/model",
- "api_key": "k-weak",
- "base_url": "https://weak.example/v1",
- },
- },
- },
- })
+ with pytest.raises(SystemExit) as error:
+ cli.main()
- # Unified ordering: stage_router virtual id first, then tier models + catalog.
- # The classifier tier is absent here AND would be skipped even if present.
- assert table.registered_models() == [
- "opus-ds-stage_router",
- "strong/model",
- "weak/model",
- "catalog/strong-extra",
- "catalog/weak-extra",
- ]
+ assert error.value.code == f"error: invalid route bundle: {missing}: file not found"
-def test_deterministic_route_hydrates_tier_catalogs(
- monkeypatch: pytest.MonkeyPatch,
+def test_serve_loads_noop_bundle(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
- """``type: deterministic`` — same catalog-hydration shape as stage_router,
- with the classifier tier intentionally skipped."""
- monkeypatch.setattr(
- "switchyard.cli.route_bundle.fetch_model_ids",
- lambda base_url, api_key: (
- ["catalog/strong-extra"]
- if base_url == "https://primary.example/v1"
- else ["catalog/weak-extra"]
- ),
- )
-
- table = build_route_bundle_table({
- "routes": {
- "opus-ds-classifier": {
- "type": "deterministic",
- "profile": "coding_agent",
- "fallback_target_on_evict": "strong",
- "classifier": {
- "model": "classifier/model",
- "api_key": "k-classifier",
- "base_url": "https://classifier.example/v1",
- },
- "strong": {
- "id": "strong",
- "model": "strong/model",
- "api_key": "k-strong",
- "base_url": "https://primary.example/v1",
- },
- "weak": {
- "id": "weak",
- "model": "weak/model",
- "api_key": "k-weak",
- "base_url": "https://weak.example/v1",
- },
- },
- },
- })
-
- # Unified ordering: virtual id first, then tier models + catalog.
- # Classifier's `classifier/model` and its catalog are not in the list:
- # discovery skips internal-only tiers.
- assert table.registered_models() == [
- "opus-ds-classifier",
- "strong/model",
- "weak/model",
- "catalog/strong-extra",
- "catalog/weak-extra",
- ]
+ path = tmp_path / "routes.yaml"
+ path.write_text("routes:\n test/noop:\n type: noop\n")
+ captured: dict[str, object] = {}
+ def fake_serve(args: object, switchyard: object, **kwargs: object) -> None:
+ captured.update(args=args, switchyard=switchyard, **kwargs)
-def test_route_bundle_keeps_first_multi_target_route_as_default() -> None:
- """StageRouter/deterministic discovery merges preserve the first YAML route."""
- table = build_route_bundle_table({
- "routes": {
- "first-stage_router": {
- "type": "stage_router",
- "picker": "capable_first",
- "fallback_target_on_evict": "strong",
- "strong": {
- "id": "strong",
- "model": "first/strong",
- "api_key": "k-strong",
- "base_url": "https://first-strong.example/v1",
- },
- "weak": {
- "id": "weak",
- "model": "first/weak",
- "api_key": "k-weak",
- "base_url": "https://first-weak.example/v1",
- },
- },
- "second-stage_router": {
- "type": "stage_router",
- "picker": "capable_first",
- "fallback_target_on_evict": "strong",
- "strong": {
- "id": "strong",
- "model": "second/strong",
- "api_key": "k-strong",
- "base_url": "https://second-strong.example/v1",
- },
- "weak": {
- "id": "weak",
- "model": "second/weak",
- "api_key": "k-weak",
- "base_url": "https://second-weak.example/v1",
- },
- },
- },
- })
-
- assert table.default_model() == "first-stage_router"
-
-
-_AFFINITY_DET_ROUTE = {
- "type": "deterministic",
- "profile": "coding_agent",
- "fallback_target_on_evict": "strong",
- "session_affinity": True,
- "affinity_max_sessions": 50,
- "classifier": {"model": "m", "api_key": "k", "base_url": "https://ls.test/v1"},
- "strong": {"model": "s", "api_key": "k", "base_url": "https://ls.test/v1"},
- "weak": {"model": "w", "api_key": "k", "base_url": "https://ls.test/v1"},
-}
-
-
[email protected]("route", [_AFFINITY_DET_ROUTE])
-def test_session_affinity_keys_accepted_by_route_bundle(route: dict[str, Any]) -> None:
- """session_affinity / affinity_max_sessions are valid keys on the deterministic route."""
- table = build_route_bundle_table({"routes": {"r": route}})
- assert isinstance(table, RouteTable)
-
-
-def test_deterministic_affinity_warmup_turns_accepted_by_route_bundle() -> None:
- """affinity_warmup_turns is a deterministic-route knob."""
- table = build_route_bundle_table({
- "routes": {"r": {**_AFFINITY_DET_ROUTE, "affinity_warmup_turns": 2}},
- })
- assert isinstance(table, RouteTable)
-
-
[email protected]("route", [_AFFINITY_DET_ROUTE])
-def test_zero_capacity_affinity_rejected_via_bundle(route: dict[str, Any]) -> None:
- """A zero cap with affinity on is rejected — proving the keys reach the config."""
- from pydantic import ValidationError
-
- with pytest.raises(ValidationError):
- build_route_bundle_table({"routes": {"r": {**route, "affinity_max_sessions": 0}}})
-
-
-def test_negative_affinity_warmup_turns_rejected_via_bundle() -> None:
- """The deterministic config validates affinity_warmup_turns as non-negative."""
- from pydantic import ValidationError
+ monkeypatch.setattr(cli, "build_and_serve", fake_serve)
+ parser = cli._build_parser()
+ args = parser.parse_args(["serve", "--routes", str(path)])
+ args.func(args)
- route = {**_AFFINITY_DET_ROUTE, "affinity_warmup_turns": -1}
- with pytest.raises(ValidationError):
- build_route_bundle_table({"routes": {"r": route}})
+ assert isinstance(captured["switchyard"], RouteTable)
+ assert captured["switchyard"].registered_models() == ["test/noop"]
diff --git a/tests/test_route_table.py b/tests/test_route_table.py
index 2bc112acc..e11b98f08 100644
--- a/tests/test_route_table.py
+++ b/tests/test_route_table.py
@@ -9,7 +9,6 @@ import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
from switchyard.lib.endpoints.anthropic_messages_endpoint import (
AnthropicMessagesEndpoint,
)
@@ -17,8 +16,6 @@ from switchyard.lib.endpoints.openai_chat_endpoint import OpenAIChatEndpoint
from switchyard.lib.endpoints.responses_endpoint import ResponsesEndpoint
from switchyard.lib.proxy_context import CTX_CALLER_API_KEY
from switchyard.lib.route_table import RouteTable
-from switchyard.lib.route_table_builders import build_passthrough_table
-from switchyard.lib.stats_accumulator import StatsAccumulator
from switchyard.lib.switchyard import Switchyard
from switchyard.server.switchyard_app import build_switchyard_app
@@ -122,7 +119,7 @@ def test_models_endpoint_lists_registered_models() -> None:
"display_name": "Switchyard random routing",
"description": "Random routes strong and weak.",
"switchyard": {
- "profile": "random_routing",
+ "algorithm": "random",
"strong_model": "strong/model",
"weak_model": "weak/model",
"strong_probability": 0.5,
@@ -149,7 +146,7 @@ def test_models_endpoint_lists_registered_models() -> None:
]
assert body["data"][0]["display_name"] == "Switchyard random routing"
assert body["data"][0]["description"] == "Random routes strong and weak."
- assert body["data"][0]["switchyard"]["profile"] == "random_routing"
+ assert body["data"][0]["switchyard"]["algorithm"] == "random"
assert body["data"][0]["capabilities"]["streaming"] is True
assert body["data"][0]["capabilities"]["supported_inbound_formats"] == [
"openai-chat-completions",
@@ -183,93 +180,6 @@ def test_models_endpoint_uses_table_default_model() -> None:
]
-def test_models_endpoint_lists_discovered_catalog_models() -> None:
- def discover(base_url: str, api_key: str) -> list[str]:
- assert base_url == "https://primary.example/v1"
- assert api_key == "k-primary"
- return ["catalog/a", "catalog/b"]
-
- table = build_passthrough_table(
- (
- LlmTarget(
- model="configured/model",
- format=BackendFormat.AUTO,
- api_key="k-primary",
- base_url="https://primary.example/v1",
- ),
- ),
- StatsAccumulator(),
- discovery_fn=discover,
- )
-
- with TestClient(build_switchyard_app(table), raise_server_exceptions=False) as client:
- response = client.get("/v1/models")
-
- assert response.status_code == 200
- body = response.json()
- assert body["model_pool"] == [
- "configured/model",
- "catalog/a",
- "catalog/b",
- ]
- assert [item["switchyard"]["source"] for item in body["data"]] == [
- "configured",
- "discovered",
- "discovered",
- ]
-
-
-def test_large_discovered_catalog_registers_stats_endpoint_once() -> None:
- discovered_models = [f"catalog/model-{index}" for index in range(340)]
- table = build_passthrough_table(
- (
- LlmTarget(
- model="configured/model",
- format=BackendFormat.AUTO,
- api_key="k-primary",
- base_url="https://primary.example/v1",
- ),
- ),
- StatsAccumulator(),
- discovery_fn=lambda _base_url, _api_key: discovered_models,
- )
-
- app = build_switchyard_app(table)
-
- for path in ("/v1/stats", "/v1/routing/stats", "/metrics"):
- assert sum(route.path == path for route in app.routes) == 1
- with TestClient(app, raise_server_exceptions=False) as client:
- assert client.get("/health").status_code == 200
-
-
-def test_models_endpoint_returns_static_models_and_warning_when_discovery_fails() -> None:
- def discover(_base_url: str, _api_key: str) -> list[str]:
- raise RuntimeError("catalog timed out")
-
- table = build_passthrough_table(
- (
- LlmTarget(
- model="configured/model",
- format=BackendFormat.AUTO,
- api_key="k-primary",
- base_url="https://primary.example/v1",
- ),
- ),
- StatsAccumulator(),
- discovery_fn=discover,
- )
-
- with TestClient(build_switchyard_app(table), raise_server_exceptions=False) as client:
- response = client.get("/v1/models")
-
- assert response.status_code == 200
- body = response.json()
- assert body["model_pool"] == ["configured/model"]
- assert body["warnings"] == [
- "Model discovery failed for https://primary.example/v1: catalog timed out"
- ]
-
-
@pytest.mark.parametrize(
("path", "body"),
[
diff --git a/tests/test_routing_log_response_processor.py b/tests/test_routing_log_response_processor.py
index 8cb95dfd0..c63d321e3 100644
--- a/tests/test_routing_log_response_processor.py
+++ b/tests/test_routing_log_response_processor.py
@@ -116,8 +116,8 @@ def _records(log_file: Path) -> list[dict]:
async def test_openai_completion_record_carries_task_and_usage(tmp_path: Path) -> None:
log_file = tmp_path / "routing_requests.jsonl"
ctx = _ctx(headers=TASK_HEADERS)
- ctx.selected_model = "configured-profile"
- ctx.metadata["_random_routing_tier"] = "weak"
+ ctx.selected_model = "configured-route"
+ ctx.selected_target = "weak"
await RoutingLogResponseProcessor(log_file).process(ctx, _openai_completion())
(record,) = _records(log_file)
@@ -288,7 +288,7 @@ def test_serve_parser_accepts_routing_log_file() -> None:
args = parser.parse_args(
[
"serve",
- "--routing-profiles",
+ "--routes",
"routes.yaml",
"--routing-log-file",
"tmp/routing.jsonl",
diff --git a/tests/test_session_affinity.py b/tests/test_session_affinity.py
deleted file mode 100644
index d06a8ac57..000000000
--- a/tests/test_session_affinity.py
+++ /dev/null
@@ -1,294 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for SessionAffinity's L1 (in-process) + optional L2 (shared) tiering.
-
-The classifier and escalation router integration tests cover affinity
-*policy*; these tests cover the store mechanics in isolation: write-through,
-read-through with L1 warming, and best-effort (fail-open) degradation when the
-L2 errors.
-"""
-
-from __future__ import annotations
-
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.session_affinity import SessionAffinity
-from switchyard_rust.core import ChatRequest
-
-
-def _request(text: str = "hi") -> ChatRequest:
- """A turn-1 request whose first user message anchors a distinct session."""
- return ChatRequest.openai_chat(
- {"model": "incoming-model", "messages": [{"role": "user", "content": text}]}
- )
-
-
-class _DictL2:
- """In-memory AffinityPinStore double with call counters."""
-
- def __init__(self) -> None:
- self.store: dict[str, str] = {}
- self.gets = 0
- self.puts = 0
-
- async def get(self, key: str) -> str | None:
- self.gets += 1
- return self.store.get(key)
-
- async def put(self, key: str, value: str) -> None:
- self.puts += 1
- self.store[key] = value
-
-
-class _BrokenL2:
- """AffinityPinStore double that always raises, to exercise fail-open."""
-
- async def get(self, key: str) -> str | None:
- raise RuntimeError("l2 unavailable")
-
- async def put(self, key: str, value: str) -> None:
- raise RuntimeError("l2 unavailable")
-
-
-async def test_no_l2_still_pins_via_l1() -> None:
- """The async interface works L1-only when no L2 is configured."""
- affinity = SessionAffinity(enabled=True)
- req = _request("task")
- await affinity.pin(ProxyContext(), req, "model-A")
- assert await affinity.pinned(ProxyContext(), req) == "model-A"
-
-
-async def test_pin_writes_through_to_l2() -> None:
- """A pin lands in both L1 and the shared L2."""
- l2 = _DictL2()
- affinity = SessionAffinity(enabled=True, l2=l2)
- req = _request("task")
- await affinity.pin(ProxyContext(), req, "model-A")
- assert l2.puts == 1
- assert list(l2.store.values()) == ["model-A"]
-
-
-async def test_l1_miss_reads_through_l2_and_warms_l1() -> None:
- """A pin written by one worker is visible to another via the shared L2,
- and the read-through warms the reader's L1 so later turns skip the round-trip."""
- l2 = _DictL2()
- worker_a = SessionAffinity(enabled=True, l2=l2)
- worker_b = SessionAffinity(enabled=True, l2=l2)
- req = _request("shared task")
-
- # Worker A serves the first turn and pins.
- await worker_a.pin(ProxyContext(), req, "model-A")
-
- # Worker B has never seen this conversation: L1 miss → L2 read-through.
- assert await worker_b.pinned(ProxyContext(), req) == "model-A"
- assert l2.gets == 1
- assert worker_b.l2_hits == 1
-
- # L1 is now warm on worker B: a second lookup does not hit L2 again.
- assert await worker_b.pinned(ProxyContext(), req) == "model-A"
- assert l2.gets == 1
- assert worker_b.l2_hits == 1
-
-
-async def test_l2_get_failure_is_fail_open() -> None:
- """An L2 read error routes as unpinned rather than raising, and is counted."""
- affinity = SessionAffinity(enabled=True, l2=_BrokenL2())
- assert await affinity.pinned(ProxyContext(), _request("task")) is None
- assert affinity.l2_errors == 1
-
-
-async def test_l2_put_failure_is_fail_open_and_keeps_l1_pin() -> None:
- """An L2 write error does not raise; the pin still holds in the local L1."""
- affinity = SessionAffinity(enabled=True, l2=_BrokenL2())
- req = _request("task")
- await affinity.pin(ProxyContext(), req, "model-A") # must not raise
- assert affinity.l2_errors == 1
- # L1 retained the pin, so a same-worker lookup still resolves (no L2 read).
- assert await affinity.pinned(ProxyContext(), req) == "model-A"
- assert affinity.l2_errors == 1 # the L1 hit never consulted the broken L2
-
-
-async def test_disabled_never_touches_l2() -> None:
- """A disabled instance no-ops without reading or writing the shared store."""
- l2 = _DictL2()
- affinity = SessionAffinity(enabled=False, l2=l2)
- req = _request("task")
- await affinity.pin(ProxyContext(), req, "model-A")
- assert await affinity.pinned(ProxyContext(), req) is None
- assert l2.gets == 0
- assert l2.puts == 0
-
-
-async def test_warmup_turn_never_touches_l2() -> None:
- """Turns inside the warmup window neither read nor write the shared store."""
- l2 = _DictL2()
- affinity = SessionAffinity(enabled=True, warmup_turns=1, l2=l2)
- req = _request("task") # turn 1, within warmup_turns=1
- await affinity.pin(ProxyContext(), req, "model-A")
- assert await affinity.pinned(ProxyContext(), req) is None
- assert l2.gets == 0
- assert l2.puts == 0
-
-
-async def test_aclose_delegates_to_l2() -> None:
- """Closing the affinity coordinator releases the shared store."""
-
- class _ClosableL2(_DictL2):
- def __init__(self) -> None:
- super().__init__()
- self.closed = False
-
- async def aclose(self) -> None:
- self.closed = True
-
- l2 = _ClosableL2()
- affinity = SessionAffinity(enabled=True, l2=l2)
- await affinity.aclose()
- assert l2.closed
-
-
-async def test_aclose_is_noop_without_l2_or_without_aclose() -> None:
- """L1-only instances and L2s lacking aclose() close without error."""
- await SessionAffinity(enabled=True).aclose()
- await SessionAffinity(enabled=True, l2=_DictL2()).aclose() # no aclose method
-
-
-async def test_aclose_swallows_l2_close_errors() -> None:
- """A failing store close is logged, never raised — teardown can't wedge."""
-
- class _BrokenClose(_DictL2):
- async def aclose(self) -> None:
- raise RuntimeError("close failed")
-
- await SessionAffinity(enabled=True, l2=_BrokenClose()).aclose() # must not raise
-
-
-# ---------------------------------------------------------------------------
-# L2 circuit breaker
-# ---------------------------------------------------------------------------
-
-
-class _Clock:
- """Manually-advanced monotonic clock for deterministic breaker tests."""
-
- def __init__(self) -> None:
- self.now = 0.0
-
- def __call__(self) -> float:
- return self.now
-
-
-class _FlakyL2(_BrokenL2):
- """Failing store that counts attempts and can be healed mid-test."""
-
- def __init__(self) -> None:
- self.attempts = 0
- self.healed = False
- self.store: dict[str, str] = {}
-
- async def get(self, key: str) -> str | None:
- self.attempts += 1
- if not self.healed:
- raise RuntimeError("l2 unavailable")
- return self.store.get(key)
-
- async def put(self, key: str, value: str) -> None:
- self.attempts += 1
- if not self.healed:
- raise RuntimeError("l2 unavailable")
- self.store[key] = value
-
-
-def _breaker_affinity(l2: _FlakyL2, clock: _Clock) -> SessionAffinity:
- return SessionAffinity(
- enabled=True,
- l2=l2,
- l2_breaker_threshold=3,
- l2_breaker_cooldown_s=10.0,
- clock=clock,
- )
-
-
-async def test_l2_breaker_opens_after_streak_and_skips_operations() -> None:
- """Threshold consecutive failures open the breaker; further operations
- never reach the store while the cooldown runs."""
- l2, clock = _FlakyL2(), _Clock()
- affinity = _breaker_affinity(l2, clock)
-
- for i in range(3):
- await affinity.pin(ProxyContext(), _request(f"t{i}"), "model-A")
- assert l2.attempts == 3
- assert affinity.l2_breaker_open
-
- # Open breaker: neither reads (distinct sessions = L1 misses) nor writes
- # touch the store.
- assert await affinity.pinned(ProxyContext(), _request("t-new")) is None
- await affinity.pin(ProxyContext(), _request("t-newer"), "model-A")
- assert l2.attempts == 3
- assert affinity.l2_errors == 3
-
-
-async def test_l2_breaker_probes_after_cooldown_and_closes_on_success() -> None:
- """The first operation after the cooldown is the probe; success closes the
- breaker and normal read/write-through resumes."""
- l2, clock = _FlakyL2(), _Clock()
- affinity = _breaker_affinity(l2, clock)
-
- for i in range(3):
- await affinity.pin(ProxyContext(), _request(f"t{i}"), "model-A")
- l2.healed = True
- clock.now = 10.0 # cooldown elapsed → next op is the probe
-
- await affinity.pin(ProxyContext(), _request("probe"), "model-A")
-
- assert not affinity.l2_breaker_open
- assert l2.attempts == 4
- # Fully closed: later operations go through again.
- await affinity.pin(ProxyContext(), _request("after"), "model-A")
- assert l2.attempts == 5
-
-
-async def test_l2_breaker_failed_probe_rearms_the_cooldown() -> None:
- """A failing probe re-opens the breaker for another full cooldown."""
- l2, clock = _FlakyL2(), _Clock()
- affinity = _breaker_affinity(l2, clock)
-
- for i in range(3):
- await affinity.pin(ProxyContext(), _request(f"t{i}"), "model-A")
- clock.now = 10.0 # still broken: the probe fails
-
- await affinity.pin(ProxyContext(), _request("probe"), "model-A")
-
- assert l2.attempts == 4
- assert affinity.l2_breaker_open
- # Mid-cooldown operations stay skipped.
- clock.now = 19.9
- await affinity.pin(ProxyContext(), _request("mid"), "model-A")
- assert l2.attempts == 4
-
-
-async def test_l2_non_consecutive_failures_do_not_open_the_breaker() -> None:
- """A success resets the streak: intermittent blips never trip the breaker."""
- l2, clock = _FlakyL2(), _Clock()
- affinity = _breaker_affinity(l2, clock)
-
- await affinity.pin(ProxyContext(), _request("t0"), "model-A")
- await affinity.pin(ProxyContext(), _request("t1"), "model-A")
- l2.healed = True
- await affinity.pin(ProxyContext(), _request("t2"), "model-A") # success resets
- l2.healed = False
- await affinity.pin(ProxyContext(), _request("t3"), "model-A")
- await affinity.pin(ProxyContext(), _request("t4"), "model-A")
-
- assert not affinity.l2_breaker_open
- assert affinity.l2_errors == 4
-
-
-async def test_l2_breaker_config_validated() -> None:
- """Breaker knobs reject nonsensical values."""
- import pytest
-
- with pytest.raises(ValueError):
- SessionAffinity(enabled=True, l2_breaker_threshold=0)
- with pytest.raises(ValueError):
- SessionAffinity(enabled=True, l2_breaker_cooldown_s=0)
diff --git a/tests/test_signal_tier_selector_request_processor.py b/tests/test_signal_tier_selector_request_processor.py
deleted file mode 100644
index 9b9249cfe..000000000
--- a/tests/test_signal_tier_selector_request_processor.py
+++ /dev/null
@@ -1,895 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for mapping LLM classifier signals to deterministic backend tiers."""
-
-from __future__ import annotations
-
-from typing import Any, cast
-
-import pytest
-
-from switchyard.lib.backends.deterministic_routing_llm_backend import (
- CTX_DETERMINISTIC_ROUTING_TIER,
- DeterministicRoutingLLMBackend,
-)
-from switchyard.lib.processors.llm_classifier import (
- CTX_DETERMINISTIC_ROUTE_SIGNALS,
- CTX_DETERMINISTIC_TIER_DECISION,
- ChannelKind,
- CodeModificationScope,
- CodingAgentRouteDecision,
- CodingAgentTurnType,
- Complexity,
- ContextDependency,
- MemoryDependency,
- OpenClawRouteDecision,
- OpenClawTurnType,
- PrecisionRequirement,
- ReasonCode,
- ReasoningDepth,
- RiskLevel,
- RouteSignals,
- RouteTier,
- SignalTierSelectorConfig,
- SignalTierSelectorRequestProcessor,
- TierSelectionDecision,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard.lib.roles import LLMBackend
-from switchyard.lib.session_affinity import SessionAffinity
-from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse
-
-
-def _stub_response() -> ChatResponse:
- return ChatResponse.openai_completion({"id": "test", "choices": []})
-
-
-class _RecordingBackend(LLMBackend):
- def __init__(self) -> None:
- self.models: list[str] = []
-
- @property
- def supported_request_types(self) -> list[ChatRequestType]:
- return [
- ChatRequestType.OPENAI_CHAT,
- ChatRequestType.OPENAI_RESPONSES,
- ChatRequestType.ANTHROPIC,
- ]
-
- async def call(self, ctx: ProxyContext, request: Any) -> ChatResponse:
- self.models.append(request.body["model"])
- return _stub_response()
-
-
-def _request(content: str = "hello", *, prior_assistant_turns: int = 0) -> ChatRequest:
- messages: list[dict[str, str]] = [{"role": "user", "content": content}]
- for i in range(prior_assistant_turns):
- messages.append({"role": "assistant", "content": f"turn {i + 1}"})
- return ChatRequest.openai_chat(cast(Any, {
- "model": "client-model",
- "messages": messages,
- }))
-
-
-def _two_tier_config() -> SignalTierSelectorConfig:
- """Config mapping COMPLEX/REASONING→strong, SIMPLE/MEDIUM/default→weak."""
- return SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "weak",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="weak",
- min_confidence=0.5,
- )
-
-
-def _signals(
- *,
- recommended_tier: RouteTier = RouteTier.COMPLEX,
- confidence: float = 0.9,
- abstain: bool = False,
-) -> RouteSignals:
- return RouteSignals(
- task_type="coding",
- complexity=Complexity.COMPLEX,
- reasoning_depth=ReasoningDepth.MULTI_STEP,
- tool_planning_required=False,
- precision_requirement=PrecisionRequirement.HIGH,
- context_dependency=ContextDependency.CONVERSATION,
- structured_output_risk=RiskLevel.LOW,
- recommended_tier=recommended_tier,
- confidence=confidence,
- reason_code=ReasonCode.CODING_COMPLEX,
- abstain=abstain,
- )
-
-
-async def test_signal_tier_selector_maps_signals_to_llm_target() -> None:
- processor = SignalTierSelectorRequestProcessor(
- SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "weak",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="weak",
- min_confidence=0.5,
- ),
- )
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(),
- })
- req = _request()
-
- returned = await processor.process(ctx, req)
-
- assert returned is req
- assert req.body["model"] == "client-model"
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- decision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert isinstance(decision, TierSelectionDecision)
- assert decision.tier == "strong"
- assert decision.source == "policy_tier"
- assert decision.policy_tier is RouteTier.COMPLEX
- assert decision.llm_recommended_tier is RouteTier.COMPLEX
-
-
-async def test_signal_tier_selector_defaults_on_low_confidence() -> None:
- processor = SignalTierSelectorRequestProcessor(
- SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "weak",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="weak",
- min_confidence=0.8,
- ),
- )
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.2),
- })
-
- await processor.process(ctx, _request())
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- decision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert isinstance(decision, TierSelectionDecision)
- assert decision.source == "low_confidence"
-
-
-async def test_signal_tier_selector_defaults_on_abstain() -> None:
- processor = SignalTierSelectorRequestProcessor(
- SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "weak",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="weak",
- ),
- )
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(abstain=True),
- })
-
- await processor.process(ctx, _request())
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- decision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert isinstance(decision, TierSelectionDecision)
- assert decision.source == "abstain"
-
-
-async def test_signal_tier_selector_accepts_dict_signals() -> None:
- processor = SignalTierSelectorRequestProcessor(
- SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "weak",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="weak",
- ),
- )
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals().model_dump(mode="json"),
- })
-
- await processor.process(ctx, _request())
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
-
-
-async def test_selector_routes_on_policy_tier_not_llm_recommended_tier() -> None:
- """When LLM picks SIMPLE but features score COMPLEX, policy wins."""
- processor = SignalTierSelectorRequestProcessor(
- SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "weak",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="strong",
- ),
- )
- # LLM emits "simple" but the features unambiguously score COMPLEX.
- signals = RouteSignals(
- task_type="coding",
- complexity=Complexity.COMPLEX,
- reasoning_depth=ReasoningDepth.MULTI_STEP,
- tool_planning_required=True,
- precision_requirement=PrecisionRequirement.HIGH,
- context_dependency=ContextDependency.CONVERSATION,
- structured_output_risk=RiskLevel.HIGH,
- recommended_tier=RouteTier.SIMPLE,
- confidence=0.95,
- reason_code=ReasonCode.CODING_COMPLEX,
- abstain=False,
- )
- ctx = ProxyContext(metadata={CTX_DETERMINISTIC_ROUTE_SIGNALS: signals})
-
- await processor.process(ctx, _request())
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- decision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert decision.source == "policy_tier"
- assert decision.policy_tier is RouteTier.COMPLEX
- # The LLM's own (overridden) vote is still captured for audit.
- assert decision.llm_recommended_tier is RouteTier.SIMPLE
-
-
-def test_route_signals_policy_tier_scores_complexity() -> None:
- simple = RouteSignals(
- task_type="chat",
- complexity=Complexity.SIMPLE,
- reasoning_depth=ReasoningDepth.NONE,
- tool_planning_required=False,
- precision_requirement=PrecisionRequirement.LOW,
- context_dependency=ContextDependency.LATEST_MESSAGE,
- structured_output_risk=RiskLevel.LOW,
- recommended_tier=RouteTier.REASONING, # LLM disagrees — ignored
- confidence=0.9,
- reason_code=ReasonCode.SIMPLE_QA,
- abstain=False,
- )
- assert simple.policy_tier() is RouteTier.SIMPLE
-
- deep = RouteSignals(
- task_type="math",
- complexity=Complexity.REASONING,
- reasoning_depth=ReasoningDepth.DEEP,
- tool_planning_required=False,
- precision_requirement=PrecisionRequirement.HIGH,
- context_dependency=ContextDependency.CONVERSATION,
- structured_output_risk=RiskLevel.LOW,
- recommended_tier=RouteTier.SIMPLE, # LLM disagrees — ignored
- confidence=0.9,
- reason_code=ReasonCode.MATH_REASONING,
- abstain=False,
- )
- assert deep.policy_tier() is RouteTier.REASONING
-
-
-def test_coding_agent_policy_tier_scales_with_scope_and_tools() -> None:
- chitchat = CodingAgentRouteDecision(
- recommended_tier=RouteTier.COMPLEX, # ignored
- confidence=0.9,
- turn_type=CodingAgentTurnType.CHITCHAT,
- code_modification_scope=CodeModificationScope.NONE,
- )
- assert chitchat.policy_tier() is RouteTier.SIMPLE
-
- refactor = CodingAgentRouteDecision(
- recommended_tier=RouteTier.SIMPLE, # ignored
- confidence=0.9,
- turn_type=CodingAgentTurnType.DEBUG,
- code_modification_scope=CodeModificationScope.CROSS_MODULE,
- tool_call_count_estimate=6,
- requires_codebase_context=True,
- )
- assert refactor.policy_tier() is RouteTier.COMPLEX
-
-
-def test_openclaw_policy_tier_handles_chat_vs_orchestration() -> None:
- chitchat = OpenClawRouteDecision(
- recommended_tier=RouteTier.COMPLEX, # ignored
- confidence=0.9,
- turn_type=OpenClawTurnType.CHITCHAT,
- channel_kind=ChannelKind.CASUAL,
- )
- assert chitchat.policy_tier() is RouteTier.SIMPLE
-
- irreversible_action = OpenClawRouteDecision(
- recommended_tier=RouteTier.SIMPLE, # ignored
- confidence=0.9,
- turn_type=OpenClawTurnType.ACTION,
- tool_call_count_estimate=2,
- memory_dependency=MemoryDependency.HEAVY,
- external_action_required=True,
- precision_requirement=PrecisionRequirement.HIGH,
- ambiguity=RiskLevel.LOW,
- channel_kind=ChannelKind.DELIBERATE,
- )
- assert irreversible_action.policy_tier() is RouteTier.COMPLEX
-
-
-async def test_selector_to_backend_handoff_dispatches_selected_tier() -> None:
- strong_backend = _RecordingBackend()
- weak_backend = _RecordingBackend()
- selector = SignalTierSelectorRequestProcessor(
- SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "weak",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="weak",
- ),
- )
- router = DeterministicRoutingLLMBackend(
- tiers={
- "strong": (strong_backend, "strong-model"),
- "weak": (weak_backend, "weak-model"),
- },
- default_tier="weak",
- )
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(),
- })
- req = _request()
-
- await selector.process(ctx, req)
- await router.call(ctx, req)
-
- assert strong_backend.models == ["strong-model"]
- assert weak_backend.models == []
- assert req.body["model"] == "strong-model"
-
-
-# --- Tool-planning escalation ----------------------------------------------
-
-
-def _coding_signals(
- *,
- tool_call_count_estimate: int = 0,
- turn_type: CodingAgentTurnType = CodingAgentTurnType.EXPLORATION,
- code_modification_scope: CodeModificationScope = CodeModificationScope.NONE,
- recommended_tier: RouteTier = RouteTier.MEDIUM,
- confidence: float = 0.9,
-) -> CodingAgentRouteDecision:
- return CodingAgentRouteDecision(
- turn_type=turn_type,
- code_modification_scope=code_modification_scope,
- tool_call_count_estimate=tool_call_count_estimate,
- requires_codebase_context=False,
- recommended_tier=recommended_tier,
- confidence=confidence,
- )
-
-
-def _config_with_escalation(escalate: bool) -> SignalTierSelectorConfig:
- return SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "weak",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="strong",
- escalate_on_tool_planning=escalate,
- )
-
-
-async def test_tool_planning_escalation_lifts_weak_to_strong() -> None:
- """RouteSignals with tool_planning_required=True escalates when flag on."""
- processor = SignalTierSelectorRequestProcessor(_config_with_escalation(escalate=True))
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: RouteSignals(
- task_type="coding",
- complexity=Complexity.SIMPLE, # would map to weak via SIMPLE
- reasoning_depth=ReasoningDepth.LIGHT,
- tool_planning_required=True,
- precision_requirement=PrecisionRequirement.LOW,
- context_dependency=ContextDependency.LATEST_MESSAGE,
- structured_output_risk=RiskLevel.LOW,
- recommended_tier=RouteTier.SIMPLE,
- confidence=0.9,
- reason_code=ReasonCode.SIMPLE_QA,
- ),
- })
-
- await processor.process(ctx, _request())
-
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- decision: TierSelectionDecision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert decision.source == "tool_planning_escalation"
- assert decision.policy_tier == RouteTier.SIMPLE
-
-
-async def test_tool_planning_flag_off_keeps_policy_tier() -> None:
- """Same signals, escalate flag off => no override; tier stays weak."""
- processor = SignalTierSelectorRequestProcessor(_config_with_escalation(escalate=False))
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: RouteSignals(
- task_type="coding",
- complexity=Complexity.SIMPLE,
- reasoning_depth=ReasoningDepth.LIGHT,
- tool_planning_required=True,
- precision_requirement=PrecisionRequirement.LOW,
- context_dependency=ContextDependency.LATEST_MESSAGE,
- structured_output_risk=RiskLevel.LOW,
- recommended_tier=RouteTier.SIMPLE,
- confidence=0.9,
- reason_code=ReasonCode.SIMPLE_QA,
- ),
- })
-
- await processor.process(ctx, _request())
-
- decision: TierSelectionDecision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert decision.source == "policy_tier"
- # policy_tier may differ from SIMPLE due to tool_planning_required +1
- # to COMPLEX; just assert the source is the normal policy path.
-
-
-async def test_tool_planning_no_escalation_when_already_default_tier() -> None:
- """If policy_tier already routes to default_tier (strong), no override fires."""
- processor = SignalTierSelectorRequestProcessor(_config_with_escalation(escalate=True))
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _coding_signals(
- tool_call_count_estimate=5,
- turn_type=CodingAgentTurnType.DEBUG, # already → COMPLEX → strong
- ),
- })
-
- await processor.process(ctx, _request())
-
- decision: TierSelectionDecision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert decision.tier == "strong"
- assert decision.source == "policy_tier" # not escalation
-
-
-async def test_tool_planning_escalation_for_coding_agent_threshold() -> None:
- """CodingAgent escalates at ``tool_count >= 3 AND scope >= FUNCTION``.
-
- Concordance rule: a tool-heavy turn must also be modifying code
- (scope >= FUNCTION) for escalation to fire. Pure-probe turns
- (``ls`` + ``cat`` + ``grep`` with scope=NONE) stay on weak even at
- high tool counts; 2-tool read+edit cycles stay on weak even with
- scope=FUNCTION.
- """
- config = _config_with_escalation(escalate=True)
- processor = SignalTierSelectorRequestProcessor(config)
-
- # scope=NONE: never escalates regardless of tool count.
- for count in (0, 1, 2, 3, 5):
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _coding_signals(
- tool_call_count_estimate=count,
- turn_type=CodingAgentTurnType.EXPLORATION,
- code_modification_scope=CodeModificationScope.NONE,
- ),
- })
- await processor.process(ctx, _request())
- actual = ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER]
- assert actual == "weak", (
- f"scope=NONE, count={count}: expected weak, got {actual}"
- )
-
- # scope=FUNCTION: escalates only at >=3 tool calls.
- for count, expected in [(0, "weak"), (1, "weak"), (2, "weak"), (3, "strong"), (5, "strong")]:
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _coding_signals(
- tool_call_count_estimate=count,
- turn_type=CodingAgentTurnType.EXPLORATION,
- code_modification_scope=CodeModificationScope.FUNCTION,
- ),
- })
- await processor.process(ctx, _request())
- actual = ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER]
- assert actual == expected, (
- f"scope=FUNCTION, count={count}: expected {expected}, got {actual}"
- )
-
-
-async def test_tool_planning_escalation_for_openclaw() -> None:
- """OpenClaw escalates on TOOL_ORCHESTRATION turn_type or 2+ tool calls."""
- processor = SignalTierSelectorRequestProcessor(_config_with_escalation(escalate=True))
-
- # Base case: CHITCHAT, no tools → weak, no escalation.
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: OpenClawRouteDecision(
- turn_type=OpenClawTurnType.CHITCHAT,
- tool_call_count_estimate=0,
- memory_dependency=MemoryDependency.NONE,
- external_action_required=False,
- precision_requirement=PrecisionRequirement.LOW,
- ambiguity=RiskLevel.LOW,
- channel_kind=ChannelKind.CASUAL,
- recommended_tier=RouteTier.SIMPLE,
- confidence=0.9,
- ),
- })
- await processor.process(ctx, _request())
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
-
- # 2+ tool calls on a non-tool turn_type → escalates.
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: OpenClawRouteDecision(
- turn_type=OpenClawTurnType.LOOKUP,
- tool_call_count_estimate=2,
- memory_dependency=MemoryDependency.NONE,
- external_action_required=False,
- precision_requirement=PrecisionRequirement.LOW,
- ambiguity=RiskLevel.LOW,
- channel_kind=ChannelKind.CASUAL,
- recommended_tier=RouteTier.SIMPLE,
- confidence=0.9,
- ),
- })
- await processor.process(ctx, _request())
- assert ctx.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
-
-
-async def test_requires_tool_planning_base_returns_false() -> None:
- """Bare RouteDecision returns False; the override is a no-op for it."""
- from switchyard.lib.processors.llm_classifier.signals import (
- RouteDecision,
- )
-
- decision = RouteDecision(recommended_tier=RouteTier.MEDIUM, confidence=0.9)
- assert decision.requires_tool_planning() is False
-
-
-async def test_coding_agent_preset_has_escalation_enabled() -> None:
- """Sanity check: coding_agent_2_tier and openclaw_2_tier opt in by default."""
- from switchyard.lib.processors.llm_classifier import LLMClassifierPresets
-
- coding = LLMClassifierPresets.coding_agent_2_tier(weak="weak", strong="strong")
- assert coding.escalate_on_tool_planning is True
- openclaw = LLMClassifierPresets.openclaw_2_tier(weak="weak", strong="strong")
- assert openclaw.escalate_on_tool_planning is True
- general = LLMClassifierPresets.general_2_tier(weak="weak", strong="strong")
- assert general.escalate_on_tool_planning is False
-
-
-# --- LLM alignment bump ----------------------------------------------------
-
-
-def _config_with_alignment(
- *,
- enabled: bool,
- min_conf: float = 0.7,
-) -> SignalTierSelectorConfig:
- return SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "weak",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="strong",
- align_with_llm_recommendation=enabled,
- alignment_min_confidence=min_conf,
- )
-
-
-async def test_alignment_bump_lifts_simple_to_complex_when_llm_confident() -> None:
- """Policy lands on SIMPLE; high-conf LLM says COMPLEX → bump to strong."""
- processor = SignalTierSelectorRequestProcessor(_config_with_alignment(enabled=True))
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _coding_signals(
- tool_call_count_estimate=0,
- turn_type=CodingAgentTurnType.CHITCHAT, # policy SIMPLE
- recommended_tier=RouteTier.COMPLEX, # LLM disagrees
- confidence=0.9,
- ),
- })
-
- await processor.process(ctx, _request())
-
- decision: TierSelectionDecision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert decision.tier == "strong"
- assert decision.source == "llm_alignment_bump"
- assert decision.policy_tier == RouteTier.COMPLEX
-
-
-async def test_alignment_bump_disabled_keeps_policy_tier() -> None:
- """Same signals, flag off → no bump."""
- processor = SignalTierSelectorRequestProcessor(_config_with_alignment(enabled=False))
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _coding_signals(
- turn_type=CodingAgentTurnType.CHITCHAT,
- recommended_tier=RouteTier.COMPLEX,
- confidence=0.9,
- ),
- })
-
- await processor.process(ctx, _request())
-
- decision: TierSelectionDecision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert decision.source == "policy_tier"
- assert decision.tier == "weak"
-
-
-async def test_alignment_bump_suppressed_below_confidence_floor() -> None:
- """LLM says COMPLEX but confidence 0.6 < threshold 0.7 → no bump."""
- processor = SignalTierSelectorRequestProcessor(
- _config_with_alignment(enabled=True, min_conf=0.7),
- )
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _coding_signals(
- turn_type=CodingAgentTurnType.CHITCHAT,
- recommended_tier=RouteTier.COMPLEX,
- confidence=0.6,
- ),
- })
-
- await processor.process(ctx, _request())
-
- decision: TierSelectionDecision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert decision.source == "policy_tier"
- assert decision.tier == "weak"
-
-
-async def test_alignment_bump_never_demotes() -> None:
- """Policy lands on COMPLEX; LLM says SIMPLE → policy wins, no demotion."""
- processor = SignalTierSelectorRequestProcessor(_config_with_alignment(enabled=True))
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _coding_signals(
- turn_type=CodingAgentTurnType.DEBUG, # policy COMPLEX
- recommended_tier=RouteTier.SIMPLE, # LLM disagrees softer
- confidence=0.95,
- ),
- })
-
- await processor.process(ctx, _request())
-
- decision: TierSelectionDecision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert decision.tier == "strong"
- assert decision.source == "policy_tier"
-
-
-# --- escalate_target_tier separated from default_tier ---------------------
-
-
-async def test_escalate_target_tier_defaults_to_default_tier() -> None:
- """When escalate_target_tier is None, effective target == default_tier."""
- config = SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "weak",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="strong",
- escalate_on_tool_planning=True,
- escalate_target_tier=None,
- )
- assert config.effective_escalate_target_tier == "strong"
-
-
-async def test_escalate_target_tier_override_three_tier_setup() -> None:
- """3-tier setup escalates SIMPLE+tool_planning to MEDIUM, not STRONG."""
- config = SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "mid",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="strong",
- escalate_on_tool_planning=True,
- escalate_target_tier="mid", # land on mid, not strong
- )
- processor = SignalTierSelectorRequestProcessor(config)
- ctx = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _coding_signals(
- # PLANNING turn_type triggers escalation unconditionally
- # (the one signal we trust on its own). Tool-count-driven
- # escalation now requires scope >= FUNCTION too — see
- # `requires_tool_planning` for the concordance rule — so
- # use PLANNING here to keep this test focused on the
- # escalate_target_tier mechanism rather than the threshold.
- turn_type=CodingAgentTurnType.PLANNING,
- ),
- })
-
- await processor.process(ctx, _request())
-
- decision: TierSelectionDecision = ctx.metadata[CTX_DETERMINISTIC_TIER_DECISION]
- assert decision.tier == "mid" # not "strong"
- assert decision.source == "tool_planning_escalation"
-
-
-async def test_escalate_target_tier_validates_against_tier_mapping() -> None:
- """Unknown target raises at config construction (validator)."""
- with pytest.raises(ValueError, match="escalate_target_tier"):
- SignalTierSelectorConfig(
- tier_mapping={
- RouteTier.SIMPLE: "weak",
- RouteTier.MEDIUM: "weak",
- RouteTier.COMPLEX: "strong",
- RouteTier.REASONING: "strong",
- },
- default_tier="strong",
- escalate_target_tier="middle", # not in mapping
- )
-
-
-async def test_coding_agent_preset_enables_alignment_bump() -> None:
- """coding_agent and openclaw opt into LLM alignment; general stays off."""
- from switchyard.lib.processors.llm_classifier import LLMClassifierPresets
-
- coding = LLMClassifierPresets.coding_agent_2_tier(weak="weak", strong="strong")
- assert coding.align_with_llm_recommendation is True
- openclaw = LLMClassifierPresets.openclaw_2_tier(weak="weak", strong="strong")
- assert openclaw.align_with_llm_recommendation is True
- general = LLMClassifierPresets.general_2_tier(weak="weak", strong="strong")
- assert general.align_with_llm_recommendation is False
-
-
-async def test_session_affinity_pins_confident_verdict() -> None:
- """A confident verdict is pinned and holds even when later signals flip."""
- processor = SignalTierSelectorRequestProcessor(
- _two_tier_config(),
- affinity=SessionAffinity(enabled=True),
- )
- req = _request() # identical body across turns → same session key
-
- # Turn 1: confident COMPLEX → "strong"; pin it.
- ctx1 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.9),
- })
- await processor.process(ctx1, req)
- assert ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
-
- # Turn 2: low confidence *would* fall back to "weak" — the pin overrides.
- ctx2 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.2),
- })
- await processor.process(ctx2, req)
- assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- assert ctx2.metadata[CTX_DETERMINISTIC_TIER_DECISION].source == "sticky"
-
-
-async def test_session_affinity_warmup_delays_pinning_until_after_threshold() -> None:
- """Warmup turns route normally, then the first confident post-warmup verdict pins."""
- processor = SignalTierSelectorRequestProcessor(
- _two_tier_config(),
- affinity=SessionAffinity(enabled=True, warmup_turns=2),
- )
-
- ctx1 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.9),
- })
- await processor.process(ctx1, _request("same task"))
- assert ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
-
- ctx2 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.2),
- })
- await processor.process(ctx2, _request("same task", prior_assistant_turns=1))
- assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert ctx2.metadata[CTX_DETERMINISTIC_TIER_DECISION].source == "low_confidence"
-
- ctx3 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.9),
- })
- await processor.process(ctx3, _request("same task", prior_assistant_turns=2))
- assert ctx3.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- assert ctx3.metadata[CTX_DETERMINISTIC_TIER_DECISION].source == "policy_tier"
-
- ctx4 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.2),
- })
- await processor.process(ctx4, _request("same task", prior_assistant_turns=3))
- assert ctx4.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- assert ctx4.metadata[CTX_DETERMINISTIC_TIER_DECISION].source == "sticky"
-
-
-async def test_session_affinity_warmup_does_not_pin_fallback_after_threshold() -> None:
- """Fallback decisions remain non-sticky even after the warmup period ends."""
- processor = SignalTierSelectorRequestProcessor(
- _two_tier_config(),
- affinity=SessionAffinity(enabled=True, warmup_turns=1),
- )
-
- ctx1 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.9),
- })
- await processor.process(ctx1, _request("same task"))
- assert ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
-
- ctx2 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(abstain=True),
- })
- await processor.process(ctx2, _request("same task", prior_assistant_turns=1))
- assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert ctx2.metadata[CTX_DETERMINISTIC_TIER_DECISION].source == "abstain"
-
- ctx3 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.9),
- })
- await processor.process(ctx3, _request("same task", prior_assistant_turns=2))
- assert ctx3.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
- assert ctx3.metadata[CTX_DETERMINISTIC_TIER_DECISION].source == "policy_tier"
-
-
-async def test_session_affinity_does_not_pin_fallback_decision() -> None:
- """A fail-open/abstain fallback is NOT pinned, so the next turn re-classifies.
-
- A single transient turn-1 classifier failure must not lock the whole task to
- the default tier."""
- processor = SignalTierSelectorRequestProcessor(
- _two_tier_config(),
- affinity=SessionAffinity(enabled=True),
- )
- req = _request()
-
- # Turn 1: classifier abstained → default "weak", but this must NOT pin.
- ctx1 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(abstain=True),
- })
- await processor.process(ctx1, req)
- assert ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
- assert ctx1.metadata[CTX_DETERMINISTIC_TIER_DECISION].source == "abstain"
-
- # Turn 2: a confident verdict re-classifies — the task is NOT locked to weak.
- ctx2 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.9),
- })
- await processor.process(ctx2, req)
- assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
-
-
-async def test_session_affinity_off_routes_per_turn() -> None:
- """Without affinity, each turn follows its own signals (no pinning)."""
- processor = SignalTierSelectorRequestProcessor(_two_tier_config())
- req = _request()
-
- ctx1 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.2),
- })
- await processor.process(ctx1, req)
- assert ctx1.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
-
- ctx2 = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.9),
- })
- await processor.process(ctx2, req)
- assert ctx2.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "strong"
-
-
-async def test_session_affinity_distinct_conversations_independent() -> None:
- """A pin for one conversation does not leak into another (distinct key)."""
- processor = SignalTierSelectorRequestProcessor(
- _two_tier_config(),
- affinity=SessionAffinity(enabled=True),
- )
-
- # Conversation A: confident → "strong"; pinned.
- await processor.process(
- ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.9),
- }),
- _request("question A"),
- )
- # Conversation B (distinct first-user) decides on its own signals → "weak",
- # proving A's strong pin did not leak across the key boundary.
- ctx_b = ProxyContext(metadata={
- CTX_DETERMINISTIC_ROUTE_SIGNALS: _signals(confidence=0.2),
- })
- await processor.process(ctx_b, _request("question B"))
- assert ctx_b.metadata[CTX_DETERMINISTIC_ROUTING_TIER] == "weak"
-
diff --git a/tests/test_stage_router_classifier.py b/tests/test_stage_router_classifier.py
deleted file mode 100644
index 5e0f4c84e..000000000
--- a/tests/test_stage_router_classifier.py
+++ /dev/null
@@ -1,212 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for :class:`TierClassifier`.
-
-The classifier is tested in isolation with a stubbed LLM client. Failure-mode
-coverage matters more than success — any real-world flakiness must fall open
-to ``None`` so the picker keeps moving.
-"""
-
-import json
-from dataclasses import dataclass
-from typing import Any
-
-from switchyard.lib.processors.stage_router.classifier import (
- CAPABLE_TIER,
- EFFICIENT_TIER,
- RECENT_MESSAGES_KEY,
- TierClassifier,
-)
-from switchyard_rust.components import DimensionCollector, get_tool_result_signal
-from switchyard_rust.core import ChatRequest, ProxyContext
-
-
-@dataclass
-class _Resp:
- content: str | None = None
- raise_exc: BaseException | None = None
- no_choices: bool = False
-
- @property
- def choices(self) -> list[Any]:
- if self.no_choices:
- return []
- return [type("M", (), {"message": type("C", (), {"content": self.content})()})()]
-
-
-class _StubClient:
- def __init__(self, response: _Resp | None) -> None:
- self._response = response
- self.calls = 0
-
- async def acompletion(self, **_kwargs: object) -> _Resp:
- self.calls += 1
- if self._response is None:
- raise RuntimeError("upstream unavailable")
- if self._response.raise_exc is not None:
- raise self._response.raise_exc
- return self._response
-
-
-async def _build_signal() -> Any:
- collector = DimensionCollector()
- ctx = ProxyContext()
- request = ChatRequest.openai_chat({
- "messages": [{"role": "user", "content": "hello"}]
- })
- await collector.process(ctx, request)
- signal = get_tool_result_signal(ctx)
- assert signal is not None
- return ctx, signal
-
-
-async def test_returns_capable_on_valid_capable_response():
- ctx, signal = await _build_signal()
- classifier = TierClassifier(
- model="m", api_key="k",
- client=_StubClient(_Resp(content=json.dumps({"tier": "capable"}))),
- )
- assert await classifier.classify(ctx, signal) == CAPABLE_TIER
-
-
-async def test_returns_efficient_on_valid_efficient_response():
- ctx, signal = await _build_signal()
- classifier = TierClassifier(
- model="m", api_key="k",
- client=_StubClient(_Resp(content=json.dumps({"tier": "efficient"}))),
- )
- assert await classifier.classify(ctx, signal) == EFFICIENT_TIER
-
-
-async def test_falls_open_on_malformed_json():
- ctx, signal = await _build_signal()
- classifier = TierClassifier(
- model="m", api_key="k",
- client=_StubClient(_Resp(content="not json at all")),
- )
- assert await classifier.classify(ctx, signal) is None
-
-
-async def test_falls_open_on_unexpected_tier_value():
- ctx, signal = await _build_signal()
- classifier = TierClassifier(
- model="m", api_key="k",
- client=_StubClient(_Resp(content=json.dumps({"tier": "elite"}))),
- )
- assert await classifier.classify(ctx, signal) is None
-
-
-async def test_falls_open_on_network_error():
- ctx, signal = await _build_signal()
- classifier = TierClassifier(
- model="m", api_key="k",
- client=_StubClient(_Resp(raise_exc=TimeoutError("upstream slow"))),
- )
- assert await classifier.classify(ctx, signal) is None
-
-
-async def test_falls_open_on_empty_choices():
- ctx, signal = await _build_signal()
- classifier = TierClassifier(
- model="m", api_key="k",
- client=_StubClient(_Resp(no_choices=True)),
- )
- assert await classifier.classify(ctx, signal) is None
-
-
-async def test_default_window_omits_recent_messages():
- """recent_turn_window=0 keeps the prompt to the aggregate state only."""
- ctx, signal = await _build_signal()
- ctx.metadata[RECENT_MESSAGES_KEY] = [
- {"role": "user", "content": "hello"},
- {"role": "assistant", "content": "world"},
- ]
- captured: dict[str, object] = {}
-
- class _Recorder:
- def __init__(self) -> None:
- self.calls = 0
-
- async def acompletion(self, **kwargs: object) -> _Resp:
- self.calls += 1
- captured["messages"] = kwargs["messages"]
- return _Resp(content=json.dumps({"tier": "capable"}))
-
- classifier = TierClassifier(
- model="m", api_key="k", recent_turn_window=0, client=_Recorder(),
- )
- await classifier.classify(ctx, signal)
- user_prompt = captured["messages"][1]["content"]
- assert "Recent turns" not in user_prompt
-
-
-async def test_disable_reasoning_passes_enable_thinking_false_extra_body():
- """Reasoning models on the NVIDIA Inference Hub misroute JSON output into
- `reasoning_content` unless `enable_thinking=False` is hinted via vLLM's
- `chat_template_kwargs`. The classifier must send this by default."""
- ctx, signal = await _build_signal()
- captured: dict[str, object] = {}
-
- class _Recorder:
- async def acompletion(self, **kwargs: object) -> _Resp:
- captured.update(kwargs)
- return _Resp(content=json.dumps({"tier": "efficient"}))
-
- classifier = TierClassifier(model="m", api_key="k", client=_Recorder())
- await classifier.classify(ctx, signal)
- assert captured["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}}
- # max_tokens must be wide enough for reasoning_content + JSON when the hint
- # is ignored upstream; canonical LLM-classifier uses 4096.
- assert captured["max_tokens"] == 4096
- assert captured["temperature"] == 0
-
-
-async def test_disable_reasoning_false_omits_extra_body():
- """Opt-out path: `disable_reasoning=False` sends no extra_body."""
- ctx, signal = await _build_signal()
- captured: dict[str, object] = {}
-
- class _Recorder:
- async def acompletion(self, **kwargs: object) -> _Resp:
- captured.update(kwargs)
- return _Resp(content=json.dumps({"tier": "capable"}))
-
- classifier = TierClassifier(
- model="m", api_key="k", disable_reasoning=False, client=_Recorder(),
- )
- await classifier.classify(ctx, signal)
- assert captured["extra_body"] is None
-
-
-async def test_window_4_appends_last_four_messages():
- """recent_turn_window=4 includes the last four messages, in order."""
- ctx, signal = await _build_signal()
- ctx.metadata[RECENT_MESSAGES_KEY] = [
- {"role": "user", "content": "msg-0"},
- {"role": "assistant", "content": "msg-1"},
- {"role": "user", "content": "msg-2"},
- {"role": "assistant", "content": "msg-3"},
- {"role": "user", "content": "msg-4"},
- {"role": "assistant", "content": "msg-5"},
- ]
- captured: dict[str, object] = {}
-
- class _Recorder:
- async def acompletion(self, **kwargs: object) -> _Resp:
- captured["messages"] = kwargs["messages"]
- return _Resp(content=json.dumps({"tier": "efficient"}))
-
- classifier = TierClassifier(
- model="m", api_key="k", recent_turn_window=4, client=_Recorder(),
- )
- await classifier.classify(ctx, signal)
- user_prompt = captured["messages"][1]["content"]
- assert "Recent turns" in user_prompt
- # last 4 only, in order — earlier messages dropped
- assert "msg-2" in user_prompt
- assert "msg-3" in user_prompt
- assert "msg-4" in user_prompt
- assert "msg-5" in user_prompt
- assert "msg-0" not in user_prompt
- assert "msg-1" not in user_prompt
diff --git a/tests/test_stage_router_handoff_notes.py b/tests/test_stage_router_handoff_notes.py
deleted file mode 100644
index 7afc4b970..000000000
--- a/tests/test_stage_router_handoff_notes.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for the optional handoff-note injector.
-
-The injector is stateless: on the capable tier it appends an escalation note
-when the decision source is a real wrong signal; on the efficient tier it
-appends a de-escalation note only when one is configured.
-"""
-
-from __future__ import annotations
-
-from switchyard.lib.processors.stage_router import CAPABLE, EFFICIENT
-from switchyard.lib.processors.stage_router.handoff_notes import (
- DEFAULT_DEESCALATION_NOTE,
- DEFAULT_ESCALATION_NOTE,
- HandoffNoteInjector,
-)
-from switchyard_rust.core import ChatRequest
-
-
-def _anthropic(task: str = "solve the task", turn: str = "next") -> ChatRequest:
- """Anthropic request whose trailing user turn carries a tool_result block."""
- return ChatRequest.anthropic({
- "system": "you are a coding agent",
- "messages": [
- {"role": "user", "content": task},
- {"role": "assistant", "content": [
- {"type": "tool_use", "id": "t1", "name": "Bash", "input": {}}]},
- {"role": "user", "content": [
- {"type": "tool_result", "tool_use_id": "t1", "content": turn}]},
- ],
- })
-
-
-def _last_user_text(request: ChatRequest) -> str:
- content = request.body["messages"][-1]["content"]
- if isinstance(content, str):
- return content
- return "".join(b.get("text", "") for b in content if isinstance(b, dict))
-
-
-def test_escalation_injects_note_as_trailing_block():
- inj = HandoffNoteInjector()
- req = _anthropic()
- assert inj.maybe_inject(req, tier=CAPABLE, source="dimensions") is True
- content = req.body["messages"][-1]["content"]
- # tool_result stays first; the note is appended after it as a text block.
- assert content[0]["type"] == "tool_result"
- assert content[-1] == {"type": "text", "text": DEFAULT_ESCALATION_NOTE}
-
-
-def test_escalation_from_ambiguous_source_gated_off_by_default():
- inj = HandoffNoteInjector()
- req = _anthropic()
- # capable_first can escalate on fall_open (ambiguous default) — the truthful
- # gate suppresses the "prior model stalled" note there.
- assert inj.maybe_inject(req, tier=CAPABLE, source="fall_open") is False
- assert DEFAULT_ESCALATION_NOTE not in _last_user_text(req)
-
-
-def test_wrong_signal_gate_can_be_disabled():
- inj = HandoffNoteInjector(only_on_wrong_signal_escalation=False)
- req = _anthropic()
- assert inj.maybe_inject(req, tier=CAPABLE, source="fall_open") is True
-
-
-def test_override_source_counts_as_wrong_signal():
- """Compaction / critical-severity escalations stamp ``override`` → note fires."""
- inj = HandoffNoteInjector()
- req = _anthropic()
- assert inj.maybe_inject(req, tier=CAPABLE, source="override") is True
-
-
-def test_deescalation_off_by_default():
- inj = HandoffNoteInjector()
- req = _anthropic()
- # No de-escalation note configured → hand-back to the weak tier injects nothing.
- assert inj.maybe_inject(req, tier=EFFICIENT, source="dimensions") is False
- assert DEFAULT_DEESCALATION_NOTE not in _last_user_text(req)
-
-
-def test_deescalation_note_injected_when_configured():
- inj = HandoffNoteInjector(deescalation_note=DEFAULT_DEESCALATION_NOTE)
- req = _anthropic()
- assert inj.maybe_inject(req, tier=EFFICIENT, source="dimensions") is True
- assert DEFAULT_DEESCALATION_NOTE in _last_user_text(req)
-
-
-def test_notes_do_not_accumulate_across_turns():
- """Each injected request holds at most one note — nothing carries over."""
- inj = HandoffNoteInjector(deescalation_note=DEFAULT_DEESCALATION_NOTE)
- up = _anthropic(turn="b")
- inj.maybe_inject(up, tier=CAPABLE, source="dimensions") # escalate
- down = _anthropic(turn="c")
- inj.maybe_inject(down, tier=EFFICIENT, source="dimensions") # de-escalate
- # Each request is built fresh, so the de-escalation request carries only the
- # de-escalation note, never the escalation one.
- text = _last_user_text(down)
- assert DEFAULT_DEESCALATION_NOTE in text
- assert DEFAULT_ESCALATION_NOTE not in text
-
-
-def test_openai_chat_appends_trailing_user_message():
- inj = HandoffNoteInjector()
- req = ChatRequest.openai_chat({
- "messages": [
- {"role": "user", "content": "task"},
- {"role": "assistant", "content": "", "tool_calls": [
- {"id": "t1", "type": "function",
- "function": {"name": "Bash", "arguments": "{}"}}]},
- {"role": "tool", "tool_call_id": "t1", "content": "err"},
- ],
- })
- assert inj.maybe_inject(req, tier=CAPABLE, source="dimensions") is True
- last = req.body["messages"][-1]
- assert last["role"] == "user"
- assert last["content"] == DEFAULT_ESCALATION_NOTE
diff --git a/tests/test_stage_router_pickers.py b/tests/test_stage_router_pickers.py
deleted file mode 100644
index dc8a49f80..000000000
--- a/tests/test_stage_router_pickers.py
+++ /dev/null
@@ -1,249 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Behavioural tests for the stage_router pickers.
-
-The two pickers share every decision path (override → tests_passed → scorer →
-classifier) and differ only in the low-confidence default tier.
-"""
-
-from dataclasses import dataclass
-from typing import Any
-
-import pytest
-
-from switchyard.lib.processors.stage_router import (
- CAPABLE,
- EFFICIENT,
- StageRouterDecisionLog,
- TierClassifier,
-)
-from switchyard.lib.processors.stage_router.decision_log import CONTEXT_KEY
-from switchyard.lib.processors.stage_router.picker import (
- pick_capable_first,
- pick_efficient_first,
-)
-from switchyard_rust.components import DimensionCollector
-from switchyard_rust.core import ChatRequest, ProxyContext
-
-
-async def _ctx(messages: list[dict]) -> ProxyContext:
- collector = DimensionCollector()
- ctx = ProxyContext()
- await collector.process(ctx, ChatRequest.openai_chat({"messages": messages}))
- return ctx
-
-
-def _msg_tool_call(name: str) -> dict:
- return {"role": "assistant", "tool_calls": [{"function": {"name": name, "arguments": "{}"}}]}
-
-
-def _msg_bash(cmd: str) -> dict:
- args = f'{{"command": "{cmd}"}}'
- return {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": args}}]}
-
-
-def _msg_tool_result(content: str) -> dict:
- return {"role": "tool", "tool_call_id": "x", "content": content}
-
-
-@dataclass
-class _StubClassifierResponse:
- tier: str | None
-
- @property
- def choices(self) -> list[Any]:
- if self.tier is None:
- return [type("M", (), {"message": type("C", (), {"content": "garbage"})()})()]
- content = f'{{"tier": "{self.tier}"}}'
- return [type("M", (), {"message": type("C", (), {"content": content})()})()]
-
-
-class _StubLLMClient:
- """In-memory classifier client; one canned response per call."""
-
- def __init__(self, tier: str | None) -> None:
- self._tier = tier
- self.calls = 0
-
- async def acompletion(
- self,
- model: str,
- messages: list[dict[str, Any]],
- temperature: int,
- response_format: dict[str, str],
- max_tokens: int,
- extra_body: dict[str, Any] | None,
- ) -> _StubClassifierResponse:
- self.calls += 1
- return _StubClassifierResponse(tier=self._tier)
-
-
-def _stub_classifier(tier: str | None) -> TierClassifier:
- return TierClassifier(model="stub", api_key="stub", client=_StubLLMClient(tier))
-
-
-async def test_critical_severity_overrides_to_capable():
- ctx = await _ctx([
- _msg_tool_result("Out of memory: cannot allocate memory"),
- {"role": "user", "content": "retry"},
- ])
- assert await pick_capable_first(ctx, confidence_threshold=0.7) == CAPABLE
- ctx = await _ctx([
- _msg_tool_result("Out of memory: cannot allocate memory"),
- {"role": "user", "content": "retry"},
- ])
- assert await pick_efficient_first(ctx, confidence_threshold=0.7) == CAPABLE
-
-
-async def test_compaction_overrides_to_capable():
- """A compacted context forces CAPABLE for both pickers, even with a production
- signal that would otherwise route EFFICIENT — a task hard enough to overflow its
- context belongs on strong, and the override holds it there."""
- msgs = [
- {"role": "user", "content":
- "This session is being continued from a previous conversation that ran out of context."},
- ] + [_msg_tool_call("Write"), _msg_tool_result("ok")] * 3
- ctx = await _ctx(msgs)
- assert await pick_efficient_first(ctx, confidence_threshold=0.7) == CAPABLE
- ctx = await _ctx(msgs)
- assert await pick_capable_first(ctx, confidence_threshold=0.7) == CAPABLE
-
-
-async def test_tests_passed_with_edits_routes_to_efficient():
- """A settled run (recent test-pass + recent edit, no error) is safe on cheap tier."""
- messages = [_msg_tool_call("Edit")] * 3 + [_msg_tool_result("all tests passed")] * 3
- messages += [{"role": "user", "content": "ok continue"}] * 12
- ctx = await _ctx(messages)
- assert await pick_capable_first(ctx, confidence_threshold=0.7) == EFFICIENT
- ctx = await _ctx(messages)
- assert await pick_efficient_first(ctx, confidence_threshold=0.7) == EFFICIENT
-
-
-async def test_tests_passed_blocked_by_windowed_error():
- """A HARD error in the window blocks the settled-run shortcut (finding G) — the
- turn falls through to the scorer instead of being swallowed as EFFICIENT."""
- log = StageRouterDecisionLog()
- messages = [_msg_tool_call("Edit")] * 3 + [
- _msg_tool_result("all tests passed"),
- _msg_tool_result("Traceback (most recent call last):\n ValueError: bad"),
- ]
- ctx = await _ctx(messages)
- await pick_capable_first(ctx, confidence_threshold=0.2, decision_log=log)
- assert ctx.metadata[CONTEXT_KEY] != "tests_passed"
-
-
-async def test_dimensions_routes_capable_on_corroborated_wrong_signals():
- """Deep pure-command cycling (spinning) that ends in an error corroborates
- severity + spinning → CAPABLE by sign, for both pickers, no classifier needed."""
- log = StageRouterDecisionLog()
- classifier = _stub_classifier(tier="efficient") # would flip if consulted
- seq: list[dict] = []
- for i in range(5):
- seq.append(_msg_bash("make"))
- seq.append(_msg_tool_result(
- "Traceback (most recent call last):\n ValueError" if i == 4 else "building..."
- ))
- seq.append({"role": "user", "content": "next"})
- ctx = await _ctx(seq)
- tier = await pick_capable_first(
- ctx, confidence_threshold=0.2, classifier=classifier, decision_log=log,
- )
- assert ctx.metadata[CONTEXT_KEY] == "dimensions"
- assert tier == CAPABLE
- assert classifier._client.calls == 0 # type: ignore[attr-defined]
- ctx = await _ctx(seq)
- assert await pick_efficient_first(ctx, confidence_threshold=0.2) == CAPABLE
-
-
-async def test_dimensions_routes_efficient_on_progress():
- """A deep, clean, write-heavy run scores negative → EFFICIENT for both pickers."""
- seq: list[dict] = []
- for _ in range(5):
- seq.append(_msg_tool_call("Write"))
- seq.append(_msg_tool_result("ok"))
- seq.append({"role": "user", "content": "next"})
- ctx = await _ctx(seq)
- assert await pick_capable_first(ctx, confidence_threshold=0.2) == EFFICIENT
- ctx = await _ctx(seq)
- assert await pick_efficient_first(ctx, confidence_threshold=0.2) == EFFICIENT
-
-
-async def test_no_signal_returns_default_tier():
- ctx = ProxyContext() # signal not stamped
- assert await pick_capable_first(ctx, confidence_threshold=0.7) == CAPABLE
- assert await pick_efficient_first(ctx, confidence_threshold=0.7) == EFFICIENT
-
-
-async def test_low_confidence_consults_classifier_when_configured():
- """Empty trajectory yields confidence 0.0 — picker must call the classifier."""
- ctx = await _ctx([{"role": "user", "content": "hi"}])
- classifier = _stub_classifier(tier="efficient")
- assert await pick_capable_first(
- ctx, confidence_threshold=0.7, classifier=classifier,
- ) == EFFICIENT
- assert classifier._client.calls == 1 # type: ignore[attr-defined]
-
-
-async def test_low_confidence_falls_back_to_default_without_classifier():
- ctx = await _ctx([{"role": "user", "content": "hi"}])
- assert await pick_capable_first(ctx, confidence_threshold=0.7) == CAPABLE
- ctx = await _ctx([{"role": "user", "content": "hi"}])
- assert await pick_efficient_first(ctx, confidence_threshold=0.7) == EFFICIENT
-
-
-async def test_classifier_fall_open_on_unknown_tier():
- """A malformed classifier verdict must not override the picker default."""
- ctx = await _ctx([{"role": "user", "content": "hi"}])
- classifier = _stub_classifier(tier=None)
- assert await pick_capable_first(
- ctx, confidence_threshold=0.7, classifier=classifier,
- ) == CAPABLE
-
-
-async def test_threshold_zero_skips_classifier_entirely():
- """``confidence_threshold=0`` accepts every scorer verdict, however weak."""
- ctx = await _ctx([{"role": "user", "content": "hi"}])
- classifier = _stub_classifier(tier="capable")
- await pick_capable_first(ctx, confidence_threshold=0.0, classifier=classifier)
- assert classifier._client.calls == 0 # type: ignore[attr-defined]
-
-
-async def test_decision_log_counts_sources():
- """Each decision path increments exactly one bucket in the shared log."""
- log = StageRouterDecisionLog()
- ctx_critical = await _ctx([
- _msg_tool_result("Out of memory: cannot allocate memory"),
- {"role": "user", "content": "retry"},
- ])
- await pick_capable_first(ctx_critical, confidence_threshold=0.7, decision_log=log)
- assert ctx_critical.metadata[CONTEXT_KEY] == "override"
-
- ctx_neutral = await _ctx([{"role": "user", "content": "hi"}])
- await pick_capable_first(ctx_neutral, confidence_threshold=0.99, decision_log=log)
- assert ctx_neutral.metadata[CONTEXT_KEY] == "fall_open"
-
- classifier = _stub_classifier(tier="efficient")
- ctx_class = await _ctx([{"role": "user", "content": "hi"}])
- await pick_capable_first(
- ctx_class, confidence_threshold=0.99, classifier=classifier, decision_log=log,
- )
- assert ctx_class.metadata[CONTEXT_KEY] == "llm-classifier"
-
- snapshot = log.snapshot()
- assert snapshot["override"] == 1
- assert snapshot["fall_open"] == 1
- assert snapshot["llm-classifier"] == 1
- assert log.total() == 3
diff --git a/tests/test_stage_router_profile.py b/tests/test_stage_router_profile.py
deleted file mode 100644
index 67b69de6b..000000000
--- a/tests/test_stage_router_profile.py
+++ /dev/null
@@ -1,145 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Unit tests for stage_router profile classifier construction."""
-
-from __future__ import annotations
-
-import json
-from types import SimpleNamespace
-from typing import Any
-
-import pytest
-
-from switchyard.lib.processors.stage_router_request_processor import StageRouterRequestProcessor
-from switchyard.lib.profiles.stage_router import StageRouterProfileConfig, _build_classifier
-from switchyard.lib.profiles.stage_router_config import ClassifierConfig, StageRouterConfig
-from switchyard.lib.stats_accumulator import StatsAccumulator
-from switchyard_rust.core import ChatRequest
-from switchyard_rust.profiles import ProfileInput
-
-
-class _ClassifierClient:
- """Fake async classifier client that returns one deterministic tier verdict."""
-
- def __init__(self, tier: str = "efficient") -> None:
- """Store the tier returned by later classifier calls."""
- self._tier = tier
- self.calls = 0
-
- async def acompletion(self, **_kwargs: object) -> object:
- """Return a LiteLLM-shaped classifier response with token usage."""
- self.calls += 1
- return SimpleNamespace(
- choices=[
- SimpleNamespace(
- message=SimpleNamespace(content=json.dumps({"tier": self._tier})),
- )
- ],
- usage=SimpleNamespace(
- prompt_tokens=11,
- completion_tokens=7,
- prompt_tokens_details=SimpleNamespace(cached_tokens=3),
- ),
- )
-
-
-def test_no_classifier_when_config_absent() -> None:
- assert _build_classifier(None) is None
-
-
-def test_bedrock_claude_classifier_disables_reasoning() -> None:
- classifier = _build_classifier(
- ClassifierConfig(model="aws/anthropic/bedrock-claude-sonnet-4-6", api_key="sk"),
- )
- assert classifier is not None
- assert classifier._disable_reasoning is False
-
-
-def test_deepseek_classifier_keeps_reasoning_disabled() -> None:
- classifier = _build_classifier(
- ClassifierConfig(model="nvidia/deepseek-ai/deepseek-v4-flash", api_key="sk"),
- )
- assert classifier is not None
- assert classifier._disable_reasoning is True
-
-
- ("picker", "expected_target"),
- [
- ("capable_first", "efficient"),
- ("efficient_first", "efficient"),
- ],
-)
-async def test_runtime_stats_reach_profile_classifier(
- picker: str,
- expected_target: str,
-) -> None:
- """Python stage_router classifier overhead is visible in shared routing stats."""
- stats = StatsAccumulator()
- config = StageRouterConfig.model_validate({
- "picker": picker,
- "confidence_threshold": 1.0,
- "fallback_target_on_evict": "capable",
- "capable": {
- "id": "capable",
- "model": "capable/model",
- "api_key": "capable-key",
- "base_url": "http://127.0.0.1:9/capable/v1",
- },
- "efficient": {
- "id": "efficient",
- "model": "efficient/model",
- "api_key": "efficient-key",
- "base_url": "http://127.0.0.1:9/efficient/v1",
- },
- "classifier": {
- "model": "classifier/model",
- "api_key": "classifier-key",
- "base_url": "http://127.0.0.1:9/classifier/v1",
- },
- })
- profile = (
- StageRouterProfileConfig.from_config(config)
- .build()
- .with_runtime_components(stats_accumulator=stats, enable_stats=True)
- )
- processor = next(
- component
- for component in profile.iter_components()
- if isinstance(component, StageRouterRequestProcessor)
- )
- classifier = processor._classifier
- assert classifier is not None
- client = _ClassifierClient(tier="efficient")
- classifier._client = client # type: ignore[assignment]
-
- processed = await profile.process(ProfileInput(_chat_request()))
-
- snapshot: dict[str, Any] = await stats.snapshot()
- assert client.calls == 1
- assert processed.selected_target == expected_target
- assert classifier._stats is stats
- assert snapshot["classifier"]["total_requests"] == 1
- assert snapshot["classifier"]["total_errors"] == 0
- model_stats = snapshot["classifier"]["models"]["classifier/model"]
- assert model_stats["calls"] == 1
- assert model_stats["prompt_tokens"] == 11
- assert model_stats["completion_tokens"] == 7
- assert model_stats["cached_tokens"] == 3
- assert model_stats["model_call_latency"]["count"] == 1
- assert snapshot["routing_decisions"]["stage_router"]["llm-classifier"] == 1
-
-
-def _chat_request() -> ChatRequest:
- """Build the OpenAI request shape passed through the stage_router profile."""
- return ChatRequest.openai_chat({
- "model": "stage_router-route",
- "messages": [
- {
- "role": "user",
- "content": "What is 2+2? Reply with just the number.",
- }
- ],
- "max_tokens": 8,
- })
diff --git a/tests/test_stage_router_request_processor.py b/tests/test_stage_router_request_processor.py
deleted file mode 100644
index 0958384fd..000000000
--- a/tests/test_stage_router_request_processor.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for :class:`StageRouterRequestProcessor`.
-
-The processor is a thin async dispatcher: it runs an async picker against the
-:class:`ToolResultSignal` stamped upstream by :class:`DimensionCollector` and
-stamps ``ctx.selected_target`` + ``ctx.selected_model`` for the downstream
-``MultiLlmBackend`` to dispatch on.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
-from switchyard.lib.processors.stage_router import pick_capable_first, pick_efficient_first
-from switchyard.lib.processors.stage_router_request_processor import StageRouterRequestProcessor
-from switchyard_rust.components import DimensionCollector
-from switchyard_rust.core import ChatRequest, ProxyContext
-
-
-def _target(label: str, model: str) -> LlmTarget:
- return LlmTarget(
- id=label,
- model=model,
- api_key="sk-test",
- base_url="https://test.invalid/v1",
- format=BackendFormat.OPENAI,
- )
-
-
-EFFICIENT = _target("efficient", "vendor/efficient-model")
-CAPABLE = _target("capable", "vendor/capable-model")
-
-
-async def _populated_ctx(messages: list[dict]) -> tuple[ProxyContext, ChatRequest]:
- collector = DimensionCollector()
- request = ChatRequest.openai_chat({"messages": messages})
- ctx = ProxyContext()
- await collector.process(ctx, request)
- return ctx, request
-
-
-async def _capable_pick(ctx: ProxyContext) -> int:
- return await pick_capable_first(ctx, confidence_threshold=0.7)
-
-
-async def _efficient_pick(ctx: ProxyContext) -> int:
- return await pick_efficient_first(ctx, confidence_threshold=0.7)
-
-
-def test_requires_exactly_two_targets():
- with pytest.raises(ValueError, match="exactly 2 targets"):
- StageRouterRequestProcessor(targets=(EFFICIENT,), picker=_capable_pick)
- with pytest.raises(ValueError, match="exactly 2 targets"):
- StageRouterRequestProcessor(targets=(EFFICIENT, CAPABLE, CAPABLE), picker=_capable_pick)
-
-
-async def test_capable_first_stamps_capable_on_first_turn_no_signal():
- """First turn: no ToolResultSignal yet → no_signal path → default tier."""
- processor = StageRouterRequestProcessor(targets=(EFFICIENT, CAPABLE), picker=_capable_pick)
- ctx, request = await _populated_ctx([{"role": "user", "content": "hi"}])
- await processor.process(ctx, request)
- assert ctx.selected_target == "capable"
- assert ctx.selected_model == "vendor/capable-model"
-
-
-async def test_efficient_first_stamps_efficient_on_first_turn_no_signal():
- """First turn: no ToolResultSignal yet → no_signal path → default tier."""
- processor = StageRouterRequestProcessor(targets=(EFFICIENT, CAPABLE), picker=_efficient_pick)
- ctx, request = await _populated_ctx([{"role": "user", "content": "hi"}])
- await processor.process(ctx, request)
- assert ctx.selected_target == "efficient"
-
-
-async def test_capable_first_falls_open_to_capable_on_low_confidence():
- """Signal present but scorer below threshold + no classifier → fall_open to default."""
- processor = StageRouterRequestProcessor(targets=(EFFICIENT, CAPABLE), picker=_capable_pick)
- # One Read + one clean tool_result + a follow-up user message. This produces
- # a non-None ToolResultSignal (so the no_signal short-circuit is bypassed)
- # but the scorer only sees a small no_error_streak penalty — confidence
- # well under 0.7, classifier not configured → fall_open returns default.
- ctx, request = await _populated_ctx([
- {"role": "assistant",
- "tool_calls": [{"function": {"name": "Read", "arguments": "{}"}}]},
- {"role": "tool", "tool_call_id": "x", "content": "ok"},
- {"role": "user", "content": "next"},
- ])
- await processor.process(ctx, request)
- assert ctx.selected_target == "capable"
-
-
-async def test_efficient_first_falls_open_to_efficient_on_low_confidence():
- """Sibling check on the efficient-first picker, same low-confidence shape."""
- processor = StageRouterRequestProcessor(targets=(EFFICIENT, CAPABLE), picker=_efficient_pick)
- ctx, request = await _populated_ctx([
- {"role": "assistant",
- "tool_calls": [{"function": {"name": "Read", "arguments": "{}"}}]},
- {"role": "tool", "tool_call_id": "x", "content": "ok"},
- {"role": "user", "content": "next"},
- ])
- await processor.process(ctx, request)
- assert ctx.selected_target == "efficient"
-
-
-async def test_critical_severity_escalates_both_pickers():
- fatal = [
- {"role": "tool", "tool_call_id": "1",
- "content": "Out of memory: cannot allocate memory"},
- {"role": "user", "content": "try again"},
- ]
- for picker in (_capable_pick, _efficient_pick):
- processor = StageRouterRequestProcessor(targets=(EFFICIENT, CAPABLE), picker=picker)
- ctx, request = await _populated_ctx(fatal)
- await processor.process(ctx, request)
- assert ctx.selected_target == "capable"
-
-
-async def test_request_is_not_mutated():
- processor = StageRouterRequestProcessor(targets=(EFFICIENT, CAPABLE), picker=_capable_pick)
- ctx, request = await _populated_ctx([{"role": "user", "content": "hi"}])
- returned = await processor.process(ctx, request)
- assert returned is request
-
-
-async def test_buggy_picker_falls_back_to_efficient():
- async def bad_picker(_ctx: ProxyContext) -> int:
- raise RuntimeError("boom")
-
- processor = StageRouterRequestProcessor(targets=(EFFICIENT, CAPABLE), picker=bad_picker)
- ctx, request = await _populated_ctx([{"role": "user", "content": "hi"}])
- await processor.process(ctx, request)
- assert ctx.selected_target == "efficient"
-
-
-async def test_picker_index_is_clamped():
- async def overshooting_picker(_ctx: ProxyContext) -> int:
- return 99
-
- processor = StageRouterRequestProcessor(targets=(EFFICIENT, CAPABLE), picker=overshooting_picker)
- ctx, request = await _populated_ctx([{"role": "user", "content": "hi"}])
- await processor.process(ctx, request)
- assert ctx.selected_target == "capable"
diff --git a/tests/test_subagent_routing.py b/tests/test_subagent_routing.py
deleted file mode 100644
index edd82e466..000000000
--- a/tests/test_subagent_routing.py
+++ /dev/null
@@ -1,137 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Serve-path sub-agent routing: the ``subagent_target`` route envelope key."""
-
-from __future__ import annotations
-
-from typing import Any
-
-import pytest
-
-from switchyard.cli.route_bundle import (
- RouteBundle,
- RouteBundleConfigError,
- build_table_from_bundle,
-)
-from switchyard.lib.profiles.subagent_override import SubagentOverrideRuntime
-from switchyard.lib.request_metadata import CTX_PROFILE_REQUEST_HEADERS
-from switchyard_rust.core import is_subagent_request
-
-
-class _RecordingRuntime:
- """A ChainRuntime stand-in that records the calls routed to it."""
-
- def __init__(self, name: str) -> None:
- self.name = name
- self.calls = 0
-
- async def call(self, request: Any, *, ctx: Any = None) -> str:
- self.calls += 1
- return self.name
-
- def iter_components(self) -> list[Any]:
- return [self.name]
-
-
-class _Ctx:
- """Minimal ProxyContext stand-in exposing only the retained header map."""
-
- def __init__(self, headers: dict[str, str] | None = None) -> None:
- self.metadata: dict[str, Any] = {}
- if headers is not None:
- self.metadata[CTX_PROFILE_REQUEST_HEADERS] = headers
-
-
-# --- detection policy -----------------------------------------------------------------
-
-
-def test_detection_matches_the_protocol_policy() -> None:
- # Claude Code child-agent lineage is delegated work.
- assert is_subagent_request(
- {"x-claude-code-session-id": "root", "x-claude-code-agent-id": "child-1"}
- )
- # Codex delegated-work kinds.
- assert is_subagent_request({"x-openai-subagent": "review"})
- # Harness maintenance is sub-agent lineage but NOT delegated work.
- assert not is_subagent_request({"x-openai-subagent": "compact"})
- # Explicit opt-out wins.
- assert not is_subagent_request({"x-switchyard-is-subagent": "false"})
- # Ordinary traffic.
- assert not is_subagent_request({})
-
-
-# --- runtime wrapper ------------------------------------------------------------------
-
-
-async def test_delegated_work_goes_to_the_worker() -> None:
- inner, worker = _RecordingRuntime("inner"), _RecordingRuntime("worker")
- runtime = SubagentOverrideRuntime(inner, worker)
-
- served = await runtime.call(
- object(),
- ctx=_Ctx({"x-claude-code-session-id": "s", "x-claude-code-agent-id": "child-1"}),
- )
- assert served == "worker"
- assert (inner.calls, worker.calls) == (0, 1)
-
-
-async def test_normal_traffic_runs_the_wrapped_runtime() -> None:
- inner, worker = _RecordingRuntime("inner"), _RecordingRuntime("worker")
- runtime = SubagentOverrideRuntime(inner, worker)
-
- assert await runtime.call(object(), ctx=_Ctx({})) == "inner"
- # A maintenance turn carries lineage but is not delegated work.
- assert await runtime.call(object(), ctx=_Ctx({"x-openai-subagent": "compact"})) == "inner"
- assert (inner.calls, worker.calls) == (2, 0)
-
-
-async def test_missing_context_routes_normally() -> None:
- inner, worker = _RecordingRuntime("inner"), _RecordingRuntime("worker")
- runtime = SubagentOverrideRuntime(inner, worker)
-
- # No context means no retained headers, so nothing identifies the request.
- assert await runtime.call(object(), ctx=None) == "inner"
- assert worker.calls == 0
-
-
-def test_components_span_both_branches() -> None:
- runtime = SubagentOverrideRuntime(_RecordingRuntime("inner"), _RecordingRuntime("worker"))
- assert runtime.iter_components() == ["inner", "worker"]
-
-
-# --- route bundle wiring --------------------------------------------------------------
-
-
-def _bundle(route: dict[str, Any]) -> RouteBundle:
- return RouteBundle(
- routes={"my-route": route},
- defaults={"api_key": "sk-test", "base_url": "https://example.invalid/v1"},
- )
-
-
-def test_subagent_target_wraps_any_route_type() -> None:
- table = build_table_from_bundle(
- _bundle({
- "type": "model",
- "target": {"model": "strong-model"},
- "subagent_target": {"model": "worker-model"},
- })
- )
- assert isinstance(table.lookup_switchyard("my-route"), SubagentOverrideRuntime)
-
-
-def test_routes_without_the_key_are_unwrapped() -> None:
- table = build_table_from_bundle(_bundle({"type": "model", "target": {"model": "strong-model"}}))
- assert not isinstance(table.lookup_switchyard("my-route"), SubagentOverrideRuntime)
-
-
-def test_unknown_route_keys_are_still_rejected() -> None:
- with pytest.raises(RouteBundleConfigError, match="subagent_targets"):
- build_table_from_bundle(
- _bundle({
- "type": "model",
- "target": {"model": "strong-model"},
- "subagent_targets": {"model": "typo"},
- })
- )
diff --git a/tests/test_translate_profile_config.py b/tests/test_translate_profile_config.py
deleted file mode 100644
index 8a1436254..000000000
--- a/tests/test_translate_profile_config.py
+++ /dev/null
@@ -1,85 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for the host-owned translate profile."""
-
-import pytest
-from anthropic.types import Message, TextBlock, Usage
-
-from switchyard import ProfileInput, TranslateProfileConfig, build_profile
-from switchyard.lib.processors.format_translate import TranslateConfig
-from switchyard.lib.proxy_context import (
- CTX_ORIGINAL_FORMAT,
- CTX_TARGET_FORMAT,
- ProxyContext,
-)
-from switchyard_rust.core import (
- ChatRequest,
- ChatRequestType,
- ChatResponse,
- ChatResponseType,
- SwitchyardBackendError,
- request_type_matches,
- response_type_matches,
-)
-
-
-class _SelectModelProcessor:
- """Test-only selector that mimics a host shim stamping the chosen model."""
-
- async def process(self, ctx: ProxyContext, request: ChatRequest) -> ChatRequest:
- """Stamp the selected model before translate lookup runs."""
- ctx.selected_model = "anthropic-target"
- return request
-
-
-def _openai_request() -> ChatRequest:
- """Build the inbound client request used by translate-profile tests."""
- return ChatRequest.openai_chat({
- "model": "client-model",
- "messages": [{"role": "user", "content": "Say ok"}],
- })
-
-
-def _anthropic_response() -> ChatResponse:
- """Build the host backend response returned between process and rprocess."""
- return ChatResponse.anthropic_completion(
- Message(
- id="msg_translate_profile",
- type="message",
- role="assistant",
- content=[TextBlock(type="text", text="ok")],
- model="anthropic-target",
- stop_reason="end_turn",
- stop_sequence=None,
- usage=Usage(input_tokens=4, output_tokens=1),
- )
- )
-
-
-async def test_translate_profile_process_and_rprocess_wrap_existing_processors() -> None:
- """Translate profile supports the host-owned process/backend/rprocess split."""
- profile = build_profile(TranslateProfileConfig(
- config=TranslateConfig(
- models=[{"model": "anthropic-target", "backend_format": "anthropic"}]
- ),
- model_selection_processors=(_SelectModelProcessor(),),
- ))
-
- processed = await profile.process(ProfileInput(_openai_request()))
- assert request_type_matches(processed.request, ChatRequestType.ANTHROPIC)
- assert processed.ctx.metadata[CTX_ORIGINAL_FORMAT] == ChatRequestType.OPENAI_CHAT
- assert processed.ctx.metadata[CTX_TARGET_FORMAT] == ChatRequestType.ANTHROPIC
-
- response = await profile.rprocess(processed, _anthropic_response())
-
- assert response_type_matches(response, ChatResponseType.OPENAI_COMPLETION)
- assert response.body["choices"][0]["message"]["content"] == "ok"
-
-
-async def test_translate_profile_run_fails_clearly_without_host_backend() -> None:
- """Translate profile is hook-only because the embedding host owns the call."""
- profile = build_profile(TranslateProfileConfig())
-
- with pytest.raises(SwitchyardBackendError, match="hook-only"):
- await profile.run(ProfileInput(_openai_request()))
diff --git a/tests/test_upstream_error_passthrough.py b/tests/test_upstream_error_passthrough.py
index 5952906da..ac61dfe48 100644
--- a/tests/test_upstream_error_passthrough.py
+++ b/tests/test_upstream_error_passthrough.py
@@ -172,7 +172,7 @@ def test_rust_openai_route_upstream_401_returns_structured_openai_error() -> Non
},
"routes": {
"bad-key": {
- "type": "model",
+ "type": "passthrough",
"target": "nvidia/nvidia/nemotron-nano-9b-v2",
}
},
@@ -212,7 +212,7 @@ def test_rust_openai_route_upstream_401_returns_same_error_shape_for_anthropic_i
},
"routes": {
"bad-key": {
- "type": "model",
+ "type": "passthrough",
"target": "nvidia/nvidia/nemotron-nano-9b-v2",
}
},
@@ -253,7 +253,7 @@ def test_rust_openai_route_upstream_401_returns_same_error_shape_for_responses_i
},
"routes": {
"bad-key": {
- "type": "model",
+ "type": "passthrough",
"target": "nvidia/nvidia/nemotron-nano-9b-v2",
}
},
@@ -297,7 +297,7 @@ def test_model_not_found_404_header_labels_switchyard_on_the_wire() -> None:
},
"routes": {
"registered": {
- "type": "model",
+ "type": "passthrough",
"target": "nvidia/nvidia/nemotron-nano-9b-v2",
}
},
diff --git a/uv.lock b/uv.lock
index 230dadf93..0c7d61866 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1744,14 +1744,10 @@ dependencies = [
]
[package.optional-dependencies]
-affinity-redis = [
- { name = "redis" },
-]
all = [
{ name = "ddtrace" },
{ name = "fastapi" },
{ name = "prompt-toolkit" },
- { name = "redis" },
{ name = "sse-starlette" },
{ name = "uvicorn", extra = ["standard"] },
]
@@ -1797,15 +1793,14 @@ requires-dist = [
{ name = "ddtrace", marker = "extra == 'tracing'", specifier = ">=2.9,<4" },
{ name = "fastapi", marker = "extra == 'server'", specifier = ">=0.136.1,<1.0" },
{ name = "httpx", specifier = ">=0.28.1,<1.0" },
- { name = "nemo-switchyard", extras = ["server", "cli", "tracing", "affinity-redis"], marker = "extra == 'all'" },
+ { name = "nemo-switchyard", extras = ["server", "cli", "tracing"], marker = "extra == 'all'" },
{ name = "openai", specifier = ">=2.7,<3.0" },
{ name = "prompt-toolkit", marker = "extra == 'cli'", specifier = ">=3.0.52,<4.0" },
{ name = "pydantic", specifier = ">=2.13.3,<3.0" },
- { name = "redis", marker = "extra == 'affinity-redis'", specifier = ">=5.0.1,<6" },
{ name = "sse-starlette", marker = "extra == 'server'", specifier = ">=3.4.1,<4.0" },
{ name = "uvicorn", extras = ["standard"], marker = "extra == 'server'", specifier = ">=0.46.0,<1.0" },
]
-provides-extras = ["server", "cli", "tracing", "affinity-redis", "all"]
+provides-extras = ["server", "cli", "tracing", "all"]
[package.metadata.requires-dev]
dev = [
@@ -2631,18 +2626,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b4/75/1b2cfc949595e22d8c05a2aa2cfc222921f7f94177d7e8a90542f3f73b33/realtime-2.30.0-py3-none-any.whl", hash = "sha256:7c93b63d2cf99aa1da4fa8826b03b00cd32f7b38abb27ff47b19eb5dcb5707c6", size = 22376, upload-time = "2026-05-06T17:35:22.568Z" },
]
-[[package]]
-name = "redis"
-version = "5.3.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "pyjwt" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/6a/cf/128b1b6d7086200c9f387bd4be9b2572a30b90745ef078bd8b235042dc9f/redis-5.3.1.tar.gz", hash = "sha256:ca49577a531ea64039b5a36db3d6cd1a0c7a60c34124d46924a45b956e8cf14c", size = 4626200, upload-time = "2025-07-25T08:06:27.778Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/7f/26/5c5fa0e83c3621db835cfc1f1d789b37e7fa99ed54423b5f519beb931aa7/redis-5.3.1-py3-none-any.whl", hash = "sha256:dc1909bd24669cc31b5f67a039700b16ec30571096c5f1f0d9d2324bff31af97", size = 272833, upload-time = "2025-07-25T08:06:26.317Z" },
-]
-
[[package]]
name = "referencing"
version = "0.37.0"