Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(profiles): nuke components-v2 and its mixed Python serving stack (#119)

nachiketb-nvidia4ccec5d371f2
diff --git a/.agents/skills/switchyard-codebase-exploration/SKILL.md b/.agents/skills/switchyard-codebase-exploration/SKILL.md
index 165c7e8d6..8057b9267 100644
--- a/.agents/skills/switchyard-codebase-exploration/SKILL.md
+++ b/.agents/skills/switchyard-codebase-exploration/SKILL.md
@@ -7,8 +7,8 @@ description: Use when modifying, debugging, reviewing, refactoring, renaming, re
> Rust crates under `crates/` are active on the Rust core branch. Inspect
> `crates/switchyard-core`, `crates/switchyard-components`,
-> `crates/switchyard-components-v2`, `crates/switchyard-server`,
-> `crates/switchyard-translation`, `crates/switchyard-py`, and `crates/*/tests`
+> `crates/switchyard-server`, `crates/switchyard-translation`,
+> `crates/switchyard-py`, and `crates/*/tests`
> directly when a change touches Rust.
> Direct PyO3 bindings for concrete components live under
> `crates/switchyard-py/src/component_bindings/` with Python lazy exports in
diff --git a/.agents/skills/switchyard-coding-agent-launchers/SKILL.md b/.agents/skills/switchyard-coding-agent-launchers/SKILL.md
index 0c7856145..099bb87cf 100644
--- a/.agents/skills/switchyard-coding-agent-launchers/SKILL.md
+++ b/.agents/skills/switchyard-coding-agent-launchers/SKILL.md
@@ -85,8 +85,8 @@ Route YAML and launchers share this model-dispatch path:
- `type: passthrough` registers the YAML key and discovered direct models.
- `type: random_routing` registers the YAML key as the routing profile plus
direct strong/weak passthrough entries.
-- `type: deterministic` and `type: stage_router` register the route key plus direct
- strong/weak passthrough entries.
+- `type: deterministic`, `type: escalation_router`, and `type: stage_router`
+ register the route key plus direct strong/weak passthrough entries.
- `type: plan_execute` registers the route key plus the executor as a direct
passthrough; the planner is internal routing logic.
- `type: latency_service` and `type: noop` register the
diff --git a/AGENTS.md b/AGENTS.md
index 3a9f7253e..73dba19e1 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -246,10 +246,8 @@ and their transitives never appear in downstream vulnerability scans.
export OPENAI_API_KEY="sk-..." # or NVIDIA_API_KEY / ANTHROPIC_API_KEY where supported
export OPENROUTER_API_KEY="sk-or-..." # pass with --api-key or save via configure
-# Serve a profile config (passthrough, random-routing, llm-routing, stage_router).
-# Endpoints, targets, and profiles live in the YAML; see docs/routing_algorithms/overview.md.
-switchyard serve --config profiles.yaml --port 4000
-switchyard serve --config profiles.yaml --inbound anthropic --port 4000
+# Serve a routing bundle. Routes live in YAML; see docs/routing_algorithms/overview.md.
+switchyard --routing-profiles routes.yaml -- serve --port 4000
# One-command launchers — single-model passthrough via --model
switchyard launch claude --model openai/gpt-4o-mini \
@@ -291,7 +289,7 @@ uv run mypy switchyard
1. Pick the right stage: request component (pre-call), response component (post-call), `LLMBackend` (rare), or Rust translation codec work.
2. Create a file with the explicit name (`snake_case` of the class name), one class per file.
3. Implement the async method for that stage (`process` for components, `call` for backends).
-4. Wire into the owning profile config.
+4. Wire into the owning programmatic profile config or route-bundle builder.
5. Add tests under `tests/`.
6. Export from the relevant `__init__.py` and from `switchyard/__init__.py`'s `__all__`.
diff --git a/Cargo.lock b/Cargo.lock
index b10531a52..8129d2df5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -912,12 +912,6 @@ version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
-[[package]]
-name = "libyaml-rs"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e126dda6f34391ab7b444f9922055facc83c07a910da3eb16f1e4d9c45dc777"
-
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
@@ -1671,38 +1665,6 @@ dependencies = [
"tracing",
]
-[[package]]
-name = "switchyard-components-v2"
-version = "0.1.0"
-dependencies = [
- "async-stream",
- "async-trait",
- "futures-core",
- "futures-util",
- "parking_lot",
- "rand 0.8.7",
- "reqwest",
- "serde",
- "serde_json",
- "switchyard-components",
- "switchyard-components-v2-macros",
- "switchyard-core",
- "switchyard-protocol",
- "tokio",
- "toml",
- "tracing",
- "yaml_serde",
-]
-
-[[package]]
-name = "switchyard-components-v2-macros"
-version = "0.1.0"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
[[package]]
name = "switchyard-core"
version = "0.1.0"
@@ -1770,7 +1732,6 @@ dependencies = [
"serde",
"serde_json",
"switchyard-components",
- "switchyard-components-v2",
"switchyard-core",
"switchyard-libsy",
"switchyard-protocol",
@@ -2357,19 +2318,6 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
-[[package]]
-name = "yaml_serde"
-version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08c7c1b1a6a7c8a6b2741a6c21a4f8918e51899b111cfa08d1288202656e3975"
-dependencies = [
- "indexmap",
- "itoa",
- "libyaml-rs",
- "ryu",
- "serde",
-]
-
[[package]]
name = "yansi"
version = "1.0.1"
diff --git a/Cargo.toml b/Cargo.toml
index 3e78719b5..886688b07 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,8 +7,6 @@ members = [
"crates/libsy",
"crates/libsy-llm-client",
"crates/switchyard-components",
- "crates/switchyard-components-v2",
- "crates/switchyard-components-v2-macros",
"crates/switchyard-core",
"crates/switchyard-py",
"crates/protocol",
diff --git a/README.md b/README.md
index 40a389115..5e1b696d4 100644
--- a/README.md
+++ b/README.md
@@ -20,8 +20,7 @@ write yourself.
**Launcher routing is explicit.** By default, launchers use the built-in
LLM-classifier router, which you tune with `--weak-model`, `--classifier-model`,
`--profile`, and `--classifier-min-confidence`. Use `--model X` for
-single-model passthrough. The `--routing-profiles FILE` path is deprecated and
-remains only for launcher-owned legacy bundles.
+single-model passthrough or `--routing-profiles FILE` for a YAML route bundle.
## Features
@@ -72,12 +71,12 @@ switchyard launch openclaw --model openai/gpt-4o-mini --api-key "$OPENROUTER_API
```
Each launcher starts a local proxy, points the agent at it, and shuts the proxy
-down when the agent exits. Use `--model` for single-model passthrough. The
-deprecated `--routing-profiles` flag remains for launcher-owned legacy bundles:
+down when the agent exits. Use `--model` for single-model passthrough or
+`--routing-profiles` for a route bundle:
```bash
switchyard launch claude --model openai/gpt-4o-mini --base-url https://openrouter.ai/api/v1 # single-model passthrough
-switchyard --routing-profiles routes.yaml -- launch claude # legacy route bundle
+switchyard --routing-profiles routes.yaml -- launch claude # route bundle
```
> **Bedrock-backed profile caveat (Claude Code + MCP):** Bedrock enforces a 64-character `toolSpec.name` cap. Claude Code's MCP bridge can auto-inject longer tool names, producing `BedrockException` 400s on tool-bearing requests. If you use a Bedrock-backed route and hit this, swap to an OpenAI-compatible model with `--model openai/gpt-4o` or a routing-profile YAML.
@@ -86,58 +85,39 @@ See [Agent Launchers](docs/guides/agent_launchers.md) for supported harness
versions, model requirements, troubleshooting, and Claude Code `/model` picker
aliasing.
-### 2. Run a standalone profile-config server
+### 2. Run a standalone Python server
-New standalone deployments use a profile config that separates provider
-connectivity, upstream targets, and client-facing profiles. A complete
-OpenRouter-backed random-routing config looks like this:
+Define one or more client-facing routes in YAML. A complete OpenRouter-backed
+random-routing bundle looks like this:
```yaml
-endpoints:
- openrouter:
- api_key: ${OPENROUTER_API_KEY}
- base_url: https://openrouter.ai/api/v1
-
-targets:
- strong:
- endpoint: openrouter
- model: openai/gpt-4o
- format: openai
- weak:
- endpoint: openrouter
- model: openai/gpt-4o-mini
- format: openai
-
-profiles:
+defaults:
+ api_key: ${OPENROUTER_API_KEY}
+ base_url: https://openrouter.ai/api/v1
+ format: openai
+
+routes:
smart:
- type: random-routing
- strong: strong
- weak: weak
+ type: random_routing
+ strong:
+ model: openai/gpt-4o
+ weak:
+ model: openai/gpt-4o-mini
strong_probability: 0.3
+ fallback_target_on_evict: strong
```
-Serve it as a proxy. The `smart` profile and both target ids are exposed as
-models; clients select one through the request's `model` field:
+Serve it as a proxy. The route name is exposed as a model ID, and clients
+select it through the request's `model` field:
```bash
-switchyard serve --config profiles.yaml --port 4000
+switchyard --routing-profiles routes.yaml -- serve --port 4000
curl http://localhost:4000/v1/models
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{"model": "smart", "messages": [{"role": "user", "content": "hi"}]}'
```
-> **Launcher compatibility:** Launcher subcommands do not accept `--config`.
-> The deprecated `--routing-profiles` flag remains for launcher-owned legacy
-> `routes:` bundles and saved bundle paths:
-
-```yaml
-routes:
- fast:
- type: model
- target: openai/gpt-4o-mini
-```
-
```bash
switchyard --routing-profiles routes.yaml -- launch claude
switchyard --routing-profiles routes.yaml -- configure --target provider \
@@ -148,6 +128,10 @@ switchyard --routing-profiles routes.yaml -- configure --target provider \
Non-interactive `configure` does not read provider credentials from the routing
bundle; pass `--api-key` explicitly when persisting the bundle for CI.
+The Rust `switchyard-server` binary has a separate explicit TOML schema for
+LLM clients, targets, and libsy algorithms. See
+[`crates/switchyard-server/README.md`](crates/switchyard-server/README.md).
+
For profile selection and full configuration examples, start with
[Routing Overview](docs/routing_algorithms/overview.md), then open the
strategy-specific page:
diff --git a/crates/switchyard-components-v2-macros/Cargo.toml b/crates/switchyard-components-v2-macros/Cargo.toml
deleted file mode 100644
index 0ea8c8e82..000000000
--- a/crates/switchyard-components-v2-macros/Cargo.toml
+++ /dev/null
@@ -1,20 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-[package]
-name = "switchyard-components-v2-macros"
-version = "0.1.0"
-description = "Macros for experimental Switchyard components-v2 profile definitions"
-authors.workspace = true
-edition.workspace = true
-license.workspace = true
-repository.workspace = true
-rust-version.workspace = true
-
-[lib]
-proc-macro = true
-
-[dependencies]
-proc-macro2 = "1"
-quote = "1"
-syn = { version = "2", features = ["full"] }
diff --git a/crates/switchyard-components-v2-macros/src/lib.rs b/crates/switchyard-components-v2-macros/src/lib.rs
deleted file mode 100644
index 9be222d92..000000000
--- a/crates/switchyard-components-v2-macros/src/lib.rs
+++ /dev/null
@@ -1,235 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Attribute macros for concise components-v2 profile config structs.
-
-use proc_macro::TokenStream;
-use quote::{format_ident, quote};
-use syn::{
- parse_macro_input, Attribute, Field, Fields, GenericArgument, ItemStruct, LitStr,
- PathArguments, Type,
-};
-
-/// Adds standard config derives, strict serde behavior, and a stable profile type.
-#[proc_macro_attribute]
-pub fn profile_config(args: TokenStream, input: TokenStream) -> TokenStream {
- let kind = parse_macro_input!(args as LitStr);
- let input = parse_macro_input!(input as ItemStruct);
- emit_profile_config_struct(input, kind).into()
-}
-
-// Re-emits the struct with Switchyard's standard profile-config surface.
-fn emit_profile_config_struct(input: ItemStruct, kind: LitStr) -> proc_macro2::TokenStream {
- let ident = input.ident.clone();
- let raw_ident = format_ident!("__{}Raw", ident);
- let named_fields = match &input.fields {
- Fields::Named(fields) => fields,
- Fields::Unnamed(_) | Fields::Unit => {
- return syn::Error::new_spanned(
- input,
- "profile_config only supports structs with named fields",
- )
- .to_compile_error();
- }
- };
-
- let mut original_fields = Vec::new();
- let mut raw_fields = Vec::new();
- let mut build_fields = Vec::new();
-
- for field in &named_fields.named {
- let Some(field_ident) = field.ident.clone() else {
- return syn::Error::new_spanned(field, "profile_config requires named fields")
- .to_compile_error();
- };
- let has_profile_target = has_profile_target_attr(&field.attrs);
- let cleaned_field = field_without_profile_target_attr(field);
- let raw_ty = if has_profile_target {
- match profile_target_raw_ty(&field.ty) {
- Ok(raw_ty) => raw_ty,
- Err(error) => return error.to_compile_error(),
- }
- } else {
- field.ty.clone()
- };
- let mut raw_field = cleaned_field.clone();
- raw_field.ty = raw_ty;
- let value = if has_profile_target {
- match profile_target_resolver(&field_ident, &field.ty) {
- Ok(value) => value,
- Err(error) => return error.to_compile_error(),
- }
- } else {
- quote! { raw.#field_ident }
- };
-
- original_fields.push(cleaned_field);
- raw_fields.push(raw_field);
- build_fields.push(quote! { #field_ident: #value });
- }
-
- let mut original = input.clone();
- original.fields = Fields::Named(syn::FieldsNamed {
- brace_token: named_fields.brace_token,
- named: original_fields.into_iter().collect(),
- });
-
- quote! {
- #[derive(
- Clone,
- Debug,
- PartialEq,
- ::serde::Serialize,
- ::serde::Deserialize,
- )]
- #[serde(deny_unknown_fields)]
- #original
-
- #[derive(::serde::Deserialize)]
- #[serde(deny_unknown_fields)]
- struct #raw_ident {
- #(#raw_fields,)*
- }
-
- impl #ident {
- /// Stable discriminator used when this profile appears in serialized config.
- pub const PROFILE_TYPE: &'static str = #kind;
-
- /// Returns the stable serialized discriminator for this profile config.
- pub fn profile_type(&self) -> &'static str {
- Self::PROFILE_TYPE
- }
- }
-
- impl ::switchyard_components_v2::__private::ProfileConfigDefinition for #ident {
- const PROFILE_TYPE: &'static str = #kind;
-
- fn parse_profile_config(
- value: ::serde_json::Value,
- env: &::switchyard_components_v2::__private::ProfileBuildEnv<'_>,
- ) -> ::switchyard_core::Result<Self> {
- let raw: #raw_ident = ::serde_json::from_value(value).map_err(|error| {
- ::switchyard_core::SwitchyardError::InvalidConfig(format!(
- "failed to parse {} profile config: {error}",
- #kind
- ))
- })?;
- Ok(Self {
- #(#build_fields,)*
- })
- }
- }
-
- const _: () = {
- fn assert_profile_config<T: ::switchyard_components_v2::ProfileConfig>() {}
- let _ = assert_profile_config::<#ident>;
- };
- }
-}
-
-// Removes the helper attribute before the real struct reaches the compiler.
-fn field_without_profile_target_attr(field: &Field) -> Field {
- let mut cleaned = field.clone();
- cleaned.attrs = field
- .attrs
- .iter()
- .filter(|attr| !is_profile_target_attr(attr))
- .cloned()
- .collect();
- cleaned
-}
-
-fn has_profile_target_attr(attrs: &[Attribute]) -> bool {
- attrs.iter().any(is_profile_target_attr)
-}
-
-fn is_profile_target_attr(attr: &Attribute) -> bool {
- attr.path().is_ident("profile_target")
-}
-
-fn profile_target_raw_ty(ty: &Type) -> syn::Result<Type> {
- if is_type_ident(ty, "LlmTarget") {
- Ok(syn::parse_quote! { ::switchyard_core::LlmTargetId })
- } else if let Some(inner) = single_generic_arg(ty, "Vec") {
- if is_type_ident(inner, "LlmTarget") {
- Ok(syn::parse_quote! { Vec<::switchyard_core::LlmTargetId> })
- } else {
- Err(syn::Error::new_spanned(
- ty,
- "#[profile_target] Vec fields must contain LlmTarget",
- ))
- }
- } else if let Some(inner) = single_generic_arg(ty, "Option") {
- if is_type_ident(inner, "LlmTarget") {
- Ok(syn::parse_quote! { Option<::switchyard_core::LlmTargetId> })
- } else {
- Err(syn::Error::new_spanned(
- ty,
- "#[profile_target] Option fields must contain LlmTarget",
- ))
- }
- } else {
- Err(syn::Error::new_spanned(
- ty,
- "#[profile_target] supports LlmTarget, Vec<LlmTarget>, and Option<LlmTarget>",
- ))
- }
-}
-
-fn profile_target_resolver(
- field_ident: &syn::Ident,
- ty: &Type,
-) -> syn::Result<proc_macro2::TokenStream> {
- if is_type_ident(ty, "LlmTarget") {
- Ok(quote! { env.target(&raw.#field_ident)?.clone() })
- } else if single_generic_arg(ty, "Vec").is_some() {
- Ok(quote! {
- raw.#field_ident
- .into_iter()
- .map(|target_id| env.target(&target_id).map(Clone::clone))
- .collect::<::switchyard_core::Result<Vec<_>>>()?
- })
- } else if single_generic_arg(ty, "Option").is_some() {
- Ok(quote! {
- match raw.#field_ident {
- Some(target_id) => Some(env.target(&target_id)?.clone()),
- None => None,
- }
- })
- } else {
- Err(syn::Error::new_spanned(
- ty,
- "#[profile_target] supports LlmTarget, Vec<LlmTarget>, and Option<LlmTarget>",
- ))
- }
-}
-
-fn is_type_ident(ty: &Type, ident: &str) -> bool {
- let Type::Path(type_path) = ty else {
- return false;
- };
- type_path.path.segments.last().is_some_and(|segment| {
- segment.ident == ident && matches!(segment.arguments, PathArguments::None)
- })
-}
-
-fn single_generic_arg<'a>(ty: &'a Type, outer: &str) -> Option<&'a Type> {
- let Type::Path(type_path) = ty else {
- return None;
- };
- let segment = type_path.path.segments.last()?;
- if segment.ident != outer {
- return None;
- }
- let PathArguments::AngleBracketed(args) = &segment.arguments else {
- return None;
- };
- let mut args = args.args.iter();
- let Some(GenericArgument::Type(inner)) = args.next() else {
- return None;
- };
- if args.next().is_some() {
- return None;
- }
- Some(inner)
-}
diff --git a/crates/switchyard-components-v2/Cargo.toml b/crates/switchyard-components-v2/Cargo.toml
deleted file mode 100644
index 8e6b06374..000000000
--- a/crates/switchyard-components-v2/Cargo.toml
+++ /dev/null
@@ -1,33 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-[package]
-name = "switchyard-components-v2"
-version = "0.1.0"
-description = "Experimental flatter Switchyard component profile runtime"
-authors.workspace = true
-edition.workspace = true
-license.workspace = true
-repository.workspace = true
-rust-version.workspace = true
-
-[dependencies]
-async-stream = "0.3"
-async-trait = "0.1"
-futures-util = "0.3"
-parking_lot = "0.12"
-rand = "0.8"
-reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] }
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
-switchyard-components = { path = "../switchyard-components" }
-switchyard-components-v2-macros = { path = "../switchyard-components-v2-macros" }
-switchyard-core = { path = "../switchyard-core" }
-switchyard-protocol = { path = "../protocol" }
-toml = "1.1"
-tracing = { version = "0.1", default-features = false, features = ["std"] }
-yaml_serde = "0.10"
-
-[dev-dependencies]
-futures-core = "0.3"
-tokio = { version = "1", features = ["macros", "rt", "sync"] }
diff --git a/crates/switchyard-components-v2/DESIGN.md b/crates/switchyard-components-v2/DESIGN.md
deleted file mode 100644
index 3da7e4511..000000000
--- a/crates/switchyard-components-v2/DESIGN.md
+++ /dev/null
@@ -1,902 +0,0 @@
-# Switchyard Profile V2 Design
-
-Audience: contributors adding or reviewing Switchyard profiles in Rust, Python, or bindings.
-
-Prerequisites: familiarity with `ChatRequest`, `ChatResponse`, `LlmTarget`, async execution, and
-the existing Switchyard request path. You do not need to know the older factory, bundle, graph, or
-processor pipeline internals to understand v2.
-
-## Summary
-
-V2 is the simpler component model for Switchyard as a whole. The first implementation lives in
-`switchyard-components-v2`, but the idea is not Rust-only: Python should converge on the same
-profile-centered model instead of keeping a separate factory/bundle architecture.
-
-V2 replaces the older habit of decomposing every behavior into separate request processors,
-backends, response processors, graph fragments, bundles, and factories with one central runtime
-unit:
-
-```text
-Profile
-```
-
-A profile owns the behavior for one addressable Switchyard mode. That can be passthrough,
-random routing, latency-aware routing, a classifier stage-router, observability-only middleware, or a
-future agent-facing policy. The profile decides how a request is prepared, which backend is called,
-what response cleanup is needed, and where observability is recorded.
-
-The goal is not to make Switchyard less capable. The goal is to stop spreading one behavior across
-many abstractions when most contributors need to reason about one thing: "what happens when this
-profile receives a request?"
-
-## Why V2 Exists
-
-The original Switchyard shape was even looser than today's Python chain: a linked list of
-"strategies" that accepted `Any` and returned `Any`. That made early experiments easy, but it also
-made behavior hard to type, validate, compose, and review. The Python chain was the first cleanup:
-split the untyped strategy chain into explicit request, backend, and response roles:
-
-```text
-request-side work -> LLMBackend -> response-side work
-```
-
-That shape fixed real problems and made incremental porting possible, but it also carried Python
-object boundaries into places where they no longer help. A single product feature could require
-edits in several places:
-
-- request processor config,
-- request processor implementation,
-- backend config,
-- backend implementation,
-- response processor config,
-- response processor implementation,
-- graph or table wiring,
-- factory or bundle glue,
-- tests that needed to know the glue shape.
-
-For new contributors, that is the wrong entry point. They should not need to learn an orchestration
-framework before implementing a routing policy, whether they are writing the implementation in Rust
-or exposing it through Python.
-
-V2 starts from the product-level object instead:
-
-```text
-one profile config -> one profile runtime -> one request lifecycle
-```
-
-This makes the code easier to read, easier to test, and harder to accidentally over-abstract.
-
-The config side follows the same principle. A config file should describe endpoints, targets, and
-profiles. It should not ask users to assemble request processors, response processors, graph nodes,
-bundles, or backend wrappers by hand.
-
-## Core Mental Model
-
-The v2 runtime has two surfaces:
-
-- an object-safe serving surface for code that only needs to run a profile;
-- a typed hook surface for code that wants to inspect or embed the profile's request and response
- hooks.
-
-In Rust, that target shape is:
-
-```rust
-#[async_trait]
-pub trait Profile: Send + Sync {
- async fn run(&self, input: ProfileInput) -> Result<ChatResponse>;
-}
-
-#[async_trait]
-pub trait ProfileHooks: Send + Sync {
- type ProcessedRequest: Send + Sync;
-
- async fn process(&self, input: ProfileInput) -> Result<Self::ProcessedRequest>;
-
- async fn rprocess(
- &self,
- processed: &Self::ProcessedRequest,
- response: ChatResponse,
- ) -> Result<ChatResponse>;
-}
-```
-
-Conceptually:
-
-```text
-process(request) -> profile-specific processed request
-run(request) -> final response
-rprocess(processed, response) -> processed response
-```
-
-`Profile` is the erased serving contract. Servers, config loaders, and generic profile tables
-can store `Box<dyn Profile>` and call `run()` without knowing a profile's private request
-state.
-
-`ProfileHooks` is the typed authoring and embedding contract. The associated `ProcessedRequest`
-type is the profile-owned wrapper around the prepared request and any request-side decision the
-profile made. It replaces untyped metadata bags and avoids forcing every profile into a shared
-decision enum.
-
-These methods are three related entry points into the same profile, not three required stages of a
-hidden pipeline:
-
-- `run()` is the authoritative serving path. It owns the complete request lifecycle and is the
- method a Switchyard server should call.
-- `process()` is the request-side library hook. It returns the prepared request plus any typed
- profile-specific state needed later, such as a random-routing decision or an initial
- latency-service target selection.
-- `rprocess()` is the response-side library hook. It receives the processed request state and the
- backend response so the profile can perform cleanup, normalization, or accounting without a
- side-channel.
-
-`run()` should normally call `process()` and `rprocess()` when those hooks express the same
-lifecycle cleanly. If `process()` seems unable to carry the state that `run()` needs, the first fix
-is to make that profile's `ProcessedRequest` type richer, not to add a generic metadata map. A
-profile should bypass its hooks only when the lifecycle genuinely cannot be represented by a
-request-side wrapper, and that reason should be documented on the concrete profile.
-
-The self-contained serving flow looks like this:
-
-```text
-HTTP / SDK wire request
- |
- | endpoint + translation layer
- v
-ChatRequest
- |
- | Profile::run()
- | - process into profile-specific request state
- | - prepare or rewrite the backend request
- | - choose target/backend
- | - call backend
- | - normalize response
- | - record stats/errors
- v
-ChatResponse
- |
- | endpoint + translation layer
- v
-HTTP / SDK wire response
-```
-
-`process()` and `rprocess()` are intentionally still present. They are not a return to the old
-pipeline hierarchy. They exist because Switchyard can be embedded as middleware in systems that
-own their own transport or model runtime:
-
-```text
-caller-owned request object
- |
- | caller converts or already has a ChatRequest
- |
- v
-ChatRequest
- |
- | Profile::process()
- v
-profile-specific processed request
- |
- | processed.request()
- v
-prepared ChatRequest
- |
- | caller-owned HTTP client, model runner, batcher, or agent runtime
- v
-ChatResponse
- |
- | Profile::rprocess(processed, response)
- v
-processed ChatResponse
- |
- | caller returns or translates response
- v
-caller-owned response object
-```
-
-In middleware mode, the reusable pieces are the profile's request hook, response hook, shared
-backend adapters, translation layer, and explicit helpers such as stats recorders. The caller owns
-transport and the actual model call. The profile still owns the request and response behavior it
-exposes through those hooks.
-
-The important rule is that `process()` must return more than a bare `ChatRequest` when the profile
-makes a meaningful request-side decision. For example, random routing should return a
-`RandomRoutingProcessedRequest` containing both the rewritten request and the routing decision.
-Latency-service routing can return a `LatencyServiceProcessedRequest` containing the rewritten
-request and the selected target candidate. Those wrappers are profile-specific types, not variants
-of one universal metadata object.
-
-Do not implement unsupported hooks with `panic!`, `unimplemented!()`, or runtime
-`NotImplementedError`. If every profile must expose the hook, make the hook total with a meaningful
-identity wrapper for simple profiles. If a future capability only applies to some profiles, split it
-into a separate trait instead of adding a method that most profiles cannot honestly implement.
-
-## What A Profile Owns
-
-A profile owns the policy boundary. If a reviewer asks, "where is this behavior implemented?", the
-answer should usually be one module under `src/profiles/`.
-
-For example:
-
-```text
-src/profiles/passthrough.rs
-src/profiles/random_routing.rs
-src/profiles/latency_service/
-```
-
-Each profile should keep these pieces close together:
-
-- the user-facing config struct,
-- validation that is specific to that config,
-- runtime construction,
-- request-side behavior,
-- the processed-request wrapper returned by `process()`,
-- any profile-specific request decision type,
-- backend selection and calls,
-- response-side behavior,
-- stats and error accounting,
-- focused tests for the profile's invariants.
-
-Shared engines are still allowed. Random routing can reuse `RandomRoutingEngine`. Latency routing
-can split health polling and selection into small private modules. The distinction is that these
-helpers are implementation details of a profile, not a second public orchestration model.
-
-## What V2 Removes From The Contributor Path
-
-V2 is designed so profile authors do not have to build or understand these concepts:
-
-- `ProxyContext` as a mutable cross-stage side channel,
-- request processor trait objects,
-- response processor trait objects,
-- graph fragments as a profile output format,
-- table builders as the primary authoring API,
-- middleware bundles,
-- factories,
-- served-model side channels.
-
-Those ideas may still exist elsewhere during migration or for compatibility. They should not be
-the design vocabulary for new profile code.
-
-For now, v2 does not treat backward compatibility as a requirement. If an older Python factory,
-bundle, route shape, or wrapper only exists to preserve the previous object hierarchy, it is a
-candidate for removal rather than preservation. Temporary aliases can be added later only when a
-release decision explicitly needs them; they should not shape the core design.
-
-## Deliberate Helpers Instead Of Implicit Middleware
-
-Helpers are still valuable. The difference is that v2 makes helper use visible at the profile
-boundary.
-
-Stats are the clearest example. In the older model, stats could be hidden behind separate request
-processors, response processors, or a stats backend wrapper. That hid the control flow:
-
-```text
-request processor records start
-backend records selection
-response processor records usage
-```
-
-The profile author then had to know which side channel connected those pieces. That side channel
-was often `ProxyContext`.
-
-In v2, a profile records stats at the semantic point where it knows the truth:
-
-```rust
-let backend_started_at = Instant::now();
-let response = match selected_backend.call(processed.request().clone()).await {
- Ok(response) => response,
- Err(error) => {
- self.record_error(&processed)?;
- return Err(error);
- }
-};
-
-let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0;
-self.record_success(&processed, Some(backend_latency_ms))?;
-
-let usage = response.body().map(usage_from_body).unwrap_or_default();
-self.record_usage(&processed, usage, Some(total_latency_ms), Some(routing_overhead_ms))?;
-```
-
-This is more explicit, and that is the point. The profile knows:
-
-- which target was selected,
-- which model should receive accounting,
-- which profile-specific decision should receive accounting,
-- which failures happened before a response existed,
-- which latency is backend time versus profile overhead.
-
-A stats helper should reduce mechanical work, not hide the lifecycle. A good helper answers,
-"how do I record this known event consistently?" A bad helper answers, "where did this event come
-from?" by forcing the reader to chase implicit middleware.
-
-## Backend Reuse
-
-V2 does not require rewriting every backend immediately. The crate currently adapts existing native
-OpenAI and Anthropic backends through a small `ProfileBackend` surface that does not expose
-`ProxyContext`.
-
-That adapter is a migration tool. The target direction is:
-
-```text
-Profile
- owns routing and lifecycle
-Backend
- owns provider call behavior
-Translation
- owns wire-format conversion
-```
-
-Backends remain a reasonable abstraction because provider calls are real reusable behavior. The
-problem v2 fixes is not "all abstractions are bad." The problem is splitting one profile's policy
-across too many orchestration objects.
-
-## Config Shape
-
-Each profile config lives beside its runtime and declares a stable serialized type. In Rust, the
-macro handles the repetitive serde and config-resolution surface:
-
-```rust
-#[profile_config("random-routing")]
-pub struct RandomRoutingProfileConfig {
- #[profile_target]
- pub strong: LlmTarget,
- #[profile_target]
- pub weak: LlmTarget,
- #[serde(default = "default_strong_probability")]
- pub strong_probability: f64,
- #[serde(default)]
- pub rng_seed: Option<u64>,
-}
-```
-
-The macro standardizes the boring parts:
-
-- clone/debug/serde derives,
-- strict unknown-field rejection,
-- a stable `PROFILE_TYPE`,
-- `profile_type()`,
-- profile-owned parsing from serialized config,
-- `#[profile_target]` resolution from config IDs into concrete `LlmTarget` values.
-
-It should not hide profile-specific meaning. Cross-field validation belongs in normal Rust when it
-is policy-specific. For example, validating duplicate latency targets is easier to review as plain
-code than as macro magic.
-
-Python should follow the same rule even if the mechanism is a decorator or class registration
-instead of a Rust proc macro. A profile's config belongs next to the profile implementation; it
-should not live in a distant central factory file.
-
-The intended config authoring rule is:
-
-```text
-shared config behavior can be macro-generated
-profile policy stays in the profile module
-```
-
-## User-Facing Config Model
-
-The v2 config loader is the user-facing entry point for profile construction. It accepts the same
-schema as YAML, JSON, or TOML, with `${VAR}` interpolation before typed deserialization:
-
-```yaml
-endpoints:
- nvidia:
- base_url: https://integrate.api.nvidia.com/v1
- api_key: ${NVIDIA_API_KEY}
- timeout_secs: 120.0
-
-# Targets are named by the model id they serve.
-targets:
- nvidia/frontier-model:
- endpoint: nvidia
- model: nvidia/frontier-model
- format: openai
-
- nvidia/fast-model:
- endpoint: nvidia
- model: nvidia/fast-model
- format: openai
-
-profiles:
- direct:
- type: passthrough
- target: nvidia/fast-model
-
- smart-stage-router:
- type: stage_router
- capable: nvidia/frontier-model
- efficient: nvidia/fast-model
- fallback_target_on_evict: nvidia/frontier-model
- picker: capable_first
- confidence_threshold: 0.7
-```
-
-The intended load path is:
-
-```text
-YAML / JSON / TOML
- |
- v
-ProfileConfigDocument
- |
- v
-ProfileConfigPlan
- |
- v
-Box<dyn Profile>
-```
-
-`ProfileConfigDocument` is the parsed file shape. It is allowed to contain config-facing
-references, such as `target: nvidia/fast-model` or `endpoint: nvidia`. It is not runtime-ready.
-
-`ProfileConfigPlan` is the resolved plan. It has concrete `LlmTarget` values and typed profile
-configs. The design deliberately does not rebuild the old graph/table/factory system. The plan is
-only the point where a parsed config has been validated enough to build one profile or all profiles.
-
-The `ProfileConfig` trait is the build contract that keeps runtime construction explicit. The macro
-can generate parsing and target-resolution glue, but every profile config still has to say how it
-turns into its runtime profile. The concrete runtime must implement the object-safe `Profile`
-runtime contract and the typed `ProfileHooks` embedding contract; config plans erase only to
-`Profile` for serving.
-
-The config split should be enforced with a trait shaped like this:
-
-```rust
-pub trait ProfileConfig: ProfileConfigDefinition {
- type Runtime: Profile + ProfileHooks + 'static;
-
- fn build(&self) -> Result<Self::Runtime>;
-
- fn build_boxed(&self) -> Result<Box<dyn Profile>> {
- Ok(Box::new(self.build()?))
- }
-}
-```
-
-That means a profile config can rely on macro-generated parsing, but it cannot skip runtime
-construction. The `#[profile_config]` macro emits a compile-time assertion that the config
-implements `ProfileConfig`, so the compiler requires each config to provide `build()`. Because
-`type Runtime: Profile + ProfileHooks`, a profile that only implements `run()` but leaves
-`process()` or `rprocess()` out does not satisfy the design; it should provide real hook behavior
-or the hook should move to a narrower capability trait.
-
-```rust
-impl ProfileConfig for RandomRoutingProfileConfig {
- type Runtime = RandomRoutingProfile;
-
- fn build(&self) -> Result<Self::Runtime> {
- let router = RandomRoutingEngine::new(
- RandomRoutingProcessorConfig::new(self.strong.clone(), self.weak.clone())
- .with_strong_probability(self.strong_probability)?
- .with_rng_seed(self.rng_seed),
- )?;
-
- Ok(RandomRoutingProfile {
- router,
- strong_backend: native_target_backend(self.strong.clone())?,
- weak_backend: native_target_backend(self.weak.clone())?,
- stats: profile_stats_accumulator(),
- })
- }
-}
-```
-
-This is deliberate. The macro handles the repetitive config surface. The profile author still owns
-the runtime decisions: validation, backend construction, poller construction, stats handles, and
-any profile-specific resources.
-
-The central config loader should own only generic work:
-
-- format detection and loading,
-- YAML/JSON/TOML parsing,
-- environment interpolation,
-- shared endpoint inheritance,
-- target resolution,
-- dispatch by profile `type`,
-- building requested profile runtimes.
-
-Profile modules own everything profile-specific:
-
-- which config fields exist,
-- which fields are target references,
-- cross-field validation,
-- backend/runtime construction,
-- profile-specific error semantics.
-
-This is why `#[profile_target]` matters. The file can say `strong: strong`, but the runtime config
-still receives a real `LlmTarget`. The profile author writes the natural runtime type, while the
-macro handles the file-facing ID indirection.
-
-The generated `profile_types!` registry is the narrow central list of supported profile config
-types. It is acceptable because it is one line per profile and only dispatches to profile-owned
-parsing/building. It should not grow into a central hand-written schema with duplicated fields.
-
-## Complete Profile Shape
-
-The Rust shape should make the full lifecycle visible in one profile module:
-
-```rust
-#[profile_config("random-routing")]
-pub struct RandomRoutingProfileConfig {
- #[profile_target]
- pub strong: LlmTarget,
- #[profile_target]
- pub weak: LlmTarget,
- pub strong_probability: f64,
- pub rng_seed: Option<u64>,
-}
-
-impl ProfileConfig for RandomRoutingProfileConfig {
- type Runtime = RandomRoutingProfile;
-
- fn build(&self) -> Result<Self::Runtime> {
- validate_probability(self.strong_probability)?;
-
- Ok(RandomRoutingProfile {
- router: RandomRoutingEngine::from_config(self)?,
- strong_backend: native_target_backend(self.strong.clone())?,
- weak_backend: native_target_backend(self.weak.clone())?,
- stats: profile_stats_accumulator(),
- })
- }
-}
-
-pub struct RandomRoutingProfile {
- router: RandomRoutingEngine,
- strong_backend: TargetBackend,
- weak_backend: TargetBackend,
- stats: StatsAccumulator,
-}
-
-pub struct RandomRoutingProcessedRequest {
- request: ChatRequest,
- decision: RandomRoutingDecision,
-}
-
-impl ProfileProcessedRequest for RandomRoutingProcessedRequest {
- fn request(&self) -> &ChatRequest {
- &self.request
- }
-}
-
-#[async_trait]
-impl ProfileHooks for RandomRoutingProfile {
- type ProcessedRequest = RandomRoutingProcessedRequest;
-
- async fn process(&self, request: ChatRequest) -> Result<Self::ProcessedRequest> {
- let (request, decision) = self.route_request(request)?;
- Ok(RandomRoutingProcessedRequest { request, decision })
- }
-
- async fn rprocess(
- &self,
- _processed: &Self::ProcessedRequest,
- response: ChatResponse,
- ) -> Result<ChatResponse> {
- Ok(response)
- }
-}
-
-#[async_trait]
-impl Profile for RandomRoutingProfile {
- async fn run(&self, request: ChatRequest) -> Result<ChatResponse> {
- let started_at = Instant::now();
- let processed = self.process(request).await?;
- let backend = self.backend_for(&processed.decision)?;
-
- let response = match backend.call(processed.request().clone()).await {
- Ok(response) => response,
- Err(error) => {
- self.record_error(&processed.decision)?;
- return Err(error);
- }
- };
-
- self.record_success_and_usage(&processed.decision, &response, started_at)?;
- self.rprocess(&processed, response).await
- }
-}
-```
-
-The equivalent Python shape should follow the same object boundary. Python may use decorators
-instead of Rust macros, but it should not return to factory dictionaries or hidden bundles:
-
-```python
-ProcessedRequestT = TypeVar("ProcessedRequestT")
-
-
-class Profile(Protocol):
- async def run(self, request: ChatRequest) -> ChatResponse: ...
-
-
-class ProfileHooks(Protocol[ProcessedRequestT]):
- async def process(self, request: ChatRequest) -> ProcessedRequestT: ...
-
- async def rprocess(
- self,
- processed: ProcessedRequestT,
- response: ChatResponse,
- ) -> ChatResponse: ...
-
-
-@profile_config("random-routing")
-@dataclass
-class RandomRoutingProfileConfig:
- strong: LlmTarget
- weak: LlmTarget
- strong_probability: float = 0.5
- rng_seed: int | None = None
-
- def build(self) -> "RandomRoutingProfile":
- validate_probability(self.strong_probability)
- return RandomRoutingProfile(
- router=RandomRoutingEngine(
- strong=self.strong,
- weak=self.weak,
- strong_probability=self.strong_probability,
- rng_seed=self.rng_seed,
- ),
- strong_backend=native_target_backend(self.strong),
- weak_backend=native_target_backend(self.weak),
- stats=profile_stats_accumulator(),
- )
-
-
-@dataclass
-class RandomRoutingProcessedRequest:
- request: ChatRequest
- decision: RandomRoutingDecision
-
-
-class RandomRoutingProfile(Profile[RandomRoutingProcessedRequest]):
- def __init__(
- self,
- *,
- router: RandomRoutingEngine,
- strong_backend: TargetBackend,
- weak_backend: TargetBackend,
- stats: StatsAccumulator,
- ) -> None:
- self.router = router
- self.strong_backend = strong_backend
- self.weak_backend = weak_backend
- self.stats = stats
-
- async def process(self, request: ChatRequest) -> RandomRoutingProcessedRequest:
- request, decision = self._route_request(request)
- return RandomRoutingProcessedRequest(request=request, decision=decision)
-
- async def run(self, request: ChatRequest) -> ChatResponse:
- started_at = monotonic()
- processed = await self.process(request)
- backend = self._backend_for(processed.decision)
-
- try:
- response = await backend.call(processed.request)
- except SwitchyardError:
- self._record_error(processed.decision)
- raise
-
- self._record_success_and_usage(processed.decision, response, started_at)
- return await self.rprocess(processed, response)
-
- async def rprocess(
- self,
- processed: "RandomRoutingProcessedRequest",
- response: ChatResponse,
- ) -> ChatResponse:
- return response
-```
-
-The language-specific syntax can differ. The architecture should not. A profile config builds one
-profile runtime, and the profile runtime owns the visible request lifecycle.
-
-## How A Request Flows In Existing Profiles
-
-### Passthrough
-
-Passthrough has no routing state, so its `run()` can be the straightforward lifecycle:
-
-```text
-process(request)
- |
- v
-PassthroughProcessedRequest { request }
- |
- v
-single target backend call
- |
- v
-record success, error, latency, and usage
- |
- v
-rprocess(processed, response)
-```
-
-This is the simplest profile and should remain the reference for a one-target policy.
-
-### Random Routing
-
-Random routing selects between configured targets. Its request-side decision belongs in its own
-processed-request type:
-
-```text
-process(request)
- |
- +--> rewritten request model
- |
- +--> RandomRoutingDecision
- |
- v
- selected backend call
- |
- v
- stats for selected decision
-```
-
-The key is that `RandomRoutingProcessedRequest` is not a generic metadata wrapper. It is the
-random-routing profile's own type. If another profile makes a different kind of request-side
-decision, it gets a different processed-request type.
-
-### Latency Service
-
-Latency-service routing owns a health cache and chooses among multiple targets:
-
-```text
-health cache
- |
- v
-process(request) selects initial target candidate
- |
- v
-call selected backend
- |
- +--> success: record usage and return
- |
- +--> failure: record error, exclude target, retry another target
-```
-
-The hot path reads cached health. Health polling is explicit through `poll_once()` or external
-health injection, which keeps serving free of accidental latency-service network calls.
-`LatencyServiceProcessedRequest` may contain the prepared request and the initial target selection,
-but the full retry loop still belongs in `run()` because retries update per-call exclusion state as
-backend calls fail.
-
-## Why This Helps Contributions
-
-V2 makes the contribution unit match the feature unit. A contributor adding a routing policy should
-be able to open one profile module and answer:
-
-- What does the config look like?
-- What gets validated?
-- How does request mutation work?
-- Which backend is called?
-- What happens on backend error?
-- What stats are recorded?
-- Can this profile be used as middleware-only?
-- What tests define the behavior?
-
-That is a smaller review surface than a graph of processors, bundles, factories, and table entries.
-
-It also makes tests more direct. Instead of testing whether factory glue assembled the right
-sequence of objects, tests can call:
-
-```rust
-let processed = profile.process(request).await?;
-profile.run(request).await?;
-profile.rprocess(&processed, response).await?;
-```
-
-This encourages adversarial tests around actual behavior:
-
-- invalid config is rejected,
-- request models are rewritten correctly,
-- selected target state does not leak across concurrent calls,
-- backend failures record stats and propagate errors,
-- retries do not hit the same failed target when alternatives remain,
-- middleware-only hooks do not accidentally call a backend.
-
-## Adding A New Profile
-
-Start with the smallest profile-owned implementation.
-
-1. Create `src/profiles/<name>.rs` or `src/profiles/<name>/mod.rs`.
-2. Define `<Name>ProfileConfig` with `#[profile_config("<serialized-type>")]`.
-3. Mark config fields that refer to targets with `#[profile_target]`.
-4. Implement the `ProfileConfig` build contract for the config type.
-5. Put profile-specific validation near `build()`.
-6. Build the runtime from fully resolved targets and existing backend helpers.
-7. Define `<Name>ProcessedRequest` for the output of `process()`.
-8. Include the prepared `ChatRequest` and any profile-specific request-side decision in that type.
-9. Implement `ProfileHooks` and `Profile` for `<Name>Profile`.
-10. Keep retry loops, backend calls, and mutable per-call state local to `run()`.
-11. Add tests for config validation, hook behavior, full `run()` behavior, stats, errors, and
- concurrency if the profile has mutable state.
-12. Add the config type to the generated profile type list.
-13. Export the config and runtime from `src/profiles/mod.rs` and `src/lib.rs`.
-
-For Python-defined profiles, the same shape applies:
-
-1. Put the profile config and runtime in a profile module, not in a global factory directory.
-2. Use the shared profile registration path so config loading can discover it.
-3. Keep validation and target resolution close to the profile.
-4. Define a typed processed-request object for the profile's `process()` output.
-5. Implement the same `process`, `run`, and `rprocess(processed, response)` lifecycle.
-6. Make config fields explicit and typed rather than accepting arbitrary factory dictionaries.
-7. Delete old factory or bundle glue instead of wrapping it indefinitely.
-
-The first version should be boring. Add private helpers only when they make the profile easier to
-read. Do not add a new public abstraction because two profiles happen to share five lines.
-
-## What Not To Add Back
-
-Avoid reintroducing old shapes under new names:
-
-- A "profile graph" that must be built before a profile can run.
-- A "profile bundle" that only wraps request and response hooks.
-- A hidden stats layer that records without being visible in the profile's `run()`.
-- A context object whose main job is carrying state between self-owned stages.
-- A generic "profile metadata" object used as a new untyped side channel.
-- A central factory registry that every new profile has to touch manually.
-
-If a new helper is needed, prefer one of these forms:
-
-- a private function inside one profile module,
-- a small shared function with no lifecycle ownership,
-- a reusable engine that performs pure policy logic,
-- a backend adapter that only owns provider call mechanics.
-
-## Migration Boundary
-
-`switchyard-components-v2` is intentionally separate from the older
-`switchyard-components` crate while the shape settles. During migration, v2 can reuse proven
-engines and backend code from the older crate, but new profile code should not adopt the older
-public shape.
-
-The same applies to Python. The Python package can keep working while profile v2 lands, but the
-target state is not "Rust has profiles while Python keeps factories." The target state is one
-profile model, with Python using bindings or Python-defined profiles that obey the same lifecycle.
-
-The practical migration stance is:
-
-```text
-preserve behavior
-do not preserve unnecessary object structure
-do not preserve backward compatibility by default
-```
-
-This means a v2 profile can reuse a mature random-routing engine, stats accumulator, or native
-backend implementation while still presenting a flatter runtime surface.
-
-## Staleness Risks
-
-This document will need updates when:
-
-- the temporary `ProfileBackend` adapter is replaced by native v2 backend ownership,
-- config loading for v2 profiles becomes the primary server path,
-- Python bindings expose v2 profiles directly or Python-defined profiles adopt the same lifecycle,
-- stats recording moves from explicit profile calls to a deliberate helper with an equally visible
- lifecycle,
-- new profiles introduce a legitimate pattern that this document does not cover.
-
-## PyO3 Binding Caveat
-
-The associated `ProcessedRequest` type is deliberately good Rust, but it is not directly
-object-safe across PyO3. Generic Python bindings should therefore split the surface the same way the
-Rust traits do.
-
-The serving path is easy to expose generically: Python can hold a `Profile` wrapper backed by
-`Arc<dyn Profile>` and call `run(request)` for any Rust profile returned by config loading.
-That is the right default for servers, launchers, and generic profile tables.
-
-The hook path needs one more layer. Python cannot call a fully generic `dyn ProfileHooks` without
-erasing the associated processed-request type. Bindings should expose either:
-
-- an opaque `ProcessedRequest` Python handle backed by an erased Rust wrapper, with a `.request`
- accessor for caller-owned model calls, or
-- macro-generated concrete Python classes for each registered profile and its concrete
- `<Name>ProcessedRequest` type.
-
-Both approaches keep the core invariant: the processed-request value is typed and profile-owned in
-Rust. Python should not receive a generic metadata dictionary, and `switchyard-components-v2`
-should not depend on PyO3. PyO3 registration belongs in `switchyard-py`, ideally generated from the
-same profile registry that lists the supported profile config types.
diff --git a/crates/switchyard-components-v2/src/backend.rs b/crates/switchyard-components-v2/src/backend.rs
deleted file mode 100644
index e61224d06..000000000
--- a/crates/switchyard-components-v2/src/backend.rs
+++ /dev/null
@@ -1,74 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Backend helpers for profile-owned runtimes.
-
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use switchyard_components::backends::{AnthropicNativeBackend, OpenAiNativeBackend};
-use switchyard_core::{
- BackendFormat, ChatRequest, ChatResponse, LlmTarget, Result, SwitchyardError,
-};
-
-/// Backend trait used by v2 profiles without exposing `ProxyContext`.
-#[async_trait]
-pub trait ProfileBackend: Send + Sync {
- /// Calls a backend with an already-prepared request.
- async fn call(&self, request: &ChatRequest) -> Result<ChatResponse>;
-}
-
-/// One target and the backend that serves it.
-#[derive(Clone)]
-pub struct TargetBackend {
- target: LlmTarget,
- backend: Arc<dyn ProfileBackend>,
-}
-
-impl TargetBackend {
- /// Creates a target/backend pair.
- pub fn new(target: LlmTarget, backend: Arc<dyn ProfileBackend>) -> Self {
- Self { target, backend }
- }
-
- /// Returns the target metadata.
- pub fn target(&self) -> &LlmTarget {
- &self.target
- }
-
- /// Calls the target backend with a request prepared by the profile.
- pub async fn call(&self, request: &ChatRequest) -> Result<ChatResponse> {
- self.backend.call(request).await
- }
-}
-
-/// Builds the existing native backend for one fully-resolved target.
-pub(crate) fn native_target_backend(target: LlmTarget) -> Result<TargetBackend> {
- let backend: Arc<dyn ProfileBackend> = match target.format {
- BackendFormat::OpenAi | BackendFormat::Responses => {
- Arc::new(OpenAiNativeBackend::new(target.clone())?)
- }
- BackendFormat::Anthropic => Arc::new(AnthropicNativeBackend::new(target.clone())?),
- BackendFormat::Auto => {
- return Err(SwitchyardError::InvalidConfig(format!(
- "target {} must have a resolved backend format",
- target.id
- )));
- }
- };
- Ok(TargetBackend::new(target, backend))
-}
-
-#[async_trait]
-impl ProfileBackend for OpenAiNativeBackend {
- async fn call(&self, request: &ChatRequest) -> Result<ChatResponse> {
- self.call_without_context(request).await
- }
-}
-
-#[async_trait]
-impl ProfileBackend for AnthropicNativeBackend {
- async fn call(&self, request: &ChatRequest) -> Result<ChatResponse> {
- self.call_without_context(request).await
- }
-}
diff --git a/crates/switchyard-components-v2/src/config.rs b/crates/switchyard-components-v2/src/config.rs
deleted file mode 100644
index 95e60ccab..000000000
--- a/crates/switchyard-components-v2/src/config.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Profile config entry point split by lifecycle stage.
-//!
-//! Parsing produces a file-facing `ProfileConfigDocument`. Resolving turns that
-//! document into an opaque `ProfileConfigPlan` with runtime-ready targets and
-//! validated profile bodies.
-
-mod loading;
-mod parsing;
-mod resolving;
-
-pub use loading::{parse_profile_config_path, ProfileConfigFormat};
-pub use parsing::{
- parse_profile_config_str, parse_profile_config_str_with_env_lookup, ProfileConfigDocument,
-};
-pub use resolving::{ProfileBuildEnv, ProfileConfigDefinition};
-pub use resolving::{ProfileConfig, ProfileConfigPlan};
diff --git a/crates/switchyard-components-v2/src/config/loading.rs b/crates/switchyard-components-v2/src/config/loading.rs
deleted file mode 100644
index a36aaf550..000000000
--- a/crates/switchyard-components-v2/src/config/loading.rs
+++ /dev/null
@@ -1,65 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Filesystem loading and format detection for profile configs.
-
-use std::fs;
-use std::path::Path;
-
-use switchyard_core::{Result, SwitchyardError};
-
-use super::parsing::ProfileConfigDocument;
-
-/// File format accepted by the v2 profile config loader.
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum ProfileConfigFormat {
- /// JSON configuration.
- Json,
- /// TOML configuration.
- Toml,
- /// YAML configuration.
- Yaml,
-}
-
-impl ProfileConfigFormat {
- /// Infers the config format from a path extension.
- pub fn from_path(path: impl AsRef<Path>) -> Result<Self> {
- let extension = path
- .as_ref()
- .extension()
- .and_then(|extension| extension.to_str())
- .ok_or_else(|| {
- SwitchyardError::InvalidConfig(format!(
- "profile config path {} has no file extension",
- path.as_ref().display()
- ))
- })?;
- match extension {
- "json" => Ok(Self::Json),
- "toml" => Ok(Self::Toml),
- "yaml" | "yml" => Ok(Self::Yaml),
- other => Err(SwitchyardError::InvalidConfig(format!(
- "unsupported profile config extension `{other}`"
- ))),
- }
- }
-}
-
-impl ProfileConfigDocument {
- /// Reads a profile config path and infers the format from its extension.
- pub fn from_path(path: impl AsRef<Path>) -> Result<Self> {
- let path = path.as_ref();
- let input = fs::read_to_string(path).map_err(|error| {
- SwitchyardError::InvalidConfig(format!(
- "failed to read profile config {}: {error}",
- path.display()
- ))
- })?;
- Self::from_str(&input, ProfileConfigFormat::from_path(path)?)
- }
-}
-
-/// Reads and parses a profile config path, inferring the format from extension.
-pub fn parse_profile_config_path(path: impl AsRef<Path>) -> Result<ProfileConfigDocument> {
- ProfileConfigDocument::from_path(path)
-}
diff --git a/crates/switchyard-components-v2/src/config/parsing.rs b/crates/switchyard-components-v2/src/config/parsing.rs
deleted file mode 100644
index e20fbb42a..000000000
--- a/crates/switchyard-components-v2/src/config/parsing.rs
+++ /dev/null
@@ -1,312 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! String parsing and file-facing schema for profile configs.
-
-use std::collections::{BTreeMap, BTreeSet};
-
-use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer};
-use serde_json::Value;
-use switchyard_core::{
- BackendFormat, EndpointConfig, EndpointId, LlmTargetId, ModelId, ProfileId, Result,
- SwitchyardError,
-};
-
-use super::loading::ProfileConfigFormat;
-
-/// User-facing profile config document produced by parsing.
-///
-/// This is intentionally not runtime-ready yet. Endpoint inheritance, target
-/// references, and profile-owned validation happen in the resolving stage.
-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct ProfileConfigDocument {
- /// Shared endpoint definitions referenced by targets.
- #[serde(default)]
- pub(super) endpoints: BTreeMap<EndpointId, EndpointConfig>,
- /// Concrete targets keyed by their stable target IDs.
- #[serde(default)]
- pub(super) targets: BTreeMap<LlmTargetId, TargetConfig>,
- /// Named serialized profile bodies keyed by the model/profile ID exposed by serving.
- #[serde(default)]
- pub(super) profiles: BTreeMap<ProfileId, SerializedProfileConfig>,
-}
-
-impl ProfileConfigDocument {
- /// Parses a profile config string and interpolates `${VAR}` environment values.
- pub fn from_str(input: &str, format: ProfileConfigFormat) -> Result<Self> {
- parse_profile_config_str_with_env_lookup(input, format, |name| std::env::var(name).ok())
- }
-
- /// Iterates parsed profile IDs before profile-owned validation.
- pub fn profile_ids(&self) -> impl Iterator<Item = &ProfileId> {
- self.profiles.keys()
- }
-
- /// Returns the serialized type discriminator for one parsed profile.
- pub fn profile_type(&self, profile_id: &ProfileId) -> Option<&str> {
- self.profiles
- .get(profile_id)
- .map(SerializedProfileConfig::profile_type)
- }
-
- /// Returns the profile-owned config body without the `type` discriminator.
- pub fn profile_body(&self, profile_id: &ProfileId) -> Option<&Value> {
- self.profiles
- .get(profile_id)
- .map(SerializedProfileConfig::body)
- }
-
- /// Returns a profile's envelope `subagent_target` reference, if configured.
- pub fn profile_subagent_target(&self, profile_id: &ProfileId) -> Option<&LlmTargetId> {
- self.profiles
- .get(profile_id)
- .and_then(SerializedProfileConfig::subagent_target)
- }
-
- /// Returns a copy of this document with the selected profiles removed.
- pub fn without_profiles(&self, profile_ids: &[ProfileId]) -> Self {
- let omitted = profile_ids.iter().collect::<BTreeSet<_>>();
- let mut document = self.clone();
- document
- .profiles
- .retain(|profile_id, _profile| !omitted.contains(profile_id));
- document
- }
-}
-
-/// Serialized profile body split into a type discriminator, common envelope
-/// fields, and profile-owned fields.
-#[derive(Clone, Debug, PartialEq)]
-pub(super) struct SerializedProfileConfig {
- profile_type: String,
- subagent_target: Option<LlmTargetId>,
- body: Value,
-}
-
-impl SerializedProfileConfig {
- /// Creates a serialized profile config after verifying the body is an object.
- fn new(
- profile_type: impl Into<String>,
- subagent_target: Option<LlmTargetId>,
- body: Value,
- ) -> Result<Self> {
- let profile_type = profile_type.into();
- if profile_type.trim().is_empty() {
- return Err(SwitchyardError::InvalidConfig(
- "profile config `type` must not be empty".to_string(),
- ));
- }
- if !body.is_object() {
- return Err(SwitchyardError::InvalidConfig(
- "profile config body must be an object".to_string(),
- ));
- }
- Ok(Self {
- profile_type,
- subagent_target,
- body,
- })
- }
-
- /// Returns the profile type discriminator from the file's `type` field.
- pub(super) fn profile_type(&self) -> &str {
- &self.profile_type
- }
-
- /// Returns the envelope `subagent_target` reference from the file, if any.
- pub(super) fn subagent_target(&self) -> Option<&LlmTargetId> {
- self.subagent_target.as_ref()
- }
-
- /// Returns the profile-owned config fields without the envelope fields.
- pub(super) fn body(&self) -> &Value {
- &self.body
- }
-}
-
-impl<'de> Deserialize<'de> for SerializedProfileConfig {
- fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
- where
- D: Deserializer<'de>,
- {
- let mut value = Value::deserialize(deserializer)?;
- let Value::Object(map) = &mut value else {
- return Err(de::Error::custom("profile config must be an object"));
- };
- let profile_type = map
- .remove("type")
- .ok_or_else(|| de::Error::custom("profile config requires a `type` field"))?;
- let Value::String(profile_type) = profile_type else {
- return Err(de::Error::custom("profile config `type` must be a string"));
- };
- // `subagent_target` is a common envelope field consumed here, like `type`,
- // so profile-owned bodies (which deny unknown fields) never see it.
- let subagent_target = match map.remove("subagent_target") {
- None => None,
- Some(Value::String(target_id)) => {
- Some(LlmTargetId::new(target_id).map_err(de::Error::custom)?)
- }
- Some(_) => {
- return Err(de::Error::custom(
- "profile config `subagent_target` must be a target id string",
- ));
- }
- };
- SerializedProfileConfig::new(profile_type, subagent_target, value)
- .map_err(de::Error::custom)
- }
-}
-
-impl Serialize for SerializedProfileConfig {
- fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
- where
- S: Serializer,
- {
- let Value::Object(mut map) = self.body.clone() else {
- return Err(ser::Error::custom("profile config body must be an object"));
- };
- map.insert("type".to_string(), Value::String(self.profile_type.clone()));
- if let Some(subagent_target) = &self.subagent_target {
- map.insert(
- "subagent_target".to_string(),
- Value::String(subagent_target.as_str().to_owned()),
- );
- }
- Value::Object(map).serialize(serializer)
- }
-}
-
-/// File-facing target config that can inherit from a shared endpoint.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct TargetConfig {
- /// Optional shared endpoint ID to inherit connection settings from.
- #[serde(default)]
- pub endpoint: Option<EndpointId>,
- /// Upstream model name sent to the provider.
- pub model: ModelId,
- /// Wire format expected by the upstream target.
- pub format: BackendFormat,
- /// Target-local base URL override.
- #[serde(default)]
- pub base_url: Option<String>,
- /// Target-local API key override.
- #[serde(default)]
- pub api_key: Option<String>,
- /// Target-local timeout override in seconds.
- #[serde(default)]
- pub timeout_secs: Option<f64>,
- /// Per-target outbound request body extensions.
- #[serde(default, skip_serializing_if = "Option::is_none")]
- pub extra_body: Option<Value>,
- /// Per-target outbound request header extensions.
- #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
- pub extra_headers: BTreeMap<String, String>,
-}
-
-/// Parses a profile config string with the requested format.
-pub fn parse_profile_config_str(
- input: &str,
- format: ProfileConfigFormat,
-) -> Result<ProfileConfigDocument> {
- ProfileConfigDocument::from_str(input, format)
-}
-
-/// Parses a profile config string using a caller-provided environment lookup.
-///
-/// This is useful for embedders and tests that need deterministic interpolation
-/// without mutating process-global environment variables.
-pub fn parse_profile_config_str_with_env_lookup(
- input: &str,
- format: ProfileConfigFormat,
- lookup: impl Fn(&str) -> Option<String>,
-) -> Result<ProfileConfigDocument> {
- parse_profile_config_str_with_lookup(input, format, lookup)
-}
-
-fn parse_profile_config_str_with_lookup(
- input: &str,
- format: ProfileConfigFormat,
- lookup: impl Fn(&str) -> Option<String>,
-) -> Result<ProfileConfigDocument> {
- let mut value = parse_value(input, format)?;
- interpolate_value(&mut value, &lookup)?;
- serde_json::from_value(value).map_err(|error| {
- SwitchyardError::InvalidConfig(format!("failed to parse profile config: {error}"))
- })
-}
-
-fn parse_value(input: &str, format: ProfileConfigFormat) -> Result<Value> {
- match format {
- ProfileConfigFormat::Json => serde_json::from_str(input).map_err(|error| {
- SwitchyardError::InvalidConfig(format!("failed to parse JSON profile config: {error}"))
- }),
- ProfileConfigFormat::Toml => {
- let value = toml::from_str::<toml::Value>(input).map_err(|error| {
- SwitchyardError::InvalidConfig(format!(
- "failed to parse TOML profile config: {error}"
- ))
- })?;
- serde_json::to_value(value).map_err(|error| {
- SwitchyardError::InvalidConfig(format!(
- "failed to normalize TOML profile config: {error}"
- ))
- })
- }
- ProfileConfigFormat::Yaml => yaml_serde::from_str(input).map_err(|error| {
- SwitchyardError::InvalidConfig(format!("failed to parse YAML profile config: {error}"))
- }),
- }
-}
-
-fn interpolate_value(value: &mut Value, lookup: &impl Fn(&str) -> Option<String>) -> Result<()> {
- match value {
- Value::String(text) => {
- *text = interpolate_string(text, lookup)?;
- }
- Value::Array(items) => {
- for item in items {
- interpolate_value(item, lookup)?;
- }
- }
- Value::Object(map) => {
- for item in map.values_mut() {
- interpolate_value(item, lookup)?;
- }
- }
- Value::Null | Value::Bool(_) | Value::Number(_) => {}
- }
- Ok(())
-}
-
-fn interpolate_string(input: &str, lookup: &impl Fn(&str) -> Option<String>) -> Result<String> {
- let mut output = String::new();
- let mut rest = input;
-
- while let Some(start) = rest.find("${") {
- output.push_str(&rest[..start]);
- let after_start = &rest[start + 2..];
- let Some(end) = after_start.find('}') else {
- return Err(SwitchyardError::InvalidConfig(format!(
- "unterminated environment variable in `{input}`"
- )));
- };
- let name = &after_start[..end];
- if name.is_empty() {
- return Err(SwitchyardError::InvalidConfig(
- "empty environment variable reference".to_string(),
- ));
- }
- let value = lookup(name).ok_or_else(|| {
- SwitchyardError::InvalidConfig(format!(
- "environment variable {name} is not set for profile config interpolation"
- ))
- })?;
- output.push_str(&value);
- rest = &after_start[end + 1..];
- }
-
- output.push_str(rest);
- Ok(output)
-}
diff --git a/crates/switchyard-components-v2/src/config/resolving.rs b/crates/switchyard-components-v2/src/config/resolving.rs
deleted file mode 100644
index b019e0c74..000000000
--- a/crates/switchyard-components-v2/src/config/resolving.rs
+++ /dev/null
@@ -1,351 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Resolution of parsed profile configs into runtime-ready objects.
-
-use std::{collections::BTreeMap, fmt};
-
-use serde_json::Value;
-use switchyard_core::{
- EndpointConfig, EndpointId, LlmTarget, LlmTargetId, ProfileId, Result, SwitchyardError,
-};
-
-use super::parsing::{ProfileConfigDocument, SerializedProfileConfig, TargetConfig};
-use crate::profiles::{ProfileConfigEntry, SubagentOverrideProfile};
-use crate::{Profile, ProfileHooks};
-
-impl ProfileConfigDocument {
- /// Resolves endpoints and validates every profile through its owning profile type.
- pub fn resolve(&self) -> Result<ProfileConfigPlan> {
- let targets = self.resolve_targets()?;
- let (profiles, profile_subagent_targets) = self.resolve_profiles(&targets)?;
- Ok(ProfileConfigPlan {
- targets,
- profiles,
- profile_subagent_targets,
- })
- }
-
- // Resolves all file-facing targets before profile-specific config expansion.
- fn resolve_targets(&self) -> Result<BTreeMap<LlmTargetId, LlmTarget>> {
- let mut targets = BTreeMap::new();
- for (target_id, target) in &self.targets {
- targets.insert(
- target_id.clone(),
- target.resolve(target_id, &self.endpoints)?,
- );
- }
- Ok(targets)
- }
-
- // Parses profile-specific config bodies into the generated typed profile enum.
- // Also collects resolved sub-agent targets for profiles that declare one.
- fn resolve_profiles(
- &self,
- targets: &BTreeMap<LlmTargetId, LlmTarget>,
- ) -> Result<(
- BTreeMap<ProfileId, ProfileConfigEntry>,
- BTreeMap<ProfileId, LlmTarget>,
- )> {
- let env = ProfileBuildEnv::new(targets);
- let mut profiles = BTreeMap::new();
- let mut subagent_targets: BTreeMap<ProfileId, LlmTarget> = BTreeMap::new();
- for (profile_id, profile) in &self.profiles {
- // The envelope-level sub-agent override must name a resolvable target;
- // an unknown reference is a startup configuration error.
- if let Some(target_id) = profile.subagent_target() {
- let target = targets.get(target_id).ok_or_else(|| {
- SwitchyardError::InvalidConfig(format!(
- "profile {profile_id}: subagent_target references unknown target {}",
- target_id.as_str()
- ))
- })?;
- subagent_targets.insert(profile_id.clone(), target.clone());
- }
- profiles.insert(
- profile_id.clone(),
- resolve_profile(profile_id, profile, &env)?,
- );
- }
- Ok((profiles, subagent_targets))
- }
-}
-
-fn resolve_profile(
- profile_id: &ProfileId,
- profile: &SerializedProfileConfig,
- env: &ProfileBuildEnv<'_>,
-) -> Result<ProfileConfigEntry> {
- crate::profiles::parse_profile_config(profile.profile_type(), profile.body().clone(), env)
- .map_err(|error| SwitchyardError::InvalidConfig(format!("profile {profile_id}: {error}")))
-}
-
-/// Target lookup environment used while profile-owned configs are parsed.
-pub struct ProfileBuildEnv<'a> {
- targets: &'a BTreeMap<LlmTargetId, LlmTarget>,
-}
-
-impl<'a> ProfileBuildEnv<'a> {
- /// Creates a build environment over already-resolved targets.
- pub fn new(targets: &'a BTreeMap<LlmTargetId, LlmTarget>) -> Self {
- Self { targets }
- }
-
- /// Resolves a profile-local target ID into a concrete runtime target.
- pub fn target(&self, target_id: &LlmTargetId) -> Result<&'a LlmTarget> {
- self.targets.get(target_id).ok_or_else(|| {
- SwitchyardError::InvalidConfig(format!("profile references unknown target {target_id}"))
- })
- }
-}
-
-/// Trait implemented by the profile config macro for profile-owned parsing.
-pub trait ProfileConfigDefinition: Sized {
- /// Stable discriminator used in the file-facing `type` field.
- const PROFILE_TYPE: &'static str;
-
- /// Parses profile-owned fields and resolves any `#[profile_target]` references.
- fn parse_profile_config(value: Value, env: &ProfileBuildEnv<'_>) -> Result<Self>;
-}
-
-/// Public contract implemented by every profile config type.
-///
-/// The `profile_config` attribute macro generates parsing metadata, but the
-/// runtime build remains explicit here so profile authors can construct
-/// backends, pollers, stats handles, and validation in normal Rust.
-pub trait ProfileConfig: ProfileConfigDefinition {
- /// Runtime profile produced by this config.
- type Runtime: Profile + ProfileHooks + 'static;
-
- /// Builds this config into its runtime profile.
- fn build(&self) -> Result<Self::Runtime>;
-
- /// Builds this config into the object-safe profile used by config plans.
- fn build_boxed(&self) -> Result<Box<dyn Profile>> {
- Ok(Box::new(self.build()?))
- }
-}
-
-impl TargetConfig {
- // Resolves endpoint inheritance while preserving the map key as the target ID.
- fn resolve(
- &self,
- target_id: &LlmTargetId,
- endpoints: &BTreeMap<EndpointId, EndpointConfig>,
- ) -> Result<LlmTarget> {
- let inherited = match &self.endpoint {
- Some(endpoint_id) => endpoints.get(endpoint_id).ok_or_else(|| {
- SwitchyardError::InvalidConfig(format!(
- "target {target_id} references unknown endpoint {endpoint_id}"
- ))
- })?,
- None => &EndpointConfig::default(),
- };
- let overrides = EndpointConfig {
- base_url: self.base_url.clone(),
- api_key: self.api_key.clone(),
- timeout_secs: self.timeout_secs,
- };
- Ok(LlmTarget {
- id: target_id.clone(),
- model: self.model.clone(),
- format: self.format,
- endpoint: inherited.with_overrides(&overrides),
- extra_body: self.extra_body.clone(),
- extra_headers: self.extra_headers.clone(),
- })
- }
-}
-
-/// Validated profile config plan ready to build runtime profiles.
-///
-/// This type contains resolved targets and typed profile config bodies. It
-/// deliberately does not contain built `Profile` runtimes; call
-/// `build_profile` or `build_profiles` when runtime ownership is needed.
-#[derive(Clone, PartialEq)]
-pub struct ProfileConfigPlan {
- /// Resolved targets keyed by the IDs used in the parsed config.
- targets: BTreeMap<LlmTargetId, LlmTarget>,
- /// Typed profile configs keyed by the user-facing profile ID.
- profiles: BTreeMap<ProfileId, ProfileConfigEntry>,
- /// Resolved sub-agent override targets for profiles that declare one.
- profile_subagent_targets: BTreeMap<ProfileId, LlmTarget>,
-}
-
-impl fmt::Debug for ProfileConfigPlan {
- fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
- // Avoid printing resolved target/profile configs because they can carry API keys.
- let profile_types = self
- .profiles
- .iter()
- .map(|(profile_id, profile)| (profile_id, profile.profile_type()))
- .collect::<BTreeMap<_, _>>();
- formatter
- .debug_struct("ProfileConfigPlan")
- .field("targets", &self.targets.keys().collect::<Vec<_>>())
- .field("profiles", &profile_types)
- .finish()
- }
-}
-
-impl ProfileConfigPlan {
- /// Returns the number of resolved targets in the plan.
- pub fn target_count(&self) -> usize {
- self.targets.len()
- }
-
- /// Returns the number of validated profile configs in the plan.
- pub fn profile_count(&self) -> usize {
- self.profiles.len()
- }
-
- /// Returns a resolved target by ID.
- pub fn target(&self, target_id: &LlmTargetId) -> Option<&LlmTarget> {
- self.targets.get(target_id)
- }
-
- /// Iterates resolved targets in deterministic target-ID order.
- pub fn targets(&self) -> impl Iterator<Item = (&LlmTargetId, &LlmTarget)> {
- self.targets.iter()
- }
-
- /// Iterates profile IDs in deterministic profile-ID order.
- pub fn profile_ids(&self) -> impl Iterator<Item = &ProfileId> {
- self.profiles.keys()
- }
-
- /// Returns the serialized type discriminator for one profile config.
- pub fn profile_type(&self, profile_id: &ProfileId) -> Option<&str> {
- self.profiles
- .get(profile_id)
- .map(ProfileConfigEntry::profile_type)
- }
-
- /// Builds one profile runtime by profile ID.
- pub fn build_profile(&self, profile_id: &ProfileId) -> Result<Box<dyn Profile>> {
- let profile = self.profiles.get(profile_id).ok_or_else(|| {
- SwitchyardError::InvalidConfig(format!("unknown profile {profile_id}"))
- })?;
- self.build_typed_profile(profile_id, profile)
- }
-
- /// Builds every profile in the document into an object-safe runtime profile.
- pub fn build_profiles(&self) -> Result<BTreeMap<ProfileId, Box<dyn Profile>>> {
- let mut profiles = BTreeMap::new();
- for (profile_id, profile) in &self.profiles {
- profiles.insert(
- profile_id.clone(),
- self.build_typed_profile(profile_id, profile)?,
- );
- }
- Ok(profiles)
- }
-
- /// Returns every resolved target as a directly addressable serving target.
- pub fn exposed_targets(&self) -> impl Iterator<Item = &LlmTarget> {
- self.targets.values()
- }
-
- // Builds one typed profile config with profile-ID context in errors.
- // Wraps with SubagentOverrideProfile when the profile declares a subagent_target.
- fn build_typed_profile(
- &self,
- profile_id: &ProfileId,
- profile: &ProfileConfigEntry,
- ) -> Result<Box<dyn Profile>> {
- let inner = profile.build_boxed().map_err(|error| {
- SwitchyardError::InvalidConfig(format!("profile {profile_id}: {error}"))
- })?;
- if let Some(target) = self.profile_subagent_targets.get(profile_id) {
- SubagentOverrideProfile::new(inner, target.clone())
- .map(|p| Box::new(p) as Box<dyn Profile>)
- .map_err(|error| {
- SwitchyardError::InvalidConfig(format!(
- "profile {profile_id}: subagent override: {error}"
- ))
- })
- } else {
- Ok(inner)
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use std::collections::BTreeMap;
-
- use async_trait::async_trait;
- use serde_json::json;
- use switchyard_core::{BackendFormat, ChatResponse, ModelId};
-
- use super::*;
-
- #[switchyard_components_v2_macros::profile_config("optional-target-test")]
- struct OptionalTargetConfig {
- /// Optional target used to test macro-generated `Option<LlmTarget>` resolution.
- #[profile_target]
- pub target: Option<LlmTarget>,
- }
-
- struct OptionalTargetProfile;
-
- impl ProfileConfig for OptionalTargetConfig {
- type Runtime = OptionalTargetProfile;
-
- fn build(&self) -> Result<Self::Runtime> {
- Ok(OptionalTargetProfile)
- }
- }
-
- #[async_trait]
- impl crate::ProfileHooks for OptionalTargetProfile {
- type ProcessedRequest = crate::ProfileInput;
-
- async fn process(&self, input: crate::ProfileInput) -> Result<Self::ProcessedRequest> {
- Ok(input)
- }
-
- async fn rprocess(
- &self,
- _processed: &Self::ProcessedRequest,
- response: ChatResponse,
- ) -> Result<ChatResponse> {
- Ok(response)
- }
- }
-
- #[async_trait]
- impl crate::Profile for OptionalTargetProfile {
- async fn run(&self, input: crate::ProfileInput) -> Result<crate::ProfileResponse> {
- Ok(ChatResponse::openai_completion(json!({
- "model": input.request.model(),
- "choices": []
- }))
- .into())
- }
- }
-
- fn target(id: &str, model: &str) -> Result<LlmTarget> {
- let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
- target.format = BackendFormat::OpenAi;
- Ok(target)
- }
-
- #[test]
- fn optional_target_fields_are_resolved_by_the_profile_config_macro() -> Result<()> {
- let weak = target("weak", "weak/model")?;
- let targets = BTreeMap::from([(weak.id.clone(), weak)]);
- let env = ProfileBuildEnv::new(&targets);
-
- let with_target =
- OptionalTargetConfig::parse_profile_config(json!({"target": "weak"}), &env)?;
- assert_eq!(
- with_target.target.map(|target| target.id),
- Some(LlmTargetId::new("weak")?)
- );
-
- let without_target = OptionalTargetConfig::parse_profile_config(json!({}), &env)?;
- assert_eq!(without_target.target, None);
- Ok(())
- }
-}
diff --git a/crates/switchyard-components-v2/src/lib.rs b/crates/switchyard-components-v2/src/lib.rs
deleted file mode 100644
index bfeaa30ae..000000000
--- a/crates/switchyard-components-v2/src/lib.rs
+++ /dev/null
@@ -1,44 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Experimental components-v2 crate for the flatter profile-owned design.
-//!
-//! This crate is intentionally separate from `switchyard-components` so the
-//! rewrite shape can evolve without contaminating the existing production config surface.
-
-extern crate self as switchyard_components_v2;
-
-mod backend;
-mod config;
-mod profile;
-pub mod profiles;
-mod stats;
-mod stats_recording;
-
-pub use config::{
- parse_profile_config_path, parse_profile_config_str, parse_profile_config_str_with_env_lookup,
- ProfileConfig, ProfileConfigDocument, ProfileConfigFormat, ProfileConfigPlan,
-};
-pub use profile::{
- Profile, ProfileHooks, ProfileInput, ProfileResponse, RequestMetadata, RoutingMetadata,
-};
-pub use profiles::{
- EndpointHealth, EndpointHealthStatus, LatencyServiceProcessedRequest, LatencyServiceProfile,
- LatencyServiceProfileConfig, LlmRoutingDecision, LlmRoutingProcessedRequest, LlmRoutingProfile,
- LlmRoutingProfileConfig, LlmRoutingTierMapping, NoopProfile, NoopProfileConfig,
- PassthroughProfile, PassthroughProfileConfig, RandomRoutingProcessedRequest,
- RandomRoutingProfile, RandomRoutingProfileConfig, SelectedTarget, StageRouterClassifierConfig,
- StageRouterDecision, StageRouterDecisionSource, StageRouterPickerMode,
- StageRouterProcessedRequest, StageRouterProfile, StageRouterProfileConfig, StageRouterTier,
-};
-pub use stats::profile_stats_accumulator;
-pub use switchyard_components_v2_macros::profile_config;
-
-/// Implementation details used by generated profile-config code.
-///
-/// This namespace is public only so proc-macro expansion has a stable path.
-/// It is not part of the user-facing components-v2 API.
-#[doc(hidden)]
-pub mod __private {
- pub use crate::config::{ProfileBuildEnv, ProfileConfigDefinition};
-}
diff --git a/crates/switchyard-components-v2/src/profile.rs b/crates/switchyard-components-v2/src/profile.rs
deleted file mode 100644
index 396ef580a..000000000
--- a/crates/switchyard-components-v2/src/profile.rs
+++ /dev/null
@@ -1,208 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Profile runtime contracts for components-v2.
-
-use async_trait::async_trait;
-use serde::{Deserialize, Serialize};
-use std::collections::BTreeMap;
-
-use switchyard_core::{ChatRequest, ChatRequestType, ChatResponse, RequestId, Result};
-
-/// Explicit per-request metadata passed to v2 profiles.
-///
-/// Header keys are stored as lowercase ASCII names. Repeated header values are
-/// preserved in the original value order because endpoints may need to pass
-/// through multi-value headers without silently collapsing them.
-#[derive(Clone, Default, Eq, PartialEq)]
-pub struct RequestMetadata {
- /// Caller-provided request identifier, when present.
- pub request_id: Option<RequestId>,
- /// Inbound wire format recorded by the endpoint, when present.
- pub inbound_format: Option<ChatRequestType>,
- /// Request headers supplied by the caller, keyed by normalized header name.
- pub headers: BTreeMap<String, Vec<String>>,
-}
-
-/// Input object handed to v2 profiles.
-///
-/// This is the only request carrier in the v2 profile path. It owns the
-/// provider-neutral [`ChatRequest`] plus endpoint-supplied metadata such as
-/// request IDs, inbound format, and headers. Policy decisions remain local to
-/// the profile implementation instead of being hidden in this input object.
-#[derive(Clone, PartialEq)]
-pub struct ProfileInput {
- /// Provider-neutral chat request entering the profile.
- pub request: ChatRequest,
- /// Endpoint-supplied request metadata.
- pub metadata: RequestMetadata,
-}
-
-/// Routing decision metadata emitted by profiles that select among targets.
-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
-pub struct RoutingMetadata {
- /// Upstream model selected by the router.
- pub selected_model: Option<String>,
- /// Routing tier selected by the router.
- pub selected_tier: Option<String>,
- /// Router confidence for the selected decision.
- pub confidence: Option<f64>,
- /// Stable router implementation/version label.
- pub router_version: Option<String>,
- /// Router tolerance, probability, or decision threshold.
- pub tolerance: Option<f64>,
- /// Short human-readable route rationale.
- pub rationale: Option<String>,
-}
-
-impl RoutingMetadata {
- /// Returns true when no metadata field is set.
- pub fn is_empty(&self) -> bool {
- self.selected_model.is_none()
- && self.selected_tier.is_none()
- && self.confidence.is_none()
- && self.router_version.is_none()
- && self.tolerance.is_none()
- && self.rationale.is_none()
- }
-}
-
-/// Full profile result returned to servers and other erased-profile callers.
-pub struct ProfileResponse {
- /// Final backend response after profile response processing.
- pub response: ChatResponse,
- /// Optional routing metadata for response headers and audits.
- pub routing_metadata: Option<RoutingMetadata>,
-}
-
-impl ProfileResponse {
- /// Creates a profile response without routing metadata.
- pub fn new(response: ChatResponse) -> Self {
- Self {
- response,
- routing_metadata: None,
- }
- }
-
- /// Creates a profile response with routing metadata.
- pub fn with_routing_metadata(
- response: ChatResponse,
- routing_metadata: RoutingMetadata,
- ) -> Self {
- Self {
- response,
- routing_metadata: (!routing_metadata.is_empty()).then_some(routing_metadata),
- }
- }
-
- /// Splits this value into the final response and optional routing metadata.
- pub fn into_parts(self) -> (ChatResponse, Option<RoutingMetadata>) {
- (self.response, self.routing_metadata)
- }
-
- /// Returns the response body when this is a buffered response.
- pub fn body(&self) -> Option<&serde_json::Value> {
- self.response.body()
- }
-}
-
-impl From<ChatResponse> for ProfileResponse {
- fn from(response: ChatResponse) -> Self {
- Self::new(response)
- }
-}
-
-/// Object-safe runtime surface for a complete profile.
-///
-/// Servers and config-built profile maps use this trait when they need to run
-/// an entire profile and receive a final response. It intentionally exposes
-/// only `run()` so dynamic dispatch does not erase profile-specific processed
-/// state returned by hook-level APIs.
-#[async_trait]
-pub trait Profile: Send + Sync {
- /// Executes the complete profile flow and returns the final response.
- async fn run(&self, input: ProfileInput) -> Result<ProfileResponse>;
-}
-
-/// Typed hook surface for profile authors and embedders.
-///
-/// These methods support middleware-style integrations that want Switchyard to
-/// prepare a request and/or process a response without owning the transport.
-/// The associated `ProcessedRequest` type is a profile-owned struct, which lets
-/// profiles expose real per-call state, such as a routing decision, without a
-/// generic side channel.
-#[async_trait]
-pub trait ProfileHooks: Send + Sync {
- /// Profile-owned request-side state.
- type ProcessedRequest: Send + Sync;
-
- /// Runs the profile's request-side hook.
- async fn process(&self, input: ProfileInput) -> Result<Self::ProcessedRequest>;
-
- /// Runs the profile's response-side hook after a backend response exists.
- async fn rprocess(
- &self,
- processed: &Self::ProcessedRequest,
- response: ChatResponse,
- ) -> Result<ChatResponse>;
-}
-
-#[cfg(test)]
-mod tests {
- use std::collections::BTreeMap;
-
- use serde_json::json;
- use switchyard_core::ChatRequestType;
-
- use super::*;
-
- #[test]
- fn request_metadata_is_plain_data() {
- let metadata = RequestMetadata {
- request_id: None,
- inbound_format: Some(ChatRequestType::OpenAiChat),
- headers: BTreeMap::from([(
- "x-switchyard-trace".to_string(),
- vec!["abc123".to_string()],
- )]),
- };
-
- assert_eq!(
- metadata
- .headers
- .get("x-switchyard-trace")
- .map(Vec::as_slice),
- Some(&["abc123".to_string()][..])
- );
- assert_eq!(metadata.inbound_format, Some(ChatRequestType::OpenAiChat));
- }
-
- #[test]
- fn profile_input_is_plain_data() {
- let input = ProfileInput {
- request: ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [],
- })),
- metadata: RequestMetadata {
- request_id: None,
- inbound_format: None,
- headers: BTreeMap::from([(
- "x-request-source".to_string(),
- vec!["unit-test".to_string()],
- )]),
- },
- };
-
- assert_eq!(input.request.model(), Some("client/model"));
- assert_eq!(
- input
- .metadata
- .headers
- .get("x-request-source")
- .map(Vec::as_slice),
- Some(&["unit-test".to_string()][..])
- );
- assert!(input.metadata.request_id.is_none());
- }
-}
diff --git a/crates/switchyard-components-v2/src/profiles/latency_service/health.rs b/crates/switchyard-components-v2/src/profiles/latency_service/health.rs
deleted file mode 100644
index ecfc6cffa..000000000
--- a/crates/switchyard-components-v2/src/profiles/latency_service/health.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Health snapshots used by the components-v2 latency-service profile.
-
-use serde::{Deserialize, Serialize};
-
-/// Endpoint states returned by the external Latency Service.
-#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub enum EndpointHealthStatus {
- /// Endpoint is eligible for preferred routing.
- Healthy,
- /// Endpoint can still receive traffic, but only after healthy and unknown pools.
- Degraded,
- /// Endpoint has no current health verdict.
- Unknown,
-}
-
-impl EndpointHealthStatus {
- /// Routing preference order for latency-service target selection.
- pub(crate) const ROUTING_ORDER: [Self; 3] = [Self::Healthy, Self::Unknown, Self::Degraded];
-
- /// Stable lowercase label used by routing metadata.
- pub fn as_str(self) -> &'static str {
- match self {
- Self::Healthy => "healthy",
- Self::Degraded => "degraded",
- Self::Unknown => "unknown",
- }
- }
-}
-
-/// Cached health snapshot for one configured target.
-#[derive(Clone, Copy, Debug, PartialEq, Serialize)]
-pub struct EndpointHealth {
- /// Service-reported health status for the endpoint.
- pub status: EndpointHealthStatus,
- /// Optional last observed latency in milliseconds.
- #[serde(skip_serializing_if = "Option::is_none")]
- pub last_latency_ms: Option<f64>,
-}
-
-impl EndpointHealth {
- /// Creates a health snapshot with no latency sample.
- pub fn new(status: EndpointHealthStatus) -> Self {
- Self {
- status,
- last_latency_ms: None,
- }
- }
-
- /// Creates a health snapshot with a latency sample.
- pub fn with_latency(status: EndpointHealthStatus, last_latency_ms: f64) -> Self {
- Self {
- status,
- last_latency_ms: Some(last_latency_ms),
- }
- }
-}
diff --git a/crates/switchyard-components-v2/src/profiles/latency_service/mod.rs b/crates/switchyard-components-v2/src/profiles/latency_service/mod.rs
deleted file mode 100644
index 2242248dc..000000000
--- a/crates/switchyard-components-v2/src/profiles/latency_service/mod.rs
+++ /dev/null
@@ -1,390 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Latency-service profile implemented as a profile-owned runtime.
-
-mod health;
-mod polling;
-mod selection;
-
-#[cfg(test)]
-mod tests;
-
-use std::collections::{BTreeMap, BTreeSet};
-use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
-use std::time::Instant;
-
-use async_trait::async_trait;
-use parking_lot::RwLock;
-use switchyard_components::stats::usage_from_body;
-use switchyard_components::StatsAccumulator;
-use switchyard_core::{ChatResponse, LlmTarget, LlmTargetId, Result, SwitchyardError};
-
-pub use self::health::{EndpointHealth, EndpointHealthStatus};
-use self::polling::{duration_from_secs, HealthPoller, HealthResponse};
-use self::selection::select_target;
-pub use self::selection::SelectedTarget;
-use crate::backend::{native_target_backend, TargetBackend};
-use crate::profile_stats_accumulator;
-use crate::{
- profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse,
- RoutingMetadata,
-};
-
-const DEFAULT_POLL_TIMEOUT_SECS: f64 = 5.0;
-const DEFAULT_MAX_RETRIES: usize = 2;
-
-/// Default HTTP timeout for latency-service health polls.
-fn default_poll_timeout_secs() -> f64 {
- DEFAULT_POLL_TIMEOUT_SECS
-}
-
-/// Default number of alternate targets to try after the first failed call.
-fn default_max_retries() -> usize {
- DEFAULT_MAX_RETRIES
-}
-
-/// Config for the flatter latency-service profile.
-///
-/// The profile owns the health cache and exposes [`LatencyServiceProfile::poll_once`]
-/// for whichever control plane embeds it. It does not spawn a background task;
-/// `run()` and `process()` perform a guarded initial refresh for multi-target
-/// profiles and then route from the profile-owned cache.
-#[profile_config("latency-service")]
-pub struct LatencyServiceProfileConfig {
- /// Base URL for the external latency service.
- pub latency_service_url: String,
- /// Concrete targets this profile may select at request time.
- #[profile_target]
- pub targets: Vec<LlmTarget>,
- /// Per-poll HTTP timeout in seconds.
- #[serde(default = "default_poll_timeout_secs")]
- pub poll_timeout_secs: f64,
- /// Number of alternate target attempts after the first failed call.
- #[serde(default = "default_max_retries")]
- pub max_retries: usize,
-}
-
-impl ProfileConfig for LatencyServiceProfileConfig {
- type Runtime = LatencyServiceProfile;
-
- /// Builds the runtime profile using existing native backend construction.
- fn build(&self) -> Result<Self::Runtime> {
- validate_config(self)?;
- let target_ids = self
- .targets
- .iter()
- .map(|target| target.id.clone())
- .collect::<Vec<_>>();
- let poll_timeout = duration_from_secs(self.poll_timeout_secs, "poll_timeout_secs")?;
- let poller =
- HealthPoller::new(&self.latency_service_url, target_ids.clone(), poll_timeout)?;
-
- let mut backends = BTreeMap::new();
- let mut health = BTreeMap::new();
- for target in &self.targets {
- backends.insert(target.id.clone(), native_target_backend(target.clone())?);
- health.insert(
- target.id.clone(),
- EndpointHealth::new(EndpointHealthStatus::Unknown),
- );
- }
-
- Ok(LatencyServiceProfile {
- poller,
- backends,
- health: RwLock::new(health),
- poll_count: AtomicU64::new(0),
- initial_refresh_in_flight: AtomicBool::new(false),
- max_retries: self.max_retries,
- stats: profile_stats_accumulator(),
- })
- }
-}
-
-/// Latency-service profile in the flatter design.
-pub struct LatencyServiceProfile {
- poller: HealthPoller,
- backends: BTreeMap<LlmTargetId, TargetBackend>,
- health: RwLock<BTreeMap<LlmTargetId, EndpointHealth>>,
- poll_count: AtomicU64,
- initial_refresh_in_flight: AtomicBool,
- max_retries: usize,
- stats: StatsAccumulator,
-}
-
-/// Processed latency-service request with the target selected for response processing.
-pub struct LatencyServiceProcessedRequest {
- /// Routed input prepared for the selected backend.
- pub profile_input: ProfileInput,
- /// Target associated with the backend response passed to `rprocess()`.
- pub selected: SelectedTarget,
-}
-
-impl LatencyServiceProfile {
- /// Returns true once at least one explicit health poll has completed successfully.
- pub fn is_ready(&self) -> bool {
- self.poll_count.load(Ordering::Relaxed) > 0
- }
-
- /// Returns a point-in-time copy of the cached health table.
- pub fn health_snapshot(&self) -> BTreeMap<LlmTargetId, EndpointHealth> {
- self.health.read().clone()
- }
-
- /// Applies externally supplied health for a configured target.
- ///
- /// This is useful for tests and control-plane integrations that already own health
- /// polling and want the profile hot path to remain network-free.
- pub fn update_health(&self, target_id: LlmTargetId, health: EndpointHealth) -> Result<()> {
- let mut cache = self.health.write();
- let Some(entry) = cache.get_mut(&target_id) else {
- return Err(SwitchyardError::InvalidConfig(format!(
- "latency_service target {target_id} is not configured"
- )));
- };
- *entry = health;
- Ok(())
- }
-
- /// Performs one latency-service health poll and updates the local cache.
- ///
- /// A failed poll resets known health to `unknown` so stale healthy readings do not
- /// keep biasing target selection after the control plane becomes unavailable.
- pub async fn poll_once(&self) -> Result<()> {
- match self
- .poller
- .fetch_health()
- .await
- .and_then(|payload| self.apply_health(payload))
- {
- Ok(()) => {
- self.poll_count.fetch_add(1, Ordering::Relaxed);
- Ok(())
- }
- Err(error) => {
- self.reset_to_unknown();
- Err(error)
- }
- }
- }
-
- async fn refresh_if_unready(&self) {
- if self.backends.len() <= 1 || self.is_ready() || self.has_health_signal() {
- return;
- }
- if self
- .initial_refresh_in_flight
- .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
- .is_err()
- {
- return;
- }
-
- if let Err(error) = self.poll_once().await {
- tracing::warn!(
- error = %error,
- "latency_service initial health refresh failed; routing with unknown health"
- );
- }
- self.initial_refresh_in_flight
- .store(false, Ordering::Release);
- }
-
- fn has_health_signal(&self) -> bool {
- self.health.read().values().any(|health| {
- health.status != EndpointHealthStatus::Unknown || health.last_latency_ms.is_some()
- })
- }
-
- // Selects the target and rewrites the request model without persisting state.
- fn route_request(&self, mut input: ProfileInput) -> Result<LatencyServiceProcessedRequest> {
- let selected = self.select(&BTreeSet::new())?;
- let backend = self.selected_backend(&selected.target_id)?;
- input.request.set_model(backend.target().model.as_str());
- Ok(LatencyServiceProcessedRequest {
- profile_input: input,
- selected,
- })
- }
-
- // Reads the health cache and applies the latency-service tiering policy.
- fn select(&self, excluded: &BTreeSet<LlmTargetId>) -> Result<SelectedTarget> {
- let snapshot = self.health.read();
- select_target(&snapshot, excluded)
- }
-
- // Finds the concrete backend for a selected target.
- fn selected_backend(&self, target_id: &LlmTargetId) -> Result<&TargetBackend> {
- self.backends.get(target_id).ok_or_else(|| {
- SwitchyardError::InvalidConfig(format!(
- "latency_service selected target {target_id} without a configured backend"
- ))
- })
- }
-
- // Resets preference signals while preserving the configured target set.
- fn reset_to_unknown(&self) {
- let mut cache = self.health.write();
- for health in cache.values_mut() {
- *health = EndpointHealth::new(EndpointHealthStatus::Unknown);
- }
- }
-
- // Applies health for known targets and resets omitted known targets to unknown.
- fn apply_health(&self, payload: HealthResponse) -> Result<()> {
- let known_ids = self
- .poller
- .target_ids()
- .iter()
- .map(LlmTargetId::as_str)
- .collect::<BTreeSet<_>>();
- let mut reported = BTreeSet::new();
- let mut cache = self.health.write();
- for (target_id, health) in payload.endpoint_health {
- if known_ids.contains(target_id.as_str()) {
- let target_id = LlmTargetId::new(target_id)?;
- if let Some(entry) = cache.get_mut(&target_id) {
- *entry = health;
- reported.insert(target_id);
- }
- }
- }
- for (target_id, health) in cache.iter_mut() {
- if !reported.contains(target_id) {
- *health = EndpointHealth::new(EndpointHealthStatus::Unknown);
- }
- }
- Ok(())
- }
-
- fn routing_metadata(&self, selected: &SelectedTarget, selected_model: &str) -> RoutingMetadata {
- RoutingMetadata {
- selected_model: Some(selected_model.to_string()),
- selected_tier: Some(selected.health_status.as_str().to_string()),
- confidence: None,
- router_version: Some("latency-service:v1".to_string()),
- tolerance: None,
- rationale: Some(format!(
- "latency service selected target {} from {} health tier",
- selected.target_id,
- selected.health_status.as_str()
- )),
- }
- }
-}
-
-#[async_trait]
-impl ProfileHooks for LatencyServiceProfile {
- type ProcessedRequest = LatencyServiceProcessedRequest;
-
- /// Performs a standalone latency-service routing rewrite for hook-level inspection.
- async fn process(&self, input: ProfileInput) -> Result<Self::ProcessedRequest> {
- self.refresh_if_unready().await;
- self.route_request(input)
- }
-
- /// Leaves the backend response unchanged after latency-aware routing completes.
- async fn rprocess(
- &self,
- _processed: &Self::ProcessedRequest,
- response: ChatResponse,
- ) -> Result<ChatResponse> {
- Ok(response)
- }
-}
-
-#[async_trait]
-impl Profile for LatencyServiceProfile {
- /// Executes latency-aware routing with retry state local to this call.
- async fn run(&self, mut input: ProfileInput) -> Result<ProfileResponse> {
- self.refresh_if_unready().await;
- let profile_started_at = Instant::now();
- let mut tried = BTreeSet::new();
- let mut last_error = None;
-
- for _ in 0..=self.max_retries {
- if tried.len() == self.backends.len() {
- break;
- }
- let selected = match self.select(&tried) {
- Ok(selected) => selected,
- Err(_) if last_error.is_some() => break,
- Err(error) => return Err(error),
- };
- tried.insert(selected.target_id.clone());
- let selected_backend = self.selected_backend(&selected.target_id)?;
- let stats_model = selected_backend.target().model.to_string();
- let routing_metadata = self.routing_metadata(&selected, &stats_model);
- input
- .request
- .set_model(selected_backend.target().model.as_str());
- let backend_started_at = Instant::now();
-
- match selected_backend.call(&input.request).await {
- Ok(response) => {
- let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0;
- self.stats.record_success(
- stats_model.clone(),
- Some(backend_latency_ms),
- None,
- )?;
- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;
- let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);
- let usage = response.body().map(usage_from_body).unwrap_or_default();
- self.stats.record_usage_after_success_attribution(
- stats_model,
- usage,
- Some(total_latency_ms),
- Some(routing_overhead_ms),
- None,
- )?;
- let processed = LatencyServiceProcessedRequest {
- profile_input: input,
- selected,
- };
- let response = self.rprocess(&processed, response).await?;
- return Ok(ProfileResponse::with_routing_metadata(
- response,
- routing_metadata,
- ));
- }
- Err(error) => {
- self.stats.record_error(stats_model, None)?;
- last_error = Some(error);
- }
- }
- }
-
- Err(last_error.unwrap_or_else(|| {
- SwitchyardError::Backend(
- "latency_service profile exhausted retries without calling a target".to_string(),
- )
- }))
- }
-}
-
-fn validate_config(config: &LatencyServiceProfileConfig) -> Result<()> {
- if config.latency_service_url.trim().is_empty() {
- return Err(SwitchyardError::InvalidConfig(
- "latency_service profile requires latency_service_url".to_string(),
- ));
- }
- if config.targets.is_empty() {
- return Err(SwitchyardError::InvalidConfig(
- "latency_service profile requires at least one target".to_string(),
- ));
- }
- duration_from_secs(config.poll_timeout_secs, "poll_timeout_secs")?;
-
- let mut seen = BTreeSet::new();
- for target in &config.targets {
- if !seen.insert(target.id.clone()) {
- return Err(SwitchyardError::InvalidConfig(format!(
- "latency_service profile has duplicate target {}",
- target.id
- )));
- }
- }
- Ok(())
-}
diff --git a/crates/switchyard-components-v2/src/profiles/latency_service/polling.rs b/crates/switchyard-components-v2/src/profiles/latency_service/polling.rs
deleted file mode 100644
index 280d09f66..000000000
--- a/crates/switchyard-components-v2/src/profiles/latency_service/polling.rs
+++ /dev/null
@@ -1,137 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Health polling helpers for the components-v2 latency-service profile.
-
-use std::collections::BTreeMap;
-use std::time::Duration;
-
-use serde::Deserialize;
-use switchyard_core::{LlmTargetId, Result, SwitchyardError};
-
-use super::{EndpointHealth, EndpointHealthStatus};
-
-/// Polling client for one latency-service profile.
-pub(crate) struct HealthPoller {
- url: String,
- target_ids: Vec<LlmTargetId>,
- client: reqwest::Client,
-}
-
-impl HealthPoller {
- /// Creates a poller for the profile's configured target IDs.
- pub(crate) fn new(
- latency_service_url: &str,
- target_ids: Vec<LlmTargetId>,
- poll_timeout: Duration,
- ) -> Result<Self> {
- let url = format!(
- "{}/v1/endpoints/health",
- latency_service_url.trim_end_matches('/')
- );
- let client = reqwest::Client::builder()
- .timeout(poll_timeout)
- .user_agent("SwitchyardLatencyServiceProfile")
- .build()
- .map_err(|error| {
- SwitchyardError::InvalidConfig(format!(
- "latency_service failed to build HTTP client: {error}"
- ))
- })?;
- Ok(Self {
- url,
- target_ids,
- client,
- })
- }
-
- /// Returns the target IDs this poller sends to the latency service.
- pub(crate) fn target_ids(&self) -> &[LlmTargetId] {
- &self.target_ids
- }
-
- /// Fetches one health payload from the external latency service.
- pub(crate) async fn fetch_health(&self) -> Result<HealthResponse> {
- let params = self
- .target_ids
- .iter()
- .map(|target_id| ("endpoint_ids", target_id.as_str()))
- .collect::<Vec<_>>();
- let response = self
- .client
- .get(&self.url)
- .query(&params)
- .send()
- .await
- .map_err(|error| {
- SwitchyardError::Upstream(format!("Latency Service health request failed: {error}"))
- })?;
- let status = response.status();
- if !status.is_success() {
- return Err(SwitchyardError::Upstream(format!(
- "Latency Service health request returned HTTP {status}"
- )));
- }
- response
- .json::<RawHealthResponse>()
- .await
- .map(HealthResponse::from)
- .map_err(|error| {
- SwitchyardError::Upstream(format!(
- "Latency Service health response was invalid JSON: {error}"
- ))
- })
- }
-}
-
-/// Parsed health response with public target health values.
-pub(crate) struct HealthResponse {
- pub(crate) endpoint_health: BTreeMap<String, EndpointHealth>,
-}
-
-#[derive(Debug, Deserialize)]
-struct RawHealthResponse {
- endpoint_health: BTreeMap<String, HealthEntry>,
-}
-
-#[derive(Debug, Deserialize)]
-struct HealthEntry {
- status: EndpointHealthStatus,
- #[serde(default)]
- last_latency_ms: Option<f64>,
-}
-
-impl From<RawHealthResponse> for HealthResponse {
- fn from(response: RawHealthResponse) -> Self {
- Self {
- endpoint_health: response
- .endpoint_health
- .into_iter()
- .map(|(target_id, health)| (target_id, health.into()))
- .collect(),
- }
- }
-}
-
-impl From<HealthEntry> for EndpointHealth {
- fn from(entry: HealthEntry) -> Self {
- Self {
- status: entry.status,
- last_latency_ms: entry.last_latency_ms,
- }
- }
-}
-
-/// Converts positive finite seconds into a Rust duration.
-pub(crate) fn duration_from_secs(value: f64, field: &'static str) -> Result<Duration> {
- if !value.is_finite() || value <= 0.0 {
- return Err(SwitchyardError::InvalidConfig(format!(
- "latency_service {field} must be finite and positive, got {value:?}"
- )));
- }
- Duration::try_from_secs_f64(value).map_err(|error| {
- SwitchyardError::InvalidConfig(format!(
- "latency_service {field} is outside the supported duration range: {error}"
- ))
- })
-}
diff --git a/crates/switchyard-components-v2/src/profiles/latency_service/selection.rs b/crates/switchyard-components-v2/src/profiles/latency_service/selection.rs
deleted file mode 100644
index 02c6bd3c1..000000000
--- a/crates/switchyard-components-v2/src/profiles/latency_service/selection.rs
+++ /dev/null
@@ -1,94 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Target selection policy for the components-v2 latency-service profile.
-
-use std::collections::{BTreeMap, BTreeSet};
-
-use rand::distributions::{Distribution, WeightedIndex};
-use rand::Rng;
-use serde::{Deserialize, Serialize};
-use switchyard_core::{LlmTargetId, Result, SwitchyardError};
-
-use super::{EndpointHealth, EndpointHealthStatus};
-
-/// Selected target ID emitted by the latency-service selector.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct SelectedTarget {
- /// Target ID selected from the health cache.
- pub target_id: LlmTargetId,
- /// Health tier that made the target selectable.
- pub health_status: EndpointHealthStatus,
-}
-
-/// Selects a target from the health cache while avoiding excluded targets when possible.
-pub(crate) fn select_target(
- snapshot: &BTreeMap<LlmTargetId, EndpointHealth>,
- excluded: &BTreeSet<LlmTargetId>,
-) -> Result<SelectedTarget> {
- for tier in EndpointHealthStatus::ROUTING_ORDER {
- let entries = candidates_for_tier(snapshot, excluded, tier);
- if !entries.is_empty() {
- return pick_from_tier(&entries, tier);
- }
- }
-
- Err(SwitchyardError::InvalidConfig(
- "latency_service profile has no selectable targets".to_string(),
- ))
-}
-
-fn candidates_for_tier<'a>(
- snapshot: &'a BTreeMap<LlmTargetId, EndpointHealth>,
- excluded: &BTreeSet<LlmTargetId>,
- tier: EndpointHealthStatus,
-) -> Vec<(&'a LlmTargetId, EndpointHealth)> {
- snapshot
- .iter()
- .filter_map(|(target_id, health)| {
- (!excluded.contains(target_id) && health.status == tier).then_some((target_id, *health))
- })
- .collect()
-}
-
-fn pick_from_tier(
- entries: &[(&LlmTargetId, EndpointHealth)],
- health_status: EndpointHealthStatus,
-) -> Result<SelectedTarget> {
- if entries.len() == 1 {
- return Ok(SelectedTarget {
- target_id: entries[0].0.clone(),
- health_status,
- });
- }
-
- let weights = entries
- .iter()
- .map(|(_, health)| {
- health
- .last_latency_ms
- .filter(|value| value.is_finite() && *value > 0.0)
- .map(|value| 1.0 / value)
- })
- .collect::<Option<Vec<_>>>();
-
- let index = if let Some(weights) = weights {
- weighted_index(&weights)?
- } else {
- rand::thread_rng().gen_range(0..entries.len())
- };
-
- Ok(SelectedTarget {
- target_id: entries[index].0.clone(),
- health_status,
- })
-}
-
-fn weighted_index(weights: &[f64]) -> Result<usize> {
- let distribution = WeightedIndex::new(weights).map_err(|error| {
- SwitchyardError::InvalidConfig(format!(
- "latency_service computed invalid target weights: {error}"
- ))
- })?;
- Ok(distribution.sample(&mut rand::thread_rng()))
-}
diff --git a/crates/switchyard-components-v2/src/profiles/latency_service/tests.rs b/crates/switchyard-components-v2/src/profiles/latency_service/tests.rs
deleted file mode 100644
index 0bdd8ed06..000000000
--- a/crates/switchyard-components-v2/src/profiles/latency_service/tests.rs
+++ /dev/null
@@ -1,893 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Unit tests for the components-v2 latency-service profile internals.
-
-use std::collections::{BTreeMap, BTreeSet};
-use std::io::{BufRead, BufReader, Write};
-use std::net::TcpListener;
-use std::pin::Pin;
-use std::sync::atomic::{AtomicBool, AtomicU64};
-use std::sync::{Arc, Mutex};
-use std::task::{Context, Poll};
-use std::thread::{self, JoinHandle};
-use std::time::Duration;
-
-use async_trait::async_trait;
-use futures_core::Stream;
-use parking_lot::RwLock;
-use serde_json::{json, Value};
-use switchyard_core::{BackendFormat, ChatRequest, LlmTargetId, ModelId, StreamEvent};
-
-use super::polling::HealthPoller;
-use super::selection::select_target;
-use super::*;
-use crate::backend::ProfileBackend;
-use crate::{ProfileInput, RequestMetadata};
-
-#[derive(Clone, Debug, PartialEq)]
-struct ObservedCall {
- backend: String,
- body: Value,
-}
-
-struct TestBackend {
- name: String,
- calls: Arc<Mutex<Vec<ObservedCall>>>,
- error: Option<&'static str>,
- response_mode: ResponseMode,
-}
-
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-enum ResponseMode {
- Completion,
- OpenAiStream,
-}
-
-struct OneEventStream {
- event: Option<Result<StreamEvent>>,
-}
-
-impl Stream for OneEventStream {
- type Item = Result<StreamEvent>;
-
- fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
- Poll::Ready(self.event.take())
- }
-}
-
-#[async_trait]
-impl ProfileBackend for TestBackend {
- async fn call(&self, request: &ChatRequest) -> Result<ChatResponse> {
- self.calls
- .lock()
- .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))?
- .push(ObservedCall {
- backend: self.name.clone(),
- body: request.body().clone(),
- });
- if let Some(error) = self.error {
- return Err(SwitchyardError::Backend(error.to_string()));
- }
- match self.response_mode {
- ResponseMode::Completion => Ok(ChatResponse::openai_completion(json!({
- "backend": self.name,
- "model": request.model(),
- "usage": {
- "prompt_tokens": 13,
- "completion_tokens": 5,
- },
- }))),
- ResponseMode::OpenAiStream => {
- Ok(ChatResponse::OpenAiStream(Box::pin(OneEventStream {
- event: Some(Ok(StreamEvent::Json(json!({
- "backend": self.name,
- "model": request.model(),
- })))),
- })))
- }
- }
- }
-}
-
-#[derive(Clone, Debug, PartialEq)]
-struct CapturedRequest {
- path: String,
-}
-
-struct OneShotServer {
- base_url: String,
- captured: Arc<Mutex<Option<CapturedRequest>>>,
- handle: Option<JoinHandle<Result<()>>>,
-}
-
-impl OneShotServer {
- fn json(status: u16, body: Value) -> Result<Self> {
- let listener = TcpListener::bind("127.0.0.1:0")
- .map_err(|error| SwitchyardError::Other(format!("bind failed: {error}")))?;
- let address = listener
- .local_addr()
- .map_err(|error| SwitchyardError::Other(format!("local addr failed: {error}")))?;
- let captured = Arc::new(Mutex::new(None));
- let captured_for_thread = Arc::clone(&captured);
- let handle = thread::spawn(move || serve_once(listener, status, body, captured_for_thread));
- Ok(Self {
- base_url: format!("http://{address}"),
- captured,
- handle: Some(handle),
- })
- }
-
- fn base_url(&self) -> String {
- self.base_url.clone()
- }
-
- fn captured(&mut self) -> Result<CapturedRequest> {
- if let Some(handle) = self.handle.take() {
- handle
- .join()
- .map_err(|_| SwitchyardError::Other("test server thread panicked".into()))??;
- }
- self.captured
- .lock()
- .map_err(|_| SwitchyardError::Other("captured mutex poisoned".into()))?
- .clone()
- .ok_or_else(|| SwitchyardError::Other("test server captured no request".into()))
- }
-}
-
-fn serve_once(
- listener: TcpListener,
- status: u16,
- body: Value,
- captured: Arc<Mutex<Option<CapturedRequest>>>,
-) -> Result<()> {
- let (mut stream, _) = listener
- .accept()
- .map_err(|error| SwitchyardError::Other(format!("accept failed: {error}")))?;
- let mut reader = BufReader::new(
- stream
- .try_clone()
- .map_err(|error| SwitchyardError::Other(format!("clone stream failed: {error}")))?,
- );
- let mut first_line = String::new();
- reader
- .read_line(&mut first_line)
- .map_err(|error| SwitchyardError::Other(format!("read request line failed: {error}")))?;
- let path = first_line
- .split_whitespace()
- .nth(1)
- .ok_or_else(|| SwitchyardError::Other("missing request path".into()))?
- .to_string();
- {
- let mut captured = captured
- .lock()
- .map_err(|_| SwitchyardError::Other("captured mutex poisoned".into()))?;
- *captured = Some(CapturedRequest { path });
- }
-
- loop {
- let mut line = String::new();
- let bytes = reader.read_line(&mut line).map_err(|error| {
- SwitchyardError::Other(format!("read request header failed: {error}"))
- })?;
- if bytes == 0 || line == "\r\n" {
- break;
- }
- }
-
- let body = serde_json::to_string(&body)
- .map_err(|error| SwitchyardError::Other(format!("json encode failed: {error}")))?;
- let reason = if status == 200 { "OK" } else { "ERROR" };
- let response = format!(
- "HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
- body.len()
- );
- stream
- .write_all(response.as_bytes())
- .map_err(|error| SwitchyardError::Other(format!("write response failed: {error}")))?;
- Ok(())
-}
-
-fn target(id: &str, model: &str) -> Result<LlmTarget> {
- let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
- target.format = BackendFormat::OpenAi;
- Ok(target)
-}
-
-fn profile_with_backends(
- targets: Vec<LlmTarget>,
- failures: BTreeMap<&'static str, &'static str>,
- max_retries: usize,
-) -> Result<(LatencyServiceProfile, Arc<Mutex<Vec<ObservedCall>>>)> {
- profile_with_backend_modes(targets, failures, BTreeMap::new(), max_retries)
-}
-
-fn profile_with_backend_modes(
- targets: Vec<LlmTarget>,
- failures: BTreeMap<&'static str, &'static str>,
- response_modes: BTreeMap<&'static str, ResponseMode>,
- max_retries: usize,
-) -> Result<(LatencyServiceProfile, Arc<Mutex<Vec<ObservedCall>>>)> {
- let calls = Arc::new(Mutex::new(Vec::new()));
- let mut backends = BTreeMap::new();
- let mut health = BTreeMap::new();
- for target in &targets {
- health.insert(
- target.id.clone(),
- EndpointHealth::new(EndpointHealthStatus::Unknown),
- );
- let backend = TestBackend {
- name: format!("{}-backend", target.id),
- calls: Arc::clone(&calls),
- error: failures.get(target.id.as_str()).copied(),
- response_mode: response_modes
- .get(target.id.as_str())
- .copied()
- .unwrap_or(ResponseMode::Completion),
- };
- backends.insert(
- target.id.clone(),
- TargetBackend::new(target.clone(), Arc::new(backend)),
- );
- }
-
- let target_ids = targets.iter().map(|target| target.id.clone()).collect();
- let profile = LatencyServiceProfile {
- poller: HealthPoller::new("http://latency.test", target_ids, Duration::from_secs(5))?,
- backends,
- health: RwLock::new(health),
- poll_count: AtomicU64::new(0),
- initial_refresh_in_flight: AtomicBool::new(false),
- max_retries,
- stats: StatsAccumulator::new(),
- };
- Ok((profile, calls))
-}
-
-fn observed(calls: &Arc<Mutex<Vec<ObservedCall>>>) -> Result<Vec<ObservedCall>> {
- calls
- .lock()
- .map(|calls| calls.clone())
- .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))
-}
-
-fn profile_input(request: ChatRequest) -> ProfileInput {
- ProfileInput {
- request,
- metadata: RequestMetadata::default(),
- }
-}
-
-#[test]
-fn profile_config_build_rejects_invalid_config() -> Result<()> {
- let fast = target("fast", "upstream-fast")?;
- let duplicate = LatencyServiceProfileConfig {
- latency_service_url: "http://latency.test".to_string(),
- targets: vec![fast.clone(), fast],
- poll_timeout_secs: 5.0,
- max_retries: 2,
- };
- match duplicate.build() {
- Err(SwitchyardError::InvalidConfig(message)) => {
- assert!(message.contains("duplicate target fast"));
- }
- Ok(_) => {
- return Err(SwitchyardError::Other(
- "duplicate targets should reject profile construction".into(),
- ));
- }
- Err(other) => {
- return Err(SwitchyardError::Other(format!(
- "expected InvalidConfig, got {other}"
- )));
- }
- }
-
- let empty_url = LatencyServiceProfileConfig {
- latency_service_url: " ".to_string(),
- targets: vec![target("fast", "upstream-fast")?],
- poll_timeout_secs: 5.0,
- max_retries: 2,
- };
- match empty_url.build() {
- Err(SwitchyardError::InvalidConfig(message)) => {
- assert!(message.contains("requires latency_service_url"));
- }
- Ok(_) => {
- return Err(SwitchyardError::Other(
- "empty URL should reject profile construction".into(),
- ));
- }
- Err(other) => {
- return Err(SwitchyardError::Other(format!(
- "expected InvalidConfig, got {other}"
- )));
- }
- }
-
- Ok(())
-}
-
-#[test]
-fn profile_config_macro_adds_type_metadata_and_strict_serde() -> Result<()> {
- let config = LatencyServiceProfileConfig {
- latency_service_url: "http://latency.test".to_string(),
- targets: vec![target("fast", "upstream-fast")?],
- poll_timeout_secs: 5.0,
- max_retries: 2,
- };
- assert_eq!(LatencyServiceProfileConfig::PROFILE_TYPE, "latency-service");
- assert_eq!(config.profile_type(), "latency-service");
-
- let unknown_field = json!({
- "latency_service_url": "http://latency.test",
- "targets": config.targets,
- "poll_timeout_secs": 5.0,
- "max_retries": 2,
- "poll_interval_secs": 1.0,
- });
- let error = serde_json::from_value::<LatencyServiceProfileConfig>(unknown_field)
- .err()
- .ok_or_else(|| SwitchyardError::Other("unknown profile field should fail".into()))?;
- assert!(error.to_string().contains("unknown field"));
- Ok(())
-}
-
-#[tokio::test]
-async fn healthy_endpoint_serves_request_and_records_stats() -> Result<()> {
- let (profile, calls) = profile_with_backends(
- vec![
- target("fast", "upstream-fast")?,
- target("slow", "upstream-slow")?,
- ],
- BTreeMap::new(),
- 0,
- )?;
- profile.update_health(
- LlmTargetId::from_static("fast"),
- EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 50.0),
- )?;
- profile.update_health(
- LlmTargetId::from_static("slow"),
- EndpointHealth::with_latency(EndpointHealthStatus::Degraded, 10.0),
- )?;
-
- let response = profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "incoming-model",
- "messages": [{"role": "user", "content": "hi"}],
- }))))
- .await?;
-
- let routing_metadata = response
- .routing_metadata
- .as_ref()
- .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?;
- assert_eq!(
- routing_metadata.selected_model.as_deref(),
- Some("upstream-fast")
- );
- assert_eq!(routing_metadata.selected_tier.as_deref(), Some("healthy"));
- assert_eq!(
- routing_metadata.router_version.as_deref(),
- Some("latency-service:v1")
- );
- let response = response.response;
-
- let calls = observed(&calls)?;
- assert_eq!(calls.len(), 1);
- assert_eq!(calls[0].backend, "fast-backend");
- assert_eq!(calls[0].body["model"], "upstream-fast");
- match response {
- ChatResponse::OpenAiCompletion(body) => {
- assert_eq!(body.body()["backend"], "fast-backend");
- }
- _ => return Err(SwitchyardError::Other("unexpected response shape".into())),
- }
-
- let snapshot = profile.stats.snapshot()?;
- let model = snapshot
- .models
- .get("upstream-fast")
- .ok_or_else(|| SwitchyardError::Other("selected model stats should exist".into()))?;
- assert_eq!(model.calls, 1);
- assert_eq!(snapshot.total_tokens.prompt, 13);
- assert_eq!(snapshot.total_tokens.completion, 5);
- Ok(())
-}
-
-#[tokio::test]
-async fn retry_avoids_failed_target_while_alternative_exists() -> Result<()> {
- let (profile, calls) = profile_with_backends(
- vec![
- target("failing", "upstream-failing")?,
- target("fallback", "upstream-fallback")?,
- ],
- BTreeMap::from([("failing", "down")]),
- 1,
- )?;
- profile.update_health(
- LlmTargetId::from_static("failing"),
- EndpointHealth::new(EndpointHealthStatus::Healthy),
- )?;
- profile.update_health(
- LlmTargetId::from_static("fallback"),
- EndpointHealth::new(EndpointHealthStatus::Degraded),
- )?;
-
- let response = profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "incoming-model",
- "messages": [],
- }))))
- .await?;
-
- let routing_metadata = response
- .routing_metadata
- .as_ref()
- .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?;
- assert_eq!(
- routing_metadata.selected_model.as_deref(),
- Some("upstream-fallback")
- );
- assert_eq!(routing_metadata.selected_tier.as_deref(), Some("degraded"));
- let response = response.response;
-
- let calls = observed(&calls)?;
- assert_eq!(calls.len(), 2);
- assert_eq!(calls[0].backend, "failing-backend");
- assert_eq!(calls[1].backend, "fallback-backend");
- match response {
- ChatResponse::OpenAiCompletion(body) => {
- assert_eq!(body.body()["backend"], "fallback-backend");
- }
- _ => return Err(SwitchyardError::Other("unexpected response shape".into())),
- }
- Ok(())
-}
-
-#[tokio::test]
-async fn exhausted_retries_return_last_target_error() -> Result<()> {
- let (profile, calls) = profile_with_backends(
- vec![
- target("first", "upstream-first")?,
- target("second", "upstream-second")?,
- ],
- BTreeMap::from([("first", "first down"), ("second", "second down")]),
- 5,
- )?;
- profile.update_health(
- LlmTargetId::from_static("first"),
- EndpointHealth::new(EndpointHealthStatus::Healthy),
- )?;
- profile.update_health(
- LlmTargetId::from_static("second"),
- EndpointHealth::new(EndpointHealthStatus::Degraded),
- )?;
-
- let error = match profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "incoming-model",
- "messages": [],
- }))))
- .await
- {
- Ok(_) => {
- return Err(SwitchyardError::Other(
- "all failing targets should return an error".into(),
- ));
- }
- Err(error) => error,
- };
-
- assert!(
- error.to_string().contains("second down"),
- "last target error should be returned, got {error:?}"
- );
- let calls = observed(&calls)?;
- assert_eq!(
- calls
- .iter()
- .map(|call| call.backend.as_str())
- .collect::<Vec<_>>(),
- vec!["first-backend", "second-backend"]
- );
- Ok(())
-}
-
-#[tokio::test]
-async fn process_only_rewrites_request_and_does_not_call_backend() -> Result<()> {
- let (profile, calls) = profile_with_backends(
- vec![
- target("fast", "upstream-fast")?,
- target("slow", "upstream-slow")?,
- ],
- BTreeMap::new(),
- 0,
- )?;
- profile.update_health(
- LlmTargetId::from_static("fast"),
- EndpointHealth::new(EndpointHealthStatus::Healthy),
- )?;
-
- let request = profile
- .process(profile_input(ChatRequest::openai_chat(json!({
- "model": "incoming-model",
- "messages": [],
- }))))
- .await?;
-
- assert_eq!(request.profile_input.request.model(), Some("upstream-fast"));
- assert_eq!(request.selected.target_id, LlmTargetId::from_static("fast"));
- assert!(observed(&calls)?.is_empty());
- Ok(())
-}
-
-#[tokio::test]
-async fn streaming_response_is_passed_through_from_selected_target() -> Result<()> {
- let (profile, calls) = profile_with_backend_modes(
- vec![target("fast", "upstream-fast")?],
- BTreeMap::new(),
- BTreeMap::from([("fast", ResponseMode::OpenAiStream)]),
- 0,
- )?;
- profile.update_health(
- LlmTargetId::from_static("fast"),
- EndpointHealth::new(EndpointHealthStatus::Healthy),
- )?;
-
- let response = profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "incoming-model",
- "messages": [],
- }))))
- .await?;
-
- let routing_metadata = response
- .routing_metadata
- .as_ref()
- .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?;
- assert_eq!(
- routing_metadata.selected_model.as_deref(),
- Some("upstream-fast")
- );
- assert_eq!(routing_metadata.selected_tier.as_deref(), Some("healthy"));
- let response = response.response;
-
- match response {
- ChatResponse::OpenAiStream(_) => {}
- _ => {
- return Err(SwitchyardError::Other(
- "expected OpenAI stream response".into(),
- ))
- }
- }
- let calls = observed(&calls)?;
- assert_eq!(calls.len(), 1);
- assert_eq!(calls[0].backend, "fast-backend");
- assert_eq!(calls[0].body["model"], "upstream-fast");
- Ok(())
-}
-
-#[test]
-fn unknown_target_health_update_is_rejected() -> Result<()> {
- let (profile, _calls) =
- profile_with_backends(vec![target("known", "upstream-known")?], BTreeMap::new(), 0)?;
-
- let error = match profile.update_health(
- LlmTargetId::from_static("missing"),
- EndpointHealth::new(EndpointHealthStatus::Healthy),
- ) {
- Ok(_) => {
- return Err(SwitchyardError::Other(
- "unknown target health update should fail".into(),
- ));
- }
- Err(error) => error,
- };
-
- assert!(error
- .to_string()
- .contains("target missing is not configured"));
- Ok(())
-}
-
-#[tokio::test]
-async fn poll_once_updates_cache_and_ignores_unknown_service_ids() -> Result<()> {
- let mut server = OneShotServer::json(
- 200,
- json!({
- "endpoint_health": {
- "fast": {"status": "healthy", "last_latency_ms": 42.0},
- "unknown-service-id": {"status": "degraded", "last_latency_ms": 999.0}
- }
- }),
- )?;
- let config = LatencyServiceProfileConfig {
- latency_service_url: server.base_url(),
- targets: vec![target("fast", "upstream-fast")?],
- poll_timeout_secs: 5.0,
- max_retries: 0,
- };
- let profile = config.build()?;
-
- profile.poll_once().await?;
- let captured = server.captured()?;
-
- assert!(captured.path.starts_with("/v1/endpoints/health?"));
- assert!(captured.path.contains("endpoint_ids=fast"));
- assert_eq!(
- profile
- .health_snapshot()
- .get(&LlmTargetId::from_static("fast")),
- Some(&EndpointHealth::with_latency(
- EndpointHealthStatus::Healthy,
- 42.0
- ))
- );
- assert!(profile.is_ready());
- Ok(())
-}
-
-#[tokio::test]
-async fn poll_once_resets_omitted_known_targets_to_unknown() -> Result<()> {
- let mut server = OneShotServer::json(
- 200,
- json!({
- "endpoint_health": {
- "fast": {"status": "healthy", "last_latency_ms": 42.0}
- }
- }),
- )?;
- let config = LatencyServiceProfileConfig {
- latency_service_url: server.base_url(),
- targets: vec![
- target("fast", "upstream-fast")?,
- target("slow", "upstream-slow")?,
- ],
- poll_timeout_secs: 5.0,
- max_retries: 0,
- };
- let profile = config.build()?;
- profile.update_health(
- LlmTargetId::from_static("slow"),
- EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 7.0),
- )?;
-
- profile.poll_once().await?;
- let _captured = server.captured()?;
-
- assert_eq!(
- profile
- .health_snapshot()
- .get(&LlmTargetId::from_static("fast")),
- Some(&EndpointHealth::with_latency(
- EndpointHealthStatus::Healthy,
- 42.0
- ))
- );
- assert_eq!(
- profile
- .health_snapshot()
- .get(&LlmTargetId::from_static("slow")),
- Some(&EndpointHealth::new(EndpointHealthStatus::Unknown))
- );
- Ok(())
-}
-
-#[tokio::test]
-async fn poll_once_tolerates_additive_health_payload_fields() -> Result<()> {
- let mut server = OneShotServer::json(
- 200,
- json!({
- "endpoint_health": {
- "fast": {
- "status": "healthy",
- "last_latency_ms": 42.0,
- "reason": "warm"
- }
- },
- "generated_at": "2026-05-19T00:00:00Z"
- }),
- )?;
- let config = LatencyServiceProfileConfig {
- latency_service_url: server.base_url(),
- targets: vec![target("fast", "upstream-fast")?],
- poll_timeout_secs: 5.0,
- max_retries: 0,
- };
- let profile = config.build()?;
-
- profile.poll_once().await?;
- let _captured = server.captured()?;
-
- assert_eq!(
- profile
- .health_snapshot()
- .get(&LlmTargetId::from_static("fast")),
- Some(&EndpointHealth::with_latency(
- EndpointHealthStatus::Healthy,
- 42.0
- ))
- );
- Ok(())
-}
-
-#[tokio::test]
-async fn poll_failure_resets_stale_health_to_unknown_without_marking_ready() -> Result<()> {
- let mut server = OneShotServer::json(500, json!({"error": "down"}))?;
- let config = LatencyServiceProfileConfig {
- latency_service_url: server.base_url(),
- targets: vec![target("fast", "upstream-fast")?],
- poll_timeout_secs: 5.0,
- max_retries: 0,
- };
- let profile = config.build()?;
- profile.update_health(
- LlmTargetId::from_static("fast"),
- EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 42.0),
- )?;
-
- let error = match profile.poll_once().await {
- Ok(_) => return Err(SwitchyardError::Other("poll should fail".into())),
- Err(error) => error,
- };
-
- assert!(error.to_string().contains("HTTP 500"));
- let _captured = server.captured()?;
- assert_eq!(
- profile
- .health_snapshot()
- .get(&LlmTargetId::from_static("fast")),
- Some(&EndpointHealth::new(EndpointHealthStatus::Unknown))
- );
- assert!(!profile.is_ready());
- Ok(())
-}
-
-#[tokio::test]
-async fn malformed_health_status_resets_to_unknown() -> Result<()> {
- let mut server = OneShotServer::json(
- 200,
- json!({
- "endpoint_health": {
- "fast": {"status": "on_fire", "last_latency_ms": 42.0}
- }
- }),
- )?;
- let config = LatencyServiceProfileConfig {
- latency_service_url: server.base_url(),
- targets: vec![target("fast", "upstream-fast")?],
- poll_timeout_secs: 5.0,
- max_retries: 0,
- };
- let profile = config.build()?;
- profile.update_health(
- LlmTargetId::from_static("fast"),
- EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 42.0),
- )?;
-
- let error = match profile.poll_once().await {
- Ok(_) => {
- return Err(SwitchyardError::Other(
- "malformed health status should fail".into(),
- ));
- }
- Err(error) => error,
- };
-
- assert!(
- error.to_string().contains("invalid JSON"),
- "malformed status should surface as invalid response, got {error:?}"
- );
- let _captured = server.captured()?;
- assert_eq!(
- profile
- .health_snapshot()
- .get(&LlmTargetId::from_static("fast")),
- Some(&EndpointHealth::new(EndpointHealthStatus::Unknown))
- );
- Ok(())
-}
-
-#[test]
-fn faster_endpoint_gets_more_traffic_within_same_tier() -> Result<()> {
- let snapshot = BTreeMap::from([
- (
- LlmTargetId::from_static("fast"),
- EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 50.0),
- ),
- (
- LlmTargetId::from_static("slow"),
- EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 500.0),
- ),
- ]);
-
- let mut fast = 0;
- let mut slow = 0;
- for _ in 0..2_000 {
- match select_target(&snapshot, &BTreeSet::new())?
- .target_id
- .as_str()
- {
- "fast" => fast += 1,
- "slow" => slow += 1,
- other => {
- return Err(SwitchyardError::Other(format!(
- "unexpected target selected: {other}"
- )));
- }
- }
- }
-
- assert!(
- fast > slow * 5,
- "expected fast endpoint to dominate inverse-latency routing; fast={fast}, slow={slow}"
- );
- Ok(())
-}
-
-#[test]
-fn missing_latency_falls_back_to_uniform_random() -> Result<()> {
- let snapshot = BTreeMap::from([
- (
- LlmTargetId::from_static("with-latency"),
- EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 50.0),
- ),
- (
- LlmTargetId::from_static("without-latency"),
- EndpointHealth::new(EndpointHealthStatus::Healthy),
- ),
- ]);
-
- let mut first = 0;
- let mut second = 0;
- for _ in 0..2_000 {
- match select_target(&snapshot, &BTreeSet::new())?
- .target_id
- .as_str()
- {
- "with-latency" => first += 1,
- "without-latency" => second += 1,
- other => {
- return Err(SwitchyardError::Other(format!(
- "unexpected target selected: {other}"
- )));
- }
- }
- }
-
- assert!(
- (850..=1150).contains(&first),
- "uniform fallback should stay near 50/50; first={first}, second={second}"
- );
- assert!(
- (850..=1150).contains(&second),
- "uniform fallback should stay near 50/50; first={first}, second={second}"
- );
- Ok(())
-}
-
-#[test]
-fn non_positive_latency_selects_without_error_or_dividing_by_zero() -> Result<()> {
- let snapshot = BTreeMap::from([
- (
- LlmTargetId::from_static("zero"),
- EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 0.0),
- ),
- (
- LlmTargetId::from_static("positive"),
- EndpointHealth::with_latency(EndpointHealthStatus::Healthy, 100.0),
- ),
- ]);
-
- for _ in 0..100 {
- let selected = select_target(&snapshot, &BTreeSet::new())?;
- assert!(
- selected.target_id == LlmTargetId::from_static("zero")
- || selected.target_id == LlmTargetId::from_static("positive")
- );
- }
- Ok(())
-}
diff --git a/crates/switchyard-components-v2/src/profiles/llm_routing.rs b/crates/switchyard-components-v2/src/profiles/llm_routing.rs
deleted file mode 100644
index f44ca5175..000000000
--- a/crates/switchyard-components-v2/src/profiles/llm_routing.rs
+++ /dev/null
@@ -1,1602 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! LLM classifier routing as a profile-owned runtime.
-
-use std::collections::BTreeSet;
-use std::time::Instant;
-
-use async_trait::async_trait;
-use serde::{Deserialize, Serialize};
-use serde_json::{json, Map, Value};
-use switchyard_components::stats::usage_from_body;
-use switchyard_components::StatsAccumulator;
-use switchyard_core::{
- BackendFormat, ChatRequest, ChatResponse, LlmTarget, LlmTargetId, Result, SwitchyardError,
-};
-
-use crate::backend::{native_target_backend, TargetBackend};
-use crate::profile_stats_accumulator;
-use crate::stats_recording::record_usage_or_wrap_stream;
-use crate::{
- profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse,
- RoutingMetadata,
-};
-
-const TIER_STRONG: &str = "strong";
-const TIER_WEAK: &str = "weak";
-const PROFILE_GENERAL: &str = "general";
-const PROFILE_CODING_AGENT: &str = "coding_agent";
-const PROFILE_OPENCLAW: &str = "openclaw";
-const DEFAULT_CLASSIFIER_MAX_TOKENS: u64 = 4096;
-const DEFAULT_ALIGNMENT_MIN_CONFIDENCE: f64 = 0.85;
-const DEFAULT_CLASSIFIER_TOOL_NAME: &str = "select_route";
-
-/// Config for the flatter LLM classifier profile.
-#[profile_config("llm-routing")]
-pub struct LlmRoutingProfileConfig {
- /// Strong target served by this profile.
- #[profile_target]
- pub strong: LlmTarget,
- /// Weak target served by this profile.
- #[profile_target]
- pub weak: LlmTarget,
- /// OpenAI-compatible classifier target used for tool-call route selection.
- #[profile_target]
- pub classifier: LlmTarget,
- /// Target used for one retry after context-window overflow.
- #[serde(default = "default_fallback_target_on_evict")]
- pub fallback_target_on_evict: LlmTargetId,
- /// Classifier policy profile: `general`, `coding_agent`, or `openclaw`.
- #[serde(default = "default_profile_name")]
- pub profile_name: String,
- /// Confidence floor below which routing falls back to the profile default.
- #[serde(default)]
- pub classifier_min_confidence: f64,
- /// Whether classifier failures fall open to the default tier.
- #[serde(default = "default_classifier_fail_open")]
- pub classifier_fail_open: bool,
- /// Number of recent turns included in the classifier request summary.
- #[serde(default = "default_recent_turn_window")]
- pub classifier_recent_turn_window: usize,
- /// Maximum tokens allowed for the classifier tool-call response.
- #[serde(default = "default_classifier_max_tokens")]
- pub classifier_max_tokens: u64,
- /// Confidence required before the classifier recommendation can bump the policy tier.
- #[serde(default = "default_alignment_min_confidence")]
- pub alignment_min_confidence: f64,
- /// Default backend tier used for abstain, low-confidence, and fail-open decisions.
- #[serde(default)]
- pub default_tier: Option<String>,
- /// Optional mapping from classifier route tiers to backend tiers.
- #[serde(default)]
- pub tier_mapping: Option<LlmRoutingTierMapping>,
- /// Optional system prompt override for the classifier request.
- #[serde(default)]
- pub classifier_system_prompt: Option<String>,
- /// Optional function/tool name override for classifier route selection.
- #[serde(default)]
- pub classifier_tool_name: Option<String>,
- /// Optional function/tool description override for classifier route selection.
- #[serde(default)]
- pub classifier_tool_description: Option<String>,
- /// Optional JSON schema override for classifier tool arguments.
- #[serde(default)]
- pub classifier_tool_parameters: Option<Value>,
-}
-
-/// Mapping from classifier route tiers to configured backend tiers.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct LlmRoutingTierMapping {
- /// Backend tier for `recommended_tier: simple`.
- pub simple: String,
- /// Backend tier for `recommended_tier: medium`.
- pub medium: String,
- /// Backend tier for `recommended_tier: complex`.
- pub complex: String,
- /// Backend tier for `recommended_tier: reasoning`.
- pub reasoning: String,
-}
-
-impl ProfileConfig for LlmRoutingProfileConfig {
- type Runtime = LlmRoutingProfile;
-
- /// Builds the runtime profile using profile-local targets and backends.
- fn build(&self) -> Result<Self::Runtime> {
- validate_config(self)?;
- let policy = ClassifierPolicy::from_name(&self.profile_name)?;
- let strong_target = llm_routing_target(self.strong.clone());
- let weak_target = llm_routing_target(self.weak.clone());
- let classifier_target = llm_routing_classifier_target(self.classifier.clone());
- Ok(LlmRoutingProfile {
- policy,
- strong_backend: native_target_backend(strong_target)?,
- weak_backend: native_target_backend(weak_target)?,
- classifier_backend: native_target_backend(classifier_target.clone())?,
- classifier_target,
- fallback_target_on_evict: self.fallback_target_on_evict.clone(),
- classifier_min_confidence: self.classifier_min_confidence,
- classifier_fail_open: self.classifier_fail_open,
- classifier_recent_turn_window: self.classifier_recent_turn_window,
- classifier_max_tokens: self.classifier_max_tokens,
- alignment_min_confidence: self.alignment_min_confidence,
- default_tier: resolve_default_tier(self.default_tier.as_deref())?,
- tier_mapping: self
- .tier_mapping
- .clone()
- .unwrap_or_else(|| policy.default_tier_mapping()),
- classifier_tool: LlmRoutingClassifierTool::from_config(policy, self),
- stats: profile_stats_accumulator(),
- })
- }
-}
-
-/// LLM classifier profile runtime.
-pub struct LlmRoutingProfile {
- policy: ClassifierPolicy,
- strong_backend: TargetBackend,
- weak_backend: TargetBackend,
- classifier_backend: TargetBackend,
- classifier_target: LlmTarget,
- fallback_target_on_evict: LlmTargetId,
- classifier_min_confidence: f64,
- classifier_fail_open: bool,
- classifier_recent_turn_window: usize,
- classifier_max_tokens: u64,
- alignment_min_confidence: f64,
- default_tier: String,
- tier_mapping: LlmRoutingTierMapping,
- classifier_tool: LlmRoutingClassifierTool,
- stats: StatsAccumulator,
-}
-
-/// Processed LLM-routing request with the profile-owned route decision.
-pub struct LlmRoutingProcessedRequest {
- /// Routed input prepared for the selected backend.
- pub profile_input: ProfileInput,
- /// Selected LLM-routing decision for this request.
- pub decision: LlmRoutingDecision,
-}
-
-/// Serializable decision record used by Rust callers and tests.
-#[derive(Clone, Debug, PartialEq, Serialize)]
-pub struct LlmRoutingDecision {
- /// Concrete tier label selected for backend dispatch.
- pub tier: String,
- /// Decision source (`policy_tier`, `low_confidence`, `abstain`, etc.).
- pub source: String,
- /// Human-readable reason for the tier selection.
- pub reason: String,
- /// Policy tier computed from classifier features.
- pub policy_tier: Option<String>,
- /// Classifier-emitted recommended tier.
- pub llm_recommended_tier: Option<String>,
- /// Classifier confidence, when a classifier output was available.
- pub confidence: Option<f64>,
- /// Selected v2 target id.
- pub selected_target: String,
- /// Selected upstream model.
- pub selected_model: String,
- /// Raw classifier signals as JSON for audit/debugging.
- pub signals: Option<Value>,
-}
-
-impl LlmRoutingProfile {
- async fn classify(&self, request: &ChatRequest) -> Result<ClassifierSignals> {
- let classifier_request = ChatRequest::openai_chat(json!({
- "model": self.classifier_target.model.as_str(),
- "messages": [
- {"role": "system", "content": self.classifier_tool.system_prompt.as_str()},
- {"role": "user", "content": summarize_request(
- request,
- self.classifier_recent_turn_window,
- )},
- ],
- "temperature": 0,
- "max_tokens": self.classifier_max_tokens,
- "tools": [self.classifier_tool.definition()],
- "tool_choice": self.classifier_tool.choice(),
- }));
-
- let started_at = Instant::now();
- let response = self.classifier_backend.call(&classifier_request).await?;
- let latency_ms = started_at.elapsed().as_secs_f64() * 1000.0;
- let raw = classifier_tool_arguments(&response, &self.classifier_tool.name)?;
- let signals = self.policy.parse_signals(&raw)?;
- validate_signals(&signals)?;
- let usage = response.body().map(usage_from_body).unwrap_or_default();
- self.stats.record_classifier_usage(
- self.classifier_target.model.as_str(),
- usage,
- Some(latency_ms),
- )?;
- Ok(signals)
- }
-
- async fn route_request(&self, mut input: ProfileInput) -> Result<LlmRoutingProcessedRequest> {
- normalize_reasoning_effort(&mut input.request);
- let decision = match self.classify(&input.request).await {
- Ok(signals) => self.select(&signals)?,
- Err(error) => {
- self.stats
- .record_classifier_error(self.classifier_target.model.as_str())?;
- if self.classifier_fail_open {
- let signals = ClassifierSignals::abstain(self.policy);
- self.default_decision(
- "classifier_error_fall_open",
- "classifier failed and classifier_fail_open routed to the default tier",
- &signals,
- )?
- } else {
- return Err(classifier_error(error));
- }
- }
- };
- input.request.set_model(&decision.selected_model);
- Ok(LlmRoutingProcessedRequest {
- profile_input: input,
- decision,
- })
- }
-
- fn select(&self, signals: &ClassifierSignals) -> Result<LlmRoutingDecision> {
- let selected = if signals.abstain {
- self.default_decision("abstain", "classifier abstained", signals)?
- } else if signals.confidence < self.classifier_min_confidence {
- self.default_decision(
- "low_confidence",
- &format!(
- "classifier confidence {:.3} < min_confidence {:.3}",
- signals.confidence, self.classifier_min_confidence
- ),
- signals,
- )?
- } else {
- let mut policy_tier = self.policy.policy_tier(signals);
- let mut source = "policy_tier";
- let mut reason = format!("policy_tier() returned {:?}", policy_tier.as_str());
- if self.policy.align_with_llm_recommendation()
- && signals.confidence >= self.alignment_min_confidence
- && matches!(policy_tier, RouteTier::Simple | RouteTier::Medium)
- && matches!(
- signals.recommended_tier,
- RouteTier::Complex | RouteTier::Reasoning
- )
- {
- policy_tier = signals.recommended_tier;
- source = "llm_alignment_bump";
- reason = format!(
- "LLM alignment bump to {:?} at confidence {:.3}",
- policy_tier.as_str(),
- signals.confidence
- );
- }
- let mut tier = self.tier_for(policy_tier);
- if self.policy.escalate_on_tool_planning()
- && tier != self.default_tier
- && self.policy.requires_tool_planning(signals)
- {
- tier = self.default_tier.clone();
- source = "tool_planning_escalation";
- reason = format!(
- "tool planning escalation from {:?} to {:?}",
- self.tier_for(policy_tier),
- tier
- );
- }
- self.decision(LlmRoutingDecisionInput {
- tier,
- source,
- reason,
- policy_tier: Some(policy_tier),
- llm_recommended_tier: Some(signals.recommended_tier),
- confidence: Some(signals.confidence),
- signals: Some(signals.raw.clone()),
- })?
- };
- Ok(selected)
- }
-
- fn default_decision(
- &self,
- source: &'static str,
- reason: &str,
- signals: &ClassifierSignals,
- ) -> Result<LlmRoutingDecision> {
- self.decision(LlmRoutingDecisionInput {
- tier: self.default_tier.clone(),
- source,
- reason: reason.to_string(),
- policy_tier: Some(self.policy.policy_tier(signals)),
- llm_recommended_tier: Some(signals.recommended_tier),
- confidence: Some(signals.confidence),
- signals: Some(signals.raw.clone()),
- })
- }
-
- fn decision(&self, input: LlmRoutingDecisionInput) -> Result<LlmRoutingDecision> {
- let backend = self.backend_for_tier(&input.tier)?;
- Ok(LlmRoutingDecision {
- tier: input.tier,
- source: input.source.to_string(),
- reason: input.reason,
- policy_tier: input.policy_tier.map(|tier| tier.as_str().to_string()),
- llm_recommended_tier: input
- .llm_recommended_tier
- .map(|tier| tier.as_str().to_string()),
- confidence: input.confidence,
- selected_target: backend.target().id.to_string(),
- selected_model: backend.target().model.to_string(),
- signals: input.signals,
- })
- }
-
- fn backend_for_tier(&self, tier: &str) -> Result<&TargetBackend> {
- match tier {
- TIER_STRONG => Ok(&self.strong_backend),
- TIER_WEAK => Ok(&self.weak_backend),
- other => Err(SwitchyardError::InvalidConfig(format!(
- "llm-routing selected unknown tier {other:?}"
- ))),
- }
- }
-
- fn tier_for(&self, tier: RouteTier) -> String {
- self.tier_mapping.tier_for(tier).to_string()
- }
-
- fn backend_for_target(&self, target_id: &LlmTargetId) -> Result<&TargetBackend> {
- if *target_id == self.strong_backend.target().id {
- Ok(&self.strong_backend)
- } else if *target_id == self.weak_backend.target().id {
- Ok(&self.weak_backend)
- } else {
- Err(SwitchyardError::InvalidConfig(format!(
- "llm-routing selected target {target_id} that is not configured for this profile"
- )))
- }
- }
-
- fn tier_for_target(&self, target_id: &LlmTargetId) -> Result<&'static str> {
- if *target_id == self.strong_backend.target().id {
- Ok(TIER_STRONG)
- } else if *target_id == self.weak_backend.target().id {
- Ok(TIER_WEAK)
- } else {
- Err(SwitchyardError::InvalidConfig(format!(
- "llm-routing target {target_id} is not configured for this profile"
- )))
- }
- }
-
- fn fallback_processed_request(
- &self,
- processed: &LlmRoutingProcessedRequest,
- ) -> Result<LlmRoutingProcessedRequest> {
- let backend = self.backend_for_target(&self.fallback_target_on_evict)?;
- let target = backend.target();
- let mut profile_input = processed.profile_input.clone();
- profile_input.request.set_model(target.model.as_str());
- let mut decision = processed.decision.clone();
- decision.tier = self.tier_for_target(&target.id)?.to_string();
- decision.source = "context_overflow_fallback".to_string();
- decision.reason = format!(
- "selected target {} exceeded its context window; retried fallback target {}",
- processed.decision.selected_target, target.id
- );
- decision.selected_target = target.id.to_string();
- decision.selected_model = target.model.to_string();
- Ok(LlmRoutingProcessedRequest {
- profile_input,
- decision,
- })
- }
-
- async fn call_selected(
- &self,
- processed: &LlmRoutingProcessedRequest,
- ) -> (Result<ChatResponse>, f64) {
- let started_at = Instant::now();
- let backend = match self.backend_for_target_id_str(&processed.decision.selected_target) {
- Ok(backend) => backend,
- Err(error) => return (Err(error), 0.0),
- };
- let result = backend.call(&processed.profile_input.request).await;
- let latency_ms = started_at.elapsed().as_secs_f64() * 1000.0;
- (result, latency_ms)
- }
-
- fn backend_for_target_id_str(&self, target_id: &str) -> Result<&TargetBackend> {
- if target_id == self.strong_backend.target().id.as_str() {
- Ok(&self.strong_backend)
- } else if target_id == self.weak_backend.target().id.as_str() {
- Ok(&self.weak_backend)
- } else {
- Err(SwitchyardError::InvalidConfig(format!(
- "llm-routing selected target {target_id:?} that is not configured for this profile"
- )))
- }
- }
-
- fn record_success(
- &self,
- decision: &LlmRoutingDecision,
- response: ChatResponse,
- profile_started_at: Instant,
- backend_latency_ms: f64,
- ) -> Result<ChatResponse> {
- self.stats.record_success(
- decision.selected_model.as_str(),
- Some(backend_latency_ms),
- Some(decision.tier.as_str()),
- )?;
- record_usage_or_wrap_stream(
- &self.stats,
- decision.selected_model.as_str(),
- Some(decision.tier.as_str()),
- profile_started_at,
- backend_latency_ms,
- response,
- )
- }
-
- fn record_error(&self, decision: &LlmRoutingDecision) -> Result<()> {
- self.stats.record_error(
- decision.selected_model.as_str(),
- Some(decision.tier.as_str()),
- )
- }
-
- fn routing_metadata(&self, decision: &LlmRoutingDecision) -> RoutingMetadata {
- RoutingMetadata {
- selected_model: Some(decision.selected_model.clone()),
- selected_tier: Some(decision.tier.clone()),
- confidence: decision.confidence,
- router_version: Some(format!("llm-routing:{}:v1", self.policy.as_str())),
- tolerance: Some(self.classifier_min_confidence),
- rationale: Some(decision.reason.clone()),
- }
- }
-}
-
-#[async_trait]
-impl ProfileHooks for LlmRoutingProfile {
- type ProcessedRequest = LlmRoutingProcessedRequest;
-
- /// Runs classifier routing and returns a prepared backend request.
- async fn process(&self, input: ProfileInput) -> Result<Self::ProcessedRequest> {
- self.route_request(input).await
- }
-
- /// Leaves the backend response unchanged after LLM routing completes.
- async fn rprocess(
- &self,
- _processed: &Self::ProcessedRequest,
- response: ChatResponse,
- ) -> Result<ChatResponse> {
- Ok(response)
- }
-}
-
-#[async_trait]
-impl Profile for LlmRoutingProfile {
- /// Executes LLM classifier routing with one context-window fallback retry.
- async fn run(&self, input: ProfileInput) -> Result<ProfileResponse> {
- let profile_started_at = Instant::now();
- let processed = self.process(input).await?;
- let (first_result, first_backend_latency_ms) = self.call_selected(&processed).await;
- match first_result {
- Ok(response) => {
- let response = self.record_success(
- &processed.decision,
- response,
- profile_started_at,
- first_backend_latency_ms,
- )?;
- let response = self.rprocess(&processed, response).await?;
- return Ok(ProfileResponse::with_routing_metadata(
- response,
- self.routing_metadata(&processed.decision),
- ));
- }
- Err(SwitchyardError::ContextWindowExceeded { .. }) => {
- let retry = self.fallback_processed_request(&processed)?;
- let (retry_result, retry_backend_latency_ms) = self.call_selected(&retry).await;
- match retry_result {
- Ok(response) => {
- let response = self.record_success(
- &retry.decision,
- response,
- profile_started_at,
- retry_backend_latency_ms,
- )?;
- let response = self.rprocess(&retry, response).await?;
- return Ok(ProfileResponse::with_routing_metadata(
- response,
- self.routing_metadata(&retry.decision),
- ));
- }
- Err(SwitchyardError::ContextWindowExceeded { target_id, .. }) => {
- self.record_error(&retry.decision)?;
- return Err(SwitchyardError::ContextPoolExhausted {
- last_target_id: target_id,
- reason: "all attempted targets returned context-window overflow"
- .to_string(),
- });
- }
- Err(error) => {
- self.record_error(&retry.decision)?;
- return Err(error);
- }
- }
- }
- Err(error) => {
- self.record_error(&processed.decision)?;
- return Err(error);
- }
- }
- }
-}
-
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-enum RouteTier {
- Simple,
- Medium,
- Complex,
- Reasoning,
-}
-
-struct LlmRoutingDecisionInput {
- tier: String,
- source: &'static str,
- reason: String,
- policy_tier: Option<RouteTier>,
- llm_recommended_tier: Option<RouteTier>,
- confidence: Option<f64>,
- signals: Option<Value>,
-}
-
-impl RouteTier {
- fn parse(raw: &str) -> Option<Self> {
- match raw {
- "simple" => Some(Self::Simple),
- "medium" => Some(Self::Medium),
- "complex" => Some(Self::Complex),
- "reasoning" => Some(Self::Reasoning),
- _ => None,
- }
- }
-
- fn as_str(self) -> &'static str {
- match self {
- Self::Simple => "simple",
- Self::Medium => "medium",
- Self::Complex => "complex",
- Self::Reasoning => "reasoning",
- }
- }
-}
-
-#[derive(Clone, Debug)]
-struct LlmRoutingClassifierTool {
- name: String,
- description: String,
- parameters: Value,
- system_prompt: String,
-}
-
-impl LlmRoutingClassifierTool {
- fn from_config(policy: ClassifierPolicy, config: &LlmRoutingProfileConfig) -> Self {
- Self {
- name: config
- .classifier_tool_name
- .clone()
- .unwrap_or_else(default_classifier_tool_name),
- description: config
- .classifier_tool_description
- .clone()
- .unwrap_or_else(|| policy.default_tool_description().to_string()),
- parameters: config
- .classifier_tool_parameters
- .clone()
- .unwrap_or_else(|| policy.schema()),
- system_prompt: config
- .classifier_system_prompt
- .clone()
- .unwrap_or_else(|| policy.default_system_prompt().to_string()),
- }
- }
-
- fn definition(&self) -> Value {
- json!({
- "type": "function",
- "function": {
- "name": self.name.as_str(),
- "description": self.description.as_str(),
- "parameters": self.parameters.clone(),
- "strict": true,
- },
- })
- }
-
- fn choice(&self) -> Value {
- json!({
- "type": "function",
- "function": {"name": self.name.as_str()},
- })
- }
-}
-
-impl LlmRoutingTierMapping {
- fn tier_for(&self, tier: RouteTier) -> &str {
- match tier {
- RouteTier::Simple => &self.simple,
- RouteTier::Medium => &self.medium,
- RouteTier::Complex => &self.complex,
- RouteTier::Reasoning => &self.reasoning,
- }
- }
-}
-
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-enum ClassifierPolicy {
- General,
- CodingAgent,
- OpenClaw,
-}
-
-impl ClassifierPolicy {
- fn from_name(name: &str) -> Result<Self> {
- match name {
- PROFILE_GENERAL => Ok(Self::General),
- PROFILE_CODING_AGENT => Ok(Self::CodingAgent),
- PROFILE_OPENCLAW => Ok(Self::OpenClaw),
- other => Err(SwitchyardError::InvalidConfig(format!(
- "unknown LLM classifier profile {other:?}; expected general, coding_agent, or openclaw"
- ))),
- }
- }
-
- fn as_str(self) -> &'static str {
- match self {
- Self::General => PROFILE_GENERAL,
- Self::CodingAgent => PROFILE_CODING_AGENT,
- Self::OpenClaw => PROFILE_OPENCLAW,
- }
- }
-
- fn default_recommended_tier(self) -> RouteTier {
- RouteTier::Medium
- }
-
- fn default_tier_mapping(self) -> LlmRoutingTierMapping {
- match self {
- Self::General => LlmRoutingTierMapping {
- simple: TIER_WEAK.to_string(),
- medium: TIER_STRONG.to_string(),
- complex: TIER_STRONG.to_string(),
- reasoning: TIER_STRONG.to_string(),
- },
- Self::CodingAgent | Self::OpenClaw => LlmRoutingTierMapping {
- simple: TIER_WEAK.to_string(),
- medium: TIER_WEAK.to_string(),
- complex: TIER_STRONG.to_string(),
- reasoning: TIER_STRONG.to_string(),
- },
- }
- }
-
- fn escalate_on_tool_planning(self) -> bool {
- matches!(self, Self::CodingAgent | Self::OpenClaw)
- }
-
- fn align_with_llm_recommendation(self) -> bool {
- matches!(self, Self::CodingAgent | Self::OpenClaw)
- }
-
- fn default_system_prompt(self) -> &'static str {
- match self {
- Self::General => include_str!("llm_routing/prompts/general.md"),
- Self::CodingAgent => include_str!("llm_routing/prompts/coding_agent.md"),
- Self::OpenClaw => include_str!("llm_routing/prompts/openclaw.md"),
- }
- }
-
- fn default_tool_description(self) -> &'static str {
- match self {
- Self::General => "Select the backend tier for a general LLM request.",
- Self::CodingAgent => "Select the backend tier for a coding-agent request.",
- Self::OpenClaw => "Select the backend tier for an OpenClaw assistant request.",
- }
- }
-
- fn schema(self) -> Value {
- match self {
- Self::General => general_schema(),
- Self::CodingAgent => coding_agent_schema(),
- Self::OpenClaw => openclaw_schema(),
- }
- }
-
- fn parse_signals(self, raw: &str) -> Result<ClassifierSignals> {
- let value: Value = serde_json::from_str(raw).map_err(|error| {
- SwitchyardError::Backend(format!("classifier returned invalid JSON: {error}"))
- })?;
- ClassifierSignals::from_value(self, value)
- }
-
- fn policy_tier(self, signals: &ClassifierSignals) -> RouteTier {
- match self {
- Self::General => general_policy_tier(signals),
- Self::CodingAgent => coding_agent_policy_tier(signals),
- Self::OpenClaw => openclaw_policy_tier(signals),
- }
- }
-
- fn requires_tool_planning(self, signals: &ClassifierSignals) -> bool {
- match self {
- Self::General => bool_field(&signals.raw, "tool_planning_required"),
- Self::CodingAgent => {
- let turn_type = str_field(&signals.raw, "turn_type");
- let scope = str_field(&signals.raw, "code_modification_scope");
- let tool_count = u64_field(&signals.raw, "tool_call_count_estimate");
- let scope_is_modifying = matches!(
- scope,
- Some("function" | "file" | "multi_file" | "cross_module")
- );
- turn_type == Some("planning") || (tool_count >= 3 && scope_is_modifying)
- }
- Self::OpenClaw => {
- u64_field(&signals.raw, "tool_call_count_estimate") >= 2
- || str_field(&signals.raw, "turn_type") == Some("tool_orchestration")
- }
- }
- }
-}
-
-#[derive(Clone, Debug, PartialEq)]
-struct ClassifierSignals {
- raw: Value,
- recommended_tier: RouteTier,
- confidence: f64,
- abstain: bool,
-}
-
-impl ClassifierSignals {
- fn from_value(policy: ClassifierPolicy, raw: Value) -> Result<Self> {
- let recommended_tier = str_field(&raw, "recommended_tier")
- .and_then(RouteTier::parse)
- .ok_or_else(|| {
- SwitchyardError::Backend(
- "classifier JSON missing valid recommended_tier".to_string(),
- )
- })?;
- let confidence = raw
- .get("confidence")
- .and_then(Value::as_f64)
- .ok_or_else(|| {
- SwitchyardError::Backend("classifier JSON missing numeric confidence".to_string())
- })?;
- let abstain = raw.get("abstain").and_then(Value::as_bool).unwrap_or(false);
- let signals = Self {
- raw,
- recommended_tier,
- confidence,
- abstain,
- };
- validate_policy_fields(policy, &signals)?;
- Ok(signals)
- }
-
- fn abstain(policy: ClassifierPolicy) -> Self {
- let recommended_tier = policy.default_recommended_tier();
- let base = json!({
- "recommended_tier": recommended_tier.as_str(),
- "confidence": 0.0,
- "abstain": true,
- });
- let raw = match policy {
- ClassifierPolicy::General => merge_json_objects(
- base,
- json!({
- "task_type": "other",
- "complexity": "medium",
- "reasoning_depth": "light",
- "tool_planning_required": false,
- "precision_requirement": "medium",
- "context_dependency": "conversation",
- "structured_output_risk": "medium",
- "reason_code": "ambiguous",
- }),
- ),
- ClassifierPolicy::CodingAgent => merge_json_objects(
- base,
- json!({
- "turn_type": "exploration",
- "code_modification_scope": "none",
- "tool_call_count_estimate": 0,
- "requires_codebase_context": false,
- }),
- ),
- ClassifierPolicy::OpenClaw => merge_json_objects(
- base,
- json!({
- "turn_type": "chitchat",
- "tool_call_count_estimate": 0,
- "memory_dependency": "none",
- "external_action_required": false,
- "precision_requirement": "low",
- "ambiguity": "low",
- "channel_kind": "casual",
- }),
- ),
- };
- Self {
- raw,
- recommended_tier,
- confidence: 0.0,
- abstain: true,
- }
- }
-}
-
-fn merge_json_objects(mut left: Value, right: Value) -> Value {
- let (Some(left), Some(right)) = (left.as_object_mut(), right.as_object()) else {
- return left;
- };
- for (key, value) in right {
- left.insert(key.clone(), value.clone());
- }
- Value::Object(left.clone())
-}
-
-fn validate_signals(signals: &ClassifierSignals) -> Result<()> {
- if signals.confidence.is_finite() && (0.0..=1.0).contains(&signals.confidence) {
- return Ok(());
- }
- Err(SwitchyardError::Backend(format!(
- "classifier confidence must be finite and in [0.0, 1.0], got {:?}",
- signals.confidence
- )))
-}
-
-fn validate_policy_fields(policy: ClassifierPolicy, signals: &ClassifierSignals) -> Result<()> {
- let required = match policy {
- ClassifierPolicy::General => &[
- "task_type",
- "complexity",
- "reasoning_depth",
- "tool_planning_required",
- "precision_requirement",
- "context_dependency",
- "structured_output_risk",
- "reason_code",
- ][..],
- ClassifierPolicy::CodingAgent => &[
- "turn_type",
- "code_modification_scope",
- "tool_call_count_estimate",
- "requires_codebase_context",
- ][..],
- ClassifierPolicy::OpenClaw => &[
- "turn_type",
- "tool_call_count_estimate",
- "memory_dependency",
- "external_action_required",
- "precision_requirement",
- "ambiguity",
- "channel_kind",
- ][..],
- };
- for field in required {
- if signals.raw.get(*field).is_none() {
- return Err(SwitchyardError::Backend(format!(
- "classifier JSON missing required field {field:?}"
- )));
- }
- }
- Ok(())
-}
-
-fn general_policy_tier(signals: &ClassifierSignals) -> RouteTier {
- let mut scores = [0_i32; 4];
- match str_field(&signals.raw, "complexity") {
- Some("simple") => scores[0] += 2,
- Some("medium") => scores[1] += 2,
- Some("complex") => scores[2] += 2,
- Some("reasoning") => scores[3] += 2,
- _ => {}
- }
- match str_field(&signals.raw, "reasoning_depth") {
- Some("deep") => scores[3] += 2,
- Some("multi_step") => scores[2] += 1,
- _ => {}
- }
- if bool_field(&signals.raw, "tool_planning_required") {
- scores[2] += 1;
- }
- if str_field(&signals.raw, "precision_requirement") == Some("high") {
- scores[2] += 1;
- }
- if str_field(&signals.raw, "structured_output_risk") == Some("high") {
- scores[2] += 1;
- }
- argmax_tier(scores)
-}
-
-fn coding_agent_policy_tier(signals: &ClassifierSignals) -> RouteTier {
- let mut scores = [0_i32; 4];
- match str_field(&signals.raw, "turn_type") {
- Some("chitchat" | "clarification" | "summarize") => scores[0] += 2,
- Some("exploration" | "explanation" | "edit") => scores[1] += 2,
- Some("planning" | "debug") => scores[2] += 2,
- _ => {}
- }
- match str_field(&signals.raw, "code_modification_scope") {
- Some("none" | "single_line") => scores[0] += 1,
- Some("function" | "file") => scores[1] += 1,
- Some("multi_file" | "cross_module") => scores[2] += 2,
- _ => {}
- }
- let tool_count = u64_field(&signals.raw, "tool_call_count_estimate");
- if tool_count >= 4 {
- scores[2] += 1;
- } else if tool_count >= 2 {
- scores[1] += 1;
- }
- if bool_field(&signals.raw, "requires_codebase_context") {
- scores[1] += 1;
- }
- argmax_tier(scores)
-}
-
-fn openclaw_policy_tier(signals: &ClassifierSignals) -> RouteTier {
- let mut scores = [0_i32; 4];
- match str_field(&signals.raw, "turn_type") {
- Some("chitchat" | "lookup" | "memory_recall" | "clarification") => scores[0] += 2,
- Some("planning" | "explanation") => scores[1] += 2,
- Some("tool_orchestration" | "action") => scores[2] += 2,
- _ => {}
- }
- let tool_count = u64_field(&signals.raw, "tool_call_count_estimate");
- if tool_count >= 4 {
- scores[2] += 1;
- } else if tool_count >= 1 {
- scores[1] += 1;
- }
- match str_field(&signals.raw, "memory_dependency") {
- Some("heavy") => scores[2] += 2,
- Some("light") => scores[1] += 1,
- _ => {}
- }
- if bool_field(&signals.raw, "external_action_required")
- && str_field(&signals.raw, "precision_requirement") == Some("high")
- {
- scores[2] += 2;
- } else if bool_field(&signals.raw, "external_action_required") {
- scores[1] += 1;
- }
- match str_field(&signals.raw, "ambiguity") {
- Some("high") => scores[2] += 1,
- Some("medium") => scores[1] += 1,
- _ => {}
- }
- if str_field(&signals.raw, "channel_kind") == Some("casual") {
- scores[0] += 1;
- } else {
- scores[2] += 1;
- }
- argmax_tier(scores)
-}
-
-fn argmax_tier(scores: [i32; 4]) -> RouteTier {
- let best = scores.into_iter().max().unwrap_or(0);
- for (index, score) in scores.into_iter().enumerate().rev() {
- if score == best {
- return match index {
- 0 => RouteTier::Simple,
- 1 => RouteTier::Medium,
- 2 => RouteTier::Complex,
- _ => RouteTier::Reasoning,
- };
- }
- }
- RouteTier::Medium
-}
-
-fn classifier_tool_arguments(response: &ChatResponse, tool_name: &str) -> Result<String> {
- let body = response.body().ok_or_else(|| {
- SwitchyardError::Backend("classifier returned a streaming response".to_string())
- })?;
- let tool_calls = body
- .get("choices")
- .and_then(Value::as_array)
- .and_then(|choices| choices.first())
- .and_then(|choice| choice.get("message"))
- .and_then(|message| message.get("tool_calls"))
- .and_then(Value::as_array)
- .ok_or_else(|| {
- SwitchyardError::Backend("classifier response did not include tool_calls".to_string())
- })?;
- let Some(tool_call) = tool_calls.iter().find(|tool_call| {
- tool_call
- .get("function")
- .and_then(|function| function.get("name"))
- .and_then(Value::as_str)
- == Some(tool_name)
- }) else {
- return Err(SwitchyardError::Backend(format!(
- "classifier response did not call required tool {tool_name:?}"
- )));
- };
- let arguments = tool_call
- .get("function")
- .and_then(|function| function.get("arguments"))
- .ok_or_else(|| {
- SwitchyardError::Backend("classifier tool call omitted arguments".to_string())
- })?;
- match arguments {
- Value::String(raw) if !raw.trim().is_empty() => Ok(raw.trim().to_string()),
- Value::Object(_) => serde_json::to_string(arguments).map_err(|error| {
- SwitchyardError::Backend(format!(
- "classifier tool arguments could not be serialized: {error}"
- ))
- }),
- _ => Err(SwitchyardError::Backend(
- "classifier tool arguments must be a non-empty JSON string or object".to_string(),
- )),
- }
-}
-
-fn summarize_request(request: &ChatRequest, recent_turn_window: usize) -> String {
- let body = request.body();
- let summary_body = body
- .as_object()
- .map(|object| condense_body(object, recent_turn_window))
- .unwrap_or_else(|| body.clone());
- let payload = json!({
- "request_type": request.request_type(),
- "body": summary_body,
- });
- let text = serde_json::to_string(&payload).unwrap_or_else(|_| "{}".to_string());
- const MAX_CHARS: usize = 16_000;
- if text.len() <= MAX_CHARS {
- text
- } else {
- format!(
- "{}...<truncated>",
- truncate_at_char_boundary(&text, MAX_CHARS - 32)
- )
- }
-}
-
-fn truncate_at_char_boundary(text: &str, max_bytes: usize) -> &str {
- if text.len() <= max_bytes {
- return text;
- }
- let mut end = max_bytes.min(text.len());
- while !text.is_char_boundary(end) {
- end -= 1;
- }
- &text[..end]
-}
-
-fn condense_body(body: &Map<String, Value>, recent_turn_window: usize) -> Value {
- let mut out = Map::new();
- for (key, value) in body {
- if !matches!(key.as_str(), "messages" | "tools" | "input" | "tool_choice") {
- out.insert(key.clone(), value.clone());
- }
- }
- if let Some(Value::Array(tools)) = body.get("tools") {
- out.insert(
- "tools".to_string(),
- Value::Array(tools.iter().map(condense_tool).collect()),
- );
- }
- if let Some(Value::Array(messages)) = body.get("messages") {
- out.insert(
- "messages".to_string(),
- Value::Array(trim_messages(messages, recent_turn_window)),
- );
- }
- if let Some(input) = body.get("input") {
- match input {
- Value::Array(items) => {
- out.insert(
- "input".to_string(),
- Value::Array(trim_messages(items, recent_turn_window)),
- );
- }
- Value::String(_) => {
- out.insert("input".to_string(), input.clone());
- }
- _ => {}
- }
- }
- Value::Object(out)
-}
-
-fn condense_tool(tool: &Value) -> Value {
- let Some(object) = tool.as_object() else {
- return tool.clone();
- };
- let mut out = object.clone();
- if let Some(Value::Object(function)) = object.get("function") {
- let mut slim = function.clone();
- slim.remove("parameters");
- out.insert("function".to_string(), Value::Object(slim));
- }
- out.remove("input_schema");
- Value::Object(out)
-}
-
-fn trim_messages(messages: &[Value], recent_turn_window: usize) -> Vec<Value> {
- let mut system = Vec::new();
- let mut first_user = None;
- let mut first_user_idx = None;
- for (idx, message) in messages.iter().enumerate() {
- let role = message
- .as_object()
- .and_then(|object| object.get("role"))
- .and_then(Value::as_str);
- match role {
- Some("system" | "developer") => system.push(message.clone()),
- Some("user") if first_user.is_none() => {
- first_user = Some(message.clone());
- first_user_idx = Some(idx);
- }
- _ => {}
- }
- }
- let Some(first_user) = first_user else {
- return system;
- };
- let tail = messages
- .iter()
- .enumerate()
- .filter(|(idx, message)| {
- *idx > first_user_idx.unwrap_or(0)
- && message
- .as_object()
- .and_then(|object| object.get("role"))
- .and_then(Value::as_str)
- .is_some_and(|role| !matches!(role, "system" | "developer"))
- })
- .map(|(_, message)| message.clone())
- .collect::<Vec<_>>();
- if recent_turn_window == 0 {
- let mut out = system;
- out.push(first_user);
- if let Some(last_user) = tail.iter().rev().find(|message| {
- message
- .as_object()
- .and_then(|object| object.get("role"))
- .and_then(Value::as_str)
- == Some("user")
- }) {
- out.push(last_user.clone());
- }
- return out;
- }
- let mut out = system;
- out.push(first_user);
- let start = tail.len().saturating_sub(recent_turn_window);
- out.extend_from_slice(&tail[start..]);
- out
-}
-
-fn base_properties(extra: Vec<(&'static str, Value)>) -> Map<String, Value> {
- let mut properties = Map::from_iter([
- (
- "recommended_tier".to_string(),
- enum_schema(&["simple", "medium", "complex", "reasoning"]),
- ),
- ("confidence".to_string(), json!({"type": "number"})),
- ("abstain".to_string(), json!({"type": "boolean"})),
- ]);
- for (name, value) in extra {
- properties.insert(name.to_string(), value);
- }
- properties
-}
-
-fn object_schema(properties: Map<String, Value>) -> Value {
- let required = properties
- .keys()
- .cloned()
- .map(Value::String)
- .collect::<Vec<_>>();
- json!({
- "type": "object",
- "properties": properties,
- "required": required,
- "additionalProperties": false,
- })
-}
-
-fn enum_schema(values: &[&str]) -> Value {
- json!({
- "type": "string",
- "enum": values,
- })
-}
-
-fn general_schema() -> Value {
- object_schema(base_properties(vec![
- (
- "task_type",
- enum_schema(&[
- "chat",
- "summarization",
- "extraction",
- "translation",
- "coding",
- "debugging",
- "math",
- "planning",
- "creative_writing",
- "agentic_task",
- "research",
- "data_analysis",
- "other",
- ]),
- ),
- (
- "complexity",
- enum_schema(&["simple", "medium", "complex", "reasoning"]),
- ),
- (
- "reasoning_depth",
- enum_schema(&["none", "light", "multi_step", "deep"]),
- ),
- ("tool_planning_required", json!({"type": "boolean"})),
- (
- "precision_requirement",
- enum_schema(&["low", "medium", "high"]),
- ),
- (
- "context_dependency",
- enum_schema(&["latest_message", "conversation", "external_context"]),
- ),
- (
- "structured_output_risk",
- enum_schema(&["low", "medium", "high"]),
- ),
- (
- "reason_code",
- enum_schema(&[
- "simple_qa",
- "summarization",
- "extraction",
- "translation",
- "coding_simple",
- "coding_complex",
- "debugging",
- "math_reasoning",
- "tool_agentic",
- "long_context",
- "structured_output",
- "creative_generation",
- "research_synthesis",
- "ambiguous",
- "other",
- ]),
- ),
- ]))
-}
-
-fn coding_agent_schema() -> Value {
- object_schema(base_properties(vec![
- (
- "turn_type",
- enum_schema(&[
- "chitchat",
- "planning",
- "exploration",
- "edit",
- "debug",
- "explanation",
- "clarification",
- "summarize",
- ]),
- ),
- (
- "code_modification_scope",
- enum_schema(&[
- "none",
- "single_line",
- "function",
- "file",
- "multi_file",
- "cross_module",
- ]),
- ),
- ("tool_call_count_estimate", json!({"type": "integer"})),
- ("requires_codebase_context", json!({"type": "boolean"})),
- ]))
-}
-
-fn openclaw_schema() -> Value {
- object_schema(base_properties(vec![
- (
- "turn_type",
- enum_schema(&[
- "chitchat",
- "lookup",
- "memory_recall",
- "planning",
- "tool_orchestration",
- "action",
- "explanation",
- "clarification",
- ]),
- ),
- ("tool_call_count_estimate", json!({"type": "integer"})),
- (
- "memory_dependency",
- enum_schema(&["none", "light", "heavy"]),
- ),
- ("external_action_required", json!({"type": "boolean"})),
- (
- "precision_requirement",
- enum_schema(&["low", "medium", "high"]),
- ),
- ("ambiguity", enum_schema(&["low", "medium", "high"])),
- ("channel_kind", enum_schema(&["casual", "deliberate"])),
- ]))
-}
-
-fn str_field<'a>(value: &'a Value, name: &str) -> Option<&'a str> {
- value.get(name).and_then(Value::as_str)
-}
-
-fn bool_field(value: &Value, name: &str) -> bool {
- value.get(name).and_then(Value::as_bool).unwrap_or(false)
-}
-
-fn u64_field(value: &Value, name: &str) -> u64 {
- value.get(name).and_then(Value::as_u64).unwrap_or(0)
-}
-
-fn classifier_error(error: SwitchyardError) -> SwitchyardError {
- SwitchyardError::Backend(format!(
- "LLM classifier failed to produce valid route signals: {error}"
- ))
-}
-
-fn validate_config(config: &LlmRoutingProfileConfig) -> Result<()> {
- ClassifierPolicy::from_name(&config.profile_name)?;
- if !(config.classifier_min_confidence.is_finite()
- && (0.0..=1.0).contains(&config.classifier_min_confidence))
- {
- return Err(SwitchyardError::InvalidConfig(format!(
- "classifier_min_confidence must be finite and in [0.0, 1.0], got {:?}",
- config.classifier_min_confidence
- )));
- }
- if !(config.alignment_min_confidence.is_finite()
- && (0.0..=1.0).contains(&config.alignment_min_confidence))
- {
- return Err(SwitchyardError::InvalidConfig(format!(
- "alignment_min_confidence must be finite and in [0.0, 1.0], got {:?}",
- config.alignment_min_confidence
- )));
- }
- if config.classifier_max_tokens == 0 {
- return Err(SwitchyardError::InvalidConfig(
- "classifier_max_tokens must be greater than 0".to_string(),
- ));
- }
- if let Some(default_tier) = &config.default_tier {
- validate_backend_tier("default_tier", default_tier)?;
- }
- if let Some(tier_mapping) = &config.tier_mapping {
- validate_backend_tier("tier_mapping.simple", &tier_mapping.simple)?;
- validate_backend_tier("tier_mapping.medium", &tier_mapping.medium)?;
- validate_backend_tier("tier_mapping.complex", &tier_mapping.complex)?;
- validate_backend_tier("tier_mapping.reasoning", &tier_mapping.reasoning)?;
- }
- if let Some(tool_name) = &config.classifier_tool_name {
- validate_classifier_tool_name(tool_name)?;
- }
- if let Some(tool_parameters) = &config.classifier_tool_parameters {
- validate_classifier_tool_parameters(tool_parameters)?;
- }
- if config.fallback_target_on_evict != config.strong.id
- && config.fallback_target_on_evict != config.weak.id
- {
- return Err(SwitchyardError::InvalidConfig(format!(
- "fallback_target_on_evict={} must match one of [{}, {}]",
- config.fallback_target_on_evict, config.weak.id, config.strong.id
- )));
- }
- if config.classifier.format != BackendFormat::OpenAi {
- return Err(SwitchyardError::InvalidConfig(
- "llm-routing classifier target must use format: openai".to_string(),
- ));
- }
- Ok(())
-}
-
-fn validate_backend_tier(field: &str, value: &str) -> Result<()> {
- match value {
- TIER_STRONG | TIER_WEAK => Ok(()),
- other => Err(SwitchyardError::InvalidConfig(format!(
- "{field} must be {TIER_STRONG:?} or {TIER_WEAK:?}, got {other:?}"
- ))),
- }
-}
-
-fn validate_classifier_tool_name(value: &str) -> Result<()> {
- let valid = !value.is_empty()
- && value
- .chars()
- .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-'));
- if valid {
- Ok(())
- } else {
- Err(SwitchyardError::InvalidConfig(format!(
- "classifier_tool_name must be non-empty and contain only ASCII letters, numbers, '_' or '-', got {value:?}"
- )))
- }
-}
-
-fn validate_classifier_tool_parameters(value: &Value) -> Result<()> {
- validate_strict_object_schema("classifier_tool_parameters", value)
-}
-
-fn validate_strict_object_schema(path: &str, value: &Value) -> Result<()> {
- if value.get("type").and_then(Value::as_str) != Some("object") {
- return Err(SwitchyardError::InvalidConfig(format!(
- "{path} must be a strict JSON schema object with type: object"
- )));
- }
- if value.get("additionalProperties").and_then(Value::as_bool) != Some(false) {
- return Err(SwitchyardError::InvalidConfig(format!(
- "{path} must set additionalProperties: false when strict tool calls are enabled"
- )));
- }
- let properties = value
- .get("properties")
- .and_then(Value::as_object)
- .ok_or_else(|| {
- SwitchyardError::InvalidConfig(format!(
- "{path} must define object properties for strict tool calls"
- ))
- })?;
- let required = value
- .get("required")
- .and_then(Value::as_array)
- .ok_or_else(|| {
- SwitchyardError::InvalidConfig(format!(
- "{path} must list all properties in required for strict tool calls"
- ))
- })?;
- let required = required
- .iter()
- .map(|item| {
- item.as_str().ok_or_else(|| {
- SwitchyardError::InvalidConfig(format!(
- "{path}.required must contain only property names"
- ))
- })
- })
- .collect::<Result<BTreeSet<_>>>()?;
- let property_names = properties
- .keys()
- .map(String::as_str)
- .collect::<BTreeSet<_>>();
- if required != property_names {
- return Err(SwitchyardError::InvalidConfig(format!(
- "{path}.required must contain exactly every key from properties for strict tool calls"
- )));
- }
- for (name, schema) in properties {
- validate_nested_strict_schema(&format!("{path}.properties.{name}"), schema)?;
- }
- Ok(())
-}
-
-fn validate_nested_strict_schema(path: &str, value: &Value) -> Result<()> {
- if value.get("type").and_then(Value::as_str) == Some("object")
- || value.get("properties").is_some()
- {
- validate_strict_object_schema(path, value)?;
- }
- if let Some(items) = value.get("items") {
- validate_nested_strict_schema(&format!("{path}.items"), items)?;
- }
- for combinator in ["anyOf", "oneOf", "allOf"] {
- if let Some(schemas) = value.get(combinator).and_then(Value::as_array) {
- for (index, schema) in schemas.iter().enumerate() {
- validate_nested_strict_schema(&format!("{path}.{combinator}[{index}]"), schema)?;
- }
- }
- }
- Ok(())
-}
-
-fn resolve_default_tier(value: Option<&str>) -> Result<String> {
- let value = value.unwrap_or(TIER_STRONG);
- validate_backend_tier("default_tier", value)?;
- Ok(value.to_string())
-}
-
-fn default_profile_name() -> String {
- PROFILE_CODING_AGENT.to_string()
-}
-
-fn default_classifier_fail_open() -> bool {
- true
-}
-
-fn default_recent_turn_window() -> usize {
- 4
-}
-
-fn default_classifier_max_tokens() -> u64 {
- DEFAULT_CLASSIFIER_MAX_TOKENS
-}
-
-fn default_alignment_min_confidence() -> f64 {
- DEFAULT_ALIGNMENT_MIN_CONFIDENCE
-}
-
-fn default_classifier_tool_name() -> String {
- DEFAULT_CLASSIFIER_TOOL_NAME.to_string()
-}
-
-fn default_fallback_target_on_evict() -> LlmTargetId {
- LlmTargetId::from_static(TIER_STRONG)
-}
-
-fn llm_routing_target(mut target: LlmTarget) -> LlmTarget {
- apply_deepseek_defaults(&mut target);
- target
-}
-
-fn llm_routing_classifier_target(mut target: LlmTarget) -> LlmTarget {
- if target.extra_body.is_none() && model_accepts_reasoning_hint(target.model.as_str()) {
- target.extra_body = Some(json!({"chat_template_kwargs": {"enable_thinking": false}}));
- }
- apply_deepseek_defaults(&mut target);
- target
-}
-
-fn apply_deepseek_defaults(target: &mut LlmTarget) {
- let model = target.model.as_str().to_ascii_lowercase();
- if target.extra_body.is_none() && model.contains("deepseek-v4") {
- target.extra_body = Some(json!({"chat_template_kwargs": {"enable_thinking": false}}));
- }
- if target.extra_headers.is_empty() && model.contains("deepseek") {
- target
- .extra_headers
- .insert("X-Inference-Priority".to_string(), "batch".to_string());
- }
-}
-
-fn model_accepts_reasoning_hint(model: &str) -> bool {
- let lowered = model.to_ascii_lowercase();
- !["anthropic", "bedrock", "claude"]
- .iter()
- .any(|tag| lowered.contains(tag))
-}
-
-fn normalize_reasoning_effort(request: &mut ChatRequest) {
- const VALID: &[&str] = &["low", "medium", "high", "max"];
- let Value::Object(body) = request.body_mut() else {
- return;
- };
- let Some(effort) = body.get("reasoning_effort").and_then(Value::as_str) else {
- return;
- };
- if VALID.contains(&effort) {
- return;
- }
- body.insert(
- "reasoning_effort".to_string(),
- Value::String("high".to_string()),
- );
-}
diff --git a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/coding_agent.md b/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/coding_agent.md
deleted file mode 100644
index 808553742..000000000
--- a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/coding_agent.md
+++ /dev/null
@@ -1 +0,0 @@
-You are a routing classifier inside a coding-agent harness. Rate the turn, then call the provided route-selection tool exactly once with the requested schema.
diff --git a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/general.md b/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/general.md
deleted file mode 100644
index b66d77890..000000000
--- a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/general.md
+++ /dev/null
@@ -1 +0,0 @@
-You are a routing classifier for an LLM proxy. Call the provided route-selection tool exactly once with the requested schema.
diff --git a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/openclaw.md b/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/openclaw.md
deleted file mode 100644
index 2abb86052..000000000
--- a/crates/switchyard-components-v2/src/profiles/llm_routing/prompts/openclaw.md
+++ /dev/null
@@ -1 +0,0 @@
-You are a routing classifier inside an OpenClaw personal-assistant agent. Rate the message, then call the provided route-selection tool exactly once with the requested schema.
diff --git a/crates/switchyard-components-v2/src/profiles/macros.rs b/crates/switchyard-components-v2/src/profiles/macros.rs
deleted file mode 100644
index 6f6bcfcb6..000000000
--- a/crates/switchyard-components-v2/src/profiles/macros.rs
+++ /dev/null
@@ -1,87 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Macros used by profile implementations and profile config wiring.
-
-macro_rules! profile_types {
- ($($config:ident),+ $(,)?) => {
- /// Resolved, strongly typed config for one profile entry.
- ///
- /// Variants intentionally mirror config type names so this macro has one
- /// input per profile and does not need a second variant-name registry.
- #[allow(clippy::enum_variant_names)]
- #[derive(Clone, Debug, PartialEq)]
- pub(crate) enum ProfileConfigEntry {
- $(
- /// Config variant generated from the profile registry.
- $config(Box<$config>),
- )+
- }
-
- impl ProfileConfigEntry {
- /// Returns the file-facing type discriminator for this resolved profile config.
- pub(crate) fn profile_type(&self) -> &'static str {
- match self {
- $(
- Self::$config(_) =>
- <$config as crate::config::ProfileConfigDefinition>::PROFILE_TYPE,
- )+
- }
- }
-
- /// Builds this resolved config into the erased runtime profile.
- pub(crate) fn build_boxed(
- &self,
- ) -> switchyard_core::Result<Box<dyn crate::Profile>> {
- match self {
- $(
- Self::$config(config) =>
- <$config as crate::ProfileConfig>::build_boxed(config.as_ref()),
- )+
- }
- }
- }
-
- /// Parses a serialized profile body by dispatching to the owning config type.
- ///
- /// The `type` discriminator is matched with `-` and `_` treated as
- /// equivalent, so a snake_case name copied from a legacy route bundle
- /// (e.g. `random_routing`) resolves to its v2 profile, and a hyphenated
- /// spelling of an underscore type (e.g. `stage-router`) resolves too.
- pub(crate) fn parse_profile_config(
- profile_type: &str,
- value: serde_json::Value,
- env: &crate::config::ProfileBuildEnv<'_>,
- ) -> switchyard_core::Result<ProfileConfigEntry> {
- // Compare treating `-` and `_` as equal without allocating. Profile
- // type names are short ASCII, so an equal-length per-byte scan that
- // folds the separators is enough. Registered names must stay unique
- // under this folding (they are today) so dispatch is unambiguous.
- fn eq_ignoring_separators(a: &str, b: &str) -> bool {
- a.len() == b.len()
- && a.bytes().zip(b.bytes()).all(|(x, y)| {
- x == y || (matches!(x, b'-' | b'_') && matches!(y, b'-' | b'_'))
- })
- }
-
- $(
- if eq_ignoring_separators(
- profile_type,
- <$config as crate::config::ProfileConfigDefinition>::PROFILE_TYPE,
- ) {
- let config =
- <$config as crate::config::ProfileConfigDefinition>::parse_profile_config(
- value,
- env,
- )?;
- return Ok(ProfileConfigEntry::$config(Box::new(config)));
- }
- )+
- Err(switchyard_core::SwitchyardError::InvalidConfig(format!(
- "unknown profile type `{profile_type}`"
- )))
- }
- };
-}
-
-pub(crate) use profile_types;
diff --git a/crates/switchyard-components-v2/src/profiles/mod.rs b/crates/switchyard-components-v2/src/profiles/mod.rs
deleted file mode 100644
index f83960023..000000000
--- a/crates/switchyard-components-v2/src/profiles/mod.rs
+++ /dev/null
@@ -1,35 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Profile implementations in the flatter components-v2 design.
-
-mod latency_service;
-mod llm_routing;
-mod macros;
-mod noop;
-mod passthrough;
-mod profile_types;
-mod random_routing;
-mod stage_router;
-mod subagent_override;
-
-pub use latency_service::{
- EndpointHealth, EndpointHealthStatus, LatencyServiceProcessedRequest, LatencyServiceProfile,
- LatencyServiceProfileConfig, SelectedTarget,
-};
-pub use llm_routing::{
- LlmRoutingDecision, LlmRoutingProcessedRequest, LlmRoutingProfile, LlmRoutingProfileConfig,
- LlmRoutingTierMapping,
-};
-pub use noop::{NoopProfile, NoopProfileConfig};
-pub use passthrough::{PassthroughProfile, PassthroughProfileConfig};
-pub(crate) use profile_types::{parse_profile_config, ProfileConfigEntry};
-pub use random_routing::{
- RandomRoutingProcessedRequest, RandomRoutingProfile, RandomRoutingProfileConfig,
-};
-pub use stage_router::{
- StageRouterClassifierConfig, StageRouterDecision, StageRouterDecisionSource,
- StageRouterPickerMode, StageRouterProcessedRequest, StageRouterProfile,
- StageRouterProfileConfig, StageRouterTier,
-};
-pub(crate) use subagent_override::SubagentOverrideProfile;
diff --git a/crates/switchyard-components-v2/src/profiles/noop.rs b/crates/switchyard-components-v2/src/profiles/noop.rs
deleted file mode 100644
index efa509ce1..000000000
--- a/crates/switchyard-components-v2/src/profiles/noop.rs
+++ /dev/null
@@ -1,78 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! No-op profile for local smoke tests and benchmark harnesses.
-
-use async_trait::async_trait;
-use serde_json::json;
-use switchyard_core::{ChatResponse, Result};
-
-use crate::{profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse};
-
-/// Config for a no-op profile.
-#[profile_config("noop")]
-pub struct NoopProfileConfig {}
-
-impl ProfileConfig for NoopProfileConfig {
- type Runtime = NoopProfile;
-
- /// Builds the no-op runtime profile.
- fn build(&self) -> Result<Self::Runtime> {
- Ok(NoopProfile {})
- }
-}
-
-/// Profile that returns a deterministic local response without calling an upstream model.
-pub struct NoopProfile {}
-
-#[async_trait]
-impl ProfileHooks for NoopProfile {
- type ProcessedRequest = ProfileInput;
-
- /// Leaves the request unchanged for hook-level inspection.
- async fn process(&self, input: ProfileInput) -> Result<Self::ProcessedRequest> {
- Ok(input)
- }
-
- /// Leaves the response unchanged after no-op response creation.
- async fn rprocess(
- &self,
- _processed: &Self::ProcessedRequest,
- response: ChatResponse,
- ) -> Result<ChatResponse> {
- Ok(response)
- }
-}
-
-#[async_trait]
-impl Profile for NoopProfile {
- /// Returns a deterministic OpenAI-compatible chat completion.
- async fn run(&self, input: ProfileInput) -> Result<ProfileResponse> {
- let processed = self.process(input).await?;
- let model = processed.request.model().unwrap_or("switchyard/noop");
- let response = self
- .rprocess(
- &processed,
- ChatResponse::openai_completion(json!({
- "id": "switchyard-noop",
- "object": "chat.completion",
- "model": model,
- "choices": [{
- "index": 0,
- "message": {
- "role": "assistant",
- "content": "ok"
- },
- "finish_reason": "stop"
- }],
- "usage": {
- "prompt_tokens": 0,
- "completion_tokens": 0,
- "total_tokens": 0
- }
- })),
- )
- .await?;
- Ok(ProfileResponse::from(response))
- }
-}
diff --git a/crates/switchyard-components-v2/src/profiles/passthrough.rs b/crates/switchyard-components-v2/src/profiles/passthrough.rs
deleted file mode 100644
index 8d9edb892..000000000
--- a/crates/switchyard-components-v2/src/profiles/passthrough.rs
+++ /dev/null
@@ -1,273 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Passthrough profile implemented as a single profile-owned runtime.
-
-use std::time::Instant;
-
-use async_trait::async_trait;
-use switchyard_components::StatsAccumulator;
-use switchyard_core::{ChatResponse, LlmTarget, Result};
-
-use crate::backend::{native_target_backend, TargetBackend};
-use crate::profile_stats_accumulator;
-use crate::stats_recording::record_usage_or_wrap_stream;
-use crate::{profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse};
-
-/// Config for the flatter passthrough profile.
-#[profile_config("passthrough")]
-pub struct PassthroughProfileConfig {
- /// Target served by this profile without routing.
- #[profile_target]
- pub target: LlmTarget,
-}
-
-impl ProfileConfig for PassthroughProfileConfig {
- type Runtime = PassthroughProfile;
-
- /// Builds the runtime profile using the existing native backend stack.
- fn build(&self) -> Result<Self::Runtime> {
- Ok(PassthroughProfile {
- backend: native_target_backend(self.target.clone())?,
- stats: profile_stats_accumulator(),
- })
- }
-}
-
-/// Passthrough profile in the flatter design.
-pub struct PassthroughProfile {
- backend: TargetBackend,
- stats: StatsAccumulator,
-}
-
-#[async_trait]
-impl ProfileHooks for PassthroughProfile {
- type ProcessedRequest = ProfileInput;
-
- /// Rewrites the request model to the single configured target.
- async fn process(&self, mut input: ProfileInput) -> Result<Self::ProcessedRequest> {
- input
- .request
- .set_model(self.backend.target().model.as_str());
- Ok(input)
- }
-
- /// Leaves the response unchanged after the backend call.
- async fn rprocess(
- &self,
- _processed: &Self::ProcessedRequest,
- response: ChatResponse,
- ) -> Result<ChatResponse> {
- Ok(response)
- }
-}
-
-#[async_trait]
-impl Profile for PassthroughProfile {
- /// Executes passthrough by composing the hook methods around one backend call.
- ///
- /// Passthrough has no per-call routing decision to preserve, so `run()` can use the
- /// straightforward `process -> backend -> rprocess` shape described by the
- /// [`Profile`] trait. Profiles that select targets dynamically may use a different
- /// internal flow while still exposing the same public lifecycle methods.
- async fn run(&self, input: ProfileInput) -> Result<ProfileResponse> {
- let profile_started_at = Instant::now();
- let processed = self.process(input).await?;
- let target_model = self.backend.target().model.clone();
- let backend_started_at = Instant::now();
- let response = match self.backend.call(&processed.request).await {
- Ok(response) => response,
- Err(error) => {
- self.stats.record_error(target_model.as_str(), None)?;
- return Err(error);
- }
- };
- let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0;
- self.stats
- .record_success(target_model.to_string(), Some(backend_latency_ms), None)?;
- let response = record_usage_or_wrap_stream(
- &self.stats,
- target_model.as_str(),
- None,
- profile_started_at,
- backend_latency_ms,
- response,
- )?;
- let response = self.rprocess(&processed, response).await?;
- Ok(ProfileResponse::from(response))
- }
-}
-
-#[cfg(test)]
-mod tests {
- use std::sync::{Arc, Mutex};
-
- use async_trait::async_trait;
- use serde_json::{json, Value};
- use switchyard_core::{BackendFormat, ChatRequest, LlmTargetId, ModelId, SwitchyardError};
-
- use crate::backend::{ProfileBackend, TargetBackend};
- use crate::RequestMetadata;
-
- use super::*;
-
- #[derive(Clone, Debug, PartialEq)]
- struct ObservedCall {
- body: Value,
- }
-
- struct TestBackend {
- calls: Arc<Mutex<Vec<ObservedCall>>>,
- }
-
- #[async_trait]
- impl ProfileBackend for TestBackend {
- async fn call(&self, request: &ChatRequest) -> Result<ChatResponse> {
- self.calls
- .lock()
- .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))?
- .push(ObservedCall {
- body: request.body().clone(),
- });
- Ok(ChatResponse::openai_completion(json!({
- "model": request.model(),
- "served_model": request.model(),
- "usage": {
- "prompt_tokens": 5,
- "completion_tokens": 3,
- },
- })))
- }
- }
-
- fn target(id: &str, model: &str) -> Result<LlmTarget> {
- let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
- target.format = BackendFormat::OpenAi;
- Ok(target)
- }
-
- fn backend(target: &LlmTarget, calls: Arc<Mutex<Vec<ObservedCall>>>) -> TargetBackend {
- TargetBackend::new(target.clone(), Arc::new(TestBackend { calls }))
- }
-
- fn observed(calls: &Arc<Mutex<Vec<ObservedCall>>>) -> Result<Vec<ObservedCall>> {
- calls
- .lock()
- .map(|calls| calls.clone())
- .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))
- }
-
- fn profile_input(request: ChatRequest) -> ProfileInput {
- ProfileInput {
- request,
- metadata: RequestMetadata::default(),
- }
- }
-
- fn profile(target: LlmTarget) -> Result<(PassthroughProfile, Arc<Mutex<Vec<ObservedCall>>>)> {
- let calls = Arc::new(Mutex::new(Vec::new()));
- let profile = PassthroughProfile {
- backend: backend(&target, calls.clone()),
- stats: StatsAccumulator::new(),
- };
- Ok((profile, calls))
- }
-
- #[tokio::test]
- async fn passthrough_profile_calls_single_target_backend() -> Result<()> {
- let (profile, calls) = profile(target("direct", "provider/model")?)?;
-
- let response = profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [{"role": "user", "content": "hi"}],
- }))))
- .await?;
-
- let response = response.response;
- let calls = observed(&calls)?;
- assert_eq!(calls.len(), 1);
- assert_eq!(calls[0].body["model"], "provider/model");
- match response {
- ChatResponse::OpenAiCompletion(body) => {
- assert_eq!(body.body()["served_model"], "provider/model");
- }
- _ => return Err(SwitchyardError::Other("unexpected response shape".into())),
- }
- Ok(())
- }
-
- #[tokio::test]
- async fn run_records_stats_for_single_target() -> Result<()> {
- let (profile, _calls) = profile(target("direct", "provider/model")?)?;
-
- let _response = profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [],
- }))))
- .await?;
-
- let snapshot = profile.stats.snapshot()?;
- assert_eq!(snapshot.total_requests, 1);
- assert_eq!(snapshot.total_tokens.prompt, 5);
- assert_eq!(snapshot.total_tokens.completion, 3);
- let model = snapshot.models.get("provider/model").ok_or_else(|| {
- SwitchyardError::Other("provider model stats should be present".into())
- })?;
- assert_eq!(model.calls, 1);
- assert_eq!(model.tier, None);
- Ok(())
- }
-
- #[tokio::test]
- async fn process_only_prepares_target_request_and_does_not_call_backend() -> Result<()> {
- let (profile, calls) = profile(target("direct", "provider/model")?)?;
- let request = ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [],
- }));
-
- let processed = profile.process(profile_input(request.clone())).await?;
-
- assert_eq!(processed.request.model(), Some("provider/model"));
- assert_eq!(
- processed.request.body()["messages"],
- request.body()["messages"]
- );
- assert!(observed(&calls)?.is_empty());
- Ok(())
- }
-
- #[tokio::test]
- async fn rprocess_only_returns_response_unchanged() -> Result<()> {
- let (profile, calls) = profile(target("direct", "provider/model")?)?;
- let response = ChatResponse::openai_completion(json!({"ok": true}));
-
- let request = profile_input(ChatRequest::openai_chat(
- json!({"model": "client/model", "messages": []}),
- ));
- let processed = profile.rprocess(&request, response).await?;
-
- match processed {
- ChatResponse::OpenAiCompletion(body) => assert_eq!(body.body()["ok"], true),
- _ => return Err(SwitchyardError::Other("unexpected response shape".into())),
- }
- assert!(observed(&calls)?.is_empty());
- Ok(())
- }
-
- #[tokio::test]
- async fn malformed_body_still_gets_target_model_for_single_target_backend() -> Result<()> {
- let (profile, calls) = profile(target("direct", "provider/model")?)?;
-
- let _response = profile
- .run(profile_input(ChatRequest::openai_chat(json!("bad-body"))))
- .await?;
-
- let calls = observed(&calls)?;
- assert_eq!(calls.len(), 1);
- assert_eq!(calls[0].body, json!({"model": "provider/model"}));
- Ok(())
- }
-}
diff --git a/crates/switchyard-components-v2/src/profiles/profile_types.rs b/crates/switchyard-components-v2/src/profiles/profile_types.rs
deleted file mode 100644
index 3126a8154..000000000
--- a/crates/switchyard-components-v2/src/profiles/profile_types.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Central profile config registry generated from concrete profile config types.
-
-use super::macros::profile_types;
-use super::{
- LatencyServiceProfileConfig, LlmRoutingProfileConfig, NoopProfileConfig,
- PassthroughProfileConfig, RandomRoutingProfileConfig, StageRouterProfileConfig,
-};
-
-profile_types! {
- StageRouterProfileConfig,
- PassthroughProfileConfig,
- RandomRoutingProfileConfig,
- LatencyServiceProfileConfig,
- LlmRoutingProfileConfig,
- NoopProfileConfig,
-}
diff --git a/crates/switchyard-components-v2/src/profiles/random_routing.rs b/crates/switchyard-components-v2/src/profiles/random_routing.rs
deleted file mode 100644
index dea63c67f..000000000
--- a/crates/switchyard-components-v2/src/profiles/random_routing.rs
+++ /dev/null
@@ -1,559 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Random-routing profile implemented as a single profile-owned runtime.
-
-use std::time::Instant;
-
-use async_trait::async_trait;
-use switchyard_components::request_processors::{
- RandomRoutingDecision, RandomRoutingEngine, RandomRoutingProcessorConfig, RandomRoutingTier,
-};
-use switchyard_components::StatsAccumulator;
-use switchyard_core::{ChatResponse, LlmTarget, Result, SwitchyardError};
-
-use crate::backend::{native_target_backend, TargetBackend};
-use crate::profile_stats_accumulator;
-use crate::stats_recording::record_usage_or_wrap_stream;
-use crate::{
- profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse,
- RoutingMetadata,
-};
-
-/// Config for the flatter random-routing profile.
-#[profile_config("random-routing")]
-pub struct RandomRoutingProfileConfig {
- /// Strong target served by this profile.
- #[profile_target]
- pub strong: LlmTarget,
- /// Weak target served by this profile.
- #[profile_target]
- pub weak: LlmTarget,
- /// Probability of selecting the strong target.
- #[serde(default = "default_strong_probability")]
- pub strong_probability: f64,
- /// Optional deterministic RNG seed for reproducible routing.
- #[serde(default)]
- pub rng_seed: Option<u64>,
-}
-
-impl ProfileConfig for RandomRoutingProfileConfig {
- type Runtime = RandomRoutingProfile;
-
- /// Builds the runtime profile using existing native backend construction.
- fn build(&self) -> Result<Self::Runtime> {
- let router_config =
- RandomRoutingProcessorConfig::new(self.strong.clone(), self.weak.clone())
- .with_strong_probability(self.strong_probability)?
- .with_rng_seed(self.rng_seed);
- Ok(RandomRoutingProfile {
- router: RandomRoutingEngine::new(router_config)?,
- strong_backend: native_target_backend(self.strong.clone())?,
- weak_backend: native_target_backend(self.weak.clone())?,
- stats: profile_stats_accumulator(),
- })
- }
-}
-
-/// Random-routing profile in the flatter design.
-pub struct RandomRoutingProfile {
- router: RandomRoutingEngine,
- strong_backend: TargetBackend,
- weak_backend: TargetBackend,
- stats: StatsAccumulator,
-}
-
-/// Processed random-routing request with the profile-owned routing decision.
-pub struct RandomRoutingProcessedRequest {
- /// Routed input prepared for the selected backend.
- pub profile_input: ProfileInput,
- /// Selected routing decision for this request.
- pub decision: RandomRoutingDecision,
-}
-
-impl RandomRoutingProfile {
- // Selects the target and rewrites the request model without side-channel state.
- fn route_request(&self, mut input: ProfileInput) -> Result<RandomRoutingProcessedRequest> {
- let decision = self
- .router
- .select(input.request.model().map(std::borrow::ToOwned::to_owned))?;
- input.request.set_model(decision.selected_model.as_str());
- Ok(RandomRoutingProcessedRequest {
- profile_input: input,
- decision,
- })
- }
-
- // Finds the routed backend by the target ID emitted by the routing engine.
- fn selected_backend(&self, decision: &RandomRoutingDecision) -> Result<&TargetBackend> {
- if decision.selected_target == self.strong_backend.target().id {
- Ok(&self.strong_backend)
- } else if decision.selected_target == self.weak_backend.target().id {
- Ok(&self.weak_backend)
- } else {
- Err(SwitchyardError::InvalidConfig(format!(
- "router selected target {} that is not configured for this profile",
- decision.selected_target
- )))
- }
- }
-
- fn routing_metadata(&self, decision: &RandomRoutingDecision) -> RoutingMetadata {
- let comparison = if decision.tier == RandomRoutingTier::Strong {
- "<"
- } else {
- ">="
- };
- RoutingMetadata {
- selected_model: Some(decision.selected_model.to_string()),
- selected_tier: Some(decision.tier.as_str().to_string()),
- confidence: None,
- router_version: Some("random-routing:v1".to_string()),
- tolerance: Some(decision.strong_probability),
- rationale: Some(format!(
- "random draw {} {comparison} strong_probability {}; selected {}",
- decision.draw,
- decision.strong_probability,
- decision.tier.as_str()
- )),
- }
- }
-}
-
-#[async_trait]
-impl ProfileHooks for RandomRoutingProfile {
- type ProcessedRequest = RandomRoutingProcessedRequest;
-
- /// Performs a standalone routing rewrite for hook-level inspection.
- async fn process(&self, input: ProfileInput) -> Result<Self::ProcessedRequest> {
- self.route_request(input)
- }
-
- /// Leaves the backend response unchanged after random routing completes.
- async fn rprocess(
- &self,
- _processed: &Self::ProcessedRequest,
- response: ChatResponse,
- ) -> Result<ChatResponse> {
- Ok(response)
- }
-}
-
-#[async_trait]
-impl Profile for RandomRoutingProfile {
- /// Executes random routing while keeping selected-target state local to this call.
- async fn run(&self, input: ProfileInput) -> Result<ProfileResponse> {
- let profile_started_at = Instant::now();
- let processed = self.process(input).await?;
- let decision = &processed.decision;
- let selected_backend = self.selected_backend(decision)?;
- let backend_started_at = Instant::now();
- let response = match selected_backend
- .call(&processed.profile_input.request)
- .await
- {
- Ok(response) => response,
- Err(error) => {
- self.stats.record_error(
- decision.selected_model.as_str(),
- Some(decision.tier.as_str()),
- )?;
- return Err(error);
- }
- };
- let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0;
- self.stats.record_success(
- decision.selected_model.as_str(),
- Some(backend_latency_ms),
- Some(decision.tier.as_str()),
- )?;
- let response = record_usage_or_wrap_stream(
- &self.stats,
- decision.selected_model.as_str(),
- Some(decision.tier.as_str()),
- profile_started_at,
- backend_latency_ms,
- response,
- )?;
- let response = self.rprocess(&processed, response).await?;
- Ok(ProfileResponse::with_routing_metadata(
- response,
- self.routing_metadata(decision),
- ))
- }
-}
-
-/// Default probability matches the existing random-routing config.
-fn default_strong_probability() -> f64 {
- 0.5
-}
-
-#[cfg(test)]
-mod tests {
- use std::sync::{Arc, Mutex};
-
- use async_trait::async_trait;
- use futures_util::StreamExt;
- use serde_json::{json, Value};
- use switchyard_core::{
- BackendFormat, ChatRequest, LlmTargetId, ModelId, StreamEvent, SwitchyardError,
- };
-
- use crate::backend::{ProfileBackend, TargetBackend};
- use crate::RequestMetadata;
-
- use super::*;
-
- #[derive(Clone, Debug, PartialEq)]
- struct ObservedCall {
- backend: &'static str,
- body: Value,
- }
-
- struct TestBackend {
- name: &'static str,
- calls: Arc<Mutex<Vec<ObservedCall>>>,
- }
-
- struct StreamingUsageBackend;
-
- #[async_trait]
- impl ProfileBackend for TestBackend {
- async fn call(&self, request: &ChatRequest) -> Result<ChatResponse> {
- self.calls
- .lock()
- .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))?
- .push(ObservedCall {
- backend: self.name,
- body: request.body().clone(),
- });
- Ok(ChatResponse::openai_completion(json!({
- "served_by": self.name,
- "model": request.model(),
- "usage": {
- "prompt_tokens": 11,
- "completion_tokens": 7,
- },
- })))
- }
- }
-
- #[async_trait]
- impl ProfileBackend for StreamingUsageBackend {
- async fn call(&self, _request: &ChatRequest) -> Result<ChatResponse> {
- Ok(ChatResponse::OpenAiStream(Box::pin(
- futures_util::stream::iter([
- Ok(StreamEvent::Json(json!({
- "choices": [{"delta": {"content": "ok"}}],
- }))),
- Ok(StreamEvent::Json(json!({
- "choices": [],
- "usage": {
- "prompt_tokens": 3,
- "completion_tokens": 4,
- "total_tokens": 7,
- },
- }))),
- ]),
- )))
- }
- }
-
- fn target(id: &str, model: &str) -> Result<LlmTarget> {
- let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
- target.format = BackendFormat::OpenAi;
- Ok(target)
- }
-
- fn config(strong: LlmTarget, weak: LlmTarget, probability: f64) -> RandomRoutingProfileConfig {
- RandomRoutingProfileConfig {
- strong,
- weak,
- strong_probability: probability,
- rng_seed: Some(7),
- }
- }
-
- fn backend(
- strong: &LlmTarget,
- weak: &LlmTarget,
- calls: Arc<Mutex<Vec<ObservedCall>>>,
- ) -> (TargetBackend, TargetBackend) {
- (
- TargetBackend::new(
- strong.clone(),
- Arc::new(TestBackend {
- name: "strong-backend",
- calls: calls.clone(),
- }),
- ),
- TargetBackend::new(
- weak.clone(),
- Arc::new(TestBackend {
- name: "weak-backend",
- calls,
- }),
- ),
- )
- }
-
- fn observed(calls: &Arc<Mutex<Vec<ObservedCall>>>) -> Result<Vec<ObservedCall>> {
- calls
- .lock()
- .map(|calls| calls.clone())
- .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))
- }
-
- fn profile_input(request: ChatRequest) -> ProfileInput {
- ProfileInput {
- request,
- metadata: RequestMetadata::default(),
- }
- }
-
- fn profile(
- strong: LlmTarget,
- weak: LlmTarget,
- probability: f64,
- ) -> Result<(RandomRoutingProfile, Arc<Mutex<Vec<ObservedCall>>>)> {
- let calls = Arc::new(Mutex::new(Vec::new()));
- let config = config(strong.clone(), weak.clone(), probability);
- let router_config =
- RandomRoutingProcessorConfig::new(config.strong.clone(), config.weak.clone())
- .with_strong_probability(config.strong_probability)?
- .with_rng_seed(config.rng_seed);
- let (strong_backend, weak_backend) = backend(&strong, &weak, calls.clone());
- let profile = RandomRoutingProfile {
- router: RandomRoutingEngine::new(router_config)?,
- strong_backend,
- weak_backend,
- stats: StatsAccumulator::new(),
- };
- Ok((profile, calls))
- }
-
- #[tokio::test]
- async fn random_routing_profile_routes_with_request_only_handoff() -> Result<()> {
- let (profile, calls) = profile(
- target("strong", "frontier/model")?,
- target("weak", "cheap/model")?,
- 1.0,
- )?;
-
- let response = profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [{"role": "user", "content": "hi"}],
- }))))
- .await?;
-
- let routing_metadata = response
- .routing_metadata
- .as_ref()
- .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?;
- assert_eq!(
- routing_metadata.selected_model.as_deref(),
- Some("frontier/model")
- );
- assert_eq!(routing_metadata.selected_tier.as_deref(), Some("strong"));
- assert_eq!(
- routing_metadata.router_version.as_deref(),
- Some("random-routing:v1")
- );
- assert_eq!(routing_metadata.tolerance, Some(1.0));
- let response = response.response;
- let calls = observed(&calls)?;
- assert_eq!(calls.len(), 1);
- assert_eq!(calls[0].backend, "strong-backend");
- assert_eq!(calls[0].body["model"], "frontier/model");
- match response {
- ChatResponse::OpenAiCompletion(body) => {
- assert_eq!(body.body()["served_by"], "strong-backend");
- assert_eq!(body.body()["model"], "frontier/model");
- }
- _ => return Err(SwitchyardError::Other("unexpected response shape".into())),
- }
- Ok(())
- }
-
- #[tokio::test]
- async fn run_records_stats_with_selected_random_tier() -> Result<()> {
- let (profile, _calls) = profile(
- target("strong", "frontier/model")?,
- target("weak", "cheap/model")?,
- 1.0,
- )?;
-
- let _response = profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [],
- }))))
- .await?;
-
- let snapshot = profile.stats.snapshot()?;
- assert_eq!(snapshot.total_requests, 1);
- assert_eq!(snapshot.total_tokens.prompt, 11);
- assert_eq!(snapshot.total_tokens.completion, 7);
- let model = snapshot.models.get("frontier/model").ok_or_else(|| {
- SwitchyardError::Other("frontier model stats should be present".into())
- })?;
- assert_eq!(model.calls, 1);
- assert_eq!(model.tier.as_deref(), Some("strong"));
- let tier = snapshot
- .tiers
- .get("strong")
- .ok_or_else(|| SwitchyardError::Other("strong tier stats should be present".into()))?;
- assert_eq!(tier.calls, 1);
- assert_eq!(tier.model, "frontier/model");
- Ok(())
- }
-
- #[tokio::test]
- async fn run_records_streaming_usage_with_selected_random_tier() -> Result<()> {
- let strong = target("strong", "frontier/model")?;
- let weak = target("weak", "cheap/model")?;
- let router_config = RandomRoutingProcessorConfig::new(strong.clone(), weak.clone())
- .with_strong_probability(0.0)?
- .with_rng_seed(Some(7));
- let profile = RandomRoutingProfile {
- router: RandomRoutingEngine::new(router_config)?,
- strong_backend: TargetBackend::new(
- strong,
- Arc::new(TestBackend {
- name: "strong-backend",
- calls: Arc::new(Mutex::new(Vec::new())),
- }),
- ),
- weak_backend: TargetBackend::new(weak, Arc::new(StreamingUsageBackend)),
- stats: StatsAccumulator::new(),
- };
-
- let response = profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [],
- "stream": true,
- }))))
- .await?;
- let ChatResponse::OpenAiStream(mut stream) = response.response else {
- return Err(SwitchyardError::Other("expected streaming response".into()));
- };
- while let Some(event) = stream.next().await {
- event?;
- }
-
- let snapshot = profile.stats.snapshot()?;
- assert_eq!(snapshot.total_requests, 1);
- assert_eq!(snapshot.total_tokens.prompt, 3);
- assert_eq!(snapshot.total_tokens.completion, 4);
- assert_eq!(snapshot.total_tokens.total, 7);
- let model = snapshot
- .models
- .get("cheap/model")
- .ok_or_else(|| SwitchyardError::Other("weak model stats should be present".into()))?;
- assert_eq!(model.calls, 1);
- assert_eq!(model.tier.as_deref(), Some("weak"));
- assert_eq!(model.total_tokens, 7);
- let tier = snapshot
- .tiers
- .get("weak")
- .ok_or_else(|| SwitchyardError::Other("weak tier stats should be present".into()))?;
- assert_eq!(tier.total_tokens, 7);
- Ok(())
- }
-
- #[tokio::test]
- async fn run_disambiguates_duplicate_target_models_without_context_state() -> Result<()> {
- let (profile, calls) = profile(
- target("strong-endpoint", "shared/model")?,
- target("weak-endpoint", "shared/model")?,
- 0.0,
- )?;
-
- let _response = profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [],
- }))))
- .await?;
-
- let calls = observed(&calls)?;
- assert_eq!(calls.len(), 1);
- assert_eq!(calls[0].backend, "weak-backend");
- assert_eq!(calls[0].body["model"], "shared/model");
- Ok(())
- }
-
- #[tokio::test]
- async fn malformed_request_body_is_recovered_without_context_state() -> Result<()> {
- let (profile, calls) = profile(
- target("strong", "frontier/model")?,
- target("weak", "cheap/model")?,
- 1.0,
- )?;
-
- let _response = profile
- .run(profile_input(ChatRequest::openai_chat(json!("bad-body"))))
- .await?;
-
- let calls = observed(&calls)?;
- assert_eq!(calls.len(), 1);
- assert_eq!(calls[0].backend, "strong-backend");
- assert_eq!(calls[0].body, json!({"model": "frontier/model"}));
- Ok(())
- }
-
- #[tokio::test]
- async fn process_only_prepares_request_and_does_not_call_backend() -> Result<()> {
- let (profile, calls) = profile(
- target("strong", "frontier/model")?,
- target("weak", "cheap/model")?,
- 1.0,
- )?;
-
- let request = profile
- .process(profile_input(ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [],
- }))))
- .await?;
-
- assert_eq!(
- request.profile_input.request.model(),
- Some("frontier/model")
- );
- assert_eq!(request.decision.selected_model.as_str(), "frontier/model");
- assert!(observed(&calls)?.is_empty());
- Ok(())
- }
-
- #[tokio::test]
- async fn rprocess_only_handles_response() -> Result<()> {
- let (profile, calls) = profile(
- target("strong", "frontier/model")?,
- target("weak", "cheap/model")?,
- 1.0,
- )?;
-
- let processed = profile
- .process(profile_input(ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [],
- }))))
- .await?;
- let response = profile
- .rprocess(
- &processed,
- ChatResponse::openai_completion(json!({"ok": true})),
- )
- .await?;
-
- match response {
- ChatResponse::OpenAiCompletion(body) => assert_eq!(body.body()["ok"], true),
- _ => return Err(SwitchyardError::Other("unexpected response shape".into())),
- }
- assert!(observed(&calls)?.is_empty());
- Ok(())
- }
-}
diff --git a/crates/switchyard-components-v2/src/profiles/stage_router.rs b/crates/switchyard-components-v2/src/profiles/stage_router.rs
deleted file mode 100644
index 6aebe709e..000000000
--- a/crates/switchyard-components-v2/src/profiles/stage_router.rs
+++ /dev/null
@@ -1,1362 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Signal stage_router profile implemented as a profile-owned Rust runtime.
-
-use std::time::{Duration, Instant};
-
-use async_trait::async_trait;
-use serde::{Deserialize, Serialize};
-use serde_json::{json, Value};
-use switchyard_components::dimension_collector::{
- extract_tool_signals_with_window, ToolResultSignal, DEFAULT_RECENT_WINDOW,
-};
-use switchyard_components::stats::{usage_from_body, TokenUsage};
-use switchyard_components::StatsAccumulator;
-use switchyard_core::{ChatRequest, ChatResponse, LlmTarget, LlmTargetId, Result, SwitchyardError};
-
-use crate::backend::{native_target_backend, TargetBackend};
-use crate::profile_stats_accumulator;
-use crate::stats_recording::record_usage_or_wrap_stream;
-use crate::{
- profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse,
- RoutingMetadata,
-};
-
-const DEFAULT_CONFIDENCE_THRESHOLD: f64 = 0.7;
-const DEFAULT_CLASSIFIER_TIMEOUT_SECS: f64 = 30.0;
-const DEFAULT_CLASSIFIER_RECENT_TURN_WINDOW: usize = 3;
-const CLASSIFIER_MAX_TOKENS: u32 = 4096;
-const DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com/v1";
-const STAGE_ROUTER_PROFILE_TYPE: &str = "stage_router";
-
-const SEVERITY_CRITICAL: f32 = 1.0;
-const CLEAN_TESTS_MIN_TURN_DEPTH: u32 = 10;
-const CLEAN_TESTS_MAX_WRITES: u32 = 1;
-const PURE_BASH_NORM: f64 = 8.0;
-
-const CLASSIFIER_SYSTEM_PROMPT: &str = include_str!("stage_router/prompts/classifier.md");
-
-const DEFAULT_WEIGHTS: &[(&str, f64)] = &[
- ("severity", 0.80),
- ("stuck_exploring", 0.70),
- ("no_progress", 0.60),
- ("tests_passed", -0.80),
- ("planning_active", -0.70),
- ("write_intensity", -0.40),
- ("edit_intensity", -0.30),
- ("recent_write_intensity", -0.30),
- ("pure_bash_intensity", -0.30),
- ("no_error_streak_intensity", -0.20),
-];
-
-/// Default picker mode names the tier used when the scorer is ambiguous.
-#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub enum StageRouterPickerMode {
- /// Default to capable unless the scorer/classifier confidently picks efficient.
- CapableFirst,
- /// Default to efficient unless the scorer/classifier confidently picks capable.
- EfficientFirst,
-}
-
-impl StageRouterPickerMode {
- fn default_tier(self) -> StageRouterTier {
- match self {
- Self::CapableFirst => StageRouterTier::Capable,
- Self::EfficientFirst => StageRouterTier::Efficient,
- }
- }
-}
-
-/// Optional LLM classifier invoked for low-confidence scorer outputs.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub struct StageRouterClassifierConfig {
- /// OpenAI-compatible classifier model.
- pub model: String,
- /// API key used for the classifier call.
- pub api_key: String,
- /// OpenAI-compatible base URL. Defaults to OpenAI's `/v1` endpoint.
- #[serde(default)]
- pub base_url: Option<String>,
- /// Per-call timeout in seconds.
- #[serde(default = "default_classifier_timeout_secs")]
- pub timeout_secs: f64,
- /// Number of trailing request messages rendered into the classifier prompt.
- #[serde(default = "default_classifier_recent_turn_window")]
- pub recent_turn_window: usize,
- /// Maximum tokens allowed for the classifier response.
- #[serde(default = "default_classifier_max_tokens")]
- pub max_tokens: u32,
- /// Optional system prompt override for the classifier request.
- #[serde(default)]
- pub system_prompt: Option<String>,
-}
-
-/// Config for a capable/efficient signal stage_router profile.
-#[profile_config("stage_router")]
-pub struct StageRouterProfileConfig {
- /// Capable target served by this profile.
- #[profile_target]
- pub capable: LlmTarget,
- /// Efficient target served by this profile.
- #[profile_target]
- pub efficient: LlmTarget,
- /// Target used for one retry after context-window overflow.
- pub fallback_target_on_evict: LlmTargetId,
- /// Picker mode controlling the low-confidence default tier.
- #[serde(default = "default_picker")]
- pub picker: StageRouterPickerMode,
- /// Scorer confidence threshold in `[0.0, 1.0]`.
- #[serde(default = "default_confidence_threshold")]
- pub confidence_threshold: f64,
- /// Sliding window for `recent_*` tool-result signal counts.
- #[serde(default = "default_signal_recent_window")]
- pub signal_recent_window: usize,
- /// Optional LLM classifier for ambiguous scorer outputs.
- #[serde(default)]
- pub classifier: Option<StageRouterClassifierConfig>,
- /// Whether to emit stats for this profile.
- #[serde(default = "default_enable_stats")]
- pub enable_stats: bool,
-}
-
-impl ProfileConfig for StageRouterProfileConfig {
- type Runtime = StageRouterProfile;
-
- /// Builds the runtime profile using native target backends.
- fn build(&self) -> Result<Self::Runtime> {
- self.validate()?;
- Ok(StageRouterProfile {
- capable_backend: native_target_backend(self.capable.clone())?,
- efficient_backend: native_target_backend(self.efficient.clone())?,
- fallback_target_on_evict: self.fallback_target_on_evict.clone(),
- picker: self.picker,
- confidence_threshold: self.confidence_threshold,
- signal_recent_window: self.signal_recent_window,
- classifier: self
- .classifier
- .as_ref()
- .map(StageRouterTierClassifier::new)
- .transpose()?,
- stats: profile_stats_accumulator(),
- enable_stats: self.enable_stats,
- })
- }
-}
-
-impl StageRouterProfileConfig {
- fn validate(&self) -> Result<()> {
- if self.capable.id == self.efficient.id {
- return Err(SwitchyardError::InvalidConfig(
- "stage_router capable and efficient targets must have distinct target ids"
- .to_string(),
- ));
- }
- if !self.confidence_threshold.is_finite()
- || !(0.0..=1.0).contains(&self.confidence_threshold)
- {
- return Err(SwitchyardError::InvalidConfig(format!(
- "confidence_threshold must be finite and in [0.0, 1.0], got {:?}",
- self.confidence_threshold
- )));
- }
- if self.signal_recent_window == 0 {
- return Err(SwitchyardError::InvalidConfig(
- "signal_recent_window must be at least 1".to_string(),
- ));
- }
- if self.fallback_target_on_evict != self.capable.id
- && self.fallback_target_on_evict != self.efficient.id
- {
- return Err(SwitchyardError::InvalidConfig(format!(
- "fallback_target_on_evict={} must match one of [{}, {}]",
- self.fallback_target_on_evict, self.efficient.id, self.capable.id
- )));
- }
- if let Some(classifier) = &self.classifier {
- classifier.validate()?;
- }
- Ok(())
- }
-}
-
-impl StageRouterClassifierConfig {
- fn validate(&self) -> Result<()> {
- if self.model.trim().is_empty() {
- return Err(SwitchyardError::InvalidConfig(
- "classifier.model must not be empty".to_string(),
- ));
- }
- if !self.timeout_secs.is_finite() || self.timeout_secs <= 0.0 {
- return Err(SwitchyardError::InvalidConfig(format!(
- "classifier.timeout_secs must be finite and > 0.0, got {:?}",
- self.timeout_secs
- )));
- }
- if self.max_tokens == 0 {
- return Err(SwitchyardError::InvalidConfig(
- "classifier.max_tokens must be greater than 0".to_string(),
- ));
- }
- if let Some(system_prompt) = &self.system_prompt {
- if system_prompt.trim().is_empty() {
- return Err(SwitchyardError::InvalidConfig(
- "classifier.system_prompt must not be empty".to_string(),
- ));
- }
- }
- Ok(())
- }
-}
-
-/// Capable/efficient stage_router profile runtime.
-pub struct StageRouterProfile {
- capable_backend: TargetBackend,
- efficient_backend: TargetBackend,
- fallback_target_on_evict: LlmTargetId,
- picker: StageRouterPickerMode,
- confidence_threshold: f64,
- signal_recent_window: usize,
- classifier: Option<StageRouterTierClassifier>,
- stats: StatsAccumulator,
- enable_stats: bool,
-}
-
-/// Processed stage_router request with profile-owned decision state.
-pub struct StageRouterProcessedRequest {
- /// Routed input prepared for the selected backend.
- pub profile_input: ProfileInput,
- /// Selected routing decision for this request.
- pub decision: StageRouterDecision,
-}
-
-/// Named side of a stage_router decision.
-#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub enum StageRouterTier {
- /// Capable tier.
- Capable,
- /// Efficient tier.
- Efficient,
-}
-
-impl StageRouterTier {
- /// Stable lowercase label used by stats.
- pub fn as_str(self) -> &'static str {
- match self {
- Self::Capable => "capable",
- Self::Efficient => "efficient",
- }
- }
-}
-
-/// Source that produced a stage_router decision.
-#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "snake_case")]
-pub enum StageRouterDecisionSource {
- /// Hard override fired.
- Override,
- /// Dimension scorer crossed `confidence_threshold`.
- Dimensions,
- /// LLM classifier returned a usable verdict.
- #[serde(rename = "llm-classifier")]
- LlmClassifier,
- /// Classifier was absent or failed; picker default tier was used.
- FallOpen,
- /// A context-window overflow retried the configured fallback target.
- ContextOverflowFallback,
-}
-
-impl StageRouterDecisionSource {
- /// Stable lowercase label used in stats JSON.
- pub fn as_str(self) -> &'static str {
- match self {
- Self::Override => "override",
- Self::Dimensions => "dimensions",
- Self::LlmClassifier => "llm-classifier",
- Self::FallOpen => "fall_open",
- Self::ContextOverflowFallback => "context_overflow_fallback",
- }
- }
-}
-
-/// StageRouter routing decision with the selected target and scorer metadata.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct StageRouterDecision {
- /// Selected capable/efficient side.
- pub tier: StageRouterTier,
- /// Selected target id.
- pub selected_target: LlmTargetId,
- /// Selected upstream model.
- pub selected_model: switchyard_core::ModelId,
- /// Client-provided model before routing.
- pub original_model: Option<String>,
- /// Decision source for observability.
- pub source: StageRouterDecisionSource,
- /// Linear scorer value in `[-1.0, 1.0]`.
- pub score: f64,
- /// Router confidence when the decision source produced one.
- pub confidence: Option<f64>,
-}
-
-impl StageRouterProfile {
- async fn route_request(&self, mut input: ProfileInput) -> Result<StageRouterProcessedRequest> {
- let signal = extract_tool_signals_with_window(&input.request, self.signal_recent_window);
- let original_model = input.request.model().map(std::borrow::ToOwned::to_owned);
- let decision = self.pick(&input.request, &signal, original_model).await?;
- input.request.set_model(decision.selected_model.as_str());
- self.record_decision_source(decision.source)?;
- Ok(StageRouterProcessedRequest {
- profile_input: input,
- decision,
- })
- }
-
- // Routing decision flow:
- // 1. Hard overrides choose capable for critical failures or efficient for clean tests.
- // 2. Dimensions scoring chooses by score sign when confidence clears the threshold.
- // 3. Low-confidence requests use the optional LLM classifier when configured.
- // 4. Missing or failed classifier output falls open to the picker default tier.
- async fn pick(
- &self,
- request: &ChatRequest,
- signal: &ToolResultSignal,
- original_model: Option<String>,
- ) -> Result<StageRouterDecision> {
- if let Some(tier) = apply_overrides(signal) {
- return self.decision_for_tier(
- tier,
- original_model,
- StageRouterDecisionSource::Override,
- 0.0,
- Some(1.0),
- );
- }
-
- let score = score_signal(signal);
- if score.confidence >= self.confidence_threshold {
- let tier = if score.score > 0.0 {
- StageRouterTier::Capable
- } else {
- StageRouterTier::Efficient
- };
- return self.decision_for_tier(
- tier,
- original_model,
- StageRouterDecisionSource::Dimensions,
- score.score,
- Some(score.confidence),
- );
- }
-
- if let Some(classifier) = &self.classifier {
- if let Some(tier) = classifier
- .classify(request, signal, self.stats_handle())
- .await
- {
- return self.decision_for_tier(
- tier,
- original_model,
- StageRouterDecisionSource::LlmClassifier,
- score.score,
- None,
- );
- }
- }
-
- self.decision_for_tier(
- self.picker.default_tier(),
- original_model,
- StageRouterDecisionSource::FallOpen,
- score.score,
- Some(score.confidence),
- )
- }
-
- fn decision_for_tier(
- &self,
- tier: StageRouterTier,
- original_model: Option<String>,
- source: StageRouterDecisionSource,
- score: f64,
- confidence: Option<f64>,
- ) -> Result<StageRouterDecision> {
- let backend = self.backend_for_tier(tier);
- let target = backend.target();
- Ok(StageRouterDecision {
- tier,
- selected_target: target.id.clone(),
- selected_model: target.model.clone(),
- original_model,
- source,
- score,
- confidence,
- })
- }
-
- fn fallback_decision(&self, decision: &StageRouterDecision) -> Result<StageRouterDecision> {
- let backend = self.backend_for_target(&self.fallback_target_on_evict)?;
- let target = backend.target();
- Ok(StageRouterDecision {
- tier: self.tier_for_target(&target.id)?,
- selected_target: target.id.clone(),
- selected_model: target.model.clone(),
- original_model: decision.original_model.clone(),
- source: StageRouterDecisionSource::ContextOverflowFallback,
- score: decision.score,
- confidence: decision.confidence,
- })
- }
-
- fn retry_processed_request(
- &self,
- processed: &StageRouterProcessedRequest,
- ) -> Result<StageRouterProcessedRequest> {
- let decision = self.fallback_decision(&processed.decision)?;
- let mut profile_input = processed.profile_input.clone();
- profile_input
- .request
- .set_model(decision.selected_model.as_str());
- Ok(StageRouterProcessedRequest {
- profile_input,
- decision,
- })
- }
-
- fn backend_for_tier(&self, tier: StageRouterTier) -> &TargetBackend {
- match tier {
- StageRouterTier::Capable => &self.capable_backend,
- StageRouterTier::Efficient => &self.efficient_backend,
- }
- }
-
- fn backend_for_target(&self, target_id: &LlmTargetId) -> Result<&TargetBackend> {
- if *target_id == self.capable_backend.target().id {
- Ok(&self.capable_backend)
- } else if *target_id == self.efficient_backend.target().id {
- Ok(&self.efficient_backend)
- } else {
- Err(SwitchyardError::InvalidConfig(format!(
- "stage_router selected target {target_id} that is not configured for this profile"
- )))
- }
- }
-
- fn tier_for_target(&self, target_id: &LlmTargetId) -> Result<StageRouterTier> {
- if *target_id == self.capable_backend.target().id {
- Ok(StageRouterTier::Capable)
- } else if *target_id == self.efficient_backend.target().id {
- Ok(StageRouterTier::Efficient)
- } else {
- Err(SwitchyardError::InvalidConfig(format!(
- "stage_router target {target_id} is not configured for this profile"
- )))
- }
- }
-
- async fn call_selected(
- &self,
- processed: &StageRouterProcessedRequest,
- ) -> (Result<ChatResponse>, f64) {
- let started_at = Instant::now();
- let backend = match self.backend_for_target(&processed.decision.selected_target) {
- Ok(backend) => backend,
- Err(error) => return (Err(error), 0.0),
- };
- let result = backend.call(&processed.profile_input.request).await;
- let latency_ms = started_at.elapsed().as_secs_f64() * 1000.0;
- (result, latency_ms)
- }
-
- fn stats_handle(&self) -> Option<&StatsAccumulator> {
- self.enable_stats.then_some(&self.stats)
- }
-
- fn record_decision_source(&self, source: StageRouterDecisionSource) -> Result<()> {
- if let Some(stats) = self.stats_handle() {
- stats.record_routing_decision(STAGE_ROUTER_PROFILE_TYPE, source.as_str())?;
- }
- Ok(())
- }
-
- fn record_success(
- &self,
- decision: &StageRouterDecision,
- response: ChatResponse,
- profile_started_at: Instant,
- backend_latency_ms: f64,
- ) -> Result<ChatResponse> {
- let Some(stats) = self.stats_handle() else {
- return Ok(response);
- };
- stats.record_success(
- decision.selected_model.as_str(),
- Some(backend_latency_ms),
- Some(decision.tier.as_str()),
- )?;
- record_usage_or_wrap_stream(
- stats,
- decision.selected_model.as_str(),
- Some(decision.tier.as_str()),
- profile_started_at,
- backend_latency_ms,
- response,
- )
- }
-
- fn record_error(&self, decision: &StageRouterDecision) -> Result<()> {
- if let Some(stats) = self.stats_handle() {
- stats.record_error(
- decision.selected_model.as_str(),
- Some(decision.tier.as_str()),
- )?;
- }
- Ok(())
- }
-
- fn routing_metadata(&self, decision: &StageRouterDecision) -> RoutingMetadata {
- RoutingMetadata {
- selected_model: Some(decision.selected_model.to_string()),
- selected_tier: Some(decision.tier.as_str().to_string()),
- confidence: decision.confidence,
- router_version: Some("stage_router:v1".to_string()),
- tolerance: Some(self.confidence_threshold),
- rationale: Some(format!(
- "stage_router source={}; score={}; selected {}",
- decision.source.as_str(),
- decision.score,
- decision.tier.as_str()
- )),
- }
- }
-}
-
-#[async_trait]
-impl ProfileHooks for StageRouterProfile {
- type ProcessedRequest = StageRouterProcessedRequest;
-
- /// Extracts signals, picks a tier, and rewrites the request model.
- async fn process(&self, input: ProfileInput) -> Result<Self::ProcessedRequest> {
- self.route_request(input).await
- }
-
- /// Leaves the backend response unchanged after stage_router routing completes.
- async fn rprocess(
- &self,
- _processed: &Self::ProcessedRequest,
- response: ChatResponse,
- ) -> Result<ChatResponse> {
- Ok(response)
- }
-}
-
-#[async_trait]
-impl Profile for StageRouterProfile {
- /// Executes stage_router routing with one context-window fallback retry.
- async fn run(&self, input: ProfileInput) -> Result<ProfileResponse> {
- let profile_started_at = Instant::now();
- let processed = self.process(input).await?;
- let (first_result, first_backend_latency_ms) = self.call_selected(&processed).await;
- match first_result {
- Ok(response) => {
- let response = self.record_success(
- &processed.decision,
- response,
- profile_started_at,
- first_backend_latency_ms,
- )?;
- let response = self.rprocess(&processed, response).await?;
- return Ok(ProfileResponse::with_routing_metadata(
- response,
- self.routing_metadata(&processed.decision),
- ));
- }
- Err(SwitchyardError::ContextWindowExceeded { .. }) => {
- let retry = self.retry_processed_request(&processed)?;
- let (retry_result, retry_backend_latency_ms) = self.call_selected(&retry).await;
- match retry_result {
- Ok(response) => {
- let response = self.record_success(
- &retry.decision,
- response,
- profile_started_at,
- retry_backend_latency_ms,
- )?;
- let response = self.rprocess(&retry, response).await?;
- return Ok(ProfileResponse::with_routing_metadata(
- response,
- self.routing_metadata(&retry.decision),
- ));
- }
- Err(SwitchyardError::ContextWindowExceeded { target_id, .. }) => {
- self.record_error(&retry.decision)?;
- return Err(SwitchyardError::ContextPoolExhausted {
- last_target_id: target_id,
- reason: "all attempted targets returned context-window overflow"
- .to_string(),
- });
- }
- Err(error) => {
- self.record_error(&retry.decision)?;
- return Err(error);
- }
- }
- }
- Err(error) => {
- self.record_error(&processed.decision)?;
- return Err(error);
- }
- }
- }
-}
-
-struct ScoreResult {
- score: f64,
- confidence: f64,
-}
-
-struct CodingAgentDimensions {
- severity: f64,
- no_error_streak_intensity: f64,
- write_intensity: f64,
- edit_intensity: f64,
- recent_write_intensity: f64,
- planning_active: f64,
- pure_bash_intensity: f64,
- stuck_exploring: f64,
- no_progress: f64,
- tests_passed: f64,
-}
-
-impl CodingAgentDimensions {
- fn value(&self, name: &str) -> f64 {
- match name {
- "severity" => self.severity,
- "no_error_streak_intensity" => self.no_error_streak_intensity,
- "write_intensity" => self.write_intensity,
- "edit_intensity" => self.edit_intensity,
- "recent_write_intensity" => self.recent_write_intensity,
- "planning_active" => self.planning_active,
- "pure_bash_intensity" => self.pure_bash_intensity,
- "stuck_exploring" => self.stuck_exploring,
- "no_progress" => self.no_progress,
- "tests_passed" => self.tests_passed,
- _ => 0.0,
- }
- }
-}
-
-fn score_signal(signal: &ToolResultSignal) -> ScoreResult {
- let dimensions = dimensions_from_signal(signal);
- let raw = DEFAULT_WEIGHTS
- .iter()
- .map(|(name, weight)| dimensions.value(name) * weight)
- .sum::<f64>();
- let score = raw.clamp(-1.0, 1.0);
- ScoreResult {
- score,
- confidence: score.abs(),
- }
-}
-
-fn dimensions_from_signal(signal: &ToolResultSignal) -> CodingAgentDimensions {
- let total_tool_ops = signal.write_count + signal.edit_count + signal.read_count;
- let recent_tool_ops =
- signal.recent_write_count + signal.recent_edit_count + signal.recent_read_count;
- let stuck = signal.turn_depth >= 8 && signal.write_count <= 1 && signal.read_count >= 5;
- let no_progress = signal.turn_depth > 60 && signal.write_count == 0;
-
- CodingAgentDimensions {
- severity: f64::from(signal.severity),
- no_error_streak_intensity: saturating(f64::from(signal.no_error_streak), 3.0),
- write_intensity: ratio(signal.write_count, total_tool_ops),
- edit_intensity: ratio(signal.edit_count, total_tool_ops),
- recent_write_intensity: ratio(signal.recent_write_count, recent_tool_ops),
- planning_active: if signal.recent_todowrite_count > 0 {
- 1.0
- } else {
- 0.0
- },
- pure_bash_intensity: saturating(f64::from(signal.pure_bash_streak), PURE_BASH_NORM),
- stuck_exploring: if stuck { 1.0 } else { 0.0 },
- no_progress: if no_progress { 1.0 } else { 0.0 },
- tests_passed: if signal.tests_passed && signal.write_count >= 3 {
- 1.0
- } else {
- 0.0
- },
- }
-}
-
-fn apply_overrides(signal: &ToolResultSignal) -> Option<StageRouterTier> {
- if signal.severity >= SEVERITY_CRITICAL {
- return Some(StageRouterTier::Capable);
- }
- if signal.tests_passed
- && signal.turn_depth >= CLEAN_TESTS_MIN_TURN_DEPTH
- && signal.write_count <= CLEAN_TESTS_MAX_WRITES
- {
- return Some(StageRouterTier::Efficient);
- }
- None
-}
-
-fn saturating(value: f64, scale: f64) -> f64 {
- if value <= 0.0 {
- 0.0
- } else {
- 1.0 - (-value / scale).exp()
- }
-}
-
-fn ratio(numerator: u32, denominator: u32) -> f64 {
- if denominator == 0 {
- 0.0
- } else {
- f64::from(numerator) / f64::from(denominator)
- }
-}
-
-struct StageRouterTierClassifier {
- config: StageRouterClassifierConfig,
- client: reqwest::Client,
- disable_reasoning: bool,
-}
-
-impl StageRouterTierClassifier {
- fn new(config: &StageRouterClassifierConfig) -> Result<Self> {
- config.validate()?;
- let client = reqwest::Client::builder()
- .timeout(Duration::from_secs_f64(config.timeout_secs))
- .build()
- .map_err(|error| {
- SwitchyardError::InvalidConfig(format!(
- "failed to build stage_router classifier HTTP client: {error}"
- ))
- })?;
- Ok(Self {
- config: config.clone(),
- client,
- disable_reasoning: model_accepts_reasoning_hint(config.model.as_str()),
- })
- }
-
- async fn classify(
- &self,
- request: &ChatRequest,
- signal: &ToolResultSignal,
- stats: Option<&StatsAccumulator>,
- ) -> Option<StageRouterTier> {
- let started_at = Instant::now();
- let response = match self
- .client
- .post(chat_completions_url(self.config.base_url.as_deref()))
- .bearer_auth(&self.config.api_key)
- .json(&self.request_body(request, signal))
- .send()
- .await
- {
- Ok(response) => response,
- Err(error) => {
- tracing::warn!(error = %error, "stage_router classifier call failed; falling open");
- record_classifier_error(stats, self.config.model.as_str());
- return None;
- }
- };
-
- if !response.status().is_success() {
- tracing::warn!(
- status = %response.status(),
- "stage_router classifier returned error status; falling open"
- );
- record_classifier_error(stats, self.config.model.as_str());
- return None;
- }
-
- let body = match response.json::<Value>().await {
- Ok(body) => body,
- Err(error) => {
- tracing::warn!(error = %error, "stage_router classifier returned invalid JSON; falling open");
- record_classifier_error(stats, self.config.model.as_str());
- return None;
- }
- };
- record_classifier_usage(
- stats,
- self.config.model.as_str(),
- usage_from_body(&body),
- started_at.elapsed().as_secs_f64() * 1000.0,
- );
- let tier = parse_classifier_tier(&body);
- if tier.is_none() {
- record_classifier_error(stats, self.config.model.as_str());
- }
- tier
- }
-
- fn request_body(&self, request: &ChatRequest, signal: &ToolResultSignal) -> Value {
- let system_prompt = self
- .config
- .system_prompt
- .as_deref()
- .unwrap_or(CLASSIFIER_SYSTEM_PROMPT);
- let mut body = json!({
- "model": self.config.model,
- "messages": [
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": summarise_signal(request, signal, self.config.recent_turn_window)},
- ],
- "temperature": 0,
- "response_format": {"type": "json_object"},
- "max_tokens": self.config.max_tokens,
- });
- if self.disable_reasoning {
- body["chat_template_kwargs"] = json!({"enable_thinking": false});
- }
- body
- }
-}
-
-fn record_classifier_usage(
- stats: Option<&StatsAccumulator>,
- model: &str,
- usage: TokenUsage,
- latency_ms: f64,
-) {
- if let Some(stats) = stats {
- if let Err(error) = stats.record_classifier_usage(model, usage, Some(latency_ms)) {
- tracing::debug!(error = %error, "failed to record stage_router classifier usage");
- }
- }
-}
-
-fn record_classifier_error(stats: Option<&StatsAccumulator>, model: &str) {
- if let Some(stats) = stats {
- if let Err(error) = stats.record_classifier_error(model) {
- tracing::debug!(error = %error, "failed to record stage_router classifier error");
- }
- }
-}
-
-fn parse_classifier_tier(body: &Value) -> Option<StageRouterTier> {
- let content = body
- .get("choices")?
- .as_array()?
- .first()?
- .get("message")?
- .get("content")?
- .as_str()?;
- let payload = serde_json::from_str::<Value>(content).ok()?;
- match payload.get("tier").and_then(Value::as_str) {
- Some("capable") => Some(StageRouterTier::Capable),
- Some("efficient") => Some(StageRouterTier::Efficient),
- _ => None,
- }
-}
-
-fn summarise_signal(
- request: &ChatRequest,
- signal: &ToolResultSignal,
- recent_window: usize,
-) -> String {
- let state_line = format!(
- "State: turn_depth={}, severity={:.1}, writes={}, edits={}, reads={}, todowrites={}, recent_writes={}, recent_edits={}, recent_reads={}, pure_bash_streak={}, no_error_streak={}, tests_passed={}",
- signal.turn_depth,
- signal.severity,
- signal.write_count,
- signal.edit_count,
- signal.read_count,
- signal.todowrite_count,
- signal.recent_write_count,
- signal.recent_edit_count,
- signal.recent_read_count,
- signal.pure_bash_streak,
- signal.no_error_streak,
- signal.tests_passed,
- );
- let recent_messages = recent_messages(request, recent_window);
- if recent_messages.is_empty() {
- return format!("Decide CAPABLE or EFFICIENT for the next call. {state_line}");
- }
-
- let mut lines = vec![
- "Decide CAPABLE or EFFICIENT for the next call.".to_string(),
- state_line,
- "Recent turns (most recent last):".to_string(),
- ];
- lines.extend(recent_messages.iter().map(format_message));
- lines.join("\n")
-}
-
-fn recent_messages(request: &ChatRequest, recent_window: usize) -> Vec<Value> {
- if recent_window == 0 {
- return Vec::new();
- }
- let Some(messages) = request
- .body()
- .as_object()
- .and_then(|body| body.get("messages"))
- .and_then(Value::as_array)
- else {
- return Vec::new();
- };
- messages
- .iter()
- .skip(messages.len().saturating_sub(recent_window))
- .cloned()
- .collect()
-}
-
-fn format_message(message: &Value) -> String {
- let Some(object) = message.as_object() else {
- let rendered = message.to_string();
- return format!("[?] {}", truncate(&rendered, 400));
- };
- let role = object.get("role").and_then(Value::as_str).unwrap_or("?");
- let body = match object.get("content") {
- Some(Value::String(text)) => text.clone(),
- Some(Value::Array(blocks)) => blocks
- .iter()
- .filter_map(format_content_block)
- .collect::<Vec<_>>()
- .join(" "),
- Some(other) => other.to_string(),
- None => "(empty)".to_string(),
- };
- format!("[{role}] {}", truncate(&body, 400))
-}
-
-fn format_content_block(block: &Value) -> Option<String> {
- let object = block.as_object()?;
- match object.get("type").and_then(Value::as_str) {
- Some("tool_use") => Some(format!(
- "<tool_use:{}>",
- object.get("name").and_then(Value::as_str).unwrap_or("?")
- )),
- Some("tool_result") => Some(format!(
- "<tool_result:{}>",
- truncate(
- &object
- .get("content")
- .map(Value::to_string)
- .unwrap_or_default(),
- 120
- )
- )),
- Some("text") => Some(
- object
- .get("text")
- .and_then(Value::as_str)
- .unwrap_or("")
- .to_string(),
- ),
- _ => None,
- }
-}
-
-fn truncate(text: &str, max_chars: usize) -> String {
- let mut chars = text.chars();
- let truncated = chars.by_ref().take(max_chars).collect::<String>();
- if chars.next().is_some() {
- format!("{truncated}...")
- } else {
- truncated
- }
-}
-
-fn chat_completions_url(base_url: Option<&str>) -> String {
- format!(
- "{}/chat/completions",
- base_url
- .unwrap_or(DEFAULT_OPENAI_BASE_URL)
- .trim_end_matches('/')
- )
-}
-
-fn model_accepts_reasoning_hint(model: &str) -> bool {
- let lowered = model.to_ascii_lowercase();
- !["anthropic", "bedrock", "claude"]
- .iter()
- .any(|tag| lowered.contains(tag))
-}
-
-fn default_picker() -> StageRouterPickerMode {
- StageRouterPickerMode::CapableFirst
-}
-
-fn default_confidence_threshold() -> f64 {
- DEFAULT_CONFIDENCE_THRESHOLD
-}
-
-fn default_signal_recent_window() -> usize {
- DEFAULT_RECENT_WINDOW
-}
-
-fn default_classifier_timeout_secs() -> f64 {
- DEFAULT_CLASSIFIER_TIMEOUT_SECS
-}
-
-fn default_classifier_recent_turn_window() -> usize {
- DEFAULT_CLASSIFIER_RECENT_TURN_WINDOW
-}
-
-fn default_classifier_max_tokens() -> u32 {
- CLASSIFIER_MAX_TOKENS
-}
-
-fn default_enable_stats() -> bool {
- true
-}
-
-#[cfg(test)]
-mod tests {
- use std::sync::{Arc, Mutex};
-
- use async_trait::async_trait;
- use serde_json::json;
- use switchyard_core::{BackendFormat, ModelId};
-
- use crate::backend::ProfileBackend;
- use crate::RequestMetadata;
-
- use super::*;
-
- #[derive(Clone, Debug, PartialEq)]
- struct ObservedCall {
- backend: &'static str,
- body: Value,
- }
-
- #[derive(Clone, Debug)]
- enum BackendAction {
- Ok,
- ContextOverflow,
- }
-
- struct TestBackend {
- name: &'static str,
- target_id: String,
- calls: Arc<Mutex<Vec<ObservedCall>>>,
- actions: Mutex<Vec<BackendAction>>,
- }
-
- #[async_trait]
- impl ProfileBackend for TestBackend {
- async fn call(&self, request: &ChatRequest) -> Result<ChatResponse> {
- self.calls
- .lock()
- .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))?
- .push(ObservedCall {
- backend: self.name,
- body: request.body().clone(),
- });
- let action = self
- .actions
- .lock()
- .map_err(|_| SwitchyardError::Other("actions mutex poisoned".to_string()))?
- .pop()
- .unwrap_or(BackendAction::Ok);
- match action {
- BackendAction::Ok => Ok(ChatResponse::openai_completion(json!({
- "served_by": self.name,
- "model": request.model(),
- "usage": {
- "prompt_tokens": 13,
- "completion_tokens": 5,
- },
- }))),
- BackendAction::ContextOverflow => Err(SwitchyardError::ContextWindowExceeded {
- target_id: self.target_id.clone(),
- model: request.model().unwrap_or("").to_string(),
- message: "prompt is too long".to_string(),
- }),
- }
- }
- }
-
- fn target(id: &str, model: &str) -> Result<LlmTarget> {
- let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
- target.format = BackendFormat::OpenAi;
- Ok(target)
- }
-
- fn target_backend(
- target: &LlmTarget,
- name: &'static str,
- calls: Arc<Mutex<Vec<ObservedCall>>>,
- actions: Vec<BackendAction>,
- ) -> TargetBackend {
- let mut actions = actions;
- actions.reverse();
- TargetBackend::new(
- target.clone(),
- Arc::new(TestBackend {
- name,
- target_id: target.id.to_string(),
- calls,
- actions: Mutex::new(actions),
- }),
- )
- }
-
- fn observed(calls: &Arc<Mutex<Vec<ObservedCall>>>) -> Result<Vec<ObservedCall>> {
- calls
- .lock()
- .map(|calls| calls.clone())
- .map_err(|_| SwitchyardError::Other("calls mutex poisoned".to_string()))
- }
-
- fn profile_input(request: ChatRequest) -> ProfileInput {
- ProfileInput {
- request,
- metadata: RequestMetadata::default(),
- }
- }
-
- fn profile(
- capable: LlmTarget,
- efficient: LlmTarget,
- picker: StageRouterPickerMode,
- confidence_threshold: f64,
- efficient_actions: Vec<BackendAction>,
- capable_actions: Vec<BackendAction>,
- ) -> Result<(StageRouterProfile, Arc<Mutex<Vec<ObservedCall>>>)> {
- let calls = Arc::new(Mutex::new(Vec::new()));
- let profile = StageRouterProfile {
- capable_backend: target_backend(
- &capable,
- "capable-backend",
- calls.clone(),
- capable_actions,
- ),
- efficient_backend: target_backend(
- &efficient,
- "efficient-backend",
- calls.clone(),
- efficient_actions,
- ),
- fallback_target_on_evict: capable.id.clone(),
- picker,
- confidence_threshold,
- signal_recent_window: DEFAULT_RECENT_WINDOW,
- classifier: None,
- stats: StatsAccumulator::new(),
- enable_stats: true,
- };
- Ok((profile, calls))
- }
-
- #[tokio::test]
- async fn stage_router_routes_critical_tool_errors_to_capable() -> Result<()> {
- let (profile, calls) = profile(
- target("capable", "frontier/model")?,
- target("efficient", "cheap/model")?,
- StageRouterPickerMode::EfficientFirst,
- 0.7,
- vec![BackendAction::Ok],
- vec![BackendAction::Ok],
- )?;
-
- let response = profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "smart-stage-router",
- "messages": [
- {"role": "assistant", "tool_calls": [{
- "type": "function",
- "function": {"name": "Bash", "arguments": "{\"command\":\"python test.py\"}"}
- }]},
- {"role": "tool", "tool_call_id": "call_1", "content": "Out of memory"}
- ],
- }))))
- .await?;
-
- let routing_metadata = response
- .routing_metadata
- .as_ref()
- .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?;
- assert_eq!(
- routing_metadata.selected_model.as_deref(),
- Some("frontier/model")
- );
- assert_eq!(routing_metadata.selected_tier.as_deref(), Some("capable"));
- assert_eq!(routing_metadata.confidence, Some(1.0));
- assert_eq!(
- routing_metadata.router_version.as_deref(),
- Some("stage_router:v1")
- );
- let response = response.response;
-
- let calls = observed(&calls)?;
- assert_eq!(calls.len(), 1);
- assert_eq!(calls[0].backend, "capable-backend");
- assert_eq!(calls[0].body["model"], "frontier/model");
- match response {
- ChatResponse::OpenAiCompletion(body) => {
- assert_eq!(body.body()["served_by"], "capable-backend");
- }
- _ => return Err(SwitchyardError::Other("unexpected response shape".into())),
- }
-
- let snapshot = profile.stats.snapshot()?;
- assert_eq!(
- snapshot
- .routing_decisions
- .get("stage_router")
- .and_then(|sources| sources.get("override")),
- Some(&1)
- );
- assert_eq!(
- snapshot
- .models
- .get("frontier/model")
- .and_then(|model| model.tier.as_deref()),
- Some("capable")
- );
- Ok(())
- }
-
- #[tokio::test]
- async fn stage_router_threshold_zero_accepts_neutral_scorer_as_efficient() -> Result<()> {
- let (profile, calls) = profile(
- target("capable", "frontier/model")?,
- target("efficient", "cheap/model")?,
- StageRouterPickerMode::CapableFirst,
- 0.0,
- vec![BackendAction::Ok],
- vec![BackendAction::Ok],
- )?;
-
- let processed = profile
- .process(profile_input(ChatRequest::openai_chat(json!({
- "model": "smart-stage-router",
- "messages": [{"role": "user", "content": "continue"}],
- }))))
- .await?;
-
- assert_eq!(processed.decision.tier, StageRouterTier::Efficient);
- assert_eq!(
- processed.decision.source,
- StageRouterDecisionSource::Dimensions
- );
- assert_eq!(processed.profile_input.request.model(), Some("cheap/model"));
- assert!(observed(&calls)?.is_empty());
- Ok(())
- }
-
- #[tokio::test]
- async fn stage_router_retries_configured_fallback_after_context_overflow() -> Result<()> {
- let (profile, calls) = profile(
- target("capable", "frontier/model")?,
- target("efficient", "cheap/model")?,
- StageRouterPickerMode::EfficientFirst,
- 0.7,
- vec![BackendAction::ContextOverflow],
- vec![BackendAction::Ok],
- )?;
-
- let response = profile
- .run(profile_input(ChatRequest::openai_chat(json!({
- "model": "smart-stage-router",
- "messages": [{"role": "user", "content": "continue"}],
- }))))
- .await?;
-
- let routing_metadata = response
- .routing_metadata
- .as_ref()
- .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?;
- assert_eq!(
- routing_metadata.selected_model.as_deref(),
- Some("frontier/model")
- );
- assert_eq!(routing_metadata.selected_tier.as_deref(), Some("capable"));
- assert!(routing_metadata
- .rationale
- .as_deref()
- .is_some_and(|reason| reason.contains("source=context_overflow_fallback")));
- let response = response.response;
-
- let calls = observed(&calls)?;
- assert_eq!(calls.len(), 2);
- assert_eq!(calls[0].backend, "efficient-backend");
- assert_eq!(calls[0].body["model"], "cheap/model");
- assert_eq!(calls[1].backend, "capable-backend");
- assert_eq!(calls[1].body["model"], "frontier/model");
- match response {
- ChatResponse::OpenAiCompletion(body) => {
- assert_eq!(body.body()["served_by"], "capable-backend");
- }
- _ => return Err(SwitchyardError::Other("unexpected response shape".into())),
- }
-
- let snapshot = profile.stats.snapshot()?;
- assert_eq!(snapshot.total_requests, 1);
- assert_eq!(
- snapshot
- .models
- .get("frontier/model")
- .and_then(|model| model.tier.as_deref()),
- Some("capable")
- );
- assert_eq!(
- snapshot
- .routing_decisions
- .get("stage_router")
- .and_then(|sources| sources.get("fall_open")),
- Some(&1)
- );
- Ok(())
- }
-
- #[test]
- fn stage_router_config_rejects_unknown_fallback_target() -> Result<()> {
- let config = StageRouterProfileConfig {
- capable: target("capable", "frontier/model")?,
- efficient: target("efficient", "cheap/model")?,
- fallback_target_on_evict: LlmTargetId::new("ghost")?,
- picker: StageRouterPickerMode::CapableFirst,
- confidence_threshold: 0.7,
- signal_recent_window: DEFAULT_RECENT_WINDOW,
- classifier: None,
- enable_stats: true,
- };
-
- let error = config
- .validate()
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected validation error".to_string());
- assert!(error.contains("fallback_target_on_evict"));
- Ok(())
- }
-
- #[test]
- fn classifier_request_uses_reasoning_hint_for_vllm_models() -> Result<()> {
- let classifier = StageRouterTierClassifier::new(&StageRouterClassifierConfig {
- model: "nvidia/deepseek-ai/deepseek-v4-flash".to_string(),
- api_key: "test-key".to_string(),
- base_url: None,
- timeout_secs: 1.0,
- recent_turn_window: 2,
- max_tokens: CLASSIFIER_MAX_TOKENS,
- system_prompt: None,
- })?;
-
- let body = classifier.request_body(
- &ChatRequest::openai_chat(json!({
- "model": "smart-stage-router",
- "messages": [{"role": "user", "content": "hi"}],
- })),
- &ToolResultSignal::default(),
- );
-
- assert_eq!(
- body["chat_template_kwargs"],
- json!({"enable_thinking": false})
- );
- assert_eq!(body["max_tokens"], CLASSIFIER_MAX_TOKENS);
- Ok(())
- }
-}
diff --git a/crates/switchyard-components-v2/src/profiles/stage_router/prompts/classifier.md b/crates/switchyard-components-v2/src/profiles/stage_router/prompts/classifier.md
deleted file mode 100644
index 25f570fcf..000000000
--- a/crates/switchyard-components-v2/src/profiles/stage_router/prompts/classifier.md
+++ /dev/null
@@ -1,5 +0,0 @@
-You are a routing classifier inside an agentic coding stage-router. Given a compact summary of the agent's recent tool activity, decide whether the next model call should go to the CAPABLE tier (frontier, expensive) or the EFFICIENT tier (cheap, less powerful).
-
-Respond with strict JSON: {"tier": "capable"} or {"tier": "efficient"}.
-
-Pick EFFICIENT when the agent shows concrete, low-friction progress (writes landing, tests passing, edits without errors). Pick CAPABLE when the agent is stalled, hitting errors, or facing a task likely to require careful reasoning.
diff --git a/crates/switchyard-components-v2/src/profiles/subagent_override.rs b/crates/switchyard-components-v2/src/profiles/subagent_override.rs
deleted file mode 100644
index b9275b4bd..000000000
--- a/crates/switchyard-components-v2/src/profiles/subagent_override.rs
+++ /dev/null
@@ -1,90 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Sub-agent override combinator for v2 profiles.
-//!
-//! Wraps any profile without changing its behavior for normal traffic. A
-//! request whose headers mark delegated sub-agent work
-//! ([`switchyard_protocol::Metadata::is_subagent_work`]) is served directly by
-//! a fixed worker target — keeping a sub-agent loop on an intentional,
-//! cache-compatible target — while every other request delegates to the wrapped
-//! profile. A worker failure surfaces as a normal target error and is never
-//! silently re-routed through the wrapped profile.
-
-use std::collections::BTreeMap;
-use std::time::Instant;
-
-use async_trait::async_trait;
-use switchyard_components::StatsAccumulator;
-use switchyard_core::{LlmTarget, Result};
-
-use crate::backend::{native_target_backend, TargetBackend};
-use crate::profile_stats_accumulator;
-use crate::stats_recording::record_usage_or_wrap_stream;
-use crate::{Profile, ProfileInput, ProfileResponse};
-
-/// Wraps a profile, routing delegated sub-agent work to a fixed worker target.
-pub(crate) struct SubagentOverrideProfile {
- inner: Box<dyn Profile>,
- worker: TargetBackend,
- stats: StatsAccumulator,
-}
-
-impl SubagentOverrideProfile {
- /// Wraps `inner`, routing recognized sub-agent work requests to `worker`.
- pub(crate) fn new(inner: Box<dyn Profile>, target: LlmTarget) -> Result<Self> {
- Ok(Self {
- inner,
- worker: native_target_backend(target)?,
- stats: profile_stats_accumulator(),
- })
- }
-}
-
-/// Returns true when the request headers signal delegated sub-agent work.
-///
-/// Normalizes multi-value headers to single-value (first occurrence wins)
-/// before calling the protocol-layer detection logic.
-fn is_subagent_work(headers: &BTreeMap<String, Vec<String>>) -> bool {
- let flat: BTreeMap<String, String> = headers
- .iter()
- .filter_map(|(k, vs)| vs.first().map(|v| (k.clone(), v.clone())))
- .collect();
- switchyard_protocol::Metadata::from_headers(&flat).is_subagent_work()
-}
-
-#[async_trait]
-impl Profile for SubagentOverrideProfile {
- /// Routes sub-agent work to the worker target; delegates everything else.
- async fn run(&self, mut input: ProfileInput) -> Result<ProfileResponse> {
- if !is_subagent_work(&input.metadata.headers) {
- return self.inner.run(input).await;
- }
-
- let profile_started_at = Instant::now();
- let target_model = self.worker.target().model.clone();
- input.request.set_model(target_model.as_str());
-
- let backend_started_at = Instant::now();
- let response = match self.worker.call(&input.request).await {
- Ok(response) => response,
- Err(error) => {
- self.stats.record_error(target_model.as_str(), None)?;
- return Err(error);
- }
- };
-
- let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0;
- self.stats
- .record_success(target_model.to_string(), Some(backend_latency_ms), None)?;
- let response = record_usage_or_wrap_stream(
- &self.stats,
- target_model.as_str(),
- None,
- profile_started_at,
- backend_latency_ms,
- response,
- )?;
- Ok(ProfileResponse::from(response))
- }
-}
diff --git a/crates/switchyard-components-v2/src/stats.rs b/crates/switchyard-components-v2/src/stats.rs
deleted file mode 100644
index 3c245b51a..000000000
--- a/crates/switchyard-components-v2/src/stats.rs
+++ /dev/null
@@ -1,14 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Shared stats surface for profile-owned runtimes.
-
-use std::sync::OnceLock;
-
-use switchyard_components::StatsAccumulator;
-
-/// Returns the process-wide stats accumulator used by v2 profiles.
-pub fn profile_stats_accumulator() -> StatsAccumulator {
- static PROFILE_STATS: OnceLock<StatsAccumulator> = OnceLock::new();
- PROFILE_STATS.get_or_init(StatsAccumulator::new).clone()
-}
diff --git a/crates/switchyard-components-v2/src/stats_recording.rs b/crates/switchyard-components-v2/src/stats_recording.rs
deleted file mode 100644
index 501d30b49..000000000
--- a/crates/switchyard-components-v2/src/stats_recording.rs
+++ /dev/null
@@ -1,209 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Records provider token usage for buffered and streaming responses.
-//!
-//! Buffered responses carry usage in the body and can be recorded synchronously.
-//! Streaming responses only emit usage inside a later event, so the response
-//! stream is wrapped and usage is recorded when the first usage-bearing event
-//! flows through — no buffering, no added latency, and the client still sees
-//! every event.
-
-use std::time::Instant;
-
-use async_stream::try_stream;
-use futures_util::StreamExt;
-use switchyard_components::stats::{
- openai_chat_usage_from_stream_event, openai_responses_usage_from_stream_event, usage_from_body,
- AnthropicStreamUsage, StatsAccumulator, TokenUsage,
-};
-use switchyard_core::{BoxResponseStream, ChatResponse, Result};
-
-/// Records usage from a buffered response, or wraps a streaming response so
-/// usage is recorded when its usage-bearing event arrives.
-pub(crate) fn record_usage_or_wrap_stream(
- stats: &StatsAccumulator,
- model: &str,
- tier: Option<&str>,
- profile_started_at: Instant,
- backend_latency_ms: f64,
- response: ChatResponse,
-) -> Result<ChatResponse> {
- // We report time-to-first-token as latency. Otherwise in streaming case we'd be including
- // the client's processing time.
- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;
- match response {
- ChatResponse::OpenAiCompletion(_)
- | ChatResponse::OpenAiResponsesCompletion(_)
- | ChatResponse::AnthropicCompletion(_) => {
- let usage = response.body().map(usage_from_body).unwrap_or_default();
- record_usage(
- stats,
- model,
- tier,
- total_latency_ms,
- backend_latency_ms,
- usage,
- )?;
- Ok(response)
- }
- ChatResponse::OpenAiStream(stream) => Ok(ChatResponse::OpenAiStream(wrap_openai_chat(
- stream,
- stats.clone(),
- model.to_string(),
- tier.map(str::to_string),
- total_latency_ms,
- backend_latency_ms,
- ))),
- ChatResponse::OpenAiResponsesStream(stream) => {
- Ok(ChatResponse::OpenAiResponsesStream(wrap_openai_responses(
- stream,
- stats.clone(),
- model.to_string(),
- tier.map(str::to_string),
- total_latency_ms,
- backend_latency_ms,
- )))
- }
- ChatResponse::AnthropicStream(stream) => Ok(ChatResponse::AnthropicStream(wrap_anthropic(
- stream,
- stats.clone(),
- model.to_string(),
- tier.map(str::to_string),
- total_latency_ms,
- backend_latency_ms,
- ))),
- }
-}
-
-/// Wraps an OpenAI chat completions stream to record usage stats.
-///
-/// Usage is typically only present in the last JSON SSE chunk, but we only
-/// record the first instance to avoid double counting in case of middle-box weirdness.
-///
-/// Pass-through, no buffering or dropping.
-fn wrap_openai_chat(
- mut stream: BoxResponseStream,
- stats: StatsAccumulator,
- model: String,
- tier: Option<String>,
- total_latency_ms: f64,
- backend_latency_ms: f64,
-) -> BoxResponseStream {
- Box::pin(try_stream! {
- let mut committed = false;
- while let Some(event) = stream.next().await {
- let event = event?;
- if !committed {
- if let Some(usage) = openai_chat_usage_from_stream_event(&event) {
- log_stream_record_result(
- record_usage(
- &stats,
- &model,
- tier.as_deref(),
- total_latency_ms,
- backend_latency_ms,
- usage,
- ),
- &model,
- );
- committed = true;
- }
- }
- yield event;
- }
- })
-}
-
-/// Wraps an OpenAI responses stream to record usage stats.
-fn wrap_openai_responses(
- mut stream: BoxResponseStream,
- stats: StatsAccumulator,
- model: String,
- tier: Option<String>,
- total_latency_ms: f64,
- backend_latency_ms: f64,
-) -> BoxResponseStream {
- Box::pin(try_stream! {
- let mut committed = false;
- while let Some(event) = stream.next().await {
- let event = event?;
- if !committed {
- if let Some(usage) = openai_responses_usage_from_stream_event(&event) {
- log_stream_record_result(
- record_usage(
- &stats,
- &model,
- tier.as_deref(),
- total_latency_ms,
- backend_latency_ms,
- usage,
- ),
- &model,
- );
- committed = true;
- }
- }
- yield event;
- }
- })
-}
-
-/// Wraps an Anthropic messages stream to record usage stats.
-fn wrap_anthropic(
- mut stream: BoxResponseStream,
- stats: StatsAccumulator,
- model: String,
- tier: Option<String>,
- total_latency_ms: f64,
- backend_latency_ms: f64,
-) -> BoxResponseStream {
- Box::pin(try_stream! {
- let mut stream_usage = AnthropicStreamUsage::default();
- while let Some(event) = stream.next().await {
- let event = event?;
- if let Some(usage) = stream_usage.observe(&event) {
- log_stream_record_result(
- record_usage(
- &stats,
- &model,
- tier.as_deref(),
- total_latency_ms,
- backend_latency_ms,
- usage,
- ),
- &model,
- );
- }
- yield event;
- }
- })
-}
-
-fn record_usage(
- stats: &StatsAccumulator,
- model: &str,
- tier: Option<&str>,
- total_latency_ms: f64,
- backend_latency_ms: f64,
- usage: TokenUsage,
-) -> Result<()> {
- let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);
- stats.record_usage_after_success_attribution(
- model.to_string(),
- usage,
- Some(total_latency_ms),
- Some(routing_overhead_ms),
- tier,
- )
-}
-
-fn log_stream_record_result(result: Result<()>, model: &str) {
- if let Err(error) = result {
- tracing::warn!(
- error = %error,
- model = %model,
- "failed to record stream usage",
- );
- }
-}
diff --git a/crates/switchyard-components-v2/tests/config.rs b/crates/switchyard-components-v2/tests/config.rs
deleted file mode 100644
index 943a34cf8..000000000
--- a/crates/switchyard-components-v2/tests/config.rs
+++ /dev/null
@@ -1,795 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Adversarial tests for profile config parsing and profile-owned resolution.
-
-use serde_json::json;
-use switchyard_components_v2::{
- parse_profile_config_str_with_env_lookup, NoopProfileConfig, ProfileConfig,
- ProfileConfigDocument, ProfileConfigFormat, ProfileHooks, ProfileInput, RequestMetadata,
-};
-use switchyard_core::{ChatRequest, ChatResponse, LlmTargetId, ProfileId, Result, SwitchyardError};
-
-const YAML_CONFIG: &str = r#"
-endpoints:
- nvidia:
- api_key: ${NVIDIA_API_KEY}
- base_url: https://inference-api.nvidia.com/v1
- timeout_secs: 120.0
-
-targets:
- strong:
- endpoint: nvidia
- model: nvidia/moonshotai/kimi-k2.5
- format: openai
- weak:
- endpoint: nvidia
- model: nvidia/nvidia/nemotron-nano-9b-v2
- format: openai
- direct-weak:
- endpoint: nvidia
- model: nvidia/nvidia/nemotron-nano-9b-v2
- format: openai
- classifier:
- endpoint: nvidia
- model: nvidia/nvidia/nemotron-nano-9b-v2
- format: openai
-
-profiles:
- direct:
- type: passthrough
- target: direct-weak
- smart-stage-router:
- type: stage_router
- capable: strong
- efficient: weak
- fallback_target_on_evict: strong
- picker: capable_first
- confidence_threshold: 0.7
- health-aware:
- type: latency-service
- latency_service_url: http://latency.local
- targets: [strong, weak]
- llm:
- type: llm-routing
- strong: strong
- weak: weak
- classifier: classifier
- profile_name: coding_agent
- stage_router:
- type: stage_router
- capable: strong
- efficient: weak
- fallback_target_on_evict: strong
- picker: capable_first
- classifier:
- model: nvidia/nvidia/nemotron-nano-9b-v2
- api_key: ${NVIDIA_API_KEY}
- base_url: https://inference-api.nvidia.com/v1
- bench:
- type: noop
-"#;
-
-// Parses YAML profile config using the test environment lookup.
-fn parse_yaml(input: &str) -> Result<ProfileConfigDocument> {
- parse_profile_config_str_with_env_lookup(input, ProfileConfigFormat::Yaml, test_env)
-}
-
-// Parses a profile config in the requested format using the test environment lookup.
-fn parse_with_format(input: &str, format: ProfileConfigFormat) -> Result<ProfileConfigDocument> {
- parse_profile_config_str_with_env_lookup(input, format, test_env)
-}
-
-// Supplies deterministic values for environment interpolation tests.
-fn test_env(name: &str) -> Option<String> {
- match name {
- "NVIDIA_API_KEY" => Some("nvapi-test".to_string()),
- "TENANT" => Some("team-a".to_string()),
- _ => None,
- }
-}
-
-// Resolving a full YAML config should inherit endpoints and validate profile-owned configs.
-#[test]
-fn yaml_config_resolves_endpoints_targets_and_profile_owned_configs() -> Result<()> {
- let config = parse_yaml(YAML_CONFIG)?;
- let plan = config.resolve()?;
-
- assert_eq!(plan.target_count(), 4);
- let direct_weak = plan
- .target(&LlmTargetId::new("direct-weak")?)
- .ok_or_else(|| SwitchyardError::Other("direct-weak target missing".to_string()))?;
- assert_eq!(direct_weak.endpoint.api_key.as_deref(), Some("nvapi-test"));
- assert_eq!(
- direct_weak.endpoint.base_url.as_deref(),
- Some("https://inference-api.nvidia.com/v1")
- );
-
- assert_eq!(
- plan.profile_type(&ProfileId::new("direct")?),
- Some("passthrough")
- );
- assert_eq!(
- plan.profile_type(&ProfileId::new("smart-stage-router")?),
- Some("stage_router")
- );
- assert_eq!(
- plan.profile_type(&ProfileId::new("health-aware")?),
- Some("latency-service")
- );
- assert_eq!(
- plan.profile_type(&ProfileId::new("llm")?),
- Some("llm-routing")
- );
- assert_eq!(
- plan.profile_type(&ProfileId::new("stage_router")?),
- Some("stage_router")
- );
-
- let profiles = plan.build_profiles()?;
- assert_eq!(profiles.len(), 6);
-
- let targets = plan
- .targets()
- .map(|(target_id, _target)| target_id.clone())
- .collect::<Vec<_>>();
- assert_eq!(
- targets,
- vec![
- LlmTargetId::new("classifier")?,
- LlmTargetId::new("direct-weak")?,
- LlmTargetId::new("strong")?,
- LlmTargetId::new("weak")?,
- ]
- );
- Ok(())
-}
-
-// JSON and TOML should normalize into equivalent resolved config plans.
-#[test]
-fn json_and_toml_parse_to_the_same_document_shape() -> Result<()> {
- let json_config = r#"
-{
- "endpoints": {
- "nvidia": {
- "api_key": "${NVIDIA_API_KEY}",
- "base_url": "https://inference-api.nvidia.com/v1",
- "timeout_secs": 120.0
- }
- },
- "targets": {
- "weak": {
- "endpoint": "nvidia",
- "model": "nvidia/nvidia/nemotron-nano-9b-v2",
- "format": "openai"
- }
- },
- "profiles": {
- "direct": {
- "type": "passthrough",
- "target": "weak"
- }
- }
-}
-"#;
- let toml_config = r#"
-[endpoints.nvidia]
-api_key = "${NVIDIA_API_KEY}"
-base_url = "https://inference-api.nvidia.com/v1"
-timeout_secs = 120.0
-
-[targets.weak]
-endpoint = "nvidia"
-model = "nvidia/nvidia/nemotron-nano-9b-v2"
-format = "openai"
-
-[profiles.direct]
-type = "passthrough"
-target = "weak"
-"#;
-
- let json = parse_with_format(json_config, ProfileConfigFormat::Json)?.resolve()?;
- let toml = parse_with_format(toml_config, ProfileConfigFormat::Toml)?.resolve()?;
-
- assert_eq!(json, toml);
- Ok(())
-}
-
-// Unknown fields inside a profile body should be rejected by that profile's config parser.
-#[test]
-fn unknown_profile_fields_are_rejected_by_owning_profile_config() -> Result<()> {
- let input = r#"
-targets:
- strong:
- model: strong/model
- format: openai
- weak:
- model: weak/model
- format: openai
-profiles:
- bad:
- type: random-routing
- strong: strong
- weak: weak
- strong_probability: 0.3
- tina: codes well
-"#;
-
- let error = parse_yaml(input)?
- .resolve()
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected resolve failure".to_string());
- assert!(error.contains("unknown field"));
- assert!(error.contains("tina"));
- assert!(error.contains("profile bad"));
- Ok(())
-}
-
-// Target-level `expose` stays rejected because every v2 target is addressable.
-#[test]
-fn target_expose_field_is_rejected_because_all_targets_are_exposed() {
- let input = r#"
-targets:
- weak:
- model: weak/model
- format: openai
- expose: true
-profiles:
- direct:
- type: passthrough
- target: weak
-"#;
-
- let error = parse_yaml(input)
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected parse failure".to_string());
- assert!(error.contains("unknown field"));
- assert!(error.contains("expose"));
-}
-
-// Target IDs should come only from map keys, not duplicated `id` fields.
-#[test]
-fn target_id_field_is_rejected_because_map_key_is_the_id() {
- let input = r#"
-targets:
- weak:
- id: duplicate
- model: weak/model
- format: openai
-profiles:
- direct:
- type: passthrough
- target: weak
-"#;
-
- let error = parse_yaml(input)
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected parse failure".to_string());
- assert!(error.contains("unknown field"));
- assert!(error.contains("id"));
-}
-
-// The top-level schema should reject stale route/table-era sections.
-#[test]
-fn unknown_top_level_fields_are_rejected() {
- let input = r#"
-endpoints: {}
-targets: {}
-profiles: {}
-routes: {}
-"#;
-
- let error = parse_yaml(input)
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected parse failure".to_string());
- assert!(error.contains("unknown field"));
- assert!(error.contains("routes"));
-}
-
-// Unknown profile types should parse as documents but fail during profile resolution.
-#[test]
-fn unknown_profile_type_is_rejected_during_resolution() -> Result<()> {
- let input = r#"
-profiles:
- bad:
- type: handmade
-"#;
-
- let error = parse_yaml(input)?
- .resolve()
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected resolve failure".to_string());
- assert!(error.contains("handmade"));
- assert!(error.contains("profile bad"));
- Ok(())
-}
-
-// `-` and `_` are equivalent in the `type` discriminator, so a name written with
-// either separator resolves to the same v2 profile: a legacy `random_routing`
-// spelling, and `stage-router` for the underscore-canonical stage router.
-#[test]
-fn profile_type_separators_are_normalized_during_resolution() -> Result<()> {
- let input = r#"
-endpoints:
- nvidia:
- api_key: ${NVIDIA_API_KEY}
- base_url: https://inference-api.nvidia.com/v1
-
-targets:
- strong:
- endpoint: nvidia
- model: nvidia/moonshotai/kimi-k2.5
- format: openai
- weak:
- endpoint: nvidia
- model: nvidia/nvidia/nemotron-nano-9b-v2
- format: openai
- classifier:
- endpoint: nvidia
- model: nvidia/nvidia/nemotron-nano-9b-v2
- format: openai
-
-profiles:
- snake-llm:
- type: llm_routing
- strong: strong
- weak: weak
- classifier: classifier
- profile_name: coding_agent
- hyphen-stage:
- type: stage-router
- capable: strong
- efficient: weak
- fallback_target_on_evict: strong
- picker: capable_first
- confidence_threshold: 0.7
- snake-latency:
- type: latency_service
- latency_service_url: http://latency.local
- targets: [strong, weak]
-"#;
-
- let plan = parse_yaml(input)?.resolve()?;
-
- // Resolved profiles report the canonical type name regardless of the
- // separator spelled in the config.
- assert_eq!(
- plan.profile_type(&ProfileId::new("snake-llm")?),
- Some("llm-routing")
- );
- assert_eq!(
- plan.profile_type(&ProfileId::new("hyphen-stage")?),
- Some("stage_router")
- );
- assert_eq!(
- plan.profile_type(&ProfileId::new("snake-latency")?),
- Some("latency-service")
- );
-
- // The profiles build, not just parse.
- assert_eq!(plan.build_profiles()?.len(), 3);
- Ok(())
-}
-
-// Missing profile type discriminators should fail while parsing the document.
-#[test]
-fn missing_profile_type_is_rejected_during_document_parse() {
- let input = r#"
-profiles:
- bad:
- target: weak
-"#;
-
- let error = parse_yaml(input)
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected parse failure".to_string());
- assert!(error.contains("type"));
-}
-
-// Empty profile type discriminators should fail while parsing the document.
-#[test]
-fn empty_profile_type_is_rejected_during_document_parse() {
- let input = r#"
-profiles:
- bad:
- type: " "
-"#;
-
- let error = parse_yaml(input)
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected parse failure".to_string());
- assert!(error.contains("must not be empty"));
-}
-
-// Missing environment variables should fail before document deserialization completes.
-#[test]
-fn missing_env_var_is_rejected() {
- let var_name = format!(
- "SWITCHYARD_COMPONENTS_V2_MISSING_ENV_{}",
- std::process::id()
- );
- let input = format!(
- r#"
-endpoints:
- missing:
- api_key: ${{{var_name}}}
-"#
- );
-
- let error =
- parse_profile_config_str_with_env_lookup(&input, ProfileConfigFormat::Yaml, test_env)
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected parse failure".to_string());
-
- assert!(error.contains(&var_name));
- assert!(error.contains("not set"));
-}
-
-// Targets that reference a missing endpoint should fail during target resolution.
-#[test]
-fn missing_endpoint_reference_is_rejected_during_resolution() -> Result<()> {
- let input = r#"
-targets:
- weak:
- endpoint: missing
- model: weak/model
- format: openai
-profiles:
- direct:
- type: passthrough
- target: weak
-"#;
-
- let error = parse_yaml(input)?
- .resolve()
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected resolve failure".to_string());
- assert!(error.contains("unknown endpoint missing"));
- Ok(())
-}
-
-// Macro-resolved profile target fields should fail when they name an unknown target.
-#[test]
-fn missing_profile_target_reference_is_rejected_by_macro_generated_resolver() -> Result<()> {
- let input = r#"
-targets:
- weak:
- model: weak/model
- format: openai
-profiles:
- direct:
- type: passthrough
- target: typo
-"#;
-
- let error = parse_yaml(input)?
- .resolve()
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected resolve failure".to_string());
- assert!(error.contains("unknown target typo"));
- assert!(error.contains("profile direct"));
- Ok(())
-}
-
-// Target-local endpoint fields should override shared endpoints and interpolate nested strings.
-#[test]
-fn target_overrides_endpoint_fields_and_interpolates_nested_strings() -> Result<()> {
- let input = r#"
-endpoints:
- nvidia:
- api_key: ${NVIDIA_API_KEY}
- base_url: https://inference-api.nvidia.com/v1
- timeout_secs: 120.0
-targets:
- weak:
- endpoint: nvidia
- model: weak/model
- format: openai
- base_url: https://override.example/v1
- timeout_secs: 30.0
- extra_body:
- metadata:
- tenant: ${TENANT}
-profiles:
- direct:
- type: passthrough
- target: weak
-"#;
- let plan = parse_yaml(input)?.resolve()?;
- let weak = plan
- .target(&LlmTargetId::new("weak")?)
- .ok_or_else(|| SwitchyardError::Other("weak target missing".to_string()))?;
-
- assert_eq!(
- weak.endpoint.base_url.as_deref(),
- Some("https://override.example/v1")
- );
- assert_eq!(weak.endpoint.timeout_secs, Some(30.0));
- assert_eq!(weak.endpoint.api_key.as_deref(), Some("nvapi-test"));
- assert_eq!(
- weak.extra_body.as_ref().and_then(|body| {
- body.get("metadata")
- .and_then(|metadata| metadata.get("tenant"))
- .and_then(serde_json::Value::as_str)
- }),
- Some("team-a")
- );
- Ok(())
-}
-
-// Debug output for resolved plans should not include inherited endpoint API keys.
-#[test]
-fn profile_config_plan_debug_redacts_target_api_keys() -> Result<()> {
- let plan = parse_yaml(YAML_CONFIG)?.resolve()?;
- let debug = format!("{plan:?}");
-
- assert!(!debug.contains("nvapi-test"));
- assert!(debug.contains("direct-weak"));
- assert!(debug.contains("passthrough"));
- Ok(())
-}
-
-// Resolved plans should build one profile or all profiles into runtime objects.
-#[test]
-fn profile_config_plan_builds_runtime_profiles() -> Result<()> {
- let plan = parse_yaml(YAML_CONFIG)?.resolve()?;
-
- let direct = plan.build_profile(&ProfileId::new("direct")?)?;
- let profiles = plan.build_profiles()?;
-
- drop(direct);
- assert_eq!(profiles.len(), 6);
- assert!(profiles.contains_key(&ProfileId::new("direct")?));
- assert!(profiles.contains_key(&ProfileId::new("smart-stage-router")?));
- assert!(profiles.contains_key(&ProfileId::new("health-aware")?));
- assert!(profiles.contains_key(&ProfileId::new("llm")?));
- assert!(profiles.contains_key(&ProfileId::new("stage_router")?));
- assert!(profiles.contains_key(&ProfileId::new("bench")?));
- Ok(())
-}
-
-// Components-v2 stage_router config should reject route-era/top-level classifier
-// knobs instead of carrying compatibility behavior forward.
-#[test]
-fn stage_router_rejects_top_level_classifier_knobs() -> Result<()> {
- let input = r#"
-targets:
- strong:
- model: strong/model
- format: openai
- weak:
- model: weak/model
- format: openai
-profiles:
- stale:
- type: stage_router
- capable: strong
- efficient: weak
- fallback_target_on_evict: strong
- classifier_max_tokens: 64
-"#;
-
- let error = parse_yaml(input)?
- .resolve()
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected resolve failure".to_string());
- assert!(error.contains("unknown field"));
- assert!(error.contains("classifier_max_tokens"));
- Ok(())
-}
-
-// Invalid stage_router picker names fail during profile config resolution.
-#[test]
-fn stage_router_resolve_rejects_unknown_picker() -> Result<()> {
- let input = r#"
-targets:
- strong:
- model: strong/model
- format: openai
- weak:
- model: weak/model
- format: openai
-profiles:
- bad:
- type: stage_router
- capable: strong
- efficient: weak
- fallback_target_on_evict: strong
- picker: not-a-picker
-"#;
-
- let error = parse_yaml(input)?
- .resolve()
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected resolve failure".to_string());
- assert!(error.contains("unknown variant"));
- assert!(error.contains("not-a-picker"));
- Ok(())
-}
-
-// Strict OpenAI classifier calls require OpenAI-format classifier targets.
-#[test]
-fn llm_routing_build_rejects_non_openai_classifier_target() -> Result<()> {
- let input = r#"
-targets:
- strong:
- model: strong/model
- format: openai
- weak:
- model: weak/model
- format: openai
- classifier:
- model: classifier/model
- format: anthropic
-profiles:
- bad:
- type: llm-routing
- strong: strong
- weak: weak
- classifier: classifier
-"#;
-
- let error = parse_yaml(input)?
- .resolve()?
- .build_profile(&ProfileId::new("bad")?)
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected build failure".to_string());
- assert!(error.contains("classifier target must use format: openai"));
- Ok(())
-}
-
-// Config-owned `build()` should return a concrete profile runtime with hook methods.
-#[tokio::test]
-async fn profile_config_build_returns_profile_with_request_and_response_hooks() -> Result<()> {
- let profile = NoopProfileConfig {}.build()?;
- let input = ProfileInput {
- request: ChatRequest::openai_chat(json!({
- "model": "unit/noop",
- "messages": [],
- })),
- metadata: RequestMetadata::default(),
- };
-
- let processed = profile.process(input).await?;
- let response = profile
- .rprocess(
- &processed,
- ChatResponse::openai_completion(json!({
- "id": "unit-response",
- "object": "chat.completion",
- "model": "unit/noop",
- "choices": [],
- })),
- )
- .await?;
-
- assert_eq!(processed.request.model(), Some("unit/noop"));
- assert_eq!(
- response
- .body()
- .and_then(|body| body.get("id"))
- .and_then(serde_json::Value::as_str),
- Some("unit-response")
- );
- Ok(())
-}
-
-// File format detection should reject unsupported config file extensions.
-#[test]
-fn profile_config_format_rejects_unknown_extension() {
- let error = ProfileConfigFormat::from_path("profiles.ini")
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected extension failure".to_string());
- assert!(error.contains("unsupported profile config extension"));
-}
-
-// The envelope-level `subagent_target` is consumed before profile-owned parsing,
-// so profile types that deny unknown fields still resolve, and the reference
-// stays readable from the parsed document.
-#[test]
-fn subagent_target_envelope_field_is_stripped_and_exposed() -> Result<()> {
- let input = r#"
-targets:
- weak:
- model: weak/model
- format: openai
-profiles:
- direct:
- type: passthrough
- target: weak
- subagent_target: weak
-"#;
- let config = parse_yaml(input)?;
- let profile_id = ProfileId::new("direct")?;
- assert_eq!(
- config.profile_subagent_target(&profile_id),
- Some(&LlmTargetId::new("weak")?)
- );
- let body_has_field = config
- .profile_body(&profile_id)
- .and_then(|body| body.get("subagent_target"))
- .is_some();
- assert!(!body_has_field);
- config.resolve()?;
- Ok(())
-}
-
-// A profile without the envelope field reports no sub-agent target.
-#[test]
-fn subagent_target_is_absent_by_default() -> Result<()> {
- let input = r#"
-targets:
- weak:
- model: weak/model
- format: openai
-profiles:
- direct:
- type: passthrough
- target: weak
-"#;
- let config = parse_yaml(input)?;
- assert_eq!(
- config.profile_subagent_target(&ProfileId::new("direct")?),
- None
- );
- Ok(())
-}
-
-// An unknown `subagent_target` reference is a startup configuration error.
-#[test]
-fn subagent_target_referencing_unknown_target_is_rejected_at_resolve() -> Result<()> {
- let input = r#"
-targets:
- weak:
- model: weak/model
- format: openai
-profiles:
- direct:
- type: passthrough
- target: weak
- subagent_target: ghost
-"#;
- let error = parse_yaml(input)?
- .resolve()
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected resolve failure".to_string());
- assert!(error.contains("profile direct"));
- assert!(error.contains("subagent_target references unknown target ghost"));
- Ok(())
-}
-
-// The envelope field must be a target id string, not a structured value.
-#[test]
-fn subagent_target_must_be_a_target_id_string() {
- let input = r#"
-targets:
- weak:
- model: weak/model
- format: openai
-profiles:
- direct:
- type: passthrough
- target: weak
- subagent_target: [weak]
-"#;
- let error = parse_yaml(input)
- .err()
- .map(|error| error.to_string())
- .unwrap_or_else(|| "expected parse failure".to_string());
- assert!(error.contains("`subagent_target` must be a target id string"));
-}
diff --git a/crates/switchyard-components-v2/tests/latency_service_profile.rs b/crates/switchyard-components-v2/tests/latency_service_profile.rs
deleted file mode 100644
index 136831294..000000000
--- a/crates/switchyard-components-v2/tests/latency_service_profile.rs
+++ /dev/null
@@ -1,453 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Public API tests for the components-v2 latency-service profile config.
-
-use std::collections::BTreeMap;
-use std::io::{Read, Write};
-use std::net::{SocketAddr, TcpListener, TcpStream};
-use std::sync::mpsc::{self, Receiver, RecvTimeoutError};
-use std::thread::{self, JoinHandle};
-use std::time::Duration;
-
-use serde_json::{json, Value};
-use switchyard_components_v2::{
- profile_stats_accumulator, LatencyServiceProfileConfig, Profile, ProfileConfig, ProfileInput,
- RequestMetadata,
-};
-use switchyard_core::{
- BackendFormat, ChatRequest, ChatResponse, EndpointConfig, LlmTarget, LlmTargetId, ModelId,
- Result, SwitchyardError,
-};
-
-fn target(id: &str, model: &str) -> Result<LlmTarget> {
- let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
- target.format = BackendFormat::OpenAi;
- Ok(target)
-}
-
-fn config(targets: Vec<LlmTarget>) -> LatencyServiceProfileConfig {
- LatencyServiceProfileConfig {
- latency_service_url: "http://latency.test".to_string(),
- targets,
- poll_timeout_secs: 5.0,
- max_retries: 2,
- }
-}
-
-fn openai_target(id: &'static str, model: &'static str, base_url: &str) -> LlmTarget {
- LlmTarget {
- id: LlmTargetId::from_static(id),
- model: ModelId::from_static(model),
- format: BackendFormat::OpenAi,
- endpoint: EndpointConfig {
- base_url: Some(base_url.to_string()),
- api_key: Some("test-key".to_string()),
- timeout_secs: Some(5.0),
- },
- extra_body: None,
- extra_headers: BTreeMap::new(),
- }
-}
-
-#[derive(Clone, Debug, Default, PartialEq)]
-struct MockCounts {
- requests: Vec<HttpRequest>,
- health: u64,
- fast: u64,
- slow: u64,
- fast_body: Option<Value>,
- slow_body: Option<Value>,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-struct HttpRequest {
- method: String,
- path: String,
- body: Value,
-}
-
-struct MockLatencyOpenAiServer {
- base_url: String,
- address: SocketAddr,
- receiver: Receiver<Result<MockCounts>>,
- handle: Option<JoinHandle<()>>,
-}
-
-impl MockLatencyOpenAiServer {
- fn spawn(requests: usize) -> Result<Self> {
- let listener = TcpListener::bind("127.0.0.1:0")
- .map_err(|error| SwitchyardError::Other(format!("bind failed: {error}")))?;
- let address = listener
- .local_addr()
- .map_err(|error| SwitchyardError::Other(format!("local addr failed: {error}")))?;
- let (sender, receiver) = mpsc::channel();
- let handle = thread::spawn(move || {
- let result = run_mock_server(listener, requests);
- let _ignored = sender.send(result);
- });
- Ok(Self {
- base_url: format!("http://{address}"),
- address,
- receiver,
- handle: Some(handle),
- })
- }
-
- fn base_url(&self) -> String {
- self.base_url.clone()
- }
-
- fn finish(&mut self) -> Result<MockCounts> {
- let result = match self.receiver.recv_timeout(Duration::from_secs(5)) {
- Ok(result) => result,
- Err(RecvTimeoutError::Timeout) => {
- self.wake();
- self.receiver
- .recv_timeout(Duration::from_secs(1))
- .map_err(|error| {
- SwitchyardError::Other(format!("mock server did not finish: {error}"))
- })?
- }
- Err(RecvTimeoutError::Disconnected) => {
- Err(SwitchyardError::Other("mock server disconnected".into()))
- }
- };
- if let Some(handle) = self.handle.take() {
- handle
- .join()
- .map_err(|_| SwitchyardError::Other("mock server thread panicked".into()))?;
- }
- result
- }
-
- fn wake(&self) {
- let _ignored = TcpStream::connect(self.address);
- }
-}
-
-impl Drop for MockLatencyOpenAiServer {
- fn drop(&mut self) {
- if self.handle.is_some() {
- self.wake();
- }
- if let Some(handle) = self.handle.take() {
- let _ignored = handle.join();
- }
- }
-}
-
-fn run_mock_server(listener: TcpListener, requests: usize) -> Result<MockCounts> {
- let mut counts = MockCounts::default();
- for _ in 0..requests {
- let (mut stream, _address) = listener
- .accept()
- .map_err(|error| SwitchyardError::Other(format!("accept failed: {error}")))?;
- stream
- .set_read_timeout(Some(Duration::from_secs(5)))
- .map_err(|error| SwitchyardError::Other(format!("set read timeout failed: {error}")))?;
- stream
- .set_write_timeout(Some(Duration::from_secs(5)))
- .map_err(|error| {
- SwitchyardError::Other(format!("set write timeout failed: {error}"))
- })?;
- let request = read_http_request(&mut stream)?;
- let body = response_for_request(request, &mut counts)?;
- write_json_response(&mut stream, 200, body)?;
- }
- Ok(counts)
-}
-
-fn read_http_request(stream: &mut TcpStream) -> Result<HttpRequest> {
- let mut bytes = Vec::new();
- let mut chunk = [0_u8; 4096];
- loop {
- let read = stream
- .read(&mut chunk)
- .map_err(|error| SwitchyardError::Other(format!("read request failed: {error}")))?;
- if read == 0 {
- break;
- }
- bytes.extend_from_slice(&chunk[..read]);
- if let Some((header_end, content_length)) = request_shape(&bytes) {
- let body_start = header_end + 4;
- if bytes.len() >= body_start + content_length {
- break;
- }
- }
- }
-
- let (header_end, content_length) = request_shape(&bytes)
- .ok_or_else(|| SwitchyardError::Other("request should contain headers".into()))?;
- let header_text = std::str::from_utf8(&bytes[..header_end])
- .map_err(|error| SwitchyardError::Other(format!("headers should be UTF-8: {error}")))?;
- let body_start = header_end + 4;
- let body_end = body_start + content_length;
-
- let mut lines = header_text.lines();
- let first_line = lines
- .next()
- .ok_or_else(|| SwitchyardError::Other("missing request line".into()))?;
- let mut parts = first_line.split_whitespace();
- let method = parts
- .next()
- .ok_or_else(|| SwitchyardError::Other("missing request method".into()))?
- .to_string();
- let path = parts
- .next()
- .ok_or_else(|| SwitchyardError::Other("missing request path".into()))?
- .to_string();
-
- let raw_body = &bytes[body_start..body_end];
- let body = if raw_body.is_empty() {
- Value::Null
- } else {
- serde_json::from_slice(raw_body)
- .map_err(|error| SwitchyardError::Other(format!("decode request body: {error}")))?
- };
- Ok(HttpRequest { method, path, body })
-}
-
-fn request_shape(bytes: &[u8]) -> Option<(usize, usize)> {
- let header_end = find_bytes(bytes, b"\r\n\r\n")?;
- let headers = std::str::from_utf8(&bytes[..header_end]).ok()?;
- let content_length = headers
- .lines()
- .filter_map(|line| line.split_once(':'))
- .find_map(|(name, value)| {
- name.eq_ignore_ascii_case("content-length")
- .then(|| value.trim().parse::<usize>().ok())
- .flatten()
- })
- .unwrap_or(0);
- Some((header_end, content_length))
-}
-
-fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option<usize> {
- haystack
- .windows(needle.len())
- .position(|window| window == needle)
-}
-
-fn response_for_request(request: HttpRequest, counts: &mut MockCounts) -> Result<Value> {
- counts.requests.push(request.clone());
- if request.method == "GET" && request.path.starts_with("/v1/endpoints/health") {
- counts.health = counts.health.saturating_add(1);
- return Ok(json!({
- "endpoint_health": {
- "fast": {"status": "healthy", "last_latency_ms": 10.0},
- "slow": {"status": "degraded", "last_latency_ms": 1.0}
- }
- }));
- }
-
- if request.method == "POST" && request.path == "/fast/v1/chat/completions" {
- counts.fast = counts.fast.saturating_add(1);
- counts.fast_body = Some(request.body.clone());
- return Ok(openai_completion("fast", request.body));
- }
-
- if request.method == "POST" && request.path == "/slow/v1/chat/completions" {
- counts.slow = counts.slow.saturating_add(1);
- counts.slow_body = Some(request.body.clone());
- return Ok(openai_completion("slow", request.body));
- }
-
- Err(SwitchyardError::Other(format!(
- "unexpected mock request {} {}",
- request.method, request.path
- )))
-}
-
-fn openai_completion(endpoint: &'static str, request_body: Value) -> Value {
- json!({
- "id": "chatcmpl-test",
- "object": "chat.completion",
- "model": request_body.get("model").cloned().unwrap_or(Value::Null),
- "mock_endpoint": endpoint,
- "choices": [{
- "index": 0,
- "message": {"role": "assistant", "content": "ok"},
- "finish_reason": "stop"
- }],
- "usage": {
- "prompt_tokens": 2,
- "completion_tokens": 1,
- "total_tokens": 3
- }
- })
-}
-
-fn write_json_response(stream: &mut std::net::TcpStream, status: u16, body: Value) -> Result<()> {
- let body = serde_json::to_string(&body)
- .map_err(|error| SwitchyardError::Other(format!("encode response body: {error}")))?;
- let response = format!(
- "HTTP/1.1 {status} OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
- body.len()
- );
- stream
- .write_all(response.as_bytes())
- .map_err(|error| SwitchyardError::Other(format!("write response failed: {error}")))
-}
-
-#[test]
-fn profile_config_build_uses_existing_native_backend_stack() -> Result<()> {
- let config = config(vec![
- target("fast", "upstream-fast")?,
- target("slow", "upstream-slow")?,
- ]);
-
- let profile = config.build()?;
-
- assert_eq!(profile.health_snapshot().len(), 2);
- assert!(!profile.is_ready());
- Ok(())
-}
-
-#[tokio::test]
-async fn profile_run_polls_and_routes_native_openai_call_to_healthy_target() -> Result<()> {
- profile_stats_accumulator().reset()?;
- let mut server = MockLatencyOpenAiServer::spawn(2)?;
- let fast = openai_target(
- "fast",
- "upstream-fast-native",
- &format!("{}/fast/v1", server.base_url()),
- );
- let slow = openai_target(
- "slow",
- "upstream-slow-native",
- &format!("{}/slow/v1", server.base_url()),
- );
- let config = LatencyServiceProfileConfig {
- latency_service_url: server.base_url(),
- targets: vec![fast, slow],
- poll_timeout_secs: 5.0,
- max_retries: 0,
- };
- let profile = config.build()?;
-
- let response = profile
- .run(ProfileInput {
- request: ChatRequest::openai_chat(json!({
- "model": "client-model",
- "messages": [{"role": "user", "content": "route me"}],
- "max_tokens": 8
- })),
- metadata: RequestMetadata::default(),
- })
- .await?;
-
- let routing_metadata = response
- .routing_metadata
- .as_ref()
- .ok_or_else(|| SwitchyardError::Other("routing metadata missing".into()))?;
- assert_eq!(
- routing_metadata.selected_model.as_deref(),
- Some("upstream-fast-native")
- );
- assert_eq!(routing_metadata.selected_tier.as_deref(), Some("healthy"));
- let response = response.response;
-
- match response {
- ChatResponse::OpenAiCompletion(body) => {
- assert_eq!(body.body()["mock_endpoint"], "fast");
- assert_eq!(body.body()["model"], "upstream-fast-native");
- }
- _ => return Err(SwitchyardError::Other("unexpected response shape".into())),
- }
- let counts = server.finish()?;
- assert_eq!(counts.health, 1);
- assert_eq!(counts.fast, 1);
- assert_eq!(counts.slow, 0);
- assert_eq!(counts.requests.len(), 2);
- assert_eq!(counts.requests[0].method, "GET");
- assert!(counts.requests[0].path.starts_with("/v1/endpoints/health?"));
- assert!(counts.requests[0].path.contains("endpoint_ids=fast"));
- assert!(counts.requests[0].path.contains("endpoint_ids=slow"));
- assert_eq!(counts.requests[1].method, "POST");
- assert_eq!(counts.requests[1].path, "/fast/v1/chat/completions");
- assert!(!counts
- .requests
- .iter()
- .any(|request| request.path == "/slow/v1/chat/completions"));
- assert_eq!(
- counts
- .fast_body
- .as_ref()
- .and_then(|body| body.get("model"))
- .and_then(Value::as_str),
- Some("upstream-fast-native")
- );
-
- let stats = profile_stats_accumulator().snapshot()?;
- let model = stats
- .models
- .get("upstream-fast-native")
- .ok_or_else(|| SwitchyardError::Other("global v2 stats should include model".into()))?;
- assert!(model.calls >= 1);
- assert!(model.prompt_tokens >= 2);
- assert!(model.completion_tokens >= 1);
- Ok(())
-}
-
-#[test]
-fn profile_config_macro_adds_type_metadata_and_strict_serde() -> Result<()> {
- let config = config(vec![target("fast", "upstream-fast")?]);
-
- assert_eq!(LatencyServiceProfileConfig::PROFILE_TYPE, "latency-service");
- assert_eq!(config.profile_type(), "latency-service");
-
- let old_poller_field = json!({
- "latency_service_url": "http://latency.test",
- "targets": config.targets,
- "poll_timeout_secs": 5.0,
- "max_retries": 2,
- "poll_interval_secs": 1.0,
- });
- let error = serde_json::from_value::<LatencyServiceProfileConfig>(old_poller_field)
- .err()
- .ok_or_else(|| SwitchyardError::Other("unknown profile field should fail".into()))?;
- assert!(error.to_string().contains("unknown field"));
- Ok(())
-}
-
-#[test]
-fn invalid_profile_config_is_rejected_by_build() -> Result<()> {
- let fast = target("fast", "upstream-fast")?;
- let duplicate = config(vec![fast.clone(), fast]);
-
- match duplicate.build() {
- Err(SwitchyardError::InvalidConfig(message)) => {
- assert!(message.contains("duplicate target fast"));
- }
- Ok(_) => {
- return Err(SwitchyardError::Other(
- "duplicate targets should reject profile construction".into(),
- ));
- }
- Err(other) => {
- return Err(SwitchyardError::Other(format!(
- "expected InvalidConfig, got {other}"
- )));
- }
- }
-
- let mut bad_timeout = config(vec![target("fast", "upstream-fast")?]);
- bad_timeout.poll_timeout_secs = f64::INFINITY;
- match bad_timeout.build() {
- Err(SwitchyardError::InvalidConfig(message)) => {
- assert!(message.contains("poll_timeout_secs"));
- }
- Ok(_) => {
- return Err(SwitchyardError::Other(
- "invalid timeout should reject profile construction".into(),
- ));
- }
- Err(other) => {
- return Err(SwitchyardError::Other(format!(
- "expected InvalidConfig, got {other}"
- )));
- }
- }
- Ok(())
-}
diff --git a/crates/switchyard-components-v2/tests/llm_routing_profile.rs b/crates/switchyard-components-v2/tests/llm_routing_profile.rs
deleted file mode 100644
index f14dcf731..000000000
--- a/crates/switchyard-components-v2/tests/llm_routing_profile.rs
+++ /dev/null
@@ -1,687 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Public API tests for the components-v2 LLM-routing profile config.
-
-use std::collections::BTreeMap;
-use std::io::{Read, Write};
-use std::net::{SocketAddr, TcpListener, TcpStream};
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex, OnceLock};
-use std::thread::{self, JoinHandle};
-use std::time::Duration;
-
-use serde_json::{json, Value};
-use switchyard_components_v2::{
- profile_stats_accumulator, LlmRoutingProfileConfig, LlmRoutingTierMapping, Profile,
- ProfileConfig, ProfileHooks, ProfileInput, RequestMetadata,
-};
-use switchyard_core::{
- BackendFormat, ChatRequest, LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError,
-};
-use tokio::sync::Mutex as AsyncMutex;
-
-#[derive(Clone, Debug, PartialEq)]
-struct ObservedRequest {
- path: String,
- headers: BTreeMap<String, String>,
- body: Value,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-enum BackendMode {
- FixedStatus(u16),
- WeakContextOverflowThenOk,
-}
-
-struct MockOpenAiServer {
- addr: SocketAddr,
- requests: Arc<Mutex<Vec<ObservedRequest>>>,
- shutdown: Arc<AtomicBool>,
- handle: Option<JoinHandle<()>>,
-}
-
-impl MockOpenAiServer {
- fn spawn(classifier_arguments: Option<Value>, backend_status: u16) -> Result<Self> {
- Self::spawn_with_backend_mode(
- classifier_arguments,
- BackendMode::FixedStatus(backend_status),
- )
- }
-
- fn spawn_with_backend_mode(
- classifier_arguments: Option<Value>,
- backend_mode: BackendMode,
- ) -> Result<Self> {
- let listener = TcpListener::bind("127.0.0.1:0")
- .map_err(|error| SwitchyardError::Other(format!("bind failed: {error}")))?;
- let addr = listener
- .local_addr()
- .map_err(|error| SwitchyardError::Other(format!("local_addr failed: {error}")))?;
- let requests = Arc::new(Mutex::new(Vec::new()));
- let shutdown = Arc::new(AtomicBool::new(false));
- let thread_requests = Arc::clone(&requests);
- let thread_shutdown = Arc::clone(&shutdown);
- let handle = thread::spawn(move || {
- for stream in listener.incoming() {
- if thread_shutdown.load(Ordering::SeqCst) {
- break;
- }
- let Ok(mut stream) = stream else {
- continue;
- };
- let Ok(request) = read_request(&mut stream) else {
- continue;
- };
- if let Ok(mut requests) = thread_requests.lock() {
- requests.push(request.clone());
- }
- let response = response_for(&request, classifier_arguments.as_ref(), backend_mode);
- let _ = write_json_response(&mut stream, response.0, response.1);
- }
- });
- Ok(Self {
- addr,
- requests,
- shutdown,
- handle: Some(handle),
- })
- }
-
- fn base_url(&self) -> String {
- format!("http://{}", self.addr)
- }
-
- fn requests(&self) -> Result<Vec<ObservedRequest>> {
- self.requests
- .lock()
- .map(|requests| requests.clone())
- .map_err(|_| SwitchyardError::Other("requests mutex poisoned".to_string()))
- }
-}
-
-impl Drop for MockOpenAiServer {
- fn drop(&mut self) {
- self.shutdown.store(true, Ordering::SeqCst);
- let _ = TcpStream::connect(self.addr);
- if let Some(handle) = self.handle.take() {
- let _ = handle.join();
- }
- }
-}
-
-fn read_request(stream: &mut TcpStream) -> Result<ObservedRequest> {
- stream
- .set_read_timeout(Some(Duration::from_secs(5)))
- .map_err(|error| SwitchyardError::Other(format!("set timeout failed: {error}")))?;
- let mut bytes = Vec::new();
- let mut buf = [0_u8; 1024];
- let header_end = loop {
- let read = stream
- .read(&mut buf)
- .map_err(|error| SwitchyardError::Other(format!("read failed: {error}")))?;
- if read == 0 {
- return Err(SwitchyardError::Other(
- "connection closed early".to_string(),
- ));
- }
- bytes.extend_from_slice(&buf[..read]);
- if let Some(header_end) = find_header_end(&bytes) {
- break header_end;
- }
- };
- let headers = String::from_utf8_lossy(&bytes[..header_end]);
- let mut lines = headers.lines();
- let request_line = lines
- .next()
- .ok_or_else(|| SwitchyardError::Other("missing request line".to_string()))?;
- let path = request_line
- .split_whitespace()
- .nth(1)
- .ok_or_else(|| SwitchyardError::Other("missing request path".to_string()))?
- .to_string();
- let headers = headers
- .lines()
- .skip(1)
- .filter_map(|line| {
- let (name, value) = line.split_once(':')?;
- Some((name.to_ascii_lowercase(), value.trim().to_string()))
- })
- .collect::<BTreeMap<_, _>>();
- let content_length = headers
- .get("content-length")
- .and_then(|value| value.parse::<usize>().ok())
- .unwrap_or(0);
- let body_start = header_end + 4;
- while bytes.len().saturating_sub(body_start) < content_length {
- let read = stream
- .read(&mut buf)
- .map_err(|error| SwitchyardError::Other(format!("body read failed: {error}")))?;
- if read == 0 {
- break;
- }
- bytes.extend_from_slice(&buf[..read]);
- }
- let body = if content_length == 0 {
- Value::Null
- } else {
- serde_json::from_slice(&bytes[body_start..body_start + content_length])
- .map_err(|error| SwitchyardError::Other(format!("decode request body: {error}")))?
- };
- Ok(ObservedRequest {
- path,
- headers,
- body,
- })
-}
-
-fn find_header_end(bytes: &[u8]) -> Option<usize> {
- bytes.windows(4).position(|window| window == b"\r\n\r\n")
-}
-
-fn response_for(
- request: &ObservedRequest,
- classifier_arguments: Option<&Value>,
- backend_mode: BackendMode,
-) -> (u16, Value) {
- if request.path.contains("/classifier/") {
- return match classifier_arguments {
- Some(arguments) => {
- let tool_name = request.body["tool_choice"]["function"]["name"]
- .as_str()
- .unwrap_or("select_route");
- (
- 200,
- json!({
- "id": "chatcmpl-llm-routing-classifier",
- "object": "chat.completion",
- "model": request.body["model"],
- "choices": [{
- "index": 0,
- "message": {
- "role": "assistant",
- "tool_calls": [{
- "id": "call_route",
- "type": "function",
- "function": {
- "name": tool_name,
- "arguments": arguments.to_string(),
- },
- }],
- },
- "finish_reason": "tool_calls",
- }],
- "usage": {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18},
- }),
- )
- }
- None => (
- 200,
- json!({
- "id": "chatcmpl-llm-routing-classifier",
- "object": "chat.completion",
- "model": request.body["model"],
- "choices": [{"message": {"role": "assistant", "content": "not a tool call"}}],
- }),
- ),
- };
- }
-
- if matches!(backend_mode, BackendMode::WeakContextOverflowThenOk)
- && request.path.contains("/weak/")
- {
- return (
- 400,
- json!({
- "error": {
- "message": "prompt is too long",
- "code": "context_length_exceeded",
- "type": "invalid_request_error"
- }
- }),
- );
- }
-
- if let BackendMode::FixedStatus(backend_status) = backend_mode {
- if backend_status >= 400 {
- return (
- backend_status,
- json!({"error": {"message": "selected backend failed", "code": "backend_failed"}}),
- );
- }
- }
- (
- 200,
- json!({
- "id": "chatcmpl-llm-routing-backend",
- "object": "chat.completion",
- "model": request.body["model"],
- "choices": [{"message": {"role": "assistant", "content": "ok"}}],
- "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
- }),
- )
-}
-
-fn write_json_response(stream: &mut TcpStream, status: u16, body: Value) -> Result<()> {
- let body = serde_json::to_string(&body)
- .map_err(|error| SwitchyardError::Other(format!("encode response body: {error}")))?;
- let response = format!(
- "HTTP/1.1 {status} OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
- body.len()
- );
- stream
- .write_all(response.as_bytes())
- .map_err(|error| SwitchyardError::Other(format!("write response failed: {error}")))
-}
-
-fn target(id: &str, model: &str, base_url: &str) -> Result<LlmTarget> {
- let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
- target.format = BackendFormat::OpenAi;
- target.endpoint.base_url = Some(base_url.to_string());
- target.endpoint.api_key = Some("test-key".to_string());
- Ok(target)
-}
-
-fn config(base_url: &str) -> Result<LlmRoutingProfileConfig> {
- Ok(LlmRoutingProfileConfig {
- strong: target("strong", "frontier/model", &format!("{base_url}/strong/v1"))?,
- weak: target("weak", "cheap/model", &format!("{base_url}/weak/v1"))?,
- classifier: target(
- "classifier",
- "classifier/model",
- &format!("{base_url}/classifier/v1"),
- )?,
- fallback_target_on_evict: LlmTargetId::new("strong")?,
- profile_name: "coding_agent".to_string(),
- classifier_min_confidence: 0.0,
- classifier_fail_open: true,
- classifier_recent_turn_window: 4,
- classifier_max_tokens: 4096,
- alignment_min_confidence: 0.85,
- default_tier: None,
- tier_mapping: None,
- classifier_system_prompt: None,
- classifier_tool_name: None,
- classifier_tool_description: None,
- classifier_tool_parameters: None,
- })
-}
-
-fn request() -> ProfileInput {
- ProfileInput {
- request: ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [{"role": "user", "content": "list files"}],
- })),
- metadata: RequestMetadata::default(),
- }
-}
-
-fn medium_signals() -> Value {
- json!({
- "recommended_tier": "medium",
- "confidence": 0.9,
- "abstain": false,
- "turn_type": "exploration",
- "code_modification_scope": "none",
- "tool_call_count_estimate": 0,
- "requires_codebase_context": false
- })
-}
-
-fn complex_signals() -> Value {
- json!({
- "recommended_tier": "complex",
- "confidence": 0.9,
- "abstain": false,
- "turn_type": "debug",
- "code_modification_scope": "cross_module",
- "tool_call_count_estimate": 4,
- "requires_codebase_context": true
- })
-}
-
-fn stats_test_lock() -> &'static AsyncMutex<()> {
- static LOCK: OnceLock<AsyncMutex<()>> = OnceLock::new();
- LOCK.get_or_init(|| AsyncMutex::new(()))
-}
-
-#[test]
-fn profile_config_macro_adds_type_metadata_and_strict_serde() -> Result<()> {
- let server = MockOpenAiServer::spawn(Some(medium_signals()), 200)?;
- let config = config(&server.base_url())?;
-
- assert_eq!(LlmRoutingProfileConfig::PROFILE_TYPE, "llm-routing");
- assert_eq!(config.profile_type(), "llm-routing");
-
- let old_stats_toggle = json!({
- "strong": config.strong,
- "weak": config.weak,
- "classifier": config.classifier,
- "stats": false,
- });
- let error = serde_json::from_value::<LlmRoutingProfileConfig>(old_stats_toggle)
- .err()
- .ok_or_else(|| SwitchyardError::Other("unknown profile field should fail".into()))?;
- assert!(error.to_string().contains("unknown field"));
- Ok(())
-}
-
-#[tokio::test]
-async fn process_routes_medium_policy_to_weak_with_required_tool_call() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(Some(medium_signals()), 200)?;
- let profile = config(&server.base_url())?.build()?;
-
- let processed = profile.process(request()).await?;
-
- assert_eq!(processed.profile_input.request.model(), Some("cheap/model"));
- assert_eq!(processed.decision.tier, "weak");
- assert_eq!(processed.decision.source, "policy_tier");
- let requests = server.requests()?;
- assert_eq!(requests.len(), 1);
- let classifier_body = &requests[0].body;
- assert_eq!(
- classifier_body["tools"][0]["function"]["name"],
- "select_route"
- );
- assert_eq!(
- classifier_body["tool_choice"]["function"]["name"],
- "select_route"
- );
- assert_eq!(classifier_body["tools"][0]["function"]["strict"], true);
- assert!(classifier_body.get("response_format").is_none());
- Ok(())
-}
-
-#[tokio::test]
-async fn yaml_overrides_classifier_tool_contract_and_tier_mapping() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(Some(medium_signals()), 200)?;
- let mut config = config(&server.base_url())?;
- config.classifier_max_tokens = 64;
- config.classifier_system_prompt = Some("Use the tool to pick a backend tier.".to_string());
- config.classifier_tool_name = Some("choose_model".to_string());
- config.classifier_tool_description = Some("Choose the target model tier.".to_string());
- config.classifier_tool_parameters = Some(json!({
- "type": "object",
- "properties": {"recommended_tier": {"type": "string"}},
- "required": ["recommended_tier"],
- "additionalProperties": false,
- }));
- config.tier_mapping = Some(LlmRoutingTierMapping {
- simple: "weak".to_string(),
- medium: "strong".to_string(),
- complex: "strong".to_string(),
- reasoning: "strong".to_string(),
- });
- let profile = config.build()?;
-
- let processed = profile.process(request()).await?;
-
- assert_eq!(
- processed.profile_input.request.model(),
- Some("frontier/model")
- );
- let requests = server.requests()?;
- let classifier_body = &requests[0].body;
- assert_eq!(classifier_body["max_tokens"], 64);
- assert_eq!(
- classifier_body["messages"][0]["content"],
- "Use the tool to pick a backend tier."
- );
- assert_eq!(
- classifier_body["tools"][0]["function"]["name"],
- "choose_model"
- );
- assert_eq!(
- classifier_body["tools"][0]["function"]["description"],
- "Choose the target model tier."
- );
- assert_eq!(
- classifier_body["tool_choice"]["function"]["name"],
- "choose_model"
- );
- Ok(())
-}
-
-#[test]
-fn config_rejects_non_strict_classifier_tool_schema() -> Result<()> {
- let server = MockOpenAiServer::spawn(Some(medium_signals()), 200)?;
- let mut config = config(&server.base_url())?;
- config.classifier_tool_parameters = Some(json!({
- "type": "object",
- "properties": {"recommended_tier": {"type": "string"}},
- "required": ["recommended_tier"],
- "additionalProperties": true,
- }));
-
- let error = config
- .build()
- .err()
- .ok_or_else(|| SwitchyardError::Other("expected strict schema validation".to_string()))?;
-
- assert!(error.to_string().contains("additionalProperties"));
- assert!(error.to_string().contains("strict tool calls"));
- Ok(())
-}
-
-#[tokio::test]
-async fn low_confidence_defaults_to_strong() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let mut signals = medium_signals();
- signals["recommended_tier"] = json!("simple");
- signals["confidence"] = json!(0.2);
- let server = MockOpenAiServer::spawn(Some(signals), 200)?;
- let mut config = config(&server.base_url())?;
- config.classifier_min_confidence = 0.8;
- let profile = config.build()?;
-
- let processed = profile.process(request()).await?;
-
- assert_eq!(
- processed.profile_input.request.model(),
- Some("frontier/model")
- );
- assert_eq!(processed.decision.source, "low_confidence");
- Ok(())
-}
-
-#[tokio::test]
-async fn classifier_failure_fails_open_with_visible_decision_source() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(None, 200)?;
- let profile = config(&server.base_url())?.build()?;
-
- let processed = profile.process(request()).await?;
-
- assert_eq!(
- processed.profile_input.request.model(),
- Some("frontier/model")
- );
- assert_eq!(processed.decision.source, "classifier_error_fall_open");
- let stats = profile_stats_accumulator().snapshot()?;
- assert_eq!(stats.classifier.total_errors, 1);
- Ok(())
-}
-
-#[tokio::test]
-async fn classifier_failure_fails_closed_when_fail_open_is_disabled() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(None, 200)?;
- let mut config = config(&server.base_url())?;
- config.classifier_fail_open = false;
- let profile = config.build()?;
-
- let error = profile
- .process(request())
- .await
- .err()
- .ok_or_else(|| SwitchyardError::Other("expected classifier failure".to_string()))?;
-
- assert!(error.to_string().contains("LLM classifier failed"));
- let stats = profile_stats_accumulator().snapshot()?;
- assert_eq!(stats.classifier.total_errors, 1);
- Ok(())
-}
-
-#[tokio::test]
-async fn run_records_backend_and_classifier_stats() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(Some(complex_signals()), 200)?;
- let profile = config(&server.base_url())?.build()?;
-
- let response = profile.run(request()).await?;
-
- let routing_metadata = response
- .routing_metadata
- .as_ref()
- .ok_or_else(|| SwitchyardError::Other("routing metadata missing".to_string()))?;
- assert_eq!(
- routing_metadata.selected_model.as_deref(),
- Some("frontier/model")
- );
- assert_eq!(routing_metadata.selected_tier.as_deref(), Some("strong"));
- assert_eq!(routing_metadata.confidence, Some(0.9));
- assert_eq!(
- routing_metadata.router_version.as_deref(),
- Some("llm-routing:coding_agent:v1")
- );
- assert_eq!(routing_metadata.tolerance, Some(0.0));
- assert_eq!(
- response.body().and_then(|body| body["model"].as_str()),
- Some("frontier/model")
- );
- let requests = server.requests()?;
- assert_eq!(requests.len(), 2);
- assert_eq!(requests[0].path, "/classifier/v1/chat/completions");
- assert_eq!(requests[1].path, "/strong/v1/chat/completions");
- let stats = profile_stats_accumulator().snapshot()?;
- assert_eq!(stats.classifier.total_requests, 1);
- assert!(stats.models.contains_key("frontier/model"));
- assert!(stats.tiers.contains_key("strong"));
- Ok(())
-}
-
-#[tokio::test]
-async fn run_retries_fallback_target_after_context_window_overflow() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn_with_backend_mode(
- Some(medium_signals()),
- BackendMode::WeakContextOverflowThenOk,
- )?;
- let profile = config(&server.base_url())?.build()?;
-
- let response = profile.run(request()).await?;
-
- let routing_metadata = response
- .routing_metadata
- .as_ref()
- .ok_or_else(|| SwitchyardError::Other("routing metadata missing".to_string()))?;
- assert_eq!(
- routing_metadata.selected_model.as_deref(),
- Some("frontier/model")
- );
- assert_eq!(routing_metadata.selected_tier.as_deref(), Some("strong"));
- assert!(routing_metadata
- .rationale
- .as_deref()
- .is_some_and(|reason| reason.contains("context window")));
- assert_eq!(
- response.body().and_then(|body| body["model"].as_str()),
- Some("frontier/model")
- );
- let requests = server.requests()?;
- assert_eq!(
- requests
- .iter()
- .map(|request| request.path.as_str())
- .collect::<Vec<_>>(),
- vec![
- "/classifier/v1/chat/completions",
- "/weak/v1/chat/completions",
- "/strong/v1/chat/completions",
- ]
- );
- assert_eq!(requests[1].body["model"], "cheap/model");
- assert_eq!(requests[2].body["model"], "frontier/model");
- Ok(())
-}
-
-#[tokio::test]
-async fn run_normalizes_reasoning_effort_and_applies_provider_defaults() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(Some(medium_signals()), 200)?;
- let mut config = config(&server.base_url())?;
- config.classifier.model = ModelId::new("nvidia/deepseek-ai/deepseek-v4-flash")?;
- config.weak.model = ModelId::new("nvidia/deepseek-ai/evals-deepseek-v4-pro")?;
- let profile = config.build()?;
- let mut input = request();
- input.request = ChatRequest::openai_chat(json!({
- "model": "client/model",
- "messages": [{"role": "user", "content": "list files"}],
- "reasoning_effort": "xhigh",
- }));
-
- let response = profile.run(input).await?;
-
- assert_eq!(
- response.body().and_then(|body| body["model"].as_str()),
- Some("nvidia/deepseek-ai/evals-deepseek-v4-pro")
- );
- let requests = server.requests()?;
- let classifier = &requests[0];
- let backend = &requests[1];
- assert_eq!(
- classifier.body["chat_template_kwargs"],
- json!({"enable_thinking": false})
- );
- assert_eq!(
- classifier.headers.get("x-inference-priority"),
- Some(&"batch".to_string())
- );
- assert!(classifier.body["messages"][1]["content"]
- .as_str()
- .is_some_and(|content| content.contains("\"reasoning_effort\":\"high\"")));
- assert_eq!(backend.body["reasoning_effort"], "high");
- assert_eq!(
- backend.body["chat_template_kwargs"],
- json!({"enable_thinking": false})
- );
- assert_eq!(
- backend.headers.get("x-inference-priority"),
- Some(&"batch".to_string())
- );
- Ok(())
-}
-
-#[tokio::test]
-async fn run_records_selected_backend_failure() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(Some(complex_signals()), 503)?;
- let profile = config(&server.base_url())?.build()?;
-
- let error =
- profile.run(request()).await.err().ok_or_else(|| {
- SwitchyardError::Other("expected selected backend failure".to_string())
- })?;
-
- assert!(error.to_string().contains("selected backend failed"));
- let stats = profile_stats_accumulator().snapshot()?;
- assert_eq!(stats.total_errors, 1);
- let model = stats
- .models
- .get("frontier/model")
- .ok_or_else(|| SwitchyardError::Other("frontier/model stats missing".to_string()))?;
- assert_eq!(model.errors, 1);
- Ok(())
-}
diff --git a/crates/switchyard-components-v2/tests/passthrough_profile.rs b/crates/switchyard-components-v2/tests/passthrough_profile.rs
deleted file mode 100644
index 0e3cbdedc..000000000
--- a/crates/switchyard-components-v2/tests/passthrough_profile.rs
+++ /dev/null
@@ -1,84 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Public API tests for the components-v2 passthrough profile config.
-
-use serde_json::json;
-use switchyard_components_v2::{PassthroughProfileConfig, ProfileConfig};
-use switchyard_core::{BackendFormat, LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError};
-
-fn target(id: &str, model: &str) -> Result<LlmTarget> {
- let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
- target.format = BackendFormat::OpenAi;
- Ok(target)
-}
-
-fn anthropic_target(id: &str, model: &str) -> Result<LlmTarget> {
- let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
- target.format = BackendFormat::Anthropic;
- Ok(target)
-}
-
-fn auto_target(id: &str, model: &str) -> Result<LlmTarget> {
- Ok(LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?))
-}
-
-fn config(target: LlmTarget) -> PassthroughProfileConfig {
- PassthroughProfileConfig { target }
-}
-
-#[test]
-fn profile_config_build_uses_existing_native_backend_stack() -> Result<()> {
- let config = config(target("direct", "provider/model")?);
-
- let _profile = config.build()?;
- Ok(())
-}
-
-#[test]
-fn profile_config_build_supports_anthropic_native_backend_stack() -> Result<()> {
- let config = config(anthropic_target("direct", "anthropic/model")?);
-
- let _profile = config.build()?;
- Ok(())
-}
-
-#[test]
-fn profile_config_build_rejects_unresolved_backend_format() -> Result<()> {
- let config = config(auto_target("direct", "provider/model")?);
-
- match config.build() {
- Err(SwitchyardError::InvalidConfig(message)) => {
- assert!(message.contains("must have a resolved backend format"));
- }
- Ok(_) => {
- return Err(SwitchyardError::Other(
- "unresolved passthrough backend format should fail".into(),
- ));
- }
- Err(other) => {
- return Err(SwitchyardError::Other(format!(
- "expected InvalidConfig, got {other}"
- )));
- }
- }
- Ok(())
-}
-
-#[test]
-fn profile_config_macro_adds_type_metadata_and_strict_serde() -> Result<()> {
- let config = config(target("direct", "provider/model")?);
-
- assert_eq!(PassthroughProfileConfig::PROFILE_TYPE, "passthrough");
- assert_eq!(config.profile_type(), "passthrough");
-
- let old_stats_toggle = json!({
- "target": config.target,
- "stats": false,
- });
- let error = serde_json::from_value::<PassthroughProfileConfig>(old_stats_toggle)
- .err()
- .ok_or_else(|| SwitchyardError::Other("unknown profile field should fail".into()))?;
- assert!(error.to_string().contains("unknown field"));
- Ok(())
-}
diff --git a/crates/switchyard-components-v2/tests/random_routing_profile.rs b/crates/switchyard-components-v2/tests/random_routing_profile.rs
deleted file mode 100644
index 66fd2cb57..000000000
--- a/crates/switchyard-components-v2/tests/random_routing_profile.rs
+++ /dev/null
@@ -1,86 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Public API tests for the components-v2 random-routing profile config.
-
-use serde_json::json;
-use switchyard_components_v2::{ProfileConfig, RandomRoutingProfileConfig};
-use switchyard_core::{BackendFormat, LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError};
-
-fn target(id: &str, model: &str) -> Result<LlmTarget> {
- let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
- target.format = BackendFormat::OpenAi;
- Ok(target)
-}
-
-fn config(strong: LlmTarget, weak: LlmTarget, probability: f64) -> RandomRoutingProfileConfig {
- RandomRoutingProfileConfig {
- strong,
- weak,
- strong_probability: probability,
- rng_seed: Some(7),
- }
-}
-
-#[test]
-fn profile_config_build_uses_existing_native_backend_stack() -> Result<()> {
- let config = config(
- target("strong", "frontier/model")?,
- target("weak", "cheap/model")?,
- 0.5,
- );
-
- let _profile = config.build()?;
- Ok(())
-}
-
-#[test]
-fn profile_config_macro_adds_type_metadata_and_strict_serde() -> Result<()> {
- let config = config(
- target("strong", "frontier/model")?,
- target("weak", "cheap/model")?,
- 0.5,
- );
-
- assert_eq!(RandomRoutingProfileConfig::PROFILE_TYPE, "random-routing");
- assert_eq!(config.profile_type(), "random-routing");
-
- let old_stats_toggle = json!({
- "strong": config.strong,
- "weak": config.weak,
- "strong_probability": 0.5,
- "rng_seed": 7,
- "stats": false,
- });
- let error = serde_json::from_value::<RandomRoutingProfileConfig>(old_stats_toggle)
- .err()
- .ok_or_else(|| SwitchyardError::Other("unknown profile field should fail".into()))?;
- assert!(error.to_string().contains("unknown field"));
- Ok(())
-}
-
-#[test]
-fn invalid_probability_is_rejected_by_profile_config_build() -> Result<()> {
- let config = config(
- target("strong", "frontier/model")?,
- target("weak", "cheap/model")?,
- 1.5,
- );
-
- match config.build() {
- Err(SwitchyardError::InvalidConfig(message)) => {
- assert!(message.contains("strong_probability must be finite and in [0.0, 1.0]"));
- }
- Ok(_) => {
- return Err(SwitchyardError::Other(
- "invalid probability should reject profile construction".into(),
- ));
- }
- Err(other) => {
- return Err(SwitchyardError::Other(format!(
- "expected InvalidConfig, got {other}"
- )));
- }
- }
- Ok(())
-}
diff --git a/crates/switchyard-components-v2/tests/stage_router_profile.rs b/crates/switchyard-components-v2/tests/stage_router_profile.rs
deleted file mode 100644
index 7f57d34fb..000000000
--- a/crates/switchyard-components-v2/tests/stage_router_profile.rs
+++ /dev/null
@@ -1,549 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Public API tests for the components-v2 stage_router profile config.
-
-use std::io::{Read, Write};
-use std::net::{SocketAddr, TcpListener, TcpStream};
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex, OnceLock};
-use std::thread::{self, JoinHandle};
-use std::time::Duration;
-
-use serde_json::{json, Value};
-use switchyard_components_v2::{
- profile_stats_accumulator, Profile, ProfileConfig, ProfileHooks, ProfileInput, RequestMetadata,
- StageRouterClassifierConfig, StageRouterDecisionSource, StageRouterPickerMode,
- StageRouterProfileConfig, StageRouterTier,
-};
-use switchyard_core::{
- BackendFormat, ChatRequest, LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError,
-};
-use tokio::sync::Mutex as AsyncMutex;
-
-#[derive(Clone, Debug, PartialEq)]
-struct ObservedRequest {
- path: String,
- body: Value,
-}
-
-enum MockResponse {
- Json(u16, Value),
- Raw(u16, &'static str),
-}
-
-const NON_JSON_CLASSIFIER_RESPONSE: &str = "__switchyard_non_json_classifier_response__";
-
-struct MockOpenAiServer {
- addr: SocketAddr,
- requests: Arc<Mutex<Vec<ObservedRequest>>>,
- shutdown: Arc<AtomicBool>,
- handle: Option<JoinHandle<()>>,
-}
-
-impl MockOpenAiServer {
- fn spawn(classifier_content: Option<String>, backend_status: u16) -> Result<Self> {
- let listener = TcpListener::bind("127.0.0.1:0")
- .map_err(|error| SwitchyardError::Other(format!("bind failed: {error}")))?;
- let addr = listener
- .local_addr()
- .map_err(|error| SwitchyardError::Other(format!("local_addr failed: {error}")))?;
- let requests = Arc::new(Mutex::new(Vec::new()));
- let shutdown = Arc::new(AtomicBool::new(false));
- let thread_requests = Arc::clone(&requests);
- let thread_shutdown = Arc::clone(&shutdown);
- let handle = thread::spawn(move || {
- for stream in listener.incoming() {
- if thread_shutdown.load(Ordering::SeqCst) {
- break;
- }
- let Ok(mut stream) = stream else {
- continue;
- };
- let Ok(request) = read_request(&mut stream) else {
- continue;
- };
- if let Ok(mut requests) = thread_requests.lock() {
- requests.push(request.clone());
- }
- let response =
- response_for(&request, classifier_content.as_deref(), backend_status);
- let _ = write_response(&mut stream, response);
- }
- });
- Ok(Self {
- addr,
- requests,
- shutdown,
- handle: Some(handle),
- })
- }
-
- fn base_url(&self) -> String {
- format!("http://{}", self.addr)
- }
-
- fn requests(&self) -> Result<Vec<ObservedRequest>> {
- self.requests
- .lock()
- .map(|requests| requests.clone())
- .map_err(|_| SwitchyardError::Other("requests mutex poisoned".to_string()))
- }
-}
-
-impl Drop for MockOpenAiServer {
- fn drop(&mut self) {
- self.shutdown.store(true, Ordering::SeqCst);
- let _ = TcpStream::connect(self.addr);
- if let Some(handle) = self.handle.take() {
- let _ = handle.join();
- }
- }
-}
-
-fn read_request(stream: &mut TcpStream) -> Result<ObservedRequest> {
- stream
- .set_read_timeout(Some(Duration::from_secs(5)))
- .map_err(|error| SwitchyardError::Other(format!("set timeout failed: {error}")))?;
- let mut bytes = Vec::new();
- let mut buf = [0_u8; 1024];
- let header_end = loop {
- let read = stream
- .read(&mut buf)
- .map_err(|error| SwitchyardError::Other(format!("read failed: {error}")))?;
- if read == 0 {
- return Err(SwitchyardError::Other(
- "connection closed early".to_string(),
- ));
- }
- bytes.extend_from_slice(&buf[..read]);
- if let Some(header_end) = find_header_end(&bytes) {
- break header_end;
- }
- };
- let headers = String::from_utf8_lossy(&bytes[..header_end]);
- let mut lines = headers.lines();
- let request_line = lines
- .next()
- .ok_or_else(|| SwitchyardError::Other("missing request line".to_string()))?;
- let path = request_line
- .split_whitespace()
- .nth(1)
- .ok_or_else(|| SwitchyardError::Other("missing request path".to_string()))?
- .to_string();
- let content_length = headers
- .lines()
- .find_map(|line| {
- let (name, value) = line.split_once(':')?;
- name.eq_ignore_ascii_case("content-length")
- .then(|| value.trim().parse::<usize>().ok())
- .flatten()
- })
- .unwrap_or(0);
- let body_start = header_end + 4;
- while bytes.len().saturating_sub(body_start) < content_length {
- let read = stream
- .read(&mut buf)
- .map_err(|error| SwitchyardError::Other(format!("body read failed: {error}")))?;
- if read == 0 {
- break;
- }
- bytes.extend_from_slice(&buf[..read]);
- }
- let body_end = body_start + content_length;
- if bytes.len() < body_end {
- return Err(SwitchyardError::Other(
- "connection closed before full body was read".to_string(),
- ));
- }
- let body = if content_length == 0 {
- Value::Null
- } else {
- serde_json::from_slice(&bytes[body_start..body_end])
- .map_err(|error| SwitchyardError::Other(format!("decode request body: {error}")))?
- };
- Ok(ObservedRequest { path, body })
-}
-
-fn find_header_end(bytes: &[u8]) -> Option<usize> {
- bytes.windows(4).position(|window| window == b"\r\n\r\n")
-}
-
-fn response_for(
- request: &ObservedRequest,
- classifier_content: Option<&str>,
- backend_status: u16,
-) -> MockResponse {
- if request.path.contains("/classifier/") {
- if classifier_content == Some(NON_JSON_CLASSIFIER_RESPONSE) {
- return MockResponse::Raw(200, "not json");
- }
- return MockResponse::Json(
- 200,
- json!({
- "id": "chatcmpl-stage_router-classifier",
- "object": "chat.completion",
- "model": request.body["model"],
- "choices": [{
- "index": 0,
- "message": {
- "role": "assistant",
- "content": classifier_content.unwrap_or("not json"),
- },
- "finish_reason": "stop",
- }],
- "usage": {"prompt_tokens": 11, "completion_tokens": 1, "total_tokens": 12},
- }),
- );
- }
-
- if backend_status >= 400 {
- return MockResponse::Json(
- backend_status,
- json!({"error": {"message": "selected backend failed", "code": "backend_failed"}}),
- );
- }
- MockResponse::Json(
- 200,
- json!({
- "id": "chatcmpl-stage_router-backend",
- "object": "chat.completion",
- "model": request.body["model"],
- "choices": [{"message": {"role": "assistant", "content": "ok"}}],
- "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7},
- }),
- )
-}
-
-fn write_response(stream: &mut TcpStream, response: MockResponse) -> Result<()> {
- match response {
- MockResponse::Json(status, body) => write_json_response(stream, status, body),
- MockResponse::Raw(status, body) => write_raw_response(stream, status, body),
- }
-}
-
-fn write_json_response(stream: &mut TcpStream, status: u16, body: Value) -> Result<()> {
- let body = serde_json::to_string(&body)
- .map_err(|error| SwitchyardError::Other(format!("encode response body: {error}")))?;
- let response = format!(
- "HTTP/1.1 {status} OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
- body.len()
- );
- stream
- .write_all(response.as_bytes())
- .map_err(|error| SwitchyardError::Other(format!("write response failed: {error}")))
-}
-
-fn write_raw_response(stream: &mut TcpStream, status: u16, body: &'static str) -> Result<()> {
- let response = format!(
- "HTTP/1.1 {status} OK\r\ncontent-type: text/plain\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
- body.len()
- );
- stream
- .write_all(response.as_bytes())
- .map_err(|error| SwitchyardError::Other(format!("write response failed: {error}")))
-}
-
-fn target(id: &str, model: &str, base_url: &str) -> Result<LlmTarget> {
- let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
- target.format = BackendFormat::OpenAi;
- target.endpoint.base_url = Some(base_url.to_string());
- target.endpoint.api_key = Some("test-key".to_string());
- Ok(target)
-}
-
-fn config(base_url: &str) -> Result<StageRouterProfileConfig> {
- Ok(StageRouterProfileConfig {
- capable: target(
- "capable",
- "frontier/model",
- &format!("{base_url}/capable/v1"),
- )?,
- efficient: target(
- "efficient",
- "cheap/model",
- &format!("{base_url}/efficient/v1"),
- )?,
- fallback_target_on_evict: LlmTargetId::new("capable")?,
- picker: StageRouterPickerMode::CapableFirst,
- confidence_threshold: 0.7,
- signal_recent_window: 3,
- classifier: Some(StageRouterClassifierConfig {
- model: "classifier/model".to_string(),
- api_key: "test-key".to_string(),
- base_url: Some(format!("{base_url}/classifier/v1")),
- timeout_secs: 1.0,
- recent_turn_window: 3,
- max_tokens: 4096,
- system_prompt: None,
- }),
- enable_stats: true,
- })
-}
-
-fn profile_input(request: ChatRequest) -> ProfileInput {
- ProfileInput {
- request,
- metadata: RequestMetadata::default(),
- }
-}
-
-fn request() -> ProfileInput {
- profile_input(ChatRequest::openai_chat(json!({
- "model": "client/stage_router",
- "messages": [{"role": "user", "content": "hello"}],
- })))
-}
-
-fn request_with_tool_result(tool_name: &str, content: &str) -> ProfileInput {
- profile_input(ChatRequest::openai_chat(json!({
- "model": "client/stage_router",
- "messages": [
- {"role": "user", "content": "work on this"},
- {"role": "assistant", "tool_calls": [{"function": {"name": tool_name}}]},
- {"role": "tool", "tool_call_id": "1", "content": content},
- ],
- })))
-}
-
-fn tier_content(tier: &str) -> String {
- json!({"tier": tier}).to_string()
-}
-
-fn stats_test_lock() -> &'static AsyncMutex<()> {
- static LOCK: OnceLock<AsyncMutex<()>> = OnceLock::new();
- LOCK.get_or_init(|| AsyncMutex::new(()))
-}
-
-#[tokio::test]
-async fn critical_severity_overrides_to_capable_without_classifier() -> Result<()> {
- let server = MockOpenAiServer::spawn(None, 200)?;
- let mut config = config(&server.base_url())?;
- config.classifier = None;
- let profile = config.build()?;
-
- let processed = profile
- .process(request_with_tool_result("Bash", "out of memory"))
- .await?;
-
- assert_eq!(
- processed.profile_input.request.model(),
- Some("frontier/model")
- );
- assert_eq!(processed.decision.tier, StageRouterTier::Capable);
- assert_eq!(
- processed.decision.source,
- StageRouterDecisionSource::Override
- );
- assert!(server.requests()?.is_empty());
- Ok(())
-}
-
-#[tokio::test]
-async fn negative_score_routes_to_efficient_without_classifier() -> Result<()> {
- let server = MockOpenAiServer::spawn(None, 200)?;
- let mut config = config(&server.base_url())?;
- config.classifier = None;
- config.confidence_threshold = 0.1;
- let profile = config.build()?;
-
- let processed = profile
- .process(request_with_tool_result("Write", "ok"))
- .await?;
-
- assert_eq!(processed.profile_input.request.model(), Some("cheap/model"));
- assert_eq!(processed.decision.tier, StageRouterTier::Efficient);
- assert_eq!(
- processed.decision.source,
- StageRouterDecisionSource::Dimensions
- );
- Ok(())
-}
-
-#[tokio::test]
-async fn low_confidence_uses_classifier_when_configured() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(Some(tier_content("efficient")), 200)?;
- let profile = config(&server.base_url())?.build()?;
-
- let processed = profile.process(request()).await?;
-
- assert_eq!(processed.profile_input.request.model(), Some("cheap/model"));
- assert_eq!(processed.decision.tier, StageRouterTier::Efficient);
- assert_eq!(
- processed.decision.source,
- StageRouterDecisionSource::LlmClassifier
- );
- let requests = server.requests()?;
- assert_eq!(requests.len(), 1);
- let classifier_body = &requests[0].body;
- assert_eq!(classifier_body["max_tokens"], 4096);
- assert!(classifier_body["messages"][0]["content"]
- .as_str()
- .is_some_and(
- |content| content.contains("routing classifier inside an agentic coding stage-router")
- ));
- assert_eq!(classifier_body["response_format"]["type"], "json_object");
- Ok(())
-}
-
-#[tokio::test]
-async fn yaml_overrides_classifier_prompt_and_max_tokens() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(Some(tier_content("efficient")), 200)?;
- let mut config = config(&server.base_url())?;
- let classifier = config
- .classifier
- .as_mut()
- .ok_or_else(|| SwitchyardError::Other("classifier config missing".to_string()))?;
- classifier.max_tokens = 64;
- classifier.system_prompt = Some("Pick a stage_router tier.".to_string());
- let profile = config.build()?;
-
- let processed = profile.process(request()).await?;
-
- assert_eq!(processed.profile_input.request.model(), Some("cheap/model"));
- let requests = server.requests()?;
- let classifier_body = &requests[0].body;
- assert_eq!(classifier_body["max_tokens"], 64);
- assert_eq!(
- classifier_body["messages"][0]["content"],
- "Pick a stage_router tier."
- );
- Ok(())
-}
-
-#[test]
-fn config_rejects_bad_classifier_options() -> Result<()> {
- let mut zero_tokens = config("http://127.0.0.1:9")?;
- zero_tokens
- .classifier
- .as_mut()
- .ok_or_else(|| SwitchyardError::Other("classifier config missing".to_string()))?
- .max_tokens = 0;
- let error = zero_tokens
- .build()
- .err()
- .ok_or_else(|| SwitchyardError::Other("expected max token validation failure".into()))?;
- assert!(error.to_string().contains("classifier.max_tokens"));
-
- let mut empty_prompt = config("http://127.0.0.1:9")?;
- empty_prompt
- .classifier
- .as_mut()
- .ok_or_else(|| SwitchyardError::Other("classifier config missing".to_string()))?
- .system_prompt = Some(" ".to_string());
- let error = empty_prompt
- .build()
- .err()
- .ok_or_else(|| SwitchyardError::Other("expected prompt validation failure".into()))?;
- assert!(error.to_string().contains("classifier.system_prompt"));
- Ok(())
-}
-
-#[tokio::test]
-async fn malformed_classifier_falls_open_to_default() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(None, 200)?;
- let profile = config(&server.base_url())?.build()?;
-
- let processed = profile.process(request()).await?;
-
- assert_eq!(
- processed.profile_input.request.model(),
- Some("frontier/model")
- );
- assert_eq!(
- processed.decision.source,
- StageRouterDecisionSource::FallOpen
- );
- let stats = profile_stats_accumulator().snapshot()?;
- assert_eq!(stats.classifier.total_errors, 1);
- Ok(())
-}
-
-#[tokio::test]
-async fn non_json_classifier_falls_open_and_records_error() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(Some(NON_JSON_CLASSIFIER_RESPONSE.to_string()), 200)?;
- let profile = config(&server.base_url())?.build()?;
-
- let processed = profile.process(request()).await?;
-
- assert_eq!(
- processed.profile_input.request.model(),
- Some("frontier/model")
- );
- assert_eq!(
- processed.decision.source,
- StageRouterDecisionSource::FallOpen
- );
- let stats = profile_stats_accumulator().snapshot()?;
- assert_eq!(stats.classifier.total_errors, 1);
- Ok(())
-}
-
-#[tokio::test]
-async fn run_records_backend_and_classifier_stats() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(Some(tier_content("capable")), 200)?;
- let profile = config(&server.base_url())?.build()?;
-
- let response = profile.run(request()).await?;
-
- let routing_metadata = response
- .routing_metadata
- .as_ref()
- .ok_or_else(|| SwitchyardError::Other("routing metadata missing".to_string()))?;
- assert_eq!(
- routing_metadata.selected_model.as_deref(),
- Some("frontier/model")
- );
- assert_eq!(routing_metadata.selected_tier.as_deref(), Some("capable"));
- assert_eq!(routing_metadata.confidence, None);
- assert!(routing_metadata
- .rationale
- .as_deref()
- .is_some_and(|reason| reason.contains("source=llm-classifier")));
- assert_eq!(
- response.body().and_then(|body| body["model"].as_str()),
- Some("frontier/model")
- );
- let requests = server.requests()?;
- assert_eq!(requests.len(), 2);
- assert_eq!(requests[0].path, "/classifier/v1/chat/completions");
- assert_eq!(requests[1].path, "/capable/v1/chat/completions");
- let stats = profile_stats_accumulator().snapshot()?;
- assert_eq!(stats.classifier.total_requests, 1);
- assert!(stats.models.contains_key("frontier/model"));
- assert!(stats.tiers.contains_key("capable"));
- Ok(())
-}
-
-#[tokio::test]
-async fn run_records_selected_backend_failure() -> Result<()> {
- let _guard = stats_test_lock().lock().await;
- profile_stats_accumulator().reset()?;
- let server = MockOpenAiServer::spawn(Some(tier_content("capable")), 503)?;
- let profile = config(&server.base_url())?.build()?;
-
- let error =
- profile.run(request()).await.err().ok_or_else(|| {
- SwitchyardError::Other("expected selected backend failure".to_string())
- })?;
-
- assert!(error.to_string().contains("selected backend failed"));
- let stats = profile_stats_accumulator().snapshot()?;
- assert_eq!(stats.total_errors, 1);
- let model = stats
- .models
- .get("frontier/model")
- .ok_or_else(|| SwitchyardError::Other("frontier/model stats missing".to_string()))?;
- assert_eq!(model.errors, 1);
- Ok(())
-}
diff --git a/crates/switchyard-components/src/lib.rs b/crates/switchyard-components/src/lib.rs
index 5bd2e7180..3b34c631b 100644
--- a/crates/switchyard-components/src/lib.rs
+++ b/crates/switchyard-components/src/lib.rs
@@ -5,7 +5,7 @@
//!
//! `switchyard-core` owns traits and wire wrappers. This crate owns built-in
//! compatibility implementations: backends, request processors, and response
-//! processors. New Rust orchestration belongs in components-v2 profiles.
+//! processors. New Rust orchestration belongs in libsy algorithms and clients.
pub mod backends;
pub mod dimension_collector;
diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.toml
index fbd9fba93..0c6fa7075 100644
--- a/crates/switchyard-py/Cargo.toml
+++ b/crates/switchyard-py/Cargo.toml
@@ -27,7 +27,6 @@ serde_json = "1"
switchyard-core = { path = "../switchyard-core" }
switchyard-protocol = { path = "../protocol" }
switchyard-components = { path = "../switchyard-components" }
-switchyard-components-v2 = { path = "../switchyard-components-v2" }
switchyard-translation = { path = "../switchyard-translation" }
tokio = { version = "1", features = ["rt", "rt-multi-thread", "sync"] }
tracing = { version = "0.1", default-features = false, features = ["std"] }
diff --git a/crates/switchyard-py/src/lib.rs b/crates/switchyard-py/src/lib.rs
index 5ef43c27a..266aa8586 100644
--- a/crates/switchyard-py/src/lib.rs
+++ b/crates/switchyard-py/src/lib.rs
@@ -7,7 +7,6 @@ mod component_bindings;
mod core_bindings;
mod errors;
mod libsy_bindings;
-mod profile_bindings;
mod py_serde;
mod translation;
@@ -18,6 +17,5 @@ fn _switchyard_rust(module: &Bound<'_, PyModule>) -> PyResult<()> {
core_bindings::register(module)?;
component_bindings::register(module)?;
libsy_bindings::register(module)?;
- profile_bindings::register(module)?;
Ok(())
}
diff --git a/crates/switchyard-py/src/profile_bindings/mod.rs b/crates/switchyard-py/src/profile_bindings/mod.rs
deleted file mode 100644
index eb0822847..000000000
--- a/crates/switchyard-py/src/profile_bindings/mod.rs
+++ /dev/null
@@ -1,334 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Python bindings for the components-v2 profile API.
-//!
-//! Two surfaces are exposed:
-//!
-//! - The erased serving surface — [`PyProfileConfigDocument`], [`PyProfileConfigPlan`],
-//! and [`PyProfile`] — loads config files and runs any profile through the
-//! object-safe `Profile` contract. It is the right default for servers,
-//! launchers, and generic profile tables that only need `run()`.
-//! - The typed request-input surface (see [`typed`]) exposes `ProfileInput` and
-//! `ProfileRequestMetadata` so Python-authored profiles can share the same
-//! request envelope as Rust-authored profiles.
-//!
-//! Concrete profile authoring now belongs to `switchyard.lib.profiles` on the
-//! Python side or `switchyard-components-v2` on the Rust side; this module only
-//! bridges config-file execution and the shared input value.
-
-mod typed;
-
-use std::path::PathBuf;
-use std::sync::Arc;
-
-use pyo3::exceptions::PyValueError;
-use pyo3::prelude::*;
-use pyo3::types::PyDict;
-use switchyard_components_v2::{
- parse_profile_config_path as parse_v2_profile_config_path,
- parse_profile_config_str as parse_v2_profile_config_str, Profile, ProfileConfigDocument,
- ProfileConfigFormat, ProfileConfigPlan, ProfileInput,
-};
-use switchyard_core::{LlmTargetId, ProfileId, SwitchyardError};
-
-use crate::component_bindings::config::PyLlmTarget;
-use crate::core_bindings::request::PyChatRequest;
-use crate::core_bindings::response::PyChatResponse;
-use crate::errors::py_core_error;
-use crate::py_serde::value_to_python;
-
-use self::typed::PyProfileRequestMetadata;
-
-/// Parsed components-v2 profile config document.
-#[pyclass(name = "ProfileConfigDocument", frozen, skip_from_py_object)]
-#[derive(Clone, Debug)]
-struct PyProfileConfigDocument {
- inner: ProfileConfigDocument,
-}
-
-#[pymethods]
-impl PyProfileConfigDocument {
- /// Return parsed profile IDs in deterministic order.
- fn profile_ids(&self) -> Vec<String> {
- self.inner
- .profile_ids()
- .map(|profile_id| profile_id.as_str().to_string())
- .collect()
- }
-
- /// Return a parsed profile's type discriminator, or `None`.
- fn profile_type(&self, profile_id: &str) -> PyResult<Option<String>> {
- let profile_id = parse_profile_id(profile_id)?;
- Ok(self
- .inner
- .profile_type(&profile_id)
- .map(std::borrow::ToOwned::to_owned))
- }
-
- /// Return a parsed profile's envelope `subagent_target` reference, or `None`.
- fn profile_subagent_target(&self, profile_id: &str) -> PyResult<Option<String>> {
- let profile_id = parse_profile_id(profile_id)?;
- Ok(self
- .inner
- .profile_subagent_target(&profile_id)
- .map(|target_id| target_id.as_str().to_owned()))
- }
-
- /// Return a parsed profile's body without the `type` discriminator.
- fn profile_body(&self, py: Python<'_>, profile_id: &str) -> PyResult<Option<Py<PyAny>>> {
- let profile_id = parse_profile_id(profile_id)?;
- self.inner
- .profile_body(&profile_id)
- .map(|body| value_to_python(py, body))
- .transpose()
- }
-
- /// Return a copy with selected profiles removed before resolution.
- fn without_profiles(&self, profile_ids: Vec<String>) -> PyResult<Self> {
- let profile_ids = profile_ids
- .iter()
- .map(|profile_id| parse_profile_id(profile_id))
- .collect::<PyResult<Vec<_>>>()?;
- Ok(Self {
- inner: self.inner.without_profiles(&profile_ids),
- })
- }
-
- /// Resolve endpoint/target references and validate profile-owned configs.
- fn resolve(&self) -> PyResult<PyProfileConfigPlan> {
- self.inner
- .resolve()
- .map(PyProfileConfigPlan::from_core)
- .map_err(py_core_error)
- }
-
- fn __repr__(&self) -> &'static str {
- "ProfileConfigDocument()"
- }
-}
-
-/// Resolved components-v2 profile config plan.
-#[pyclass(name = "ProfileConfigPlan", frozen, skip_from_py_object)]
-#[derive(Clone)]
-struct PyProfileConfigPlan {
- inner: ProfileConfigPlan,
-}
-
-impl PyProfileConfigPlan {
- fn from_core(inner: ProfileConfigPlan) -> Self {
- Self { inner }
- }
-}
-
-#[pymethods]
-impl PyProfileConfigPlan {
- /// Return resolved profile IDs in deterministic order.
- fn profile_ids(&self) -> Vec<String> {
- self.inner
- .profile_ids()
- .map(|profile_id| profile_id.as_str().to_string())
- .collect()
- }
-
- /// Return resolved target IDs in deterministic order.
- fn target_ids(&self) -> Vec<String> {
- self.inner
- .targets()
- .map(|(target_id, _target)| target_id.as_str().to_string())
- .collect()
- }
-
- /// Return the serialized profile type for one profile ID, or `None`.
- fn profile_type(&self, profile_id: &str) -> PyResult<Option<String>> {
- let profile_id = parse_profile_id(profile_id)?;
- Ok(self
- .inner
- .profile_type(&profile_id)
- .map(std::borrow::ToOwned::to_owned))
- }
-
- /// Return a resolved target by ID, or `None`.
- fn target(&self, target_id: &str) -> PyResult<Option<PyLlmTarget>> {
- let target_id = parse_target_id(target_id)?;
- Ok(self
- .inner
- .target(&target_id)
- .cloned()
- .map(PyLlmTarget::from_core))
- }
-
- /// Build one profile runtime by profile ID.
- fn build_profile(&self, profile_id: &str) -> PyResult<PyProfile> {
- let profile_id = parse_profile_id(profile_id)?;
- let profile = self
- .inner
- .build_profile(&profile_id)
- .map_err(py_core_error)?;
- Ok(PyProfile::from_boxed(
- profile_id.as_str().to_string(),
- profile,
- ))
- }
-
- /// Build all profile runtimes in deterministic profile-ID order.
- fn build_profiles<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
- let profiles = self.inner.build_profiles().map_err(py_core_error)?;
- let out = PyDict::new(py);
- for (profile_id, profile) in profiles {
- out.set_item(
- profile_id.as_str(),
- Py::new(
- py,
- PyProfile::from_boxed(profile_id.as_str().to_string(), profile),
- )?,
- )?;
- }
- Ok(out)
- }
-
- fn __repr__(&self) -> String {
- format!(
- "ProfileConfigPlan(profiles={}, targets={})",
- self.inner.profile_count(),
- self.inner.target_count(),
- )
- }
-}
-
-/// Object-safe components-v2 profile serving runtime.
-#[pyclass(name = "Profile", frozen, skip_from_py_object)]
-#[derive(Clone)]
-struct PyProfile {
- profile_id: String,
- inner: Arc<dyn Profile>,
-}
-
-impl PyProfile {
- fn from_boxed(profile_id: String, profile: Box<dyn Profile>) -> Self {
- Self {
- profile_id,
- inner: Arc::from(profile),
- }
- }
-}
-
-#[pymethods]
-impl PyProfile {
- /// Profile ID this runtime was built from.
- #[getter]
- fn profile_id(&self) -> String {
- self.profile_id.clone()
- }
-
- /// Execute the full profile-owned request lifecycle.
- #[pyo3(signature = (request, metadata=None))]
- fn run<'py>(
- &self,
- py: Python<'py>,
- request: PyRef<'_, PyChatRequest>,
- metadata: Option<PyRef<'_, PyProfileRequestMetadata>>,
- ) -> PyResult<Bound<'py, PyAny>> {
- let profile = Arc::clone(&self.inner);
- let request = request.clone_core();
- let metadata = metadata
- .map(|metadata| metadata.clone_core())
- .unwrap_or_default();
- pyo3_async_runtimes::tokio::future_into_py(py, async move {
- // The erased profile is the metadata-free generic entry point used
- // by config plans and servers. Python-authored profiles can still
- // carry metadata through the shared `ProfileInput` binding.
- let profile_response = profile
- .run(ProfileInput { request, metadata })
- .await
- .map_err(py_core_error)?;
- let (response, _routing_metadata) = profile_response.into_parts();
- Python::attach(|py| {
- Py::new(py, PyChatResponse::from_core(py, response)?)
- .map(|response| response.into_any())
- })
- })
- }
-
- fn __repr__(&self) -> String {
- format!("Profile(profile_id={:?})", self.profile_id)
- }
-}
-
-/// Parse a components-v2 profile config string.
-#[pyfunction(name = "parse_profile_config_str")]
-#[pyo3(signature = (input, format = "yaml"))]
-fn py_parse_profile_config_str(input: &str, format: &str) -> PyResult<PyProfileConfigDocument> {
- let format = profile_config_format_from_str(format)?;
- parse_v2_profile_config_str(input, format)
- .map(|inner| PyProfileConfigDocument { inner })
- .map_err(py_core_error)
-}
-
-/// Parse a components-v2 profile config file, inferring the format from extension.
-#[pyfunction(name = "parse_profile_config_path")]
-fn py_parse_profile_config_path(path: PathBuf) -> PyResult<PyProfileConfigDocument> {
- parse_v2_profile_config_path(path)
- .map(|inner| PyProfileConfigDocument { inner })
- .map_err(py_core_error)
-}
-
-/// Report whether normalized request headers mark delegated sub-agent work.
-///
-/// A thin binding over the canonical detection and routing policy owned by
-/// the protocol crate: `switchyard_protocol::Metadata::from_headers` for the
-/// lineage fact (explicit `x-switchyard-is-subagent`, Claude Code agent
-/// lineage, Codex/relay markers) and `Metadata::is_subagent_work` for the
-/// work-vs-maintenance kind policy, so Codex `compact` /
-/// `memory_consolidation` turns stay on normal profile routing.
-#[pyfunction]
-fn is_subagent_request(headers: &Bound<'_, PyAny>) -> PyResult<bool> {
- let headers = typed::metadata_headers_from_python(headers)?;
- // Lineage headers are single-valued; keep the first value when repeated.
- let headers: std::collections::BTreeMap<String, String> = headers
- .into_iter()
- .filter_map(|(name, values)| values.into_iter().next().map(|value| (name, value)))
- .collect();
- Ok(switchyard_protocol::Metadata::from_headers(&headers).is_subagent_work())
-}
-
-/// Parse and resolve a components-v2 profile config file.
-#[pyfunction]
-fn load_profile_config(path: PathBuf) -> PyResult<PyProfileConfigPlan> {
- parse_v2_profile_config_path(path)
- .and_then(|document| document.resolve())
- .map(PyProfileConfigPlan::from_core)
- .map_err(py_core_error)
-}
-
-fn profile_config_format_from_str(raw: &str) -> PyResult<ProfileConfigFormat> {
- match raw {
- "json" => Ok(ProfileConfigFormat::Json),
- "toml" => Ok(ProfileConfigFormat::Toml),
- "yaml" | "yml" => Ok(ProfileConfigFormat::Yaml),
- other => Err(PyValueError::new_err(format!(
- "unknown profile config format {other:?}; expected 'yaml', 'json', or 'toml'"
- ))),
- }
-}
-
-fn parse_profile_id(value: &str) -> PyResult<ProfileId> {
- ProfileId::new(value).map_err(|error| py_core_error(SwitchyardError::from(error)))
-}
-
-fn parse_target_id(value: &str) -> PyResult<LlmTargetId> {
- LlmTargetId::new(value).map_err(|error| py_core_error(SwitchyardError::from(error)))
-}
-
-/// Registers profile bindings with the native Python module.
-pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
- module.add_class::<PyProfileConfigDocument>()?;
- module.add_class::<PyProfileConfigPlan>()?;
- module.add_class::<PyProfile>()?;
- module.add_function(wrap_pyfunction!(py_parse_profile_config_str, module)?)?;
- module.add_function(wrap_pyfunction!(py_parse_profile_config_path, module)?)?;
- module.add_function(wrap_pyfunction!(load_profile_config, module)?)?;
- module.add_function(wrap_pyfunction!(is_subagent_request, module)?)?;
- typed::register(module)?;
- Ok(())
-}
diff --git a/crates/switchyard-py/src/profile_bindings/typed.rs b/crates/switchyard-py/src/profile_bindings/typed.rs
deleted file mode 100644
index 2a905bee2..000000000
--- a/crates/switchyard-py/src/profile_bindings/typed.rs
+++ /dev/null
@@ -1,223 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Typed request input bindings shared by Rust and Python profile runtimes.
-//!
-//! Concrete profile config/runtime bindings intentionally live outside this
-//! module now. Python owns its profile authoring API, while the profile-config
-//! loader exposes Rust-defined profiles through the erased `Profile` runtime in
-//! `mod.rs`.
-
-use std::collections::BTreeMap;
-
-use pyo3::exceptions::{PyTypeError, PyValueError};
-use pyo3::prelude::*;
-use pyo3::types::{PyDict, PyType};
-use switchyard_components_v2::{ProfileInput, RequestMetadata};
-use switchyard_core::RequestId;
-
-use crate::core_bindings::request::{request_type_from_python, request_type_object, PyChatRequest};
-use crate::py_serde::value_to_python;
-
-/// Endpoint-style metadata for direct Python profile calls.
-#[pyclass(name = "ProfileRequestMetadata", frozen, skip_from_py_object)]
-#[derive(Clone, Default)]
-pub(crate) struct PyProfileRequestMetadata {
- inner: RequestMetadata,
-}
-
-impl PyProfileRequestMetadata {
- fn from_core(inner: RequestMetadata) -> Self {
- Self { inner }
- }
-
- pub(crate) fn clone_core(&self) -> RequestMetadata {
- self.inner.clone()
- }
-}
-
-#[pymethods]
-impl PyProfileRequestMetadata {
- #[new]
- #[pyo3(signature = (request_id=None, inbound_format=None, headers=None))]
- fn new(
- request_id: Option<String>,
- inbound_format: Option<&Bound<'_, PyAny>>,
- headers: Option<&Bound<'_, PyAny>>,
- ) -> PyResult<Self> {
- Ok(Self {
- inner: RequestMetadata {
- request_id: parse_request_id(request_id)?,
- inbound_format: inbound_format
- .filter(|value| !value.is_none())
- .map(request_type_from_python)
- .transpose()?,
- headers: match headers {
- Some(headers) if !headers.is_none() => metadata_headers_from_python(headers)?,
- _ => BTreeMap::new(),
- },
- },
- })
- }
-
- /// Build metadata from headers, inferring `request_id` from `x-request-id`.
- #[classmethod]
- #[pyo3(signature = (headers, inbound_format=None))]
- fn from_headers(
- _cls: &Bound<'_, PyType>,
- headers: &Bound<'_, PyAny>,
- inbound_format: Option<&Bound<'_, PyAny>>,
- ) -> PyResult<Self> {
- let headers = metadata_headers_from_python(headers)?;
- let request_id = headers
- .get("x-request-id")
- .and_then(|values| values.first())
- .cloned();
- Ok(Self {
- inner: RequestMetadata {
- request_id: parse_request_id(request_id)?,
- inbound_format: inbound_format
- .filter(|value| !value.is_none())
- .map(request_type_from_python)
- .transpose()?,
- headers,
- },
- })
- }
-
- #[getter]
- fn request_id(&self) -> Option<String> {
- self.inner
- .request_id
- .as_ref()
- .map(|request_id| request_id.as_str().to_string())
- }
-
- #[getter]
- fn inbound_format(&self, py: Python<'_>) -> PyResult<Option<Py<PyAny>>> {
- self.inner
- .inbound_format
- .map(|request_type| request_type_object(py, request_type))
- .transpose()
- }
-
- #[getter]
- fn headers(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
- value_to_python(
- py,
- &serde_json::to_value(&self.inner.headers)
- .map_err(|error| PyValueError::new_err(error.to_string()))?,
- )
- }
-
- /// Convert metadata into a plain Python dictionary.
- fn to_dict(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
- let dict = PyDict::new(py);
- dict.set_item("request_id", self.request_id())?;
- dict.set_item(
- "inbound_format",
- self.inner
- .inbound_format
- .map(crate::core_bindings::request::request_type_name),
- )?;
- dict.set_item("headers", self.headers(py)?)?;
- Ok(dict.unbind().into_any())
- }
-
- fn __repr__(&self) -> String {
- format!("ProfileRequestMetadata(request_id={:?})", self.request_id())
- }
-}
-
-/// Rust-owned request input for direct Python profile calls.
-#[pyclass(name = "ProfileInput", frozen, skip_from_py_object)]
-#[derive(Clone)]
-pub(crate) struct PyProfileInput {
- inner: ProfileInput,
-}
-
-#[pymethods]
-impl PyProfileInput {
- #[new]
- #[pyo3(signature = (request, metadata=None))]
- fn new(
- request: PyRef<'_, PyChatRequest>,
- metadata: Option<PyRef<'_, PyProfileRequestMetadata>>,
- ) -> Self {
- Self {
- inner: ProfileInput {
- request: request.clone_core(),
- metadata: metadata_to_core(metadata),
- },
- }
- }
-
- /// Provider-neutral chat request carried by this profile input.
- #[getter]
- fn request(&self) -> PyChatRequest {
- PyChatRequest::from_core(self.inner.request.clone())
- }
-
- /// Endpoint-style metadata carried by this profile input.
- #[getter]
- fn metadata(&self) -> PyProfileRequestMetadata {
- PyProfileRequestMetadata::from_core(self.inner.metadata.clone())
- }
-
- fn __repr__(&self) -> String {
- format!(
- "ProfileInput(request_model={:?}, request_id={:?})",
- self.inner.request.model(),
- self.inner
- .metadata
- .request_id
- .as_ref()
- .map(|request_id| request_id.as_str())
- )
- }
-}
-
-fn metadata_to_core(metadata: Option<PyRef<'_, PyProfileRequestMetadata>>) -> RequestMetadata {
- metadata
- .map(|metadata| metadata.clone_core())
- .unwrap_or_default()
-}
-
-fn parse_request_id(request_id: Option<String>) -> PyResult<Option<RequestId>> {
- request_id
- .map(RequestId::new)
- .transpose()
- .map_err(|error| PyValueError::new_err(format!("invalid request_id: {error}")))
-}
-
-pub(crate) fn metadata_headers_from_python(
- headers: &Bound<'_, PyAny>,
-) -> PyResult<BTreeMap<String, Vec<String>>> {
- let dict = PyDict::new(headers.py());
- dict.call_method1("update", (headers,))?;
- let mut out = BTreeMap::<String, Vec<String>>::new();
- for (key, value) in dict.iter() {
- let key = key.extract::<String>()?.to_ascii_lowercase();
- let values = metadata_header_values_from_python(&value)?;
- out.entry(key).or_default().extend(values);
- }
- Ok(out)
-}
-
-fn metadata_header_values_from_python(value: &Bound<'_, PyAny>) -> PyResult<Vec<String>> {
- if let Ok(value) = value.extract::<String>() {
- return Ok(vec![value]);
- }
- value.extract::<Vec<String>>().map_err(|_| {
- PyTypeError::new_err(
- "ProfileRequestMetadata headers must map strings to strings or lists of strings",
- )
- })
-}
-
-/// Registers typed request input bindings with the native Python module.
-pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
- module.add_class::<PyProfileRequestMetadata>()?;
- module.add_class::<PyProfileInput>()?;
- Ok(())
-}
diff --git a/docs/cli_reference.md b/docs/cli_reference.md
index e02fc8d1d..864225a0d 100644
--- a/docs/cli_reference.md
+++ b/docs/cli_reference.md
@@ -6,7 +6,7 @@ This page is the canonical reference for every `switchyard` subcommand. It mirro
| Verb | Audience | What it does |
|---|---|---|
-| [`serve`](#switchyard-serve) | Ops | Long-running proxy server. Serve a v2 profile config with `serve --config`; each profile id and target id appears on `GET /v1/models`, and clients select one by setting the request `model`. |
+| [`serve`](#switchyard-serve) | Ops | Long-running proxy server. Serve a YAML route bundle selected with the global `--routing-profiles` flag. |
| [`launch claude`](#switchyard-launch-claude) | Dev | Spawns Claude Code against a local proxy. Auto-picks a free port, sets `ANTHROPIC_*` env vars, tears the proxy down when Claude exits. `--smoke` runs a one-shot harness round-trip and exits. |
| [`launch codex`](#switchyard-launch-codex) | Dev | Spawns OpenAI Codex CLI against a local proxy and injects a transient `switchyard` provider via repeated `-c` flags. `--smoke` runs a one-shot Codex round-trip. |
| [`launch openclaw`](#switchyard-launch-openclaw) | Dev | Spawns `openclaw chat` against a local proxy using a transient `openclaw.json`; the user's OpenClaw configuration is untouched. `--smoke` runs a one-shot agent turn. |
@@ -20,8 +20,8 @@ These apply to the top-level `switchyard` command, before any verb.
| Flag | Purpose |
|---|---|
| `--version` | Print the installed Switchyard version (`switchyard X.Y.Z`) and exit. Reads the version from the installed package metadata. |
-| `--routing-profiles PATH` / `-c PATH` | Deprecated legacy [Routing](#routing) bundle applied to `serve`, `launch`, and `configure`. Pass before the verb; separate with `--` for clarity. |
-| `--enable-rl-logging` | Write local [RL trace logs](#rl-trace-logging) (one `message_history` JSON file per turn) for `launch` and `serve` route-bundle sessions. Pass before the verb: `switchyard --enable-rl-logging launch claude`. Rejected by `serve --config`. |
+| `--routing-profiles PATH` / `-c PATH` | [Routing](#routing) bundle applied to `serve`, `launch`, and `configure`. Pass before the verb; separate with `--` for clarity. |
+| `--enable-rl-logging` | Write local [RL trace logs](#rl-trace-logging) (one `message_history` JSON file per turn) for `launch` and `serve` route-bundle sessions. Pass before the verb: `switchyard --enable-rl-logging launch claude`. |
| `--rl-log-dir DIR` | Output directory for `--enable-rl-logging` traces (default: `./rl_data`). No effect without `--enable-rl-logging`. |
## Cross-cutting flag families
@@ -60,22 +60,20 @@ startup does not require capability probes.
### Routing
-Legacy routing policies that used to be standalone CLI verbs live in
-routing-profile YAML files. Route bundles are deprecated; use v2 profile
-configs for new `serve` setups. Two flags drive routing on `serve` and the
-launchers:
+Routing policies live in routing-profile YAML files. Two flags drive routing
+on the launchers; `serve` uses `--routing-profiles`:
| Flag | Purpose |
|---|---|
| `--model ID` | Single-model passthrough. Every request is rewritten to `model=ID` and forwarded to `--base-url`. |
-| `--routing-profiles PATH` | Deprecated path to a routing-profile YAML bundle. Each entry under `routes:` builds its own chain. Public route types are `model`, `passthrough`, `random_routing`, `stage_router`, and `deterministic`. Falls back to the path persisted by `switchyard --routing-profiles PATH -- configure` when omitted. |
+| `--routing-profiles PATH` | Path to a routing-profile YAML bundle. Each entry under `routes:` builds its own chain. Public route types are `model`, `passthrough`, `random_routing`, `stage_router`, `escalation_router`, and `deterministic`. Falls back to the path persisted by `switchyard --routing-profiles PATH -- configure` when omitted. |
On the launchers, the two flags are mutually exclusive: pass one or the other, not both.
- `--model ID`: single-model passthrough. Any model id from `GET /v1/models` is accepted; every request is rewritten to `model=ID` and forwarded upstream.
- `--routing-profiles PATH`: loads a multi-chain YAML bundle; the first declared route becomes the initial model.
-For legacy route-bundle `type: deterministic` routes, the `classifier:` block also accepts:
+For `type: deterministic` routes, the `classifier:` block also accepts:
| Key | Purpose |
|---|---|
@@ -85,7 +83,8 @@ For legacy route-bundle `type: deterministic` routes, the `classifier:` block al
Benchmark runs started through `benchmark/run-baseline.sh --routing-profiles` record the effective classifier prompt, prompt SHA-256, `max_request_chars`, and `recent_turn_window` in `run_manifest.json` under `server.classifier_prompts`.
-**Selecting a v2 profile by id.** A `serve --config` proxy exposes every profile id (and every target id) as a model on `GET /v1/models`. There is no separate "profile" flag. A client, or a launcher using `--model <profile-id>`, selects a profile simply by naming its id as the model. This is the v2 counterpart to picking a route from a bundle. This selection pattern is specific to v2 profile configs; legacy route-bundle configs still use route names as model IDs.
+Each route name becomes a model ID on `GET /v1/models`. Clients select a route
+by setting the request's `model` field to that name.
### Intake sink (serve and launchers)
@@ -147,7 +146,7 @@ matches the pre-1.0 trace format:
```
Turns without an assistant choice (e.g. upstream errors) are skipped. The flag
-works on `launch` and on `serve` with a route bundle; `serve --config` rejects it.
+works on `launch` and on `serve` with a route bundle.
### Transport (server verbs)
@@ -204,14 +203,15 @@ because the built-in default is `https://openrouter.ai/api/v1`.
## `switchyard serve`
-Serve a long-running proxy from a Switchyard v2 **profile config**: one YAML/JSON/TOML file declaring `endpoints`, `targets`, and `profiles`. Rust-defined and Python-defined profiles run through the same Python HTTP application and expose the same endpoint paths. Each profile id and each target id is exposed as a model on `GET /v1/models`, so a client selects a profile by setting the request `model` to that id.
+Serve a long-running proxy from a YAML route bundle. Each entry under `routes:`
+builds a runnable chain and is exposed as a model on `GET /v1/models`.
The server exposes the OpenAI Chat Completions (`/v1/chat/completions`), Anthropic Messages (`/v1/messages`), and OpenAI Responses (`/v1/responses`) APIs on the same host and port.
**Synopsis**
```text
-switchyard [--routing-profiles PATH] serve [--config PATH]
+switchyard [--routing-profiles PATH] serve
[--host HOST] [--port PORT] [--workers N]
[--reload] [--inbound FORMAT]
[--intake-enabled|--enable-intake [INTAKE OVERRIDES]]
@@ -221,41 +221,23 @@ switchyard [--routing-profiles PATH] serve [--config PATH]
| Flag | Source |
|---|---|
-| `--routing-profiles PATH` / `-c PATH` | Deprecated legacy [Routing](#routing) path. Global flag; pass before `serve`. Falls back to the saved path from `switchyard --routing-profiles PATH -- configure`. |
-| `--config PATH` | Switchyard v2 profile-config YAML/JSON/TOML entrypoint. This is the primary serve path. Mutually exclusive with `--routing-profiles`. |
+| `--routing-profiles PATH` / `-c PATH` | [Routing](#routing) path. Global flag; pass before `serve`. Falls back to the saved path from `switchyard --routing-profiles PATH -- configure`. |
| `--host`, `--port`/`-p`, `--reload` | [Transport](#transport-server-verbs). |
-| `--inbound FORMAT` | Valid only for legacy route-bundle serve (`--routing-profiles`); `serve --config` actively rejects it with an error. For legacy serve, the flag is a no-op — all request APIs are always registered regardless of the value (accepted for backwards compat only). |
+| `--inbound FORMAT` | Backwards-compatible no-op; all request APIs are always registered. |
| `--workers` / `-w` | uvicorn worker count. |
-| `--routing-log-file PATH` | Append one JSONL routing record per request (task, session, selected model, tier, token usage) to PATH. Route-bundle serve only; rejected by `serve --config`. |
+| `--routing-log-file PATH` | Append one JSONL routing record per request (task, session, selected model, tier, token usage) to PATH. |
| `--intake-enabled` / `--enable-intake`, `--intake-base-url`, `--intake-workspace`, `--intake-api-key`, `--intake-target-url` | [Intake sink](#intake-sink-serve-and-launchers). |
**Notes**
- `serve` always registers `POST /v1/chat/completions`, `POST /v1/messages`, `POST /v1/responses`, `GET /v1/models`, and `GET /health`. There is no flag to expose just one request API.
-- `GET /v1/stats` and `GET /v1/routing/stats` are available on both serve paths.
-- The deprecated route-bundle path accepts `--inbound` for compatibility but ignores it; all supported request APIs are always registered.
-- `serve --config` does not support `--reload`, `--workers > 1`, Intake options, `--enable-rl-logging`, `--routing-log-file`, or any explicit `--inbound` value.
-- Rust-defined and Python-defined profiles use the same profile-config schema. The profile `type` decides which implementation builds that profile.
-- Python-defined profiles are registered via `@profile_config`; the shipped `header-routing` profile is an example. A config can mix a Rust-defined profile and a Python-defined profile, and both profile ids are routable on the same served host and port.
+- `GET /v1/stats` and `GET /v1/routing/stats` expose route statistics.
+- `--inbound` is retained as a compatibility no-op; all supported request APIs are always registered.
**Examples**
```bash
-# v2 profile config (primary): each profile id + target id is a model on /v1/models
-switchyard serve --config examples/profiles.yaml --port 4000
-# select a profile by id (the `model` field):
-curl localhost:4000/v1/chat/completions \
- -H 'Content-Type: application/json' \
- -d '{"model": "smart-stage-router", "messages": [{"role": "user", "content": "hi"}]}'
-
-# mixed Rust/Python profile config on the same request APIs
-switchyard serve --config examples/python_profile.yaml --port 4000
-curl localhost:4000/v1/chat/completions \
- -H 'Content-Type: application/json' \
- -H 'x-switchyard-tier: strong' \
- -d '{"model": "smart", "messages": [{"role": "user", "content": "hi"}]}'
-
-# Legacy route bundle on port 4000
+# Route bundle on port 4000
switchyard --routing-profiles routes.yaml -- serve --port 4000
# Persist a bundle for later runs; no-TTY configure requires explicit provider credentials
@@ -264,7 +246,7 @@ switchyard --routing-profiles routes.yaml -- configure --target provider \
--base-url https://openrouter.ai/api/v1 --no-tui --no-model-discovery
switchyard serve --port 4000
-# Multi-worker uvicorn (route-bundle path)
+# Multi-worker uvicorn
SWITCHYARD_WORKERS=4 switchyard --routing-profiles routes.yaml -- serve
```
diff --git a/docs/core_concepts.md b/docs/core_concepts.md
index 4c6e768ac..739c70a73 100644
--- a/docs/core_concepts.md
+++ b/docs/core_concepts.md
@@ -1,98 +1,89 @@
# Core Concepts
-This page explains the vocabulary the rest of the documentation takes for
-granted. Read it once and the configuration examples elsewhere will make sense.
-It is not a setup guide: to install and run Switchyard, see
-[Getting Started](getting_started.md), and to follow a request through the
-system, see [Architecture](architecture.md).
+This page explains the vocabulary used throughout the documentation. For setup,
+see [Getting Started](getting_started.md). For the request lifecycle, see
+[Architecture](architecture.md).
## How it fits together
-Switchyard is a proxy. Clients (coding agents, SDKs, your own services) talk to
-it on one side, and model backends (hosted providers, private endpoints, local
-models) sit on the other. A client never asks for a backend by name. It sends a
-model ID, and the configuration behind that ID decides what actually runs.
+Switchyard is a proxy. Clients such as coding agents and SDKs talk to it on one
+side, while hosted providers, private endpoints, and local model servers sit on
+the other. A client sends a model ID, and the route registered under that ID
+decides what runs.
```mermaid
flowchart LR
client["Client<br/>picks a model ID"]
- profile["Profile<br/>routing policy"]
+ route["Route<br/>routing policy"]
target["Target<br/>upstream model"]
- endpoint["Endpoint<br/>provider connection"]
backend["Model backend"]
- client -->|"model field"| profile
- profile -->|"selects"| target
- target -->|"uses"| endpoint
- endpoint --> backend
+ client -->|"model field"| route
+ route -->|"selects"| target
+ target --> backend
```
-## Endpoints, targets, and profiles
+## Routes and targets
-A standalone deployment is described by a profile config: one file with three
-sections that keep provider connectivity, upstream models, and client-facing
-policy apart. Three terms carry most of the weight.
+The Python CLI loads a YAML bundle with a `routes:` section. Each route has a
+client-facing name and a `type` that chooses its behavior. Depending on that
+type, the route owns one target, two routing tiers, or a larger endpoint list.
-| Term | What it is | YAML section |
-|---|---|---|
-| Endpoint | A provider connection: a `base_url` and its credentials. Many targets can share one. | `endpoints:` |
-| Target | A single upstream model you can call. It names an endpoint, a `model`, and a wire `format`. | `targets:` |
-| Profile | A client-facing routing policy over one or more targets. Its `type` is the routing strategy. | `profiles:` |
-
-The three nest. An endpoint is reused by targets, and targets are referenced by
-profiles. You set credentials in one place, add a model once, and build as many
-policies on top as you need. The [Routing Overview](routing_algorithms/overview.md)
-has a complete, runnable config and the full schema.
+Shared `defaults:` can provide a base URL, API key, and backend format without
+repeating them in every target. The [Routing Overview](routing_algorithms/overview.md)
+has a complete runnable bundle.
## Model IDs
-Clients choose what happens through the `model` field on a request. Switchyard
-registers three kinds of model IDs and lists them all on `GET /v1/models`:
-profile IDs, which apply a routing policy; target IDs, which skip routing and
-call that one model; and the upstream model name itself, registered as an alias
-whenever it differs from its target ID. Send a profile ID when you want routing,
-or a target or upstream name when you want to pin a single model.
+Every route name is registered as a model ID and listed on `GET /v1/models`.
+Clients select a route by putting that name in the request's `model` field.
+Some route types also discover or register direct upstream model IDs.
## Tiers and routing strategies
Most routing strategies divide traffic between two tiers: a strong target that
is more capable and more expensive, and a weak target that is cheaper and
-faster. A tier is a role you hand to a target rather than a fixed property of it,
-so the same target can be the strong tier in one profile and the weak tier in
-another.
-
-A profile's `type` sets the strategy. `passthrough` sends everything to one
-target with no routing. `random-routing` splits traffic on a fixed probability.
-`llm-routing` asks a classifier model to pick a tier for each turn. `stage_router`
-escalates from weak to strong when request signals call for it. The
-[Routing Overview](routing_algorithms/overview.md) covers when to use each and
-how to tune it.
-
-!!! warning "There is no `type: model`"
- The current profile config does not have a `type: model`. To expose a single
- model, point clients at a target ID directly, or add a `passthrough` profile
- when you want a second name for it. `type: model` survives only in the
- deprecated `--routing-profiles` bundles used by the launcher compatibility
- path.
-
-Session affinity, or sticky routing, pins a conversation to one tier so later
-turns reuse it instead of being classified again. It belongs to `llm-routing`
-and is not a strategy of its own; random and stage-router routing decide every
-request on its own merits. [Sticky Routing](routing_algorithms/sticky_routing.md)
-covers it in full.
+faster. A tier is a role assigned inside a route, not a fixed property of a
+model.
+
+A route's `type` sets the strategy. `model` and `passthrough` call one target.
+`random_routing` splits traffic on a fixed probability. `deterministic` asks a
+classifier model to pick a tier. `stage_router` uses agent-progress signals,
+with an optional classifier fallback. `escalation_router` starts on the weak
+tier and uses an LLM judge to escalate when needed. The
+[Routing Overview](routing_algorithms/overview.md) explains when to use each.
+
+Session affinity pins a conversation to one tier so later turns reuse it
+instead of being classified again. It belongs to deterministic classifier
+routing and is not a strategy of its own. Random and stage-router routing make
+a decision for every request. [Sticky Routing](routing_algorithms/sticky_routing.md)
+covers affinity in full.
## Formats and translation
-Clients reach Switchyard in one of three inbound formats: OpenAI Chat
-Completions, Anthropic Messages, or OpenAI Responses. Each target has its own
-backend format, set by its `format:` field, which is one of `openai`,
-`anthropic`, `responses`, or `auto`. When the two differ, Switchyard translates
-the request on the way out and the response on the way back.
+Clients reach Switchyard through OpenAI Chat Completions, Anthropic Messages,
+or OpenAI Responses. Each target has a backend format: `openai`, `anthropic`,
+`responses`, or `auto`. When the inbound and backend formats differ,
+Switchyard translates the request on the way out and the response on the way
+back.
+
+That translation lets Claude Code, which speaks Anthropic Messages, run against
+an OpenAI-compatible model. The [Architecture](architecture.md) page documents
+the supported formats and request lifecycle.
+
+## Programmatic Python profiles
+
+Embedded Python callers can construct typed profile configs and runtimes
+directly instead of loading a route bundle. `ProfileInput` carries the request
+and request metadata, while the profile protocols define `run`, `process`, and
+`rprocess`. This programmatic API is separate from CLI configuration.
+
+## Rust server configuration
-That translation is what lets Claude Code, which speaks Anthropic Messages, run
-against an OpenAI-compatible model, and the reverse. The
-[Architecture](architecture.md) page documents every backend format, the `auto`
-probe, and the neutral representation used to convert between them.
+The `switchyard-server` binary is built directly on libsy. Its TOML file
+explicitly defines LLM clients, targets, and algorithm routes; it does not load
+Python route bundles. See the
+[Rust server README](../crates/switchyard-server/README.md) for details.
## Where to go next
diff --git a/docs/guides/agent_launchers.md b/docs/guides/agent_launchers.md
index 50e860199..a48ee9001 100644
--- a/docs/guides/agent_launchers.md
+++ b/docs/guides/agent_launchers.md
@@ -28,13 +28,12 @@ switchyard launch openclaw
Each launcher reads saved provider credentials from `~/.config/switchyard/`.
Interactive Claude Code and Codex sessions show live request and token totals
-in the status footer. A saved legacy route bundle is still honored for
-compatibility, but route bundles and `--routing-profiles` are deprecated.
+in the status footer. A saved route bundle is used when configured.
## Default routing behavior
The built-in LLM-as-classifier route is the default only when there is no
-`--model`, no CLI `--routing-profiles`, no saved legacy route bundle, and no
+`--model`, no CLI `--routing-profiles`, no saved route bundle, and no
saved model default. Each turn is classified and dispatched to a weak or
strong tier using the validated coding-agent trio:
@@ -46,8 +45,8 @@ strong tier using the validated coding-agent trio:
`--weak-model`, `--classifier-model`, `--profile`, and
`--classifier-min-confidence` tune only this built-in route. They are ignored
-when route resolution selects an explicit or saved legacy route bundle or
-single model.
+when route resolution selects an explicit or saved route bundle or single
+model.
## Override the default route
@@ -57,8 +56,8 @@ Use `--model` for a single-model passthrough session:
switchyard launch claude --model openai/gpt-4o-mini
```
-For launcher-owned legacy route-bundle routing, whether the bundle contains one
-route or several, use:
+For launcher route-bundle routing, whether the bundle contains one route or
+several, use:
```bash
switchyard --routing-profiles routes.yaml -- launch claude
@@ -72,12 +71,8 @@ For route-bundle sessions, the footer reports active-model and aggregate
request/token counts, including errors. `/v1/routing/stats` aggregates usage
across the registered chains.
-!!! note "Profile-config boundary"
- `--config profiles.yaml` belongs to `switchyard serve` and is the primary
- profile configuration path. Launcher subcommands do not accept `--config`
- today. Use the deprecated `--routing-profiles` flag only for launcher-owned
- legacy route-bundle routing; use `switchyard serve --config profiles.yaml`
- for standalone deployments.
+The same `--routing-profiles` bundle can be served directly with
+`switchyard --routing-profiles routes.yaml -- serve`.
## Model requirements
@@ -143,7 +138,7 @@ always has either an existing `claude`/`anthropic` prefix or the generated
This aliasing applies only to the Claude launcher. `switchyard launch codex`,
`switchyard launch openclaw`, and `switchyard serve` expose route ids verbatim.
-You can also write the `claude-` prefix directly in legacy route YAML if you
+You can also write the `claude-` prefix directly in route YAML if you
want the route ids to match exactly what appears in `/model`:
```yaml
diff --git a/docs/index.md b/docs/index.md
index fd07e2210..3801dd0ad 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -81,42 +81,33 @@ For source installs, non-interactive configuration, and a curl sanity check, use
## Configuration Model
-Standalone deployments start with a profile config that separates provider
-connectivity, upstream targets, and client-facing profiles:
+Python CLI deployments define client-facing routes in a YAML bundle:
```yaml
-endpoints:
- openrouter:
- api_key: ${OPENROUTER_API_KEY}
- base_url: https://openrouter.ai/api/v1
-
-targets:
- strong:
- endpoint: openrouter
- model: openai/gpt-4o
- format: openai
- weak:
- endpoint: openrouter
- model: openai/gpt-4o-mini
- format: openai
-
-profiles:
+defaults:
+ api_key: ${OPENROUTER_API_KEY}
+ base_url: https://openrouter.ai/api/v1
+ format: openai
+
+routes:
smart:
- type: random-routing
- strong: strong
- weak: weak
+ type: random_routing
+ strong:
+ model: openai/gpt-4o
+ weak:
+ model: openai/gpt-4o-mini
strong_probability: 0.3
+ fallback_target_on_evict: strong
```
-Run it as a long-lived proxy. Profile and target ids appear as models on
-`GET /v1/models`, and clients select one with the request's `model` field:
+Run it as a long-lived proxy. Route names appear as models on `GET /v1/models`,
+and clients select one with the request's `model` field:
```bash
-switchyard serve --config profiles.yaml --port 4000
+switchyard --routing-profiles routes.yaml -- serve --port 4000
```
-The deprecated `--routing-profiles` flag is retained only for launcher-owned
-legacy bundles and saved bundle paths:
+The same bundle can drive a launcher or be saved as the default:
```bash
switchyard --routing-profiles routes.yaml -- launch claude
@@ -128,8 +119,9 @@ switchyard --routing-profiles routes.yaml -- configure --target provider \
Non-interactive `configure` does not read provider credentials from the routing
bundle; pass `--api-key` explicitly when persisting the bundle for CI.
-Profile ids, direct targets, legacy launcher compatibility, and persistence are
-covered in [Routing Overview](routing_algorithms/overview.md).
+Route types, launcher use, and persistence are covered in
+[Routing Overview](routing_algorithms/overview.md). The Rust server uses its
+own explicit TOML configuration for LLM clients, targets, and libsy algorithms.
## Routing Reference
diff --git a/docs/internal/latency_service_routing.md b/docs/internal/latency_service_routing.md
index 9df6ffac1..d57b731b3 100644
--- a/docs/internal/latency_service_routing.md
+++ b/docs/internal/latency_service_routing.md
@@ -275,10 +275,6 @@ policy-specific CLI flags. Run it with:
switchyard --routing-profiles my_routes.yaml -- serve --port 4100
```
-The Rust `type: latency-service` profile loaded by `switchyard serve --config`
-is a separate schema and does not yet expose `session_affinity` or
-`affinity_max_sessions`.
-
### Config schema
`LatencyServiceBackendConfig`
diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md
index 3dcdc18f4..cdba2a318 100644
--- a/docs/internal/metrics_reference.md
+++ b/docs/internal/metrics_reference.md
@@ -14,11 +14,8 @@ a drop-in scrape config and starter alert rules.
| Auth | None. Designed as a public scrape endpoint |
| Default scrape interval | 15s (the Latency Service poll cycle is 10s by default, so finer scrape resolution captures no extra state) |
-!!! note "Availability: legacy route-bundle serve only"
- `GET /metrics` is served by the **legacy route-bundle path**
- (`switchyard --routing-profiles PATH serve`). The v2 profile-config path
- (`switchyard serve --config`) returns **404** for `/metrics` — use `GET /v1/stats`
- (or its alias `GET /v1/routing/stats`) on that path instead.
+`GET /metrics` is served by the Python route-bundle server started with
+`switchyard --routing-profiles PATH -- serve`.
A JSON variant of the same underlying data lives at `GET /v1/stats`, with
`GET /v1/routing/stats` as a backwards-compatible alias.
diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md
index d92fd36a4..696512c3a 100644
--- a/docs/routing_algorithms/escalation_router_routing.md
+++ b/docs/routing_algorithms/escalation_router_routing.md
@@ -71,10 +71,8 @@ without clearing an existing confirmation streak.
## Configure an escalation route
-Escalation routing is currently available through the legacy `routes:` bundle
-used by `--routing-profiles`. This path remains necessary for launcher-owned
-routing; route bundles and `--routing-profiles` are otherwise deprecated in
-favor of `switchyard serve --config` profiles.
+Configure escalation routing in the `routes:` bundle loaded by
+`--routing-profiles`.
```yaml
routes:
diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md
index 7993ab53f..f513589c5 100644
--- a/docs/routing_algorithms/llm_classifier_routing.md
+++ b/docs/routing_algorithms/llm_classifier_routing.md
@@ -12,66 +12,50 @@ policies default to `strong`.
## Choose a policy
-Set `profile_name` for the traffic you expect:
+Set `profile` for the traffic you expect:
-| `profile_name` | Use for | Default tier mapping |
+| `profile` | Use for | Default tier mapping |
|---|---|---|
| `general` | Mixed chat or API traffic | `simple` uses `weak`; all higher tiers use `strong`. |
| `coding_agent` | Claude Code, Codex, Cursor-style agents | `simple` and `medium` use `weak`; `complex` and `reasoning` use `strong`. Tool-planning turns can escalate. |
| `openclaw` | OpenClaw personal-assistant traffic | `simple` and `medium` use `weak`; `complex` and `reasoning` use `strong`. Tool orchestration and high-risk external actions can escalate. |
-For coding-agent traffic, start with `profile_name: coding_agent`.
+For coding-agent traffic, start with `profile: coding_agent`.
-## Configure a classifier profile
+## Configure a classifier route
-Define the strong, weak, and classifier models as targets, then reference those
-target IDs from an `llm-routing` profile:
+Define the strong, weak, and classifier models in a `deterministic` route:
```yaml
-endpoints:
- openrouter:
- api_key: ${OPENROUTER_API_KEY}
- base_url: https://openrouter.ai/api/v1
-
-targets:
- strong:
- endpoint: openrouter
- model: openai/gpt-4o
- format: openai
- weak:
- endpoint: openrouter
- model: openai/gpt-4o-mini
- format: openai
- classifier:
- endpoint: openrouter
- model: nvidia/nemotron-3-nano-30b-a3b
- format: openai
- extra_body:
- reasoning:
- enabled: false
-
-profiles:
+defaults:
+ api_key: ${OPENROUTER_API_KEY}
+ base_url: https://openrouter.ai/api/v1
+ format: openai
+
+routes:
smart:
- type: llm-routing
- profile_name: coding_agent
- strong: strong
- weak: weak
- classifier: classifier
+ type: deterministic
+ profile: coding_agent
+ classifier:
+ model: nvidia/nemotron-3-nano-30b-a3b
+ min_confidence: 0.6
+ fail_open: true
+ recent_turn_window: 4
+ strong:
+ model: openai/gpt-4o
+ weak:
+ model: openai/gpt-4o-mini
fallback_target_on_evict: strong
- classifier_min_confidence: 0.6
- classifier_fail_open: true
- classifier_recent_turn_window: 4
```
-The classifier target must use `format: openai`. Start the profile server with:
+Start the server with:
```bash
-switchyard serve --config profiles.yaml --port 4000
+switchyard --routing-profiles routes.yaml -- serve --port 4000
```
-The profile ID (`smart`) is the model ID clients select for classifier-based
-routing. The target IDs remain directly selectable when a client needs to
-bypass the classifier.
+The route ID (`smart`) is the model ID clients select for classifier-based
+routing.
Try the profile with representative requests:
@@ -92,12 +76,11 @@ prompt determine the verdict.
## Useful options
-| Option | Use it when |
+| Configuration path | Use it when |
|---|---|
-| `classifier_min_confidence` | Low-confidence results should use `default_tier` instead of the classifier policy. |
-| `classifier_fail_open` | Classifier errors should use `default_tier` rather than fail the client request. |
-| `classifier_recent_turn_window` | The classifier needs more or less recent conversation and tool context. |
-| `classifier_max_tokens` | You need to cap the classifier tool-call response. |
+| `classifier.min_confidence` | Low-confidence results should use the default tier instead of the classifier policy. |
+| `classifier.fail_open` | Classifier errors should use the default tier rather than fail the client request. |
+| `classifier.recent_turn_window` | The classifier needs more or less recent conversation and tool context. |
| `alignment_min_confidence` | A classifier recommendation should only raise the policy tier above this confidence. |
| `default_tier` | Abstain, low-confidence, and fail-open decisions should use a tier other than the default `strong`. |
| `tier_mapping` | The four classifier policy tiers need a custom mapping to `weak` or `strong`. |
@@ -115,10 +98,8 @@ store between the classifier and tier selector. After any configured
reuse that tier before classification, so they skip the classifier call;
abstain, low-confidence, missing-signal, and fail-open decisions do not pin.
-The CLI currently exposes these fields on a `type: deterministic` entry in a
-`routes:` bundle loaded with `--routing-profiles`. The Rust `llm-routing`
-profile loaded by `switchyard serve --config` does not yet expose them. See
-[Session Affinity](sticky_routing.md) for YAML and
+Configure these fields on the `type: deterministic` entry in the `routes:`
+bundle. See [Session Affinity](sticky_routing.md) for YAML and
[How session affinity composes](overview.md#how-session-affinity-composes) for
the interaction with routing decisions.
diff --git a/docs/routing_algorithms/overview.md b/docs/routing_algorithms/overview.md
index dc7440c3d..28ff6b356 100644
--- a/docs/routing_algorithms/overview.md
+++ b/docs/routing_algorithms/overview.md
@@ -1,198 +1,113 @@
# Routing Overview
-Switchyard profile configs register profiles and targets as model IDs that
-clients can select through OpenAI Chat Completions, Anthropic Messages, or
-OpenAI Responses API requests. Start a profile config with:
+The Python CLI loads routing policies from a YAML bundle. Each key under
+`routes:` becomes a model ID available through OpenAI Chat Completions,
+Anthropic Messages, and OpenAI Responses requests:
```bash
-switchyard serve --config profiles.yaml --port 4000
+switchyard --routing-profiles routes.yaml -- serve --port 4000
```
-Use this page to choose a routing strategy first, then open its detailed page
-for configuration and tuning.
+Use this page to choose a routing strategy, then open its detailed page for
+configuration and tuning.
## Choose a strategy
-| Strategy | Use it when | Profile `type` |
+| Strategy | Use it when | Route `type` |
|---|---|---|
-| [Random Routing](random_routing.md) | You need a fixed strong/weak split for A/B tests, baselines, or cost experiments. | `random-routing` |
-| [LLM Classifier Routing](llm_classifier_routing.md) | Request content should decide whether a turn needs the weak or strong tier. | `llm-routing` |
+| [Random Routing](random_routing.md) | You need a fixed strong/weak split for A/B tests, baselines, or cost experiments. | `random_routing` |
+| [LLM Classifier Routing](llm_classifier_routing.md) | Request content should decide whether a turn needs the weak or strong tier. | `deterministic` |
| [Stage-Router Routing](stage_router_routing.md) | Tool-result and agent-progress signals should route most turns without an extra classifier call. | `stage_router` |
-| [Escalation-Router Routing](escalation_router_routing.md) | Start every task on the weak tier and let an LLM judge escalate to strong — one-way per task — when the run is in trouble. | `escalation_router` (`routes:` bundle) |
+| [Escalation-Router Routing](escalation_router_routing.md) | Start every task on the weak tier and escalate to strong when an LLM judge detects trouble. | `escalation_router` |
[Session Affinity (Sticky Routing)](sticky_routing.md) is an opt-in feature of
-LLM classifier routing, not a standalone routing strategy. The classifier
-integrates the pin into its decision path. See
-[How session affinity composes](#how-session-affinity-composes) for the exact
-behavior.
+LLM classifier routing, not a standalone routing strategy.
-## Common profile shape
+## Common route shape
-Profile configs separate provider connectivity, upstream targets, and
-client-facing profiles:
+Provider defaults can be shared across all routes, while each route owns its
+target configuration:
```yaml
-endpoints:
- openrouter:
- api_key: ${OPENROUTER_API_KEY}
- base_url: https://openrouter.ai/api/v1
-
-targets:
- strong:
- endpoint: openrouter
- model: openai/gpt-4o
- format: openai
- weak:
- endpoint: openrouter
- model: openai/gpt-4o-mini
- format: openai
+defaults:
+ api_key: ${OPENROUTER_API_KEY}
+ base_url: https://openrouter.ai/api/v1
+ format: openai
+
+routes:
+ fast:
+ type: model
+ target: openai/gpt-4o-mini
-profiles:
smart:
- type: random-routing
- strong: strong
- weak: weak
+ type: random_routing
+ strong:
+ model: openai/gpt-4o
+ weak:
+ model: openai/gpt-4o-mini
strong_probability: 0.3
+ fallback_target_on_evict: strong
```
-The profile ID (`smart`) is the model ID clients send when they want the
-routing policy. Target IDs (`strong` and `weak`) are also directly selectable.
-When an upstream model ID differs from its target ID, the profile server
-registers that model ID as an additional direct alias.
+Use the route name (`fast` or `smart`) as the request's model ID. A single
+bundle can serve multiple routes on the same host and port.
The examples use model IDs from the
[OpenRouter model catalog](https://openrouter.ai/api/v1/models). Select IDs
available to your account before deploying; catalog availability can change.
-## Multiple profiles
-
-A single file can declare multiple profiles over the same targets. Each
-profile and target appears on `GET /v1/models`:
-
-```yaml
-profiles:
- fast:
- type: passthrough
- target: weak
-
- smart:
- type: random-routing
- strong: strong
- weak: weak
- strong_probability: 0.3
-```
-
-Use the profile ID to select policy behavior (`fast` or `smart`) and a target ID
-to bypass routing (`weak` or `strong`).
+## Model and passthrough routes
-## Sub-agent override (`subagent_target`)
+- `type: model` registers one explicit model alias without model discovery.
+- `type: passthrough` queries the upstream model catalog and registers the
+ discovered models.
-Any profile can name an optional `subagent_target` in its common envelope,
-alongside `type`. When a request carries a recognized sub-agent signal —
-Codex delegated-work kinds (`x-openai-subagent: collab_spawn` or `review`),
-Claude Code agent lineage headers, or an explicit
-`x-switchyard-is-subagent: true` — it bypasses the profile's routing and runs
-as a direct passthrough to that target:
-
-```yaml
-profiles:
- smart:
- type: random-routing
- strong: strong
- weak: weak
- strong_probability: 0.3
- subagent_target: weak
-```
-
-This keeps a sub-agent loop on one intentional, cache-compatible target
-instead of re-routing every worker turn. Codex harness-maintenance turns
-(`compact`, `memory_consolidation`) and unrecognized kinds stay on normal
-profile routing, as does everything else when the field is absent. An unknown
-target reference fails at startup, and a sub-agent target failure surfaces as
-a normal target error — it is never silently re-routed through the profile.
-
-To suppress sub-agent routing for a request that carries a recognized signal,
-send `x-switchyard-is-subagent: false`. This explicit header overrides Codex
-and Claude Code lineage signals in either direction: `false` keeps the request
-on normal profile routing even when delegated-work headers are present, and
-`true` marks a request as a sub-agent even when no harness headers appear.
-
-## Direct targets and passthrough aliases
-
-For new profile configs, use one public model concept:
-
-- Select a target ID directly when its configured name is the client-facing
- name you want.
-- Add a `type: passthrough` profile only when you need another stable alias for
- that target, such as `fast` above.
-
-Both choices call the same single target. There is no `type: model` profile in
-the current profile schema.
-
-The deprecated `--routing-profiles` route-bundle format has a separate legacy
-distinction: `type: model` registers one explicit alias without model discovery,
-while `type: passthrough` queries the upstream model catalog and registers the
-discovered models too. That distinction applies only to legacy `routes:`
-bundles used by the launcher compatibility path.
+Both create direct, single-target chains. Use a routing policy when requests
+must be split or classified across targets.
## Self-hosted targets
-Any profile target can point at an OpenAI-compatible model server you operate.
+Any route target can point at an OpenAI-compatible model server you operate.
For example, start a local vLLM server:
```bash
vllm serve ./my-rl-qwen --served-model-name my-rl-qwen --port 8000
```
-Then declare it as a normal endpoint and target:
+Then configure it as a normal route:
```yaml
-endpoints:
+routes:
local:
+ type: model
+ target: my-rl-qwen
base_url: http://localhost:8000/v1
api_key: dummy
-
-targets:
- local-weak:
- endpoint: local
- model: my-rl-qwen
format: openai
-
-profiles:
- fast-local:
- type: passthrough
- target: local-weak
```
-Reference `local-weak` from any routing profile field that accepts a target ID,
-including `strong`, `weak`, `target`, or `targets`. Switchyard does not start or
-manage the model server; it only sends requests to the configured endpoint.
+Switchyard does not start or manage the model server; it only sends requests
+to the configured endpoint.
## How session affinity composes
-Session affinity is configured directly on the LLM classifier router. It is not
-a generic wrapper applied after every routing strategy. After the configured
-warmup, the first confident policy, tool-planning, or alignment verdict pins
-the tier. Abstain, low-confidence, missing-signal, and fail-open decisions never
-pin. The classifier and tier selector share one affinity store, and later turns
-check that store before classification, reuse the tier, and skip the classifier
-call.
+Session affinity is configured directly on the LLM classifier router. After
+the configured warmup, the first confident policy, tool-planning, or alignment
+verdict pins the tier. Abstain, low-confidence, missing-signal, and fail-open
+decisions never pin. Later turns reuse the tier and skip the classifier call.
Pins use a bounded in-process LRU keyed from the stable conversation prefix.
They are not shared across workers or restarts. See
[Sticky Routing](sticky_routing.md) for configuration and key derivation.
-Random and stage-router routing do not expose session-affinity settings; they
-continue to make a routing decision for each request.
+Random and stage-router routing make a fresh routing decision for each request.
+The escalation router instead uses affinity as a one-way escalation latch: only
+the strong tier is pinned, and the judge runs until that latch fires.
-The escalation router uses the same affinity store with the opposite policy:
-affinity is always on (the pin *is* the escalation latch), only the strong
-tier is ever pinned, and once the judge starts (a configurable minimum turn),
-it keeps running each judged turn until the latch fires. See
-[Escalation-Router Routing](escalation_router_routing.md).
+## Rust server configuration
-!!! note "CLI schema availability"
- The CLI currently accepts these settings in a `deterministic` entry in a
- `routes:` bundle loaded with `--routing-profiles`. The Rust `llm-routing`
- profile loaded by `switchyard serve --config` does not yet expose
- session-affinity fields.
+The `switchyard-server` binary has a separate TOML schema that explicitly
+constructs LLM clients, targets, and libsy algorithms. It does not load Python
+route bundles. See the
+[Rust server README](../../crates/switchyard-server/README.md) for its supported
+configuration.
diff --git a/docs/routing_algorithms/random_routing.md b/docs/routing_algorithms/random_routing.md
index 360b10310..d7b85baea 100644
--- a/docs/routing_algorithms/random_routing.md
+++ b/docs/routing_algorithms/random_routing.md
@@ -30,40 +30,7 @@ tier model instead of the routing model.
## Enable It
-Use `serve --config` with a profile config:
-
-```yaml
-targets:
- strong:
- model: openai/gpt-4o
- format: openai
- base_url: https://openrouter.ai/api/v1
- api_key: ${OPENROUTER_API_KEY}
- weak:
- model: openai/gpt-4o-mini
- format: openai
- base_url: https://openrouter.ai/api/v1
- api_key: ${OPENROUTER_API_KEY}
-
-profiles:
- ab-test:
- type: random-routing
- strong: strong
- weak: weak
- strong_probability: 0.3
-```
-
-Run it with:
-
-```bash
-switchyard serve --config routes.yaml --port 4000
-```
-
-The profile id (`ab-test`) is the model id clients select when they want the
-weighted split.
-
-For routing-profile YAML used by launchers, use `type: random_routing` under
-`routes:`:
+Use `type: random_routing` under `routes:`:
```yaml
defaults:
@@ -82,3 +49,12 @@ routes:
rng_seed: 42
fallback_target_on_evict: strong
```
+
+Run it with:
+
+```bash
+switchyard --routing-profiles routes.yaml -- serve --port 4000
+```
+
+The route id (`ab-test`) is the model id clients select when they want the
+weighted split.
diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md
index a9ec02383..45c9fe047 100644
--- a/docs/routing_algorithms/stage_router_routing.md
+++ b/docs/routing_algorithms/stage_router_routing.md
@@ -89,23 +89,14 @@ clears the threshold to escalate to capable.
(If you add the optional classifier, sub-threshold turns go to it instead of
staying on the default tier.)
-**Set `0.5` explicitly.** It's the recommended starting point and is what the
-example below uses. The default when you omit the field depends on which config
-path you take:
-
-| Configuration path | Default when omitted |
-|---|---|
-| Profile config (`switchyard serve --config`) | `0.7` |
-| Deprecated route bundle (`--routing-profiles`) | `0.5` |
-
-Rather than rely on either default, set `confidence_threshold: 0.5` yourself.
+**Start with `0.5`.** It is the default and the recommended starting point.
| `confidence_threshold` | Include `classifier:` block? | Typical use |
|---|---|---|
| `0.0` | no | Cost/latency-sensitive. Every signal-based verdict is accepted; no per-turn LLM call. Critical-error signals still escalate to capable. |
| `0.5` | no | Recommended starting point. A single strong signal clears the threshold on its own; derived from SWE-Bench Pro calibration. |
-| `0.7` - `0.9` | yes | Classifier-assisted. Low-confidence turns go to the LLM classifier before falling back to the default tier. `0.7` is the profile-config default when the field is omitted. |
-| `1.0` | yes (required) | Classifier-driven. Equivalent to the legacy `coding_agent` profile. |
+| `0.7` - `0.9` | yes | Classifier-assisted. Low-confidence turns go to the LLM classifier before falling back to the default tier. |
+| `1.0` | yes (required) | Classifier-driven. Equivalent to classifier-only `coding_agent` routing. |
The signal-vs-classifier split is dataset-dependent. Measure it in
production via `routing_decisions.stage_router` on `/v1/stats` rather than relying on
@@ -187,46 +178,37 @@ In stage-router, the efficient model may inherit partial context from the capabl
fresh, so RESCUE is a conservative lower bound. Efficient performs at least as
well in stage-router as it does alone.
-## Profile configuration
+## Route configuration
```yaml
-endpoints:
- openrouter:
- base_url: https://openrouter.ai/api/v1
- api_key: ${OPENROUTER_API_KEY}
-
-# Targets are named by the model id they serve.
-targets:
- openai/gpt-4o:
- endpoint: openrouter
- model: openai/gpt-4o
- format: openai
- openai/gpt-4o-mini:
- endpoint: openrouter
- model: openai/gpt-4o-mini
- format: openai
-
-profiles:
+defaults:
+ base_url: https://openrouter.ai/api/v1
+ api_key: ${OPENROUTER_API_KEY}
+ format: openai
+
+routes:
smart-stage-router:
type: stage_router
- picker: capable_first # or efficient_first
- confidence_threshold: 0.5 # recommended; range [0.0, 1.0]
- signal_recent_window: 3 # sliding window for recent signal counts
- fallback_target_on_evict: openai/gpt-4o # required; see Context-Window Handling
- capable: openai/gpt-4o # capable tier target id
- efficient: openai/gpt-4o-mini # efficient tier target id
- enable_stats: true # default true
+ picker: capable_first
+ confidence_threshold: 0.5
+ signal_recent_window: 3
+ fallback_target_on_evict: strong
+ strong:
+ id: strong
+ model: openai/gpt-4o
+ weak:
+ id: weak
+ model: openai/gpt-4o-mini
+ enable_stats: true
```
-Save the file as `profiles.yaml` and start it with:
+Save the file as `routes.yaml` and start it with:
```bash
-switchyard serve --config profiles.yaml --port 4000
+switchyard --routing-profiles routes.yaml -- serve --port 4000
```
This is the recommended default: routing on tool signals alone, no classifier.
-If you omit `confidence_threshold`, the profile-config default of `0.7` applies;
-the example sets `0.5` explicitly.
`fallback_target_on_evict` is required and must reference one of the
declared target ids. See [Context-Window Handling](../operations/context_window.md) for
@@ -252,10 +234,7 @@ Give the classifier its own credential or quota bucket where you can. Sharing
one provider bucket with the efficient tier adds a request per classified turn
and can cause sustained 429s at scale.
-**Launcher compatibility.** Launcher subcommands don't accept `--config`. A
-launcher-owned stage-router still needs the deprecated `--routing-profiles` path
-and its legacy `routes:` schema. Use the same `type: stage_router`, picker,
-target, classifier, and explicit `confidence_threshold: 0.5` values there.
+The same route bundle works with the launchers.
## Observability
@@ -271,9 +250,8 @@ harnesses usually capture this same snapshot to a file (see below).
### Decision-source metadata (stage-router-specific)
-The profile counts why each turn was routed the way it was, under
-`routing_decisions.stage_router` in the stats JSON. For the `serve --config`
-profile the values are:
+The route counts why each turn was routed the way it was under
+`routing_decisions.stage_router` in the stats JSON:
| Source | When |
|---|---|
@@ -281,11 +259,7 @@ profile the values are:
| `dimensions` | The signals crossed `confidence_threshold` and picked the tier. |
| `llm-classifier` | The signals were ambiguous and the classifier returned a verdict. |
| `fall_open` | The signals were ambiguous and the classifier failed or wasn't configured, so the default tier was used. |
-| `context_overflow_fallback` | A context-window overflow rerouted the turn to `fallback_target_on_evict`. |
-
-The deprecated `--routing-profiles` path reports the same sources except
-`context_overflow_fallback`, plus `no_signal` for turns that arrive before any
-tool-result history exists.
+| `no_signal` | The request arrived before any tool-result history existed. |
To capture a snapshot for a batch run, redirect the endpoint to a file:
@@ -295,9 +269,9 @@ curl -s http://localhost:4000/v1/stats > routing_stats_final.json
## When *not* to use stage-router
-- **Single-model deployments.** Use a plain passthrough profile instead.
+- **Single-model deployments.** Use a `model` or `passthrough` route instead.
- **Probabilistic A/B splits.** Use
- [Random Routing](random_routing.md) (`type: random-routing` in profile configs).
+ [Random Routing](random_routing.md) (`type: random_routing`).
The stage-router's signals are wasted on a fixed traffic ratio.
- **No tool-result history.** Stage-router needs meaningful tool-call traffic to
populate the tool-result signal. For pure chat-completion workloads every
diff --git a/docs/routing_algorithms/sticky_routing.md b/docs/routing_algorithms/sticky_routing.md
index 0b0feb734..404d798aa 100644
--- a/docs/routing_algorithms/sticky_routing.md
+++ b/docs/routing_algorithms/sticky_routing.md
@@ -24,10 +24,8 @@ Opt in with `session_affinity: true` on the route. `affinity_max_sessions`
(default `10000`) caps the number of pinned conversations.
These fields are part of the deterministic router configuration, not a
-standalone route type. In CLI YAML, configure them under a `deterministic`
-entry in a `routes:` bundle loaded with `--routing-profiles`. The Rust
-`profiles:` schema loaded by `switchyard serve --config` does not yet expose
-them.
+standalone route type. Configure them under a `deterministic` entry in the
+`routes:` bundle loaded with `--routing-profiles`.
For deterministic routes, `affinity_warmup_turns` controls how many initial
turns remain non-sticky. The default is `0`, which preserves the historical
diff --git a/examples/profiles.yaml b/examples/profiles.yaml
deleted file mode 100644
index 018ed7fcb..000000000
--- a/examples/profiles.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-# Switchyard v2 profile config — the primary serve surface.
-#
-# switchyard serve --config examples/profiles.yaml
-#
-# Each profile id and each target id is exposed as a model on GET /v1/models, so
-# a client (or `switchyard launch claude --model <id>`) selects a profile by id:
-# fast passthrough to the efficient model
-# smart-stage-router signal-based stage-router routing between the efficient and capable models
-# the model ids the targets, addressable directly
-#
-# All profile types here are Rust-defined, so this file is served by the Rust
-# profile server. Files that include Python-defined profiles (see
-# examples/python_profile.yaml) use the Python FastAPI adapter.
-
-endpoints:
- openrouter:
- base_url: https://openrouter.ai/api/v1
- api_key: ${OPENROUTER_API_KEY}
-
-# Targets are named by the model id they serve, so the profile below reads as
-# "capable = <this model>, efficient = <that model>".
-targets:
- anthropic/claude-opus-4.7:
- endpoint: openrouter
- model: anthropic/claude-opus-4.7
- format: openai
- moonshotai/kimi-k2.6:
- endpoint: openrouter
- model: moonshotai/kimi-k2.6
- format: openai
-
-profiles:
- fast:
- type: passthrough
- target: moonshotai/kimi-k2.6
-
- smart-stage-router:
- type: stage_router
- capable: anthropic/claude-opus-4.7
- efficient: moonshotai/kimi-k2.6
- fallback_target_on_evict: anthropic/claude-opus-4.7
- picker: capable_first
- confidence_threshold: 0.7
- signal_recent_window: 3
- classifier:
- model: google/gemini-3.5-flash
- api_key: ${OPENROUTER_API_KEY}
- base_url: https://openrouter.ai/api/v1
diff --git a/examples/python_profile.yaml b/examples/python_profile.yaml
deleted file mode 100644
index ce60423f4..000000000
--- a/examples/python_profile.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-# Switchyard v2 profile config mixing a Rust profile and a Python-defined one.
-#
-# Load it in-process with the Python API:
-#
-# from switchyard import load_profiles
-# profiles = load_profiles("examples/python_profile.yaml")
-# await profiles["smart"].run(ProfileInput(request, metadata))
-#
-# Or serve both profile ids over the same HTTP API paths:
-#
-# switchyard serve --config examples/python_profile.yaml
-#
-# `direct` is a Rust passthrough; `smart` is the shipped Python `header-routing`
-# profile (switchyard/lib/profiles/header_routing.py), which picks a tier from the
-# x-switchyard-tier request header and delegates the call to a Rust target
-# backend. Both share the same resolved targets.
-
-endpoints:
- openrouter:
- base_url: https://openrouter.ai/api/v1
- api_key: ${OPENROUTER_API_KEY}
-
-targets:
- strong:
- endpoint: openrouter
- model: anthropic/claude-opus-4.7
- format: openai
- weak:
- endpoint: openrouter
- model: moonshotai/kimi-k2.6
- format: openai
-
-profiles:
- direct:
- type: passthrough
- target: weak
-
- smart:
- type: header-routing
- strong: strong
- weak: weak
diff --git a/switchyard/__init__.py b/switchyard/__init__.py
index ce9bbb556..927385ac5 100644
--- a/switchyard/__init__.py
+++ b/switchyard/__init__.py
@@ -80,7 +80,6 @@ from switchyard.lib.profiles import (
StageRouterProfileConfig,
TranslateProfileConfig,
build_profile,
- load_profiles,
profile_config,
profile_config_type,
)
@@ -178,7 +177,6 @@ __all__ = [
"RandomRoutingProfileConfig",
"TranslateProfileConfig",
"build_profile",
- "load_profiles",
"profile_config",
"profile_config_type",
"AnthropicNativeBackend",
diff --git a/switchyard/cli/config/user_config.py b/switchyard/cli/config/user_config.py
index d7c13a40b..3d370c04e 100644
--- a/switchyard/cli/config/user_config.py
+++ b/switchyard/cli/config/user_config.py
@@ -126,7 +126,7 @@ class UserConfig:
"""Non-secret Switchyard defaults stored under the user config dir.
``routing_profiles`` is a parsed routing-profile YAML bundle saved by
- ``switchyard configure --routing-profiles PATH``. Stored as a parsed
+ ``switchyard --routing-profiles PATH configure``. Stored as a parsed
JSON object inline (not as a path — paths rot when files move); env
var references inside the bundle are preserved verbatim and
re-expanded on each load. Consumed by ``switchyard serve`` (and as a
diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py
index 50b5c4db7..82e14254e 100644
--- a/switchyard/cli/route_bundle.py
+++ b/switchyard/cli/route_bundle.py
@@ -499,7 +499,12 @@ def _parse_route_bundle_dict(raw: object) -> RouteBundle:
validation runs inside :func:`build_table_from_bundle` so callers
that construct a bundle programmatically still benefit.
"""
- bundle = _require_mapping(_expand_env(raw), "route bundle")
+ return _validate_route_bundle_dict(_expand_env(raw))
+
+
+def _validate_route_bundle_dict(raw: object) -> RouteBundle:
+ """Validate bundle structure without expanding environment references."""
+ bundle = _require_mapping(raw, "route bundle")
_validate_allowed_keys(bundle, frozenset({"defaults", "routes"}), "route bundle")
defaults = _optional_mapping(bundle.get("defaults", {}), "defaults")
_validate_allowed_keys(defaults, _TARGET_DEFAULT_KEYS, "defaults")
@@ -1221,8 +1226,8 @@ def _stage_router_switchyard(
allowed_keys=_STAGE_ROUTER_CLASSIFIER_KEYS,
where=f"{model_id}.classifier",
)
- # The deprecated bundle keeps the shared strong/weak tier keys (also used by
- # deterministic); map them onto StageRouterConfig's capable/efficient fields.
+ # The YAML schema shares strong/weak tier keys with deterministic routing;
+ # map them onto StageRouterConfig's capable/efficient fields.
resolved = _route_config(route, target_defaults, ("strong", "weak"))
resolved["capable"] = resolved.pop("strong")
resolved["efficient"] = resolved.pop("weak")
diff --git a/switchyard/cli/switchyard_cli.py b/switchyard/cli/switchyard_cli.py
index bbb5576d2..5a2d9e260 100644
--- a/switchyard/cli/switchyard_cli.py
+++ b/switchyard/cli/switchyard_cli.py
@@ -7,8 +7,7 @@
Exposed as the ``switchyard`` console script.
Subcommands:
- serve Serve a v2 profile config (--config) as model-keyed
- profiles.
+ serve Serve a routing-profile bundle.
launch claude Spawn Claude Code against a proxy. Single model
(``--model``) or full routing via ``--routing-profiles``.
Pair with ``--smoke`` to run a one-shot harness round-trip
@@ -26,12 +25,7 @@ Subcommands:
Examples::
- # v2 profile config (primary serve path): each profile id + target id
- # becomes a model on GET /v1/models, selectable with --model <id>.
- switchyard serve --config profiles.yaml --port 4000
-
- # Deprecated: --routing-profiles is a global flag
- # (use -- to separate from the subcommand)
+ # --routing-profiles is a global flag (use -- to separate it from the subcommand).
switchyard --routing-profiles routes.yaml -- serve --port 4000
switchyard --routing-profiles dev.yaml -- serve --port 4001
switchyard --routing-profiles profiles.yaml -- launch claude
@@ -49,10 +43,10 @@ Examples::
# Forwarding args to the launched tool (second -- after the subcommand)
switchyard --routing-profiles profiles.yaml -- launch claude -- --no-auto-approve
-Legacy routing policies that used to be top-level CLI verbs (``passthrough``,
+Routing policies that used to be top-level CLI verbs (``passthrough``,
``random-routing``, ``latency-service``) and launcher flags
(``--routing``, ``--weak-model``, ``--strong-probability``, ``--preset``)
-are expressed in deprecated routing-profile YAML files. ``serve`` and
+are expressed in routing-profile YAML files. ``serve`` and
launchers still parse the YAML into profile-backed runtimes.
"""
@@ -64,7 +58,7 @@ from collections.abc import Sequence
from dataclasses import dataclass
from importlib.metadata import PackageNotFoundError, version
from inspect import signature
-from typing import Any, cast
+from typing import Any
from switchyard.cli.command_utils import (
quiet_dependency_loggers as _quiet_dependency_loggers,
@@ -107,42 +101,11 @@ logger = logging.getLogger(__name__)
_DEFAULT_OPENROUTER_BASE_URL = DEFAULT_OPENROUTER_BASE_URL
_CANONICAL_INTAKE_ENABLE_FLAG = "--intake-enabled"
_DEPRECATED_INTAKE_ENABLE_FLAG = "--enable-intake"
-_DEPRECATED_ROUTING_PROFILES_FLAG = "--routing-profiles"
_ARGPARSE_ACTION_SUPPORTS_DEPRECATED = (
"deprecated" in signature(argparse.Action.__init__).parameters
)
-def _print_deprecation_warning(
- subject: str,
- details: tuple[str, ...] = (),
-) -> None:
- """Print a concise, readable CLI deprecation warning to stderr."""
- print(f"warning: {subject} is deprecated.", file=sys.stderr)
- for detail in details:
- print(f" {detail}", file=sys.stderr)
-
-
-def _warn_deprecated_routing_profiles() -> None:
- _print_deprecation_warning(
- _DEPRECATED_ROUTING_PROFILES_FLAG,
- details=(
- "Prefer v2 profile configs with `switchyard serve --config PATH`.",
- "Legacy route bundles still run for now, but this flag will be removed in a future release.",
- ),
- )
-
-
-def _warn_deprecated_saved_route_bundle() -> None:
- _print_deprecation_warning(
- "saved routing-profile bundle",
- details=(
- "Prefer v2 profile configs with `switchyard serve --config PATH`.",
- "Clear the saved bundle with `switchyard --routing-profiles '' -- configure`.",
- ),
- )
-
-
class _IntakeEnabledAction(argparse.Action):
"""Store the normalized Intake enable flag and warn on the deprecated alias."""
@@ -396,15 +359,11 @@ def _cmd_launch_openclaw(args: argparse.Namespace) -> None:
def _cmd_serve(args: argparse.Namespace) -> None:
- """Serve a v2 profile config or a legacy route bundle."""
+ """Serve a routing-profile bundle."""
from switchyard.cli.config.user_config import load_user_config
from switchyard.cli.route_bundle import build_route_bundle_table
routing_profiles = args.routing_profiles
- if args.config:
- _cmd_serve_profile_config(args)
- return
-
# Intake sink + local RL trace logging both attach as chain processors;
# combine them so a single serve invocation can run either or both.
intake_request, intake_response = _resolve_intake_processors(args)
@@ -429,14 +388,13 @@ def _cmd_serve(args: argparse.Namespace) -> None:
if not saved:
raise SystemExit(
"serve: no routing-profiles given. Pass --routing-profiles "
- "PATH or run `switchyard configure --routing-profiles PATH` "
+ "PATH or run `switchyard --routing-profiles PATH configure` "
"to save one."
)
# Saved bundles are stored as parsed dicts, so we can skip the
# YAML parse step and feed them straight into the dict-driven
# entrypoint. Env-var references inside the dict expand inside
# build_route_bundle_table on each run.
- _warn_deprecated_saved_route_bundle()
table = build_route_bundle_table(
saved,
pre_routing_request_processors=request_processors,
@@ -461,100 +419,6 @@ def _cmd_serve(args: argparse.Namespace) -> None:
build_and_serve(args, table, inbound_default="both", strategy_summary=strategy_summary)
-def _cmd_serve_profile_config(args: argparse.Namespace) -> None:
- """Serve a v2 profile config through the Python HTTP application."""
- if args.routing_profiles:
- raise SystemExit(
- "serve --config cannot be combined with --routing-profiles; "
- "use exactly one config surface."
- )
- if args.inbound is not None:
- raise SystemExit(
- "serve --config always exposes "
- "OpenAI, Anthropic, and Responses endpoints; omit --inbound."
- )
- if args.reload:
- raise SystemExit("serve --config does not support --reload.")
- if args.workers != 1:
- raise SystemExit("serve --config does not support --workers.")
- unsupported_intake = any((
- args.intake_enabled,
- args.intake_base_url,
- args.intake_workspace,
- args.intake_api_key,
- args.intake_target_url,
- ))
- if unsupported_intake:
- raise SystemExit("serve --config does not support Intake options yet.")
- if getattr(args, "enable_rl_logging", False):
- raise SystemExit(
- "serve --config does not support --enable-rl-logging. "
- "Use serve --routing-profiles for local RL trace logging."
- )
- if getattr(args, "routing_log_file", None):
- raise SystemExit(
- "serve --config does not support --routing-log-file. "
- "Use serve --routing-profiles for per-request routing logs."
- )
-
- table = _profile_config_route_table(args.config)
- logger.info(
- "Switchyard profile config loaded from %s",
- args.config,
- )
- build_and_serve(args, table, inbound_default="both")
-
-
-def _profile_config_route_table(config_path: str) -> Any:
- from switchyard.lib.profiles import PassthroughProfileConfig, ProfileSwitchyard
- from switchyard.lib.profiles.loader import load_profiles_and_targets
- from switchyard.lib.route_table import RouteTable
-
- profiles, targets = load_profiles_and_targets(config_path)
- table = RouteTable()
- for profile_id, profile in profiles.items():
- _register_profile_config_model(
- table,
- profile_id,
- ProfileSwitchyard(cast(Any, profile)),
- kind="profile",
- )
- for target_id, target in targets.items():
- target_switchyard = ProfileSwitchyard(PassthroughProfileConfig(target=target).build())
- _register_profile_config_model(
- table,
- target_id,
- target_switchyard,
- kind="target",
- )
- if target.model != target_id:
- _register_profile_config_model(
- table,
- target.model,
- target_switchyard,
- kind="target-model",
- )
- return table
-
-
-def _register_profile_config_model(
- table: Any,
- model_id: str,
- switchyard: Any,
- kind: str,
-) -> None:
- if model_id in table.registered_models():
- raise SystemExit(
- f"serve --config: duplicate public model id {model_id!r} while "
- "registering profile config routes."
- )
- table.register(
- model_id,
- switchyard,
- metadata={"switchyard": {"source": "profile-config", "kind": kind}},
- )
-
-
# ---------------------------------------------------------------------------
# Subcommand: verify {proxy,claude,codex}
# ---------------------------------------------------------------------------
@@ -847,7 +711,7 @@ def _build_parser() -> argparse.ArgumentParser:
default=None,
metavar="PATH",
help=(
- "Deprecated path to a routing-profiles YAML file. Applies to "
+ "Path to a routing-profiles YAML file. Applies to "
"serve, launch, and configure (saves it as the default). Separate "
"from the subcommand with -- for clarity: "
"switchyard --routing-profiles dev.yaml -- launch claude"
@@ -879,24 +743,14 @@ def _build_parser() -> argparse.ArgumentParser:
# -- serve --
serve = subparsers.add_parser(
"serve",
- help="Serve a v2 profile config (--config)",
+ help="Serve a routing-profile bundle",
description=(
- "Serve a Switchyard v2 profile config via serve --config: one "
- "YAML/JSON/TOML file of endpoints, targets, and profiles; each "
- "profile id and target id is exposed on GET /v1/models."
+ "Serve a routing-profile YAML bundle selected with the global "
+ "--routing-profiles option."
),
)
add_transport_args(serve)
_add_intake_args(serve)
- serve.add_argument(
- "--config",
- dest="config",
- default=None,
- metavar="PATH",
- help=(
- "Path to a Switchyard v2 profile config (YAML, JSON, or TOML)."
- ),
- )
serve.add_argument(
"--routing-log-file",
dest="routing_log_file",
@@ -1435,9 +1289,6 @@ def main() -> None:
pass
args = parser.parse_args(argv)
- if args.routing_profiles is not None:
- _warn_deprecated_routing_profiles()
-
if not hasattr(args, "func"):
parser.print_help()
raise SystemExit(1)
diff --git a/switchyard/lib/__init__.py b/switchyard/lib/__init__.py
index f4b76bf97..237c5767c 100644
--- a/switchyard/lib/__init__.py
+++ b/switchyard/lib/__init__.py
@@ -11,6 +11,6 @@ of the library:
- ``translation`` — pure format-conversion functions and typed translation engines
- ``processors`` — reusable request/response components used by profiles
- ``backends`` — LLM backend implementations (OpenAI, Anthropic, multi-tier routing)
-- ``profiles`` — profile-owned runtime/config abstractions for the flatter v2 shape
+- ``profiles`` — profile-owned runtime and programmatic config abstractions
- ``roles`` — backend role definitions and translation response aliases
"""
diff --git a/switchyard/lib/profiles/__init__.py b/switchyard/lib/profiles/__init__.py
index be52e5582..b4faa58bf 100644
--- a/switchyard/lib/profiles/__init__.py
+++ b/switchyard/lib/profiles/__init__.py
@@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Python profile abstractions matching the components-v2 design."""
+"""Python profile abstractions for programmatic routing profiles."""
from switchyard.lib.profiles.deterministic_routing_config import (
DeterministicRoutingConfig,
@@ -22,7 +22,6 @@ from switchyard.lib.profiles.header_routing import (
HeaderRoutingProfile,
)
from switchyard.lib.profiles.latency_service import LatencyServiceProfileConfig
-from switchyard.lib.profiles.loader import load_profiles
from switchyard.lib.profiles.noop import NoopProfile, NoopProfileConfig
from switchyard.lib.profiles.passthrough import PassthroughProfileConfig
from switchyard.lib.profiles.plan_execute import PlanExecuteProfileConfig
@@ -86,7 +85,6 @@ __all__ = [
"RandomRoutingProfileConfig",
"TranslateProfileConfig",
"build_profile",
- "load_profiles",
"profile_config",
"profile_config_type",
]
diff --git a/switchyard/lib/profiles/header_routing.py b/switchyard/lib/profiles/header_routing.py
index f470e360a..448017ed4 100644
--- a/switchyard/lib/profiles/header_routing.py
+++ b/switchyard/lib/profiles/header_routing.py
@@ -1,16 +1,9 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""A header-routing Python profile: pick a tier from a request header.
+"""A programmatic Python profile that selects a tier from a request header."""
-A small, real example of a Python-defined v2 profile. It exercises the pieces
-the v2 profile system enables: a typed config with ``profile_target`` fields, a
-profile-owned processed-request type, and delegation of the actual backend call
-through a passthrough profile per target. The architecture mirrors the Rust
-profiles in ``switchyard-components-v2``; only the authoring language differs.
-"""
-
-from dataclasses import dataclass, field
+from dataclasses import dataclass
from switchyard.lib.backends.llm_target import LlmTarget
from switchyard.lib.profiles.chain import ComponentChainProfile
@@ -29,12 +22,11 @@ class HeaderRoutingConfig:
"""Route to ``strong`` or ``weak`` based on a request header.
A header value of ``strong`` selects the strong target; anything else, or a
- missing header, selects ``weak``. Both targets are Rust-owned ``LlmTarget``s
- resolved by the shared config loader, exactly like a Rust profile's targets.
+ missing header, selects ``weak``.
"""
- strong: LlmTarget = field(metadata={"profile_target": True})
- weak: LlmTarget = field(metadata={"profile_target": True})
+ strong: LlmTarget
+ weak: LlmTarget
header: str = _DEFAULT_HEADER
def build(self) -> "HeaderRoutingProfile":
diff --git a/switchyard/lib/profiles/loader.py b/switchyard/lib/profiles/loader.py
deleted file mode 100644
index ed9b2a491..000000000
--- a/switchyard/lib/profiles/loader.py
+++ /dev/null
@@ -1,216 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Load one v2 profile config into runnable Rust- and Python-defined profiles.
-
-The YAML/JSON/TOML schema is shared. Rust remains the source of truth for file
-format detection, environment interpolation, endpoint inheritance, target
-resolution, and Rust-defined profile validation. Python only classifies profile
-types registered through :func:`profile_config` and builds those runtimes from
-Rust-parsed profile bodies.
-"""
-
-import importlib
-from dataclasses import fields, is_dataclass
-from pathlib import Path
-from typing import Any, cast
-
-from switchyard.lib.profiles.protocols import ProfileInput, ProfileRunner
-from switchyard.lib.profiles.subagent_override import SubagentOverrideProfile
-from switchyard.lib.profiles.table import (
- ProfileConfigError,
- build_profile,
- lookup_profile_config,
- registered_profile_config_types,
-)
-from switchyard_rust.components import LlmTarget
-from switchyard_rust.core import ChatResponse
-from switchyard_rust.profiles import (
- ProfileConfigPlan,
- parse_profile_config_path,
-)
-
-__all__ = ["load_profiles", "load_profiles_and_targets"]
-
-_PROFILE_TARGET = "profile_target"
-
-
-def load_profiles(path: str | Path) -> dict[str, ProfileRunner]:
- """Build every profile in ``path`` into a runnable profile keyed by profile ID.
-
- Rust-defined profiles (``passthrough``, ``random-routing``, ...) and
- Python-defined profiles registered through :func:`profile_config` are built
- through the same call, sharing one resolved set of targets. Each returned
- value exposes ``async run(input)`` (see :class:`ProfileRunner`).
-
- Raises :class:`ProfileConfigError` or ``switchyard_rust`` config errors with
- profile-level context when the config is invalid.
- """
- profiles, _targets = load_profiles_and_targets(path)
- return profiles
-
-
-def load_profiles_and_targets(
- path: str | Path,
-) -> tuple[dict[str, ProfileRunner], dict[str, LlmTarget]]:
- """Build profiles and return the resolved target map used by those profiles."""
- _register_shipped_python_profiles()
- document: Any = parse_profile_config_path(path)
- python_profiles = _python_profiles(document)
- plan = document.without_profiles(list(python_profiles)).resolve()
-
- built: dict[str, ProfileRunner] = {}
- for profile_id in plan.profile_ids():
- built[profile_id] = _RustProfileRunner(plan.build_profile(profile_id))
- for profile_id, (profile_type, body) in python_profiles.items():
- built[profile_id] = _build_python_profile(profile_id, profile_type, body, plan)
- for profile_id in list(built):
- built[profile_id] = _wrap_subagent_override(profile_id, built[profile_id], document, plan)
- return built, _targets(plan)
-
-
-def _wrap_subagent_override(
- profile_id: str,
- profile: ProfileRunner,
- document: Any,
- plan: ProfileConfigPlan,
-) -> ProfileRunner:
- """Wrap a built profile when its envelope names a ``subagent_target``.
-
- Rust profiles already apply :class:`SubagentOverrideProfile` internally at
- build time, so Python-level wrapping is skipped for them. Python-defined
- profiles receive the Python wrapper here, which adds sub-agent routing
- without Rust-level stats (stats for Python profiles remain in the
- Python stats pipeline).
-
- An unknown target reference is a startup configuration error.
- """
- target_id = document.profile_subagent_target(profile_id)
- if target_id is None:
- return profile
- # Rust profiles handle subagent routing internally; skip double-wrapping.
- if isinstance(profile, _RustProfileRunner):
- return profile
- target = plan.target(target_id)
- if target is None:
- raise ProfileConfigError(
- f"profile {profile_id}: subagent_target references unknown target {target_id!r}"
- )
- from switchyard.lib.profiles.passthrough import PassthroughProfileConfig
-
- return SubagentOverrideProfile(profile, PassthroughProfileConfig(target=target).build())
-
-
-class _RustProfileRunner:
- """Adapts an erased Rust ``Profile`` to the ``run(input)`` runner contract."""
-
- def __init__(self, profile: Any) -> None:
- self._profile = profile
-
- @property
- def profile_id(self) -> str:
- return cast(str, self._profile.profile_id)
-
- async def run(self, input: ProfileInput) -> ChatResponse:
- return cast(ChatResponse, await self._profile.run(input.request, input.metadata))
-
-
-def _build_python_profile(
- profile_id: str,
- profile_type: str,
- body: dict[str, Any],
- plan: ProfileConfigPlan,
-) -> ProfileRunner:
- """Construct one Python-defined profile, resolving its target references."""
- config_cls: Any = lookup_profile_config(profile_type)
- if not is_dataclass(config_cls):
- raise ProfileConfigError(
- f"profile {profile_id}: {config_cls.__qualname__} must be a dataclass"
- )
-
- field_names = {field.name for field in fields(config_cls)}
- unknown = sorted(set(body) - field_names - {"type"})
- if unknown:
- raise ProfileConfigError(
- f"profile {profile_id}: unknown field(s) {unknown} for profile type {profile_type!r}"
- )
-
- kwargs: dict[str, Any] = {}
- for field in fields(config_cls):
- if field.name not in body:
- continue # let the dataclass default apply
- value = body[field.name]
- if field.metadata.get(_PROFILE_TARGET):
- value = _resolve_targets(profile_id, value, plan)
- kwargs[field.name] = value
-
- # config_cls is a registered dataclass; bind it through Any so mypy does not
- # treat it as the (non-instantiable) DataclassInstance protocol.
- ctor: Any = config_cls
- try:
- config = ctor(**kwargs)
- except TypeError as exc: # missing required field, wrong arity, ...
- raise ProfileConfigError(f"profile {profile_id}: {exc}") from exc
- return cast(ProfileRunner, build_profile(config))
-
-
-def _register_shipped_python_profiles() -> None:
- """Import shipped Python profile modules so their config types register."""
- importlib.import_module("switchyard.lib.profiles.header_routing")
-
-
-def _python_profiles(
- document: Any,
-) -> dict[str, tuple[str, dict[str, Any]]]:
- python_types = set(registered_profile_config_types())
- profiles: dict[str, tuple[str, dict[str, Any]]] = {}
- for profile_id in document.profile_ids():
- profile_type = document.profile_type(profile_id)
- if profile_type is None or profile_type not in python_types:
- continue
- body = document.profile_body(profile_id)
- if not isinstance(body, dict):
- raise ProfileConfigError(f"profile {profile_id}: profile body must be a mapping")
- profiles[profile_id] = (profile_type, body)
- return profiles
-
-
-def _targets(plan: ProfileConfigPlan) -> dict[str, LlmTarget]:
- targets: dict[str, LlmTarget] = {}
- for target_id in plan.target_ids():
- target = plan.target(target_id)
- if target is not None:
- targets[target_id] = target
- return targets
-
-
-def _resolve_targets(
- profile_id: str, value: Any, plan: ProfileConfigPlan
-) -> LlmTarget | list[LlmTarget] | None:
- """Resolve a ``profile_target`` field's id reference(s) into ``LlmTarget``s.
-
- The value shape selects the arity, mirroring Rust ``#[profile_target]``
- support for single, optional, and list-valued target fields.
- """
- if value is None:
- return None
- if isinstance(value, str):
- return _require_target(profile_id, value, plan)
- if isinstance(value, list):
- return [_require_target(profile_id, item, plan) for item in value]
- raise ProfileConfigError(
- f"profile {profile_id}: target reference must be a target id string or "
- f"list of ids, got {type(value).__name__}"
- )
-
-
-def _require_target(profile_id: str, target_id: Any, plan: ProfileConfigPlan) -> LlmTarget:
- if not isinstance(target_id, str):
- raise ProfileConfigError(
- f"profile {profile_id}: target reference must be a string, got "
- f"{type(target_id).__name__}"
- )
- target = plan.target(target_id)
- if target is None:
- raise ProfileConfigError(f"profile {profile_id}: references unknown target {target_id!r}")
- return target
diff --git a/switchyard/lib/profiles/protocols.py b/switchyard/lib/profiles/protocols.py
index 8d6c5a644..78c4984a2 100644
--- a/switchyard/lib/profiles/protocols.py
+++ b/switchyard/lib/profiles/protocols.py
@@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Python profile protocols matching the components-v2 runtime shape."""
+"""Python-owned profile request values and runtime protocols."""
from typing import Any, ClassVar, Protocol, TypeVar, runtime_checkable
@@ -69,11 +69,11 @@ class ProfileLifecycle(Protocol):
@runtime_checkable
class ContextAwareProfile(Profile[ProcessedRequestT], Protocol[ProcessedRequestT]):
- """Optional legacy bridge surface for profiles that need ``ProxyContext``.
+ """Optional component bridge surface for profiles that need ``ProxyContext``.
- Pure v2 profiles can implement only ``run`` / ``process`` / ``rprocess``.
- Compatibility profiles that still compose existing Python components also
- expose these methods so endpoint adapters can reuse the caller's context.
+ Profiles can implement only ``run`` / ``process`` / ``rprocess``. Profiles
+ that compose existing Python components also expose these methods so
+ endpoint adapters can reuse the caller's context.
"""
async def process_with_context(
diff --git a/switchyard/lib/profiles/stage_router_config.py b/switchyard/lib/profiles/stage_router_config.py
index ae199d158..ed90e321d 100644
--- a/switchyard/lib/profiles/stage_router_config.py
+++ b/switchyard/lib/profiles/stage_router_config.py
@@ -49,7 +49,7 @@ class StageRouterConfig(BaseModel):
#: Scorer confidence in ``[0, 1]`` below which the picker consults the
#: classifier (if configured) or returns its default tier. ``0.0`` forces
#: pure-deterministic routing; ``1.0`` forces every turn through the
- #: classifier (equivalent to the legacy ``coding_agent`` profile).
+ #: classifier (equivalent to classifier-only ``coding_agent`` routing).
confidence_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
#: Sliding-window size for the Rust signal extractor's ``recent_*``
#: counts (``recent_write_count``, ``recent_edit_count``, etc.).
diff --git a/switchyard/lib/profiles/subagent_override.py b/switchyard/lib/profiles/subagent_override.py
deleted file mode 100644
index 794932693..000000000
--- a/switchyard/lib/profiles/subagent_override.py
+++ /dev/null
@@ -1,103 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Sub-agent override wrapper routing delegated worker requests to a fixed target."""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import Any
-
-from switchyard.lib.profiles.protocols import (
- ContextAwareProfile,
- ProfileLifecycle,
- ProfileRunner,
-)
-from switchyard.lib.proxy_context import ProxyContext
-from switchyard_rust.core import ChatResponse
-from switchyard_rust.profiles import ProfileInput, is_subagent_request
-
-
-@dataclass(slots=True)
-class SubagentProcessedRequest:
- """Pairs the branch selected for one request with its request-side state."""
-
- branch: Any
- processed: Any
-
-
-class SubagentOverrideProfile:
- """Route recognized sub-agent requests to a fixed override branch.
-
- Wraps a built profile without changing its behavior for normal traffic. A
- request whose headers carry a delegated sub-agent signal runs through the
- override branch (a passthrough to the profile's configured
- ``subagent_target``); every other request runs the wrapped profile
- unchanged. The override never rewrites the request or response, and an
- override-branch failure surfaces as a normal target error — it is not
- re-routed through the wrapped profile.
- """
-
- def __init__(self, inner: ProfileRunner, override: ProfileRunner) -> None:
- """Wrap ``inner``, sending sub-agent requests to ``override`` instead."""
- self._inner = inner
- self._override = override
-
- def iter_components(self) -> list[object]:
- """Return lifecycle components of both branches in startup order."""
- return [*_components(self._inner), *_components(self._override)]
-
- def _branch(self, input: ProfileInput) -> Any:
- return self._override if is_subagent_request(input.metadata.headers) else self._inner
-
- async def run(self, input: ProfileInput) -> ChatResponse:
- """Execute the branch selected by the request's sub-agent signal."""
- response = await self._branch(input).run(input)
- return response # type: ignore[no-any-return]
-
- async def run_with_context(
- self,
- input: ProfileInput,
- ctx: ProxyContext,
- ) -> ChatResponse:
- """Execute the selected branch, preserving the caller-owned context."""
- branch = self._branch(input)
- if isinstance(branch, ContextAwareProfile):
- return await branch.run_with_context(input, ctx)
- response = await branch.run(input)
- return response # type: ignore[no-any-return]
-
- async def process(self, input: ProfileInput) -> SubagentProcessedRequest:
- """Run the selected branch's request side, remembering the branch."""
- branch = self._branch(input)
- return SubagentProcessedRequest(branch, await branch.process(input))
-
- async def process_with_context(
- self,
- input: ProfileInput,
- ctx: ProxyContext,
- ) -> SubagentProcessedRequest:
- """Run the selected branch's request side with the caller's context."""
- branch = self._branch(input)
- if isinstance(branch, ContextAwareProfile):
- return SubagentProcessedRequest(branch, await branch.process_with_context(input, ctx))
- return SubagentProcessedRequest(branch, await branch.process(input))
-
- async def rprocess(
- self,
- processed: SubagentProcessedRequest,
- response: ChatResponse,
- ) -> ChatResponse:
- """Run the response side of the branch that processed the request."""
- result = await processed.branch.rprocess(processed.processed, response)
- return result # type: ignore[no-any-return]
-
-
-def _components(profile: object) -> list[object]:
- """Return a branch's lifecycle components, mirroring ``ProfileSwitchyard``."""
- if isinstance(profile, ProfileLifecycle):
- return profile.iter_components()
- return [profile]
-
-
-__all__ = ["SubagentOverrideProfile", "SubagentProcessedRequest"]
diff --git a/switchyard_rust/core.py b/switchyard_rust/core.py
index a7859405e..59cbd0a5a 100644
--- a/switchyard_rust/core.py
+++ b/switchyard_rust/core.py
@@ -9,7 +9,6 @@ import importlib
import importlib.metadata
import os
from collections.abc import ItemsView, Iterable, Iterator, KeysView, Mapping, ValuesView
-from os import PathLike
from typing import TYPE_CHECKING, Any, ClassVar, Protocol, TypeAlias, cast
JsonScalar: TypeAlias = bool | int | float | str | None
@@ -177,47 +176,6 @@ class _Switchyard(Protocol):
) -> Any: ...
-class _ProfileConfigDocument(Protocol):
- """Rust-owned profile config document exposed through PyO3."""
-
- def resolve(self) -> _ProfileConfigPlan: ...
-
-
-class _ProfileConfigPlan(Protocol):
- """Rust-owned resolved profile config plan exposed through PyO3."""
-
- def profile_ids(self) -> list[str]: ...
- def target_ids(self) -> list[str]: ...
- def profile_type(self, profile_id: str) -> str | None: ...
- def target(self, target_id: str) -> object | None: ...
- def build_profile(self, profile_id: str) -> _Profile: ...
- def build_profiles(self) -> dict[str, _Profile]: ...
-
-
-class _Profile(Protocol):
- """Rust-owned profile runtime exposed through PyO3."""
-
- profile_id: str
-
- async def run(self, request: _ChatRequest) -> _ChatResponse: ...
-
-
-class _ParseProfileConfigStr(Protocol):
- def __call__(
- self,
- input: str,
- format: str = "yaml",
- ) -> _ProfileConfigDocument: ...
-
-
-class _ParseProfileConfigPath(Protocol):
- def __call__(self, path: str | PathLike[str]) -> _ProfileConfigDocument: ...
-
-
-class _LoadProfileConfig(Protocol):
- def __call__(self, path: str | PathLike[str]) -> _ProfileConfigPlan: ...
-
-
class _NativeModule(Protocol):
ChatRequest: type[_ChatRequest]
ChatRequestType: type[_ChatRequestType]
@@ -239,15 +197,6 @@ class _NativeModule(Protocol):
SwitchyardUpstreamError: type[RuntimeError]
SwitchyardContextWindowExceededError: type[RuntimeError]
SwitchyardContextPoolExhaustedError: type[RuntimeError]
- ProfileConfigDocument: type[_ProfileConfigDocument]
- ProfileConfigPlan: type[_ProfileConfigPlan]
- Profile: type[_Profile]
- load_profile_config: _LoadProfileConfig
- parse_profile_config_path: _ParseProfileConfigPath
- parse_profile_config_str: _ParseProfileConfigStr
- # Shared profile input bindings re-exported through `switchyard_rust.profiles`.
- ProfileRequestMetadata: type[Any]
- ProfileInput: type[Any]
# Session-affinity primitives.
SessionCache: type[Any]
session_key_from_body: Any
diff --git a/switchyard_rust/profiles.py b/switchyard_rust/profiles.py
index 7da35c46c..aa1dac500 100644
--- a/switchyard_rust/profiles.py
+++ b/switchyard_rust/profiles.py
@@ -1,37 +1,106 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
-"""Components-v2 profile config loading and shared request input bindings.
-
-Exposes two Rust-owned surfaces:
-
-* the erased serving API (``ProfileConfigDocument``, ``ProfileConfigPlan``,
- ``Profile``) for loading config files and running any profile through
- ``run``; and
-* shared request input classes (``ProfileInput`` and
- ``ProfileRequestMetadata``) used by both Rust and Python profile runtimes.
-
-Names are resolved lazily from the native extension on first access.
-"""
-
-from switchyard_rust.core import _load_native
-
-_PROFILE_EXPORTS = (
- # Erased serving surface.
- "ProfileConfigDocument",
- "ProfileConfigPlan",
- "Profile",
- "load_profile_config",
- "parse_profile_config_path",
- "parse_profile_config_str",
- "ProfileInput",
- "ProfileRequestMetadata",
- "is_subagent_request",
-)
-__all__ = _PROFILE_EXPORTS
-
-
-def __getattr__(name: str) -> object:
- if name in _PROFILE_EXPORTS:
- return getattr(_load_native(), name)
- raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+"""Python-owned request values shared by profile implementations."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from switchyard_rust.core import request_type_enum, request_type_value
+
+if TYPE_CHECKING:
+ from collections.abc import Mapping
+
+ from switchyard_rust.core import ChatRequest, ChatRequestType
+
+
+@dataclass(frozen=True, slots=True, init=False)
+class ProfileRequestMetadata:
+ """Request metadata available to Python profile runtimes."""
+
+ request_id: str | None
+ inbound_format: ChatRequestType | None
+ headers: dict[str, list[str]]
+
+ def __init__(
+ self,
+ request_id: str | None = None,
+ inbound_format: ChatRequestType | str | None = None,
+ headers: Mapping[str, str | list[str]] | None = None,
+ ) -> None:
+ if request_id is not None and not request_id.strip():
+ raise ValueError("invalid request_id: RequestId must not be empty")
+ object.__setattr__(self, "request_id", request_id)
+ object.__setattr__(
+ self,
+ "inbound_format",
+ request_type_enum(inbound_format) if inbound_format is not None else None,
+ )
+ object.__setattr__(self, "headers", _normalize_headers(headers))
+
+ @classmethod
+ def from_headers(
+ cls,
+ headers: Mapping[str, str | list[str]],
+ inbound_format: ChatRequestType | str | None = None,
+ ) -> ProfileRequestMetadata:
+ """Build metadata from headers and infer the request ID when present."""
+ normalized = _normalize_headers(headers)
+ request_ids = normalized.get("x-request-id", [])
+ return cls(
+ request_id=request_ids[0] if request_ids else None,
+ inbound_format=inbound_format,
+ headers=normalized,
+ )
+
+ def to_dict(self) -> dict[str, object]:
+ """Return a plain dictionary representation."""
+ return {
+ "request_id": self.request_id,
+ "inbound_format": (
+ request_type_value(self.inbound_format)
+ if self.inbound_format is not None
+ else None
+ ),
+ "headers": {name: list(values) for name, values in self.headers.items()},
+ }
+
+
+@dataclass(frozen=True, slots=True, init=False)
+class ProfileInput:
+ """Provider request and metadata passed to a Python profile."""
+
+ request: ChatRequest
+ metadata: ProfileRequestMetadata
+
+ def __init__(
+ self,
+ request: ChatRequest,
+ metadata: ProfileRequestMetadata | None = None,
+ ) -> None:
+ object.__setattr__(self, "request", request)
+ object.__setattr__(self, "metadata", metadata or ProfileRequestMetadata())
+
+
+def _normalize_headers(
+ headers: Mapping[str, str | list[str]] | None,
+) -> dict[str, list[str]]:
+ normalized: dict[str, list[str]] = {}
+ for name, value in (headers or {}).items():
+ if not isinstance(name, str):
+ raise TypeError("ProfileRequestMetadata header names must be strings")
+ if isinstance(value, str):
+ values = [value]
+ elif isinstance(value, list) and all(isinstance(item, str) for item in value):
+ values = list(value)
+ else:
+ raise TypeError(
+ "ProfileRequestMetadata headers must map strings to strings or lists of strings"
+ )
+ normalized.setdefault(name.lower(), []).extend(values)
+ return normalized
+
+
+__all__ = ["ProfileInput", "ProfileRequestMetadata"]
diff --git a/switchyard_rust/profiles.pyi b/switchyard_rust/profiles.pyi
index 7c00a0d61..984d7568a 100644
--- a/switchyard_rust/profiles.pyi
+++ b/switchyard_rust/profiles.pyi
@@ -2,41 +2,8 @@
# SPDX-License-Identifier: Apache-2.0
from collections.abc import Mapping
-from os import PathLike
-from typing import Literal
-from switchyard_rust.components import LlmTarget
-from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse
-
-ProfileConfigFormat = Literal["yaml", "yml", "json", "toml"]
-
-# --- Erased serving surface ---------------------------------------------------
-
-class ProfileConfigDocument:
- def resolve(self) -> ProfileConfigPlan: ...
-
-class ProfileConfigPlan:
- def profile_ids(self) -> list[str]: ...
- def target_ids(self) -> list[str]: ...
- def profile_type(self, profile_id: str) -> str | None: ...
- def target(self, target_id: str) -> LlmTarget | None: ...
- def build_profile(self, profile_id: str) -> Profile: ...
- def build_profiles(self) -> dict[str, Profile]: ...
-
-class Profile:
- profile_id: str
-
- async def run(self, request: ChatRequest) -> ChatResponse: ...
-
-def parse_profile_config_str(
- input: str,
- format: ProfileConfigFormat = "yaml",
-) -> ProfileConfigDocument: ...
-def parse_profile_config_path(path: str | PathLike[str]) -> ProfileConfigDocument: ...
-def load_profile_config(path: str | PathLike[str]) -> ProfileConfigPlan: ...
-def is_subagent_request(headers: Mapping[str, str | list[str]]) -> bool: ...
-
-# --- Metadata for direct typed profile calls ----------------------------------
+from switchyard_rust.core import ChatRequest, ChatRequestType
class ProfileInput:
request: ChatRequest
@@ -57,12 +24,12 @@ class ProfileRequestMetadata:
self,
request_id: str | None = None,
inbound_format: ChatRequestType | str | None = None,
- headers: dict[str, str | list[str]] | None = None,
+ headers: Mapping[str, str | list[str]] | None = None,
) -> None: ...
@classmethod
def from_headers(
cls,
- headers: dict[str, str | list[str]],
+ headers: Mapping[str, str | list[str]],
inbound_format: ChatRequestType | str | None = None,
) -> ProfileRequestMetadata: ...
def to_dict(self) -> dict[str, object]: ...
diff --git a/tests/readme/test_readme.py b/tests/readme/test_readme.py
index d39c7de84..6af37699c 100644
--- a/tests/readme/test_readme.py
+++ b/tests/readme/test_readme.py
@@ -7,8 +7,7 @@ Companion to ``tests/getting_started/``. Three guards:
* the "Use as a Python library" snippet executes (via ``--markdown-docs`` +
the passthrough→noop fixture in ``conftest.py``);
-* README route examples validate against the route-bundle schema;
-* canonical routing pages still build stage_router and LLM-routing profile examples;
+* README and routing-guide examples validate against the route-bundle schema;
* every CLI subcommand / flag the README names still exists.
"""
@@ -23,14 +22,14 @@ from markdown_it import MarkdownIt
from switchyard.cli import route_bundle as rb
from switchyard.cli.switchyard_cli import _build_parser
-from switchyard_rust.profiles import parse_profile_config_str
REPO_ROOT = Path(__file__).resolve().parents[2]
README_PATH = REPO_ROOT / "README.md"
ROUTING_DOC_PATHS = (
- REPO_ROOT / "docs" / "routing_algorithms" / "stage_router_routing.md",
- REPO_ROOT / "docs" / "routing_algorithms" / "llm_classifier_routing.md",
REPO_ROOT / "docs" / "routing_algorithms" / "overview.md",
+ REPO_ROOT / "docs" / "routing_algorithms" / "random_routing.md",
+ REPO_ROOT / "docs" / "routing_algorithms" / "llm_classifier_routing.md",
+ REPO_ROOT / "docs" / "routing_algorithms" / "stage_router_routing.md",
)
@@ -69,72 +68,67 @@ def test_python_snippet_tripwire(readme_text: str) -> None:
)
-def test_all_yaml_route_blocks_in_readme_validate_against_the_schema(
- readme_text: str,
-) -> None:
- # Schema/key validation, not a full chain build: building README's routes
+def _validate_route_blocks(text: str, source: Path) -> int:
+ # Schema/key validation, not a full chain build: building documented routes
# is NOT hermetic — `passthrough` with `discover: true` does a live catalog
# fetch and `latency_service` polls. The
# schema layer (route type + per-type key allowlist) is what we can check
# offline, and it catches the likeliest drift: a renamed `type:` or a key
# that no longer exists on that type.
- blocks = _code_blocks(readme_text, "yaml")
- assert blocks, "README unexpectedly has no yaml blocks"
-
+ blocks = _code_blocks(text, "yaml")
validated_routes = 0
for idx, block in enumerate(blocks):
payload = pyyaml.safe_load(block)
if not isinstance(payload, dict) or "routes" not in payload:
continue
- for model_id, route_raw in payload["routes"].items():
+ try:
+ bundle = rb._validate_route_bundle_dict(payload)
+ except rb.RouteBundleConfigError as exc:
+ raise AssertionError(
+ f"YAML block {idx} in {source} failed bundle schema "
+ f"validation: {exc}\n\nBlock:\n{block}"
+ ) from exc
+ for model_id, route_raw in bundle.routes.items():
route = rb._normalize_route(model_id, route_raw)
try:
route_type = rb._route_type(model_id, route)
rb._validate_route_keys(model_id, route, route_type)
except rb.RouteBundleConfigError as exc:
raise AssertionError(
- f"YAML block {idx} route {model_id!r} in README.md failed "
+ f"YAML block {idx} route {model_id!r} in {source} failed "
f"schema validation: {exc}\n\nBlock:\n{block}"
) from exc
validated_routes += 1
+ return validated_routes
- assert validated_routes, "no README yaml block parsed as a route bundle"
+
+def test_route_block_validation_rejects_invalid_bundle_defaults() -> None:
+ text = """```yaml
+defaults:
+ unsupported: true
+routes:
+ noop:
+ type: noop
+```
+"""
+ with pytest.raises(AssertionError, match="unknown key.*defaults: unsupported"):
+ _validate_route_blocks(text, Path("example.md"))
-def test_stage_router_and_llm_routing_yaml_blocks_in_canonical_docs_build(
- monkeypatch: pytest.MonkeyPatch,
+def test_all_yaml_route_blocks_in_readme_validate_against_the_schema(
+ readme_text: str,
) -> None:
- monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-readme")
-
- built_profiles = 0
- for path in ROUTING_DOC_PATHS:
- for idx, block in enumerate(_code_blocks(path.read_text(), "yaml")):
- payload = pyyaml.safe_load(block)
- if not isinstance(payload, dict) or not {
- "endpoints",
- "targets",
- "profiles",
- }.issubset(payload):
- continue
- relevant_profile_ids = [
- profile_id
- for profile_id, profile in payload["profiles"].items()
- if profile.get("type") in {"stage_router", "llm-routing"}
- ]
- if not relevant_profile_ids:
- continue
- try:
- plan = parse_profile_config_str(block).resolve()
- for profile_id in relevant_profile_ids:
- plan.build_profile(profile_id)
- except Exception as exc:
- raise AssertionError(
- f"YAML block {idx} profiles {relevant_profile_ids!r} in {path} "
- f"failed profile-config build: {exc}\n\nBlock:\n{block}"
- ) from exc
- built_profiles += len(relevant_profile_ids)
+ assert _validate_route_blocks(readme_text, README_PATH), (
+ "no README yaml block parsed as a route bundle"
+ )
+
+
+def test_canonical_routing_docs_validate_against_the_route_bundle_schema() -> None:
+ validated_routes = sum(
+ _validate_route_blocks(path.read_text(), path) for path in ROUTING_DOC_PATHS
+ )
- assert built_profiles >= 2
+ assert validated_routes >= len(ROUTING_DOC_PATHS)
def _subparsers(parser: argparse.ArgumentParser) -> dict[str, argparse.ArgumentParser]:
diff --git a/tests/test_cli_reference_docs.py b/tests/test_cli_reference_docs.py
index 46eebdeb7..119e65bc9 100644
--- a/tests/test_cli_reference_docs.py
+++ b/tests/test_cli_reference_docs.py
@@ -99,15 +99,15 @@ def test_cli_reference_omits_unsupported_skill_distillation_flags() -> None:
)
-def test_cli_reference_serve_overview_points_to_config_path() -> None:
+def test_cli_reference_serve_overview_points_to_route_bundle() -> None:
serve_row = next(
line
for line in CLI_REFERENCE.read_text().splitlines()
if line.startswith("| [`serve`](#switchyard-serve)")
)
- assert "`serve --config`" in serve_row
- assert "--routing-profiles" not in serve_row
+ assert "--routing-profiles" in serve_row
+ assert "`serve --config`" not in serve_row
assert "inbound format" not in serve_row.lower()
diff --git a/tests/test_launch_claude.py b/tests/test_launch_claude.py
index 3acc96792..f9b07fbeb 100644
--- a/tests/test_launch_claude.py
+++ b/tests/test_launch_claude.py
@@ -1195,11 +1195,11 @@ class TestServeRoutingProfilesFallback:
args = parser.parse_args(["serve", "--port", "4000"])
with pytest.raises(SystemExit) as excinfo:
_cmd_serve(args)
- assert "routing-profiles" in str(excinfo.value)
+ assert "switchyard --routing-profiles PATH configure" in str(excinfo.value)
class TestConfigurePersistsRoutingProfiles:
- """`switchyard configure --routing-profiles PATH` parses + snapshots the bundle."""
+ """`switchyard --routing-profiles PATH configure` snapshots the bundle."""
def test_cli_path_persists_parsed_bundle(self, monkeypatch, tmp_path):
from switchyard.cli.config.user_config import load_user_config
diff --git a/tests/test_profile_migration.py b/tests/test_profile_migration.py
deleted file mode 100644
index 58446567c..000000000
--- a/tests/test_profile_migration.py
+++ /dev/null
@@ -1,535 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Migration tests for replacing legacy serving paths with components-v2 profiles."""
-
-import errno
-import json
-import threading
-from collections.abc import Iterator
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from typing import Any
-
-import pytest
-from fastapi.testclient import TestClient
-from pytest_mock import MockerFixture
-
-from switchyard.cli.route_bundle import build_route_bundle_table
-from switchyard.server.switchyard_app import build_switchyard_app
-from switchyard_rust.core import ChatRequest, SwitchyardUpstreamError
-from switchyard_rust.profiles import Profile, parse_profile_config_str
-
-
-class _OpenAICompatStub:
- def __init__(self) -> None:
- self._server: ThreadingHTTPServer | None = None
- self._thread: threading.Thread | None = None
- self._lock = threading.Lock()
- self._requests: list[dict[str, Any]] = []
- self._responses: list[tuple[int, dict[str, object]]] = []
-
- def __enter__(self) -> "_OpenAICompatStub":
- owner = self
-
- class Handler(BaseHTTPRequestHandler):
- protocol_version = "HTTP/1.1"
-
- def do_POST(self) -> None:
- length = int(self.headers.get("content-length", "0"))
- raw = self.rfile.read(length)
- body = json.loads(raw.decode("utf-8")) if raw else {}
- with owner._lock:
- owner._requests.append({"path": self.path, "body": body})
- if owner._responses:
- status, payload = owner._responses.pop(0)
- else:
- status = 200
- payload = _completion_payload(
- str(body.get("model", "missing-model")), self.path, "ok"
- )
-
- content = json.dumps(payload).encode("utf-8")
- self.send_response(status)
- self.send_header("content-type", "application/json")
- self.send_header("content-length", str(len(content)))
- self.send_header("connection", "close")
- self.end_headers()
- self.wfile.write(content)
-
- def log_message(self, _format: str, _line=None, _status=None, _size=None) -> None:
- return None
-
- try:
- self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
- except OSError as exc:
- if exc.errno in {errno.EACCES, errno.EPERM, errno.EADDRNOTAVAIL}:
- pytest.skip(f"loopback socket binding is unavailable in this sandbox: {exc}")
- raise
- self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
- self._thread.start()
- return self
-
- def __exit__(self, _exc_type: object, _exc: object, _traceback: object) -> None:
- if self._server is not None:
- self._server.shutdown()
- self._server.server_close()
- if self._thread is not None:
- self._thread.join(timeout=5)
-
- @property
- def base_url(self) -> str:
- if self._server is None:
- raise RuntimeError("stub server is not running")
- host, port = self._server.server_address
- return f"http://{host}:{port}"
-
- @property
- def requests(self) -> list[dict[str, Any]]:
- with self._lock:
- return list(self._requests)
-
- def respond_json(self, status: int, payload: dict[str, object]) -> None:
- with self._lock:
- self._responses.append((status, payload))
-
-
-def openai_stub() -> Iterator[_OpenAICompatStub]:
- with _OpenAICompatStub() as stub:
- yield stub
-
-
[email protected](autouse=True)
-def _disable_catalog_discovery(mocker: MockerFixture) -> None:
- """Keep legacy route-bundle tests offline and focused on configured routes."""
- mocker.patch(
- "switchyard.cli.route_bundle.fetch_model_ids",
- return_value=[],
- )
-
-
-def _completion_payload(model: str, path: str, content: str) -> dict[str, object]:
- return {
- "id": "chatcmpl-profile-migration",
- "object": "chat.completion",
- "created": 1700000000,
- "model": model,
- "mock_path": path,
- "choices": [
- {
- "index": 0,
- "message": {"role": "assistant", "content": content},
- "finish_reason": "stop",
- }
- ],
- "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
- }
-
-
-def _tool_call_payload(
- model: str,
- path: str,
- tool_name: str,
- arguments: dict[str, object],
-) -> dict[str, object]:
- return {
- "id": "chatcmpl-profile-migration",
- "object": "chat.completion",
- "created": 1700000000,
- "model": model,
- "mock_path": path,
- "choices": [
- {
- "index": 0,
- "message": {
- "role": "assistant",
- "tool_calls": [
- {
- "id": "call_route",
- "type": "function",
- "function": {
- "name": tool_name,
- "arguments": json.dumps(arguments),
- },
- }
- ],
- },
- "finish_reason": "tool_calls",
- }
- ],
- "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
- }
-
-
-def _chat_payload(model: str) -> dict[str, object]:
- return {"model": model, "messages": [{"role": "user", "content": "hello"}]}
-
-
-def _legacy_app(route_bundle: dict[str, object]) -> TestClient:
- table = build_route_bundle_table(route_bundle)
- return TestClient(build_switchyard_app(table), raise_server_exceptions=False)
-
-
-def _profile_runner(config: str, profile_id: str) -> Profile:
- return parse_profile_config_str(config).resolve().build_profile(profile_id)
-
-
-def _passthrough_profile_config(base_url: str) -> str:
- return f"""
-targets:
- direct:
- model: provider/direct
- format: openai
- base_url: "{base_url}/v2-direct/v1"
- api_key: test-key
-profiles:
- direct-profile:
- type: passthrough
- target: direct
-"""
-
-
-def _random_profile_config(base_url: str, strong_probability: float) -> str:
- return f"""
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: "{base_url}/v2-random/v1"
- api_key: test-key
- weak:
- model: provider/weak
- format: openai
- base_url: "{base_url}/v2-random/v1"
- api_key: test-key
-profiles:
- random-profile:
- type: random-routing
- strong: strong
- weak: weak
- strong_probability: {strong_probability}
- rng_seed: 7
-"""
-
-
-def _latency_profile_config(base_url: str, upstream_path: str = "v2-latency") -> str:
- return f"""
-targets:
- latency:
- model: provider/latency
- format: openai
- base_url: "{base_url}/{upstream_path}/v1"
- api_key: test-key
-profiles:
- latency-profile:
- type: latency-service
- latency_service_url: "http://latency.invalid"
- targets: [latency]
- max_retries: 0
-"""
-
-
-def _llm_routing_profile_config(base_url: str) -> str:
- return f"""
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: "{base_url}/v2-llm-routing/v1"
- api_key: test-key
- weak:
- model: provider/weak
- format: openai
- base_url: "{base_url}/v2-llm-routing/v1"
- api_key: test-key
- classifier:
- model: provider/classifier
- format: openai
- base_url: "{base_url}/v2-llm-routing-classifier/v1"
- api_key: test-key
-profiles:
- llm-profile:
- type: llm-routing
- strong: strong
- weak: weak
- classifier: classifier
- profile_name: coding_agent
-"""
-
-
-def _stage_router_profile_config(base_url: str) -> str:
- return f"""
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: "{base_url}/v2-stage_router/v1"
- api_key: test-key
- weak:
- model: provider/weak
- format: openai
- base_url: "{base_url}/v2-stage_router/v1"
- api_key: test-key
- classifier:
- model: provider/classifier
- format: openai
- base_url: "{base_url}/v2-stage_router-classifier/v1"
- api_key: test-key
-profiles:
- stage_router-profile:
- type: stage_router
- capable: strong
- efficient: weak
- fallback_target_on_evict: strong
- picker: capable_first
- confidence_threshold: 0.7
- classifier:
- model: provider/classifier
- api_key: test-key
- base_url: "{base_url}/v2-stage_router-classifier/v1"
-"""
-
-
-async def test_passthrough_profile_matches_legacy_single_target_serving(
- openai_stub: _OpenAICompatStub,
-) -> None:
- legacy = _legacy_app(
- {
- "routes": {
- "provider/direct": {
- "type": "model",
- "target": {
- "model": "provider/direct",
- "format": "openai",
- "base_url": f"{openai_stub.base_url}/legacy-direct/v1",
- "api_key": "test-key",
- },
- }
- }
- }
- )
- profile = _profile_runner(
- _passthrough_profile_config(openai_stub.base_url),
- "direct-profile",
- )
-
- legacy_response = legacy.post(
- "/v1/chat/completions",
- json=_chat_payload("provider/direct"),
- )
- profile_response = await profile.run(
- ChatRequest.openai_chat(_chat_payload("direct-profile")),
- )
-
- assert legacy_response.status_code == 200, legacy_response.text
- assert legacy_response.json()["choices"][0]["message"]["content"] == "ok"
- assert profile_response.body["choices"][0]["message"]["content"] == "ok"
- assert [call["body"]["model"] for call in openai_stub.requests] == [
- "provider/direct",
- "provider/direct",
- ]
- assert [call["path"] for call in openai_stub.requests] == [
- "/legacy-direct/v1/chat/completions",
- "/v2-direct/v1/chat/completions",
- ]
-
-
- ("strong_probability", "expected_model"),
- [(1.0, "provider/strong"), (0.0, "provider/weak")],
-)
-async def test_random_routing_profile_matches_legacy_strong_and_weak_selection(
- openai_stub: _OpenAICompatStub,
- strong_probability: float,
- expected_model: str,
-) -> None:
- legacy = _legacy_app(
- {
- "routes": {
- "legacy-random": {
- "type": "random-routing",
- "strong": {
- "model": "provider/strong",
- "format": "openai",
- "base_url": f"{openai_stub.base_url}/legacy-random/v1",
- "api_key": "test-key",
- },
- "weak": {
- "model": "provider/weak",
- "format": "openai",
- "base_url": f"{openai_stub.base_url}/legacy-random/v1",
- "api_key": "test-key",
- },
- "fallback_target_on_evict": "strong",
- "strong_probability": strong_probability,
- "rng_seed": 7,
- }
- }
- }
- )
- profile = _profile_runner(
- _random_profile_config(openai_stub.base_url, strong_probability),
- "random-profile",
- )
-
- legacy_response = legacy.post(
- "/v1/chat/completions",
- json=_chat_payload("legacy-random"),
- )
- profile_response = await profile.run(
- ChatRequest.openai_chat(_chat_payload("random-profile")),
- )
-
- assert legacy_response.status_code == 200, legacy_response.text
- assert profile_response.body["choices"][0]["message"]["content"] == "ok"
- assert [call["body"]["model"] for call in openai_stub.requests] == [
- expected_model,
- expected_model,
- ]
- assert [call["path"] for call in openai_stub.requests] == [
- "/legacy-random/v1/chat/completions",
- "/v2-random/v1/chat/completions",
- ]
-
-
-async def test_latency_service_profile_routes_to_configured_target(
- openai_stub: _OpenAICompatStub,
-) -> None:
- profile = _profile_runner(
- _latency_profile_config(openai_stub.base_url),
- "latency-profile",
- )
-
- response = await profile.run(
- ChatRequest.openai_chat(_chat_payload("latency-profile")),
- )
-
- assert response.body["model"] == "provider/latency"
- assert [call["path"] for call in openai_stub.requests] == ["/v2-latency/v1/chat/completions"]
- assert [call["body"]["model"] for call in openai_stub.requests] == ["provider/latency"]
-
-
-async def test_latency_service_profile_preserves_upstream_error(
- openai_stub: _OpenAICompatStub,
-) -> None:
- openai_stub.respond_json(
- 503,
- {"error": {"message": "upstream unavailable", "code": "unavailable"}},
- )
- runner = (
- parse_profile_config_str(_latency_profile_config(openai_stub.base_url))
- .resolve()
- .build_profile("latency-profile")
- )
-
- with pytest.raises(SwitchyardUpstreamError, match="upstream unavailable"):
- await runner.run(ChatRequest.openai_chat(_chat_payload("latency-profile")))
-
- assert [call["body"]["model"] for call in openai_stub.requests] == ["provider/latency"]
-
-
-async def test_llm_routing_profile_routes_with_strict_classifier(
- openai_stub: _OpenAICompatStub,
-) -> None:
- openai_stub.respond_json(
- 200,
- _tool_call_payload(
- "provider/classifier",
- "/v2-llm-routing-classifier/v1/chat/completions",
- "select_route",
- {
- "recommended_tier": "medium",
- "confidence": 0.9,
- "abstain": False,
- "turn_type": "exploration",
- "code_modification_scope": "none",
- "tool_call_count_estimate": 0,
- "requires_codebase_context": False,
- },
- ),
- )
- profile = _profile_runner(
- _llm_routing_profile_config(openai_stub.base_url),
- "llm-profile",
- )
-
- response = await profile.run(
- ChatRequest.openai_chat(_chat_payload("llm-profile")),
- )
-
- assert response.body["model"] == "provider/weak"
- assert [call["path"] for call in openai_stub.requests] == [
- "/v2-llm-routing-classifier/v1/chat/completions",
- "/v2-llm-routing/v1/chat/completions",
- ]
- assert [call["body"]["model"] for call in openai_stub.requests] == [
- "provider/classifier",
- "provider/weak",
- ]
- classifier_body = openai_stub.requests[0]["body"]
- assert classifier_body["tools"][0]["function"]["name"] == "select_route"
- assert classifier_body["tools"][0]["function"]["strict"] is True
- assert classifier_body["tool_choice"]["function"]["name"] == "select_route"
- assert "response_format" not in classifier_body
-
-
-async def test_stage_router_profile_routes_with_classifier(
- openai_stub: _OpenAICompatStub,
-) -> None:
- openai_stub.respond_json(
- 200,
- _completion_payload(
- "provider/classifier",
- "/v2-stage_router-classifier/v1/chat/completions",
- json.dumps({"tier": "efficient"}),
- ),
- )
- profile = _profile_runner(
- _stage_router_profile_config(openai_stub.base_url),
- "stage_router-profile",
- )
-
- response = await profile.run(
- ChatRequest.openai_chat(_chat_payload("stage_router-profile")),
- )
-
- assert response.body["model"] == "provider/weak"
- assert [call["path"] for call in openai_stub.requests] == [
- "/v2-stage_router-classifier/v1/chat/completions",
- "/v2-stage_router/v1/chat/completions",
- ]
- assert [call["body"]["model"] for call in openai_stub.requests] == [
- "provider/classifier",
- "provider/weak",
- ]
- response_format = openai_stub.requests[0]["body"]["response_format"]
- assert response_format["type"] == "json_object"
-
-
-def test_python_profile_plan_builds_expected_runner_and_target(
- openai_stub: _OpenAICompatStub,
-) -> None:
- plan = parse_profile_config_str(_passthrough_profile_config(openai_stub.base_url)).resolve()
- profiles = plan.build_profiles()
- target = plan.target("direct")
-
- assert plan.profile_ids() == ["direct-profile"]
- assert plan.target_ids() == ["direct"]
- assert profiles["direct-profile"].profile_id == "direct-profile"
- assert target is not None
- assert target.model == "provider/direct"
-
-
-async def test_profile_runner_uses_same_profile_config(
- openai_stub: _OpenAICompatStub,
-) -> None:
- runner = _profile_runner(
- _passthrough_profile_config(openai_stub.base_url),
- "direct-profile",
- )
-
- response = await runner.run(ChatRequest.openai_chat(_chat_payload("direct-profile")))
-
- assert response.body["model"] == "provider/direct"
- assert response.body["mock_path"] == "/v2-direct/v1/chat/completions"
- assert [call["body"]["model"] for call in openai_stub.requests] == ["provider/direct"]
diff --git a/tests/test_python_profile_abstractions.py b/tests/test_python_profile_abstractions.py
index c6e9263d3..637bb90d1 100644
--- a/tests/test_python_profile_abstractions.py
+++ b/tests/test_python_profile_abstractions.py
@@ -157,6 +157,29 @@ def _request(*, msg: str = "hi") -> ChatRequest:
})
+def test_profile_request_metadata_normalizes_headers() -> None:
+ metadata = ProfileRequestMetadata.from_headers(
+ {
+ "X-Request-ID": "req-123",
+ "X-Switchyard-Trace": ["trace-a", "trace-b"],
+ },
+ inbound_format=ChatRequestType.OPENAI_CHAT,
+ )
+
+ assert metadata.request_id == "req-123"
+ assert metadata.inbound_format == ChatRequestType.OPENAI_CHAT
+ assert metadata.headers == {
+ "x-request-id": ["req-123"],
+ "x-switchyard-trace": ["trace-a", "trace-b"],
+ }
+ assert metadata.to_dict()["inbound_format"] == "openai_chat"
+
+
+def test_profile_request_metadata_rejects_empty_request_id() -> None:
+ with pytest.raises(ValueError, match="RequestId must not be empty"):
+ ProfileRequestMetadata(request_id=" ")
+
+
async def test_profile_config_decorator_builds_dataclass_and_profile_runtime() -> None:
@profile_config("unit-python-static", register=False)
class StaticProfileConfig:
diff --git a/tests/test_python_profile_loader.py b/tests/test_python_profile_loader.py
deleted file mode 100644
index e406bba78..000000000
--- a/tests/test_python_profile_loader.py
+++ /dev/null
@@ -1,227 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for the unified Python/Rust v2 profile loader.
-
-The headline behavior: one config file builds and runs *both* a Rust-defined
-profile and a Python-defined profile through the same loading path
-(``load_profiles``), sharing one resolved set of targets.
-"""
-
-import json
-import threading
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from pathlib import Path
-from typing import Any
-
-import pytest
-
-from switchyard import load_profiles
-from switchyard.lib.profiles import ProfileConfigError
-from switchyard_rust.core import ChatRequest, SwitchyardConfigError
-from switchyard_rust.profiles import ProfileInput, ProfileRequestMetadata
-
-
-class _MockOpenAIServer(ThreadingHTTPServer):
- calls: list[dict[str, Any]]
-
- def __init__(self) -> None:
- super().__init__(("127.0.0.1", 0), _MockOpenAIHandler)
- self.calls = []
-
- @property
- def base_url(self) -> str:
- host, port = self.server_address
- return f"http://{host}:{port}"
-
-
-class _MockOpenAIHandler(BaseHTTPRequestHandler):
- server: _MockOpenAIServer
-
- def do_POST(self) -> None:
- content_length = int(self.headers.get("content-length", "0"))
- raw_body = self.rfile.read(content_length)
- body = json.loads(raw_body.decode("utf-8")) if raw_body else {}
- self.server.calls.append({"path": self.path, "body": body})
- response = {
- "id": "chatcmpl-loader",
- "object": "chat.completion",
- "model": body.get("model"),
- "mock_path": self.path,
- "choices": [
- {
- "index": 0,
- "message": {"role": "assistant", "content": "ok"},
- "finish_reason": "stop",
- }
- ],
- "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
- }
- payload = json.dumps(response).encode("utf-8")
- self.send_response(200)
- self.send_header("content-type", "application/json")
- self.send_header("content-length", str(len(payload)))
- self.end_headers()
- self.wfile.write(payload)
-
- def log_message(self, _format: str, *_args: object) -> None:
- return
-
-
-def mock_openai_server() -> _MockOpenAIServer:
- try:
- server = _MockOpenAIServer()
- except PermissionError as exc:
- pytest.skip(f"loopback socket binding is unavailable in this sandbox: {exc}")
- thread = threading.Thread(target=server.serve_forever, daemon=True)
- thread.start()
- try:
- yield server
- finally:
- server.shutdown()
- thread.join(timeout=5)
- server.server_close()
-
-
-def _mixed_config(base_url: str) -> str:
- """A config with a Rust passthrough and the Python header-routing profile."""
- return f"""
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: "{base_url}/strong/v1"
- api_key: test-key
- weak:
- model: provider/weak
- format: openai
- base_url: "{base_url}/weak/v1"
- api_key: test-key
-
-profiles:
- direct:
- type: passthrough
- target: weak
- smart:
- type: header-routing
- strong: strong
- weak: weak
-"""
-
-
-def _write(tmp_path: Path, text: str, name: str = "profiles.yaml") -> Path:
- path = tmp_path / name
- path.write_text(text, encoding="utf-8")
- return path
-
-
-def _request(model: str) -> ChatRequest:
- return ChatRequest.openai_chat(
- {"model": model, "messages": [{"role": "user", "content": "hello"}]}
- )
-
-
-def test_load_profiles_builds_rust_and_python_profiles(tmp_path: Path) -> None:
- path = _write(tmp_path, _mixed_config("http://127.0.0.1:9"))
- profiles = load_profiles(path)
- # One Rust-defined profile (passthrough) and one Python-defined profile
- # (header-routing) built through the same path.
- assert sorted(profiles) == ["direct", "smart"]
- assert all(hasattr(runner, "run") for runner in profiles.values())
-
-
-async def test_loaded_rust_and_python_profiles_run_against_mock(
- mock_openai_server: _MockOpenAIServer,
- tmp_path: Path,
-) -> None:
- path = _write(tmp_path, _mixed_config(mock_openai_server.base_url))
- profiles = load_profiles(path)
-
- # Rust passthrough -> weak target.
- direct = await profiles["direct"].run(ProfileInput(_request("client/x")))
- assert direct.body["model"] == "provider/weak"
- assert direct.body["mock_path"] == "/weak/v1/chat/completions"
-
- # Python header-routing: header selects strong, delegating to the Rust backend.
- strong_meta = ProfileRequestMetadata(headers={"x-switchyard-tier": "strong"})
- strong = await profiles["smart"].run(ProfileInput(_request("client/x"), strong_meta))
- assert strong.body["model"] == "provider/strong"
- assert strong.body["mock_path"] == "/strong/v1/chat/completions"
-
- # Missing/other header falls back to weak.
- weak = await profiles["smart"].run(ProfileInput(_request("client/x")))
- assert weak.body["model"] == "provider/weak"
- assert weak.body["mock_path"] == "/weak/v1/chat/completions"
-
-
-def test_load_profiles_accepts_json(tmp_path: Path) -> None:
- document = {
- "targets": {
- "weak": {
- "model": "provider/weak",
- "format": "openai",
- "base_url": "http://127.0.0.1:9/weak/v1",
- "api_key": "test-key",
- }
- },
- "profiles": {"direct": {"type": "passthrough", "target": "weak"}},
- }
- path = _write(tmp_path, json.dumps(document), name="profiles.json")
- assert sorted(load_profiles(path)) == ["direct"]
-
-
-def test_unknown_profile_type_is_rejected(tmp_path: Path) -> None:
- path = _write(
- tmp_path,
- "profiles:\n bad:\n type: does-not-exist\n",
- )
- with pytest.raises(SwitchyardConfigError, match="does-not-exist"):
- load_profiles(path)
-
-
-def test_python_profile_missing_target_is_rejected(tmp_path: Path) -> None:
- config = """
-targets:
- weak:
- model: provider/weak
- format: openai
- base_url: http://127.0.0.1:9/weak/v1
- api_key: test-key
-
-profiles:
- smart:
- type: header-routing
- strong: ghost
- weak: weak
-"""
- path = _write(tmp_path, config)
- with pytest.raises(ProfileConfigError, match="unknown target 'ghost'"):
- load_profiles(path)
-
-
-def test_python_profile_unknown_field_is_rejected(tmp_path: Path) -> None:
- config = """
-targets:
- weak:
- model: provider/weak
- format: openai
- base_url: http://127.0.0.1:9/weak/v1
- api_key: test-key
-
-profiles:
- smart:
- type: header-routing
- strong: weak
- weak: weak
- bogus: 1
-"""
- path = _write(tmp_path, config)
- with pytest.raises(ProfileConfigError, match="unknown field"):
- load_profiles(path)
-
-
-def test_unknown_format_is_rejected(tmp_path: Path) -> None:
- path = _write(tmp_path, "profiles: {}\n", name="profiles.ini")
- with pytest.raises(SwitchyardConfigError, match="unsupported profile config extension"):
- load_profiles(path)
diff --git a/tests/test_rl_logging.py b/tests/test_rl_logging.py
index bb93ecdc6..0788413e6 100644
--- a/tests/test_rl_logging.py
+++ b/tests/test_rl_logging.py
@@ -339,7 +339,7 @@ def test_serve_attaches_rl_logging_processors(monkeypatch, tmp_path: Path) -> No
monkeypatch.setattr(cli, "build_and_serve", lambda *a, **k: None)
args = argparse.Namespace(
- config=None, routing_profiles="profiles.yaml",
+ routing_profiles="profiles.yaml",
enable_rl_logging=True, rl_log_dir=str(tmp_path),
intake_enabled=False, intake_base_url=None, intake_workspace=None,
intake_api_key=None, intake_target_url=None,
@@ -348,18 +348,3 @@ def test_serve_attaches_rl_logging_processors(monkeypatch, tmp_path: Path) -> No
assert [type(p).__name__ for p in captured["request"]] == ["RlLoggingRequestProcessor"]
assert [type(p).__name__ for p in captured["response"]] == ["RlLoggingResponseProcessor"]
-
-
-def test_serve_config_rejects_rl_logging() -> None:
- """The Rust profile-server path has no Python chain, so it must reject the flag."""
- from switchyard.cli.switchyard_cli import _cmd_serve_profile_config
-
- args = argparse.Namespace(
- config="profiles.yaml", routing_profiles=None, inbound=None,
- reload=False, workers=1,
- intake_enabled=False, intake_base_url=None, intake_workspace=None,
- intake_api_key=None, intake_target_url=None,
- enable_rl_logging=True, rl_log_dir="./rl_data",
- )
- with pytest.raises(SystemExit, match="enable-rl-logging"):
- _cmd_serve_profile_config(args)
diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py
index cafe39e10..684b25ac2 100644
--- a/tests/test_route_bundle.py
+++ b/tests/test_route_bundle.py
@@ -421,54 +421,6 @@ def test_serve_subcommand_hands_table_to_server(
assert captured["inbound_default"] == "both"
-def test_serve_config_and_routing_profiles_are_mutually_exclusive(
- tmp_path,
-) -> None:
- config_path = tmp_path / "profiles.yaml"
- routes_path = tmp_path / "routes.yaml"
- config_path.write_text("profiles:\n bench:\n type: noop\n")
- routes_path.write_text("routes:\n bench:\n type: noop\n")
-
- args = cli._build_parser().parse_args([
- "--routing-profiles",
- str(routes_path),
- "serve",
- "--config",
- str(config_path),
- ])
-
- with pytest.raises(SystemExit, match="cannot be combined"):
- args.func(args)
-
-
- "serve_args, match",
- [
- (["--reload"], "--reload"),
- (["--workers", "2"], "--workers"),
- (["--inbound", "openai"], "--inbound"),
- (["--inbound", "both"], "--inbound"),
- (["--intake-enabled"], "Intake"),
- ],
-)
-def test_serve_config_rejects_python_only_options(
- tmp_path,
- serve_args: list[str],
- match: str,
-) -> None:
- config_path = tmp_path / "profiles.yaml"
- config_path.write_text("profiles:\n bench:\n type: noop\n")
- args = cli._build_parser().parse_args([
- "serve",
- "--config",
- str(config_path),
- *serve_args,
- ])
-
- with pytest.raises(SystemExit, match=match):
- args.func(args)
-
-
def test_main_reports_route_bundle_config_error_without_traceback(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
@@ -554,7 +506,7 @@ def test_main_reports_non_utf8_route_bundle_without_traceback(
assert "\n" not in str(excinfo.value.code)
-def test_main_warns_when_routing_profiles_flag_is_used(
+def test_main_accepts_routing_profiles_flag(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
capsys: pytest.CaptureFixture[str],
@@ -578,13 +530,10 @@ def test_main_warns_when_routing_profiles_flag_is_used(
cli.main()
- stderr = capsys.readouterr().err
- assert "warning: --routing-profiles is deprecated." in stderr
- assert "switchyard serve --config PATH" in stderr
- assert "removed in a future release" in stderr
+ assert capsys.readouterr().err == ""
-def test_serve_warns_when_saved_route_bundle_is_used(
+def test_serve_accepts_saved_route_bundle(
monkeypatch: pytest.MonkeyPatch,
tmp_path,
capsys: pytest.CaptureFixture[str],
@@ -607,10 +556,7 @@ def test_serve_warns_when_saved_route_bundle_is_used(
cli._cmd_serve(args)
- stderr = capsys.readouterr().err
- assert "warning: saved routing-profile bundle is deprecated." in stderr
- assert "switchyard serve --config PATH" in stderr
- assert "Clear the saved bundle" in stderr
+ assert capsys.readouterr().err == ""
def test_serve_subcommand_enables_intake_from_cli_args(
diff --git a/tests/test_serve_profile_config.py b/tests/test_serve_profile_config.py
deleted file mode 100644
index 299cb4efa..000000000
--- a/tests/test_serve_profile_config.py
+++ /dev/null
@@ -1,236 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for serving a v2 profile config via `serve --config`."""
-
-import argparse
-import json
-import threading
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from pathlib import Path
-from typing import Any
-
-import pytest
-from fastapi.testclient import TestClient
-
-from switchyard.cli.switchyard_cli import _cmd_serve_profile_config, _profile_config_route_table
-from switchyard.server.switchyard_app import build_switchyard_app
-
-_PYTHON_CONFIG = """
-targets:
- weak:
- model: provider/weak
- format: openai
- base_url: http://127.0.0.1:9/weak/v1
- api_key: test-key
-
-profiles:
- smart:
- type: header-routing
- strong: weak
- weak: weak
-"""
-
-
-def _mixed_config(base_url: str) -> str:
- return f"""
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: "{base_url}/strong/v1"
- api_key: test-key
- weak:
- model: provider/weak
- format: openai
- base_url: "{base_url}/weak/v1"
- api_key: test-key
-
-profiles:
- direct:
- type: passthrough
- target: weak
- smart:
- type: header-routing
- strong: strong
- weak: weak
-"""
-
-
-class _MockOpenAIServer(ThreadingHTTPServer):
- calls: list[dict[str, Any]]
-
- def __init__(self) -> None:
- super().__init__(("127.0.0.1", 0), _MockOpenAIHandler)
- self.calls = []
-
- @property
- def base_url(self) -> str:
- host, port = self.server_address
- return f"http://{host}:{port}"
-
-
-class _MockOpenAIHandler(BaseHTTPRequestHandler):
- server: _MockOpenAIServer
-
- def do_POST(self) -> None:
- content_length = int(self.headers.get("content-length", "0"))
- raw_body = self.rfile.read(content_length)
- body = json.loads(raw_body.decode("utf-8")) if raw_body else {}
- self.server.calls.append({"path": self.path, "body": body})
- response = {
- "id": "chatcmpl-serve-profile-config",
- "object": "chat.completion",
- "model": body.get("model"),
- "choices": [
- {
- "index": 0,
- "message": {"role": "assistant", "content": "ok"},
- "finish_reason": "stop",
- }
- ],
- }
- payload = json.dumps(response).encode("utf-8")
- self.send_response(200)
- self.send_header("content-type", "application/json")
- self.send_header("content-length", str(len(payload)))
- self.end_headers()
- self.wfile.write(payload)
-
-
-def mock_openai_server() -> _MockOpenAIServer:
- try:
- server = _MockOpenAIServer()
- except PermissionError as exc:
- pytest.skip(f"loopback socket binding is unavailable in this sandbox: {exc}")
- thread = threading.Thread(target=server.serve_forever, daemon=True)
- thread.start()
- try:
- yield server
- finally:
- server.shutdown()
- thread.join(timeout=5)
- server.server_close()
-
-
-def _write(tmp_path: Path, text: str, name: str = "profiles.yaml") -> Path:
- path = tmp_path / name
- path.write_text(text, encoding="utf-8")
- return path
-
-
-def test_serve_config_uses_fastapi_profile_table(
- monkeypatch: pytest.MonkeyPatch,
- tmp_path: Path,
-) -> None:
- path = _write(tmp_path, _PYTHON_CONFIG)
- args = _serve_args(path)
- captured: dict[str, Any] = {}
-
- def capture_build_and_serve(
- _args: argparse.Namespace,
- table: Any,
- inbound_default: str = "openai",
- disable_backend_streaming: bool = False,
- extra_endpoints: list[Any] | None = None,
- ) -> None:
- captured["models"] = table.registered_models()
- captured["inbound_default"] = inbound_default
- captured["disable_backend_streaming"] = disable_backend_streaming
- captured["extra_endpoints"] = extra_endpoints
-
- monkeypatch.setattr(
- "switchyard.cli.switchyard_cli.build_and_serve",
- capture_build_and_serve,
- )
-
- _cmd_serve_profile_config(args)
-
- assert captured["models"] == ["smart", "weak", "provider/weak"]
- assert captured["inbound_default"] == "both"
-
-
-def test_profile_config_route_table_serves_mixed_profiles(
- mock_openai_server: _MockOpenAIServer,
- tmp_path: Path,
-) -> None:
- path = _write(tmp_path, _mixed_config(mock_openai_server.base_url))
- table = _profile_config_route_table(str(path))
-
- assert table.registered_models() == [
- "direct",
- "smart",
- "strong",
- "provider/strong",
- "weak",
- "provider/weak",
- ]
-
- with TestClient(build_switchyard_app(table), raise_server_exceptions=False) as client:
- models_response = client.get("/v1/models")
- assert models_response.status_code == 200
- assert models_response.json()["model_pool"] == [
- "direct",
- "smart",
- "strong",
- "provider/strong",
- "weak",
- "provider/weak",
- ]
-
- responses = {
- path: client.post(
- path,
- headers={"x-switchyard-tier": "strong"},
- json=body,
- )
- for path, body in (
- (
- "/v1/chat/completions",
- {
- "model": "smart",
- "messages": [{"role": "user", "content": "hi"}],
- },
- ),
- (
- "/v1/messages",
- {
- "model": "smart",
- "max_tokens": 16,
- "messages": [{"role": "user", "content": "hi"}],
- },
- ),
- ("/v1/responses", {"model": "smart", "input": "hi"}),
- )
- }
-
- assert [response.status_code for response in responses.values()] == [200, 200, 200]
- assert [call["path"] for call in mock_openai_server.calls] == [
- "/strong/v1/chat/completions",
- "/strong/v1/chat/completions",
- "/strong/v1/chat/completions",
- ]
- assert responses["/v1/chat/completions"].json()["model"] == "provider/strong"
- assert responses["/v1/messages"].json()["content"][0]["text"] == "ok"
- assert (
- responses["/v1/responses"].json()["output"][0]["content"][0]["text"]
- == "ok"
- )
-
-
-def _serve_args(path: Path) -> argparse.Namespace:
- return argparse.Namespace(
- config=str(path),
- routing_profiles=None,
- inbound=None,
- reload=False,
- workers=1,
- intake_enabled=False,
- intake_base_url=None,
- intake_workspace=None,
- intake_api_key=None,
- intake_target_url=None,
- host="127.0.0.1",
- port=4000,
- )
diff --git a/tests/test_subagent_routing.py b/tests/test_subagent_routing.py
deleted file mode 100644
index 310f98172..000000000
--- a/tests/test_subagent_routing.py
+++ /dev/null
@@ -1,319 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for envelope-level sub-agent routing (``subagent_target``).
-
-A profile config may name a ``subagent_target`` in its common envelope; the
-loader then wraps the built profile so recognized sub-agent requests run
-through a passthrough to that target while all other traffic keeps the
-profile's own routing.
-"""
-
-import json
-import threading
-from collections.abc import Iterator
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from pathlib import Path
-from typing import Any
-
-import pytest
-
-from switchyard import load_profiles
-from switchyard.lib.profiles import ProfileConfigError
-from switchyard.lib.profiles.subagent_override import SubagentOverrideProfile
-from switchyard_rust.core import ChatRequest, SwitchyardConfigError
-from switchyard_rust.profiles import (
- ProfileInput,
- ProfileRequestMetadata,
- is_subagent_request,
-)
-
-SUBAGENT_HEADERS = {
- "x-claude-code-session-id": "root-session",
- "x-claude-code-agent-id": "worker-1",
-}
-
-
-class _MockOpenAIServer(ThreadingHTTPServer):
- calls: list[dict[str, Any]]
-
- def __init__(self) -> None:
- super().__init__(("127.0.0.1", 0), _MockOpenAIHandler)
- self.calls = []
-
- @property
- def base_url(self) -> str:
- host, port = self.server_address
- return f"http://{host}:{port}"
-
-
-class _MockOpenAIHandler(BaseHTTPRequestHandler):
- server: _MockOpenAIServer
-
- def do_POST(self) -> None:
- content_length = int(self.headers.get("content-length", "0"))
- raw_body = self.rfile.read(content_length)
- body = json.loads(raw_body.decode("utf-8")) if raw_body else {}
- self.server.calls.append({"path": self.path, "body": body})
- response = {
- "id": "chatcmpl-subagent",
- "object": "chat.completion",
- "model": body.get("model"),
- "mock_path": self.path,
- "choices": [
- {
- "index": 0,
- "message": {"role": "assistant", "content": "ok"},
- "finish_reason": "stop",
- }
- ],
- "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
- }
- payload = json.dumps(response).encode("utf-8")
- self.send_response(200)
- self.send_header("content-type", "application/json")
- self.send_header("content-length", str(len(payload)))
- self.end_headers()
- self.wfile.write(payload)
-
- def log_message(self, _format: str, *_args: object) -> None:
- return
-
-
-def mock_openai_server() -> Iterator[_MockOpenAIServer]:
- try:
- server = _MockOpenAIServer()
- except PermissionError as exc:
- pytest.skip(f"loopback socket binding is unavailable in this sandbox: {exc}")
- thread = threading.Thread(target=server.serve_forever, daemon=True)
- thread.start()
- try:
- yield server
- finally:
- server.shutdown()
- thread.join(timeout=5)
- server.server_close()
-
-
-def _config(base_url: str) -> str:
- """Rust and Python profiles that both name a ``subagent_target``."""
- return f"""
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: "{base_url}/strong/v1"
- api_key: test-key
- worker:
- model: provider/worker
- format: openai
- base_url: "{base_url}/worker/v1"
- api_key: test-key
-
-profiles:
- direct:
- type: passthrough
- target: strong
- subagent_target: worker
- smart:
- type: header-routing
- strong: strong
- weak: strong
- subagent_target: worker
- plain:
- type: passthrough
- target: strong
-"""
-
-
-def _write(tmp_path: Path, text: str, name: str = "profiles.yaml") -> Path:
- path = tmp_path / name
- path.write_text(text, encoding="utf-8")
- return path
-
-
-def _request(model: str = "client/x") -> ChatRequest:
- return ChatRequest.openai_chat(
- {"model": model, "messages": [{"role": "user", "content": "hello"}]}
- )
-
-
-def _subagent_input() -> ProfileInput:
- return ProfileInput(
- _request(),
- ProfileRequestMetadata(headers=dict(SUBAGENT_HEADERS)),
- )
-
-
-# --- Detection ------------------------------------------------------------
-
-
- ("headers", "expected"),
- [
- # No signal at all.
- ({}, False),
- # Claude Code lineage: any non-empty agent id marks a child agent.
- (SUBAGENT_HEADERS, True),
- # Agent id alone (no session header) is still a child agent.
- ({"x-claude-code-agent-id": "child-1"}, True),
- # Session alone — root agent, no agent-id sent.
- ({"x-claude-code-session-id": "s"}, False),
- # Codex delegated-work kinds route as sub-agent work.
- ({"x-openai-subagent": "review"}, True),
- ({"x-openai-subagent": "collab_spawn"}, True),
- # Codex harness maintenance and unknown kinds stay on normal routing.
- ({"x-openai-subagent": "compact"}, False),
- ({"x-openai-subagent": "memory_consolidation"}, False),
- ({"x-openai-subagent": "brand_new_kind"}, False),
- # The explicit Switchyard header decides the lineage fact, in both
- # directions; the work-kind policy still applies on top of it.
- ({"x-switchyard-is-subagent": "true"}, True),
- ({"x-switchyard-is-subagent": "false", "x-openai-subagent": "review"}, False),
- ({"x-switchyard-is-subagent": "true", "x-openai-subagent": "compact"}, False),
- # Header names are case-insensitive and values are trimmed.
- ({"X-OpenAI-Subagent": " review "}, True),
- # Bug #2 fix: operator label x-switchyard-agent-kind must not suppress
- # routing signals from the harness (x-openai-subagent, x-switchyard-is-subagent).
- ({"x-openai-subagent": "review", "x-switchyard-agent-kind": "researcher"}, True),
- ({"x-switchyard-is-subagent": "true", "x-switchyard-agent-kind": "researcher"}, True),
- # Bug #1 fix: correlation-only headers are not routing signals.
- ({"x-switchyard-parent-agent-id": "parent"}, False),
- ({"x-nemo-relay-subagent-id": "relay-child"}, False),
- ({"x-dynamo-parent-session-id": "dynamo-parent"}, False),
- ({"x-session-id": "s", "x-parent-session-id": "p"}, False),
- ],
-)
-def test_is_subagent_request_detection(headers: dict[str, str], expected: bool) -> None:
- assert is_subagent_request(headers) is expected
-
-
-# --- Loader wiring ----------------------------------------------------------
-
-
-def test_loader_wraps_profiles_with_subagent_target(tmp_path: Path) -> None:
- path = _write(tmp_path, _config("http://127.0.0.1:9"))
- profiles = load_profiles(path)
- # Rust profiles (direct / passthrough) handle subagent routing at the Rust level;
- # the Python loader must not double-wrap them with SubagentOverrideProfile.
- assert not isinstance(profiles["direct"], SubagentOverrideProfile)
- # Python profiles (smart / header-routing) still receive the Python wrapper.
- assert isinstance(profiles["smart"], SubagentOverrideProfile)
- assert not isinstance(profiles["plain"], SubagentOverrideProfile)
-
-
-async def test_rust_profile_routes_subagent_requests_to_target(
- mock_openai_server: _MockOpenAIServer,
- tmp_path: Path,
-) -> None:
- path = _write(tmp_path, _config(mock_openai_server.base_url))
- profiles = load_profiles(path)
-
- normal = await profiles["direct"].run(ProfileInput(_request()))
- assert normal.body["model"] == "provider/strong"
- assert normal.body["mock_path"] == "/strong/v1/chat/completions"
-
- subagent = await profiles["direct"].run(_subagent_input())
- assert subagent.body["model"] == "provider/worker"
- assert subagent.body["mock_path"] == "/worker/v1/chat/completions"
-
-
-async def test_python_profile_routes_subagent_requests_to_target(
- mock_openai_server: _MockOpenAIServer,
- tmp_path: Path,
-) -> None:
- path = _write(tmp_path, _config(mock_openai_server.base_url))
- profiles = load_profiles(path)
-
- normal = await profiles["smart"].run(ProfileInput(_request()))
- assert normal.body["model"] == "provider/strong"
-
- subagent = await profiles["smart"].run(_subagent_input())
- assert subagent.body["model"] == "provider/worker"
- assert subagent.body["mock_path"] == "/worker/v1/chat/completions"
-
-
-async def test_maintenance_kinds_keep_normal_routing(
- mock_openai_server: _MockOpenAIServer,
- tmp_path: Path,
-) -> None:
- path = _write(tmp_path, _config(mock_openai_server.base_url))
- profiles = load_profiles(path)
-
- metadata = ProfileRequestMetadata(headers={"x-openai-subagent": "compact"})
- response = await profiles["direct"].run(ProfileInput(_request(), metadata))
- assert response.body["model"] == "provider/strong"
-
-
-def test_rust_profile_unknown_subagent_target_is_rejected(tmp_path: Path) -> None:
- config = """
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: http://127.0.0.1:9/v1
- api_key: test-key
-
-profiles:
- direct:
- type: passthrough
- target: strong
- subagent_target: ghost
-"""
- path = _write(tmp_path, config)
- with pytest.raises(SwitchyardConfigError, match="unknown target ghost"):
- load_profiles(path)
-
-
-def test_python_profile_unknown_subagent_target_is_rejected(tmp_path: Path) -> None:
- config = """
-targets:
- strong:
- model: provider/strong
- format: openai
- base_url: http://127.0.0.1:9/v1
- api_key: test-key
-
-profiles:
- smart:
- type: header-routing
- strong: strong
- weak: strong
- subagent_target: ghost
-"""
- path = _write(tmp_path, config)
- with pytest.raises(ProfileConfigError, match="unknown target 'ghost'"):
- load_profiles(path)
-
-
-# --- Wrapper behavior -------------------------------------------------------
-
-
-class _StubRunner:
- def __init__(self, label: str) -> None:
- self.label = label
- self.calls = 0
-
- async def run(self, input: ProfileInput) -> Any:
- self.calls += 1
- return self.label
-
-
-def test_iter_components_spans_both_branches() -> None:
- inner = _StubRunner("inner")
- override = _StubRunner("override")
- wrapper = SubagentOverrideProfile(inner, override)
- assert wrapper.iter_components() == [inner, override]
-
-
-async def test_override_failure_is_not_rerouted_to_the_wrapped_profile() -> None:
- class _FailingRunner:
- async def run(self, input: ProfileInput) -> Any:
- raise RuntimeError("worker target unavailable")
-
- inner = _StubRunner("inner")
- wrapper = SubagentOverrideProfile(inner, _FailingRunner())
- with pytest.raises(RuntimeError, match="worker target unavailable"):
- await wrapper.run(_subagent_input())
- assert inner.calls == 0
diff --git a/tests/test_switchyard_rust_profile_bindings.py b/tests/test_switchyard_rust_profile_bindings.py
deleted file mode 100644
index 1ff8aadab..000000000
--- a/tests/test_switchyard_rust_profile_bindings.py
+++ /dev/null
@@ -1,429 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-
-"""Tests for components-v2 profile config and runtime bindings."""
-
-import json
-import threading
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-from pathlib import Path
-from typing import Any
-
-import pytest
-
-from switchyard_rust.components import LlmTarget
-from switchyard_rust.core import ChatRequest, ChatRequestType, SwitchyardConfigError
-from switchyard_rust.profiles import (
- Profile,
- ProfileConfigDocument,
- ProfileConfigPlan,
- ProfileInput,
- ProfileRequestMetadata,
- load_profile_config,
- parse_profile_config_path,
- parse_profile_config_str,
-)
-
-
-class _MockOpenAIServer(ThreadingHTTPServer):
- calls: list[dict[str, Any]]
-
- def __init__(self) -> None:
- super().__init__(("127.0.0.1", 0), _MockOpenAIHandler)
- self.calls = []
-
- @property
- def base_url(self) -> str:
- host, port = self.server_address
- return f"http://{host}:{port}"
-
-
-class _MockOpenAIHandler(BaseHTTPRequestHandler):
- server: _MockOpenAIServer
-
- def do_POST(self) -> None:
- content_length = int(self.headers.get("content-length", "0"))
- raw_body = self.rfile.read(content_length)
- body = json.loads(raw_body.decode("utf-8")) if raw_body else {}
- self.server.calls.append({"path": self.path, "body": body})
-
- message: dict[str, Any] = {"role": "assistant", "content": "ok"}
- finish_reason = "stop"
- if body.get("tools"):
- tool_name = body["tool_choice"]["function"]["name"]
- message = {
- "role": "assistant",
- "tool_calls": [
- {
- "id": "call_route",
- "type": "function",
- "function": {
- "name": tool_name,
- "arguments": json.dumps(
- {
- "recommended_tier": "medium",
- "confidence": 0.9,
- "abstain": False,
- "turn_type": "exploration",
- "code_modification_scope": "none",
- "tool_call_count_estimate": 0,
- "requires_codebase_context": False,
- }
- ),
- },
- }
- ],
- }
- finish_reason = "tool_calls"
- elif body.get("response_format", {}).get("type") == "json_schema":
- schema_name = body["response_format"]["json_schema"]["name"]
- if schema_name == "StageRouterTierDecision":
- message = {"role": "assistant", "content": json.dumps({"tier": "efficient"})}
- elif (
- body.get("response_format", {}).get("type") == "json_object"
- and body.get("model") == "provider/classifier"
- ):
- message = {"role": "assistant", "content": json.dumps({"tier": "efficient"})}
-
- response = {
- "id": "chatcmpl-profile-bindings",
- "object": "chat.completion",
- "model": body.get("model"),
- "mock_path": self.path,
- "choices": [
- {
- "index": 0,
- "message": message,
- "finish_reason": finish_reason,
- }
- ],
- "usage": {
- "prompt_tokens": 2,
- "completion_tokens": 1,
- "total_tokens": 3,
- },
- }
- payload = json.dumps(response).encode("utf-8")
- self.send_response(200)
- self.send_header("content-type", "application/json")
- self.send_header("content-length", str(len(payload)))
- self.end_headers()
- self.wfile.write(payload)
-
- def log_message(
- self,
- _format: str,
- _requestline: str | None = None,
- _code: str | None = None,
- _size: str | None = None,
- ) -> None:
- return
-
-
-def mock_openai_server() -> _MockOpenAIServer:
- try:
- server = _MockOpenAIServer()
- except PermissionError as exc:
- pytest.skip(f"loopback socket binding is unavailable in this sandbox: {exc}")
- thread = threading.Thread(target=server.serve_forever, daemon=True)
- thread.start()
- try:
- yield server
- finally:
- server.shutdown()
- thread.join(timeout=5)
- server.server_close()
-
-
-def _profile_config(base_url: str) -> str:
- return f"""
-targets:
- direct:
- model: provider/direct
- format: openai
- base_url: "{base_url}/direct/v1"
- api_key: test-key
- strong:
- model: provider/strong
- format: openai
- base_url: "{base_url}/strong/v1"
- api_key: test-key
- weak:
- model: provider/weak
- format: openai
- base_url: "{base_url}/weak/v1"
- api_key: test-key
- latency:
- model: provider/latency
- format: openai
- base_url: "{base_url}/latency/v1"
- api_key: test-key
- classifier:
- model: provider/classifier
- format: openai
- base_url: "{base_url}/classifier/v1"
- api_key: test-key
-
-profiles:
- direct:
- type: passthrough
- target: direct
- random:
- type: random-routing
- strong: strong
- weak: weak
- strong_probability: 1.0
- rng_seed: 7
- latency:
- type: latency-service
- latency_service_url: "http://latency.invalid"
- targets: [latency]
- max_retries: 0
- llm:
- type: llm-routing
- strong: strong
- weak: weak
- classifier: classifier
- profile_name: coding_agent
- stage_router:
- type: stage_router
- capable: strong
- efficient: weak
- fallback_target_on_evict: strong
- picker: capable_first
- confidence_threshold: 0.7
- classifier:
- model: provider/classifier
- api_key: test-key
- base_url: "{base_url}/classifier/v1"
- bench:
- type: noop
-"""
-
-
-def _plan(base_url: str) -> ProfileConfigPlan:
- document = parse_profile_config_str(_profile_config(base_url))
- assert isinstance(document, ProfileConfigDocument)
- return document.resolve()
-
-
-def _offline_plan() -> ProfileConfigPlan:
- return _plan("http://127.0.0.1:9")
-
-
-def _request(model: str) -> ChatRequest:
- return ChatRequest.openai_chat(
- {
- "model": model,
- "messages": [{"role": "user", "content": "hello"}],
- }
- )
-
-
-def test_profile_config_plan_is_inspectable() -> None:
- document = parse_profile_config_str(_profile_config("http://127.0.0.1:9"))
-
- assert document.profile_ids() == [
- "bench",
- "direct",
- "latency",
- "llm",
- "random",
- "stage_router",
- ]
- assert document.profile_type("direct") == "passthrough"
- assert document.profile_body("random") == {
- "strong": "strong",
- "weak": "weak",
- "strong_probability": 1.0,
- "rng_seed": 7,
- }
- rust_document = document.without_profiles(["random"])
- assert rust_document.profile_ids() == ["bench", "direct", "latency", "llm", "stage_router"]
-
- plan = document.resolve()
-
- assert isinstance(plan, ProfileConfigPlan)
- assert plan.profile_ids() == [
- "bench",
- "direct",
- "latency",
- "llm",
- "random",
- "stage_router",
- ]
- assert plan.target_ids() == ["classifier", "direct", "latency", "strong", "weak"]
- assert plan.profile_type("direct") == "passthrough"
- assert plan.profile_type("random") == "random-routing"
- assert plan.profile_type("latency") == "latency-service"
- assert plan.profile_type("llm") == "llm-routing"
- assert plan.profile_type("stage_router") == "stage_router"
- assert plan.profile_type("bench") == "noop"
- assert plan.profile_type("missing") is None
-
- target = plan.target("direct")
- assert isinstance(target, LlmTarget)
- assert target.id == "direct"
- assert target.model == "provider/direct"
- assert target.base_url == "http://127.0.0.1:9/direct/v1"
- assert plan.target("missing") is None
-
- profiles = plan.build_profiles()
- assert sorted(profiles) == [
- "bench",
- "direct",
- "latency",
- "llm",
- "random",
- "stage_router",
- ]
- assert all(isinstance(profile, Profile) for profile in profiles.values())
- assert profiles["direct"].profile_id == "direct"
-
-
-def test_profile_config_can_be_loaded_from_path(
- tmp_path: Path,
-) -> None:
- path = tmp_path / "profiles.yaml"
- path.write_text(_profile_config("http://127.0.0.1:9"), encoding="utf-8")
-
- document = parse_profile_config_path(path)
- assert isinstance(document.resolve(), ProfileConfigPlan)
-
- plan = load_profile_config(path)
- assert plan.profile_ids() == [
- "bench",
- "direct",
- "latency",
- "llm",
- "random",
- "stage_router",
- ]
-
-
-async def test_noop_profile_returns_local_response() -> None:
- profile = _offline_plan().build_profile("bench")
-
- assert isinstance(profile, Profile)
- assert profile.profile_id == "bench"
- response = await profile.run(_request("client/noop"))
-
- assert response.body["id"] == "switchyard-noop"
- assert response.body["model"] == "client/noop"
- assert response.body["choices"][0]["message"]["content"] == "ok"
-
-
-async def test_native_profiles_run_against_local_openai_mock(
- mock_openai_server: _MockOpenAIServer,
-) -> None:
- plan = _plan(mock_openai_server.base_url)
-
- direct_response = await plan.build_profile("direct").run(_request("client/direct"))
- random_response = await plan.build_profile("random").run(_request("client/random"))
- latency_response = await plan.build_profile("latency").run(_request("client/latency"))
- llm_response = await plan.build_profile("llm").run(_request("client/llm-routing"))
- stage_router_response = await plan.build_profile("stage_router").run(_request("client/stage_router"))
-
- assert direct_response.body["model"] == "provider/direct"
- assert direct_response.body["mock_path"] == "/direct/v1/chat/completions"
- assert random_response.body["model"] == "provider/strong"
- assert random_response.body["mock_path"] == "/strong/v1/chat/completions"
- assert latency_response.body["model"] == "provider/latency"
- assert latency_response.body["mock_path"] == "/latency/v1/chat/completions"
- assert llm_response.body["model"] == "provider/weak"
- assert llm_response.body["mock_path"] == "/weak/v1/chat/completions"
- assert stage_router_response.body["model"] == "provider/weak"
- assert stage_router_response.body["mock_path"] == "/weak/v1/chat/completions"
- assert [call["body"]["model"] for call in mock_openai_server.calls] == [
- "provider/direct",
- "provider/strong",
- "provider/latency",
- "provider/classifier",
- "provider/weak",
- "provider/classifier",
- "provider/weak",
- ]
- llm_classifier_body = mock_openai_server.calls[3]["body"]
- assert llm_classifier_body["tools"][0]["function"]["strict"] is True
- assert llm_classifier_body["tool_choice"]["function"]["name"] == "select_route"
- assert "response_format" not in llm_classifier_body
- stage_router_classifier_body = mock_openai_server.calls[5]["body"]
- assert stage_router_classifier_body["response_format"]["type"] == "json_object"
-
-
-def test_profile_binding_errors_map_to_switchyard_exceptions() -> None:
- with pytest.raises(SwitchyardConfigError, match="unknown field.*routes"):
- parse_profile_config_str("routes: {}\n")
-
- unknown_target = """
-targets: {}
-profiles:
- bad:
- type: passthrough
- target: missing
-"""
- with pytest.raises(SwitchyardConfigError, match="profile bad:.*unknown target missing"):
- parse_profile_config_str(unknown_target).resolve()
-
- with pytest.raises(SwitchyardConfigError, match="unknown profile missing"):
- parse_profile_config_str("profiles:\n bench:\n type: noop\n").resolve().build_profile(
- "missing"
- )
-
-
-def test_profile_request_metadata_normalizes_headers() -> None:
- metadata = ProfileRequestMetadata.from_headers(
- {
- "X-Request-ID": "req-123",
- "X-Switchyard-Trace": ["trace-a", "trace-b"],
- },
- inbound_format=ChatRequestType.OPENAI_CHAT,
- )
- assert metadata.request_id == "req-123"
- assert metadata.inbound_format == ChatRequestType.OPENAI_CHAT
- assert metadata.headers == {
- "x-request-id": ["req-123"],
- "x-switchyard-trace": ["trace-a", "trace-b"],
- }
-
-
-def test_profile_input_binding_wraps_request_and_metadata() -> None:
- metadata = ProfileRequestMetadata(
- request_id="req-profile-input",
- inbound_format=ChatRequestType.OPENAI_CHAT,
- headers={"X-Switchyard-Trace": "trace-profile-input"},
- )
-
- input = ProfileInput(_request("client/profile-input"), metadata=metadata)
-
- assert input.request.model == "client/profile-input"
- assert input.metadata.request_id == "req-profile-input"
- assert input.metadata.inbound_format == ChatRequestType.OPENAI_CHAT
- assert input.metadata.headers == {"x-switchyard-trace": ["trace-profile-input"]}
-
-
-def test_concrete_profile_trios_are_not_reexported_from_rust_profiles() -> None:
- import switchyard_rust.profiles as profiles
-
- for name in (
- "PassthroughProfileConfig",
- "PassthroughProfile",
- "PassthroughProcessedRequest",
- "RandomRoutingProfileConfig",
- "RandomRoutingProfile",
- "RandomRoutingProcessedRequest",
- "LatencyServiceProfileConfig",
- "LatencyServiceProfile",
- "LatencyServiceProcessedRequest",
- "LlmRoutingProfileConfig",
- "LlmRoutingProfile",
- "LlmRoutingProcessedRequest",
- "StageRouterProfileConfig",
- "StageRouterProfile",
- "StageRouterProcessedRequest",
- "NoopProfileConfig",
- "NoopProfile",
- "NoopProcessedRequest",
- ):
- assert not hasattr(profiles, name)