Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
refactor(server): move token counting out of algorithms (#316)
nachiketb-nvidia70aeb1f8e7c1
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex 67f778af6..925dbdc6b 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -111,23 +111,6 @@ impl TranslatingLlmClient {})}- /// A configured Anthropic backend to forward a direct `count_tokens` call to,- /// paired with its upstream model id (the id the inbound route name is- /// restamped to). `None` when this client has no Anthropic backend;- /// `count_tokens` is Anthropic-only.- fn anthropic_backend(&self) -> Option<(&str, &Backend)> {- self.model_to_config.values().find_map(|config| {- if config.default_backend.is_anthropic() {- return Some((config.model_name.as_str(), &config.default_backend));- }- config- .other_backends- .as_ref()- .and_then(|backends| backends.iter().find(|backend| backend.is_anthropic()))- .map(|backend| (config.model_name.as_str(), backend))- })- }-/// The backend serving `model` over `format` — the default backend when its/// format matches, otherwise a matching entry in `other_backends`; `None` when/// the model is unknown or has no backend for `format`.@@ -144,6 +127,50 @@ impl TranslatingLlmClient {})}+ /// Whether `model` has an Anthropic backend that supports token counting.+ pub fn supports_count_tokens(&self, model: &str) -> bool {+ self.backend_for(model, WireFormat::AnthropicMessages)+ .is_some()+ }++ /// Counts input tokens with `model`'s Anthropic backend.+ ///+ /// Returns an error when the model has no Anthropic backend or the upstream+ /// request fails or returns invalid JSON.+ pub async fn count_tokens(&self, model: &str, request: Request) -> Result<Value> {+ let backend = self+ .backend_for(model, WireFormat::AnthropicMessages)+ .ok_or_else(|| LlmClientError::Configuration {+ message: format!("model {model} has no Anthropic backend for count_tokens"),+ })?;+ let Request {+ llm_request,+ metadata,+ ..+ } = request;+ let http_response = self+ .send_encoded(+ backend,+ WireFormat::AnthropicMessages,+ llm_request,+ metadata.as_ref(),+ model,+ UpstreamEndpoint::CountTokens,+ )+ .await?;+ let body = match http_response {+ EncodedResponse::Buffered { body, .. } => body,+ EncodedResponse::Streaming(_) => {+ return Err(LlmClientError::InvalidRequest {+ message: "count_tokens does not support streaming requests".to_string(),+ });+ }+ };+ serde_json::from_slice(&body).map_err(|error| LlmClientError::InvalidResponse {+ source: Box::new(error),+ })+ }+/// Encode `llm_request` (its model restamped to `model`) for `wire_format`,/// POST it to `url` with the request's forwarded headers plus the backend's/// static headers and auth, and return the successful upstream response. A@@ -153,8 +180,8 @@ impl TranslatingLlmClient {/// overflow via the backend's provider rules. Shared by/// [`call_rewrite_model`](Self::call_rewrite_model) (which POSTs to the/// backend's completion URL and decodes a response) and- /// [`count_tokens`](RoutedLlmClient::count_tokens) (which POSTs to the- /// `count_tokens` URL and returns the raw JSON).+ /// [`count_tokens`](Self::count_tokens) (which POSTs to the `count_tokens`+ /// URL and returns the raw JSON).async fn send_encoded(&self,backend: &Backend,@@ -488,49 +515,6 @@ impl RoutedLlmClient for TranslatingLlmClient {let model_name = Some(decision.selected_model());self.call_rewrite_model(ctx, request, model_name).await}-- fn supports_count_tokens(&self) -> bool {- self.anthropic_backend().is_some()- }-- async fn count_tokens(&self, request: Request) -> Result<Value> {- // Direct passthrough: forward straight to this client's Anthropic- // backend's count_tokens endpoint. Not routed — no decision.- let (model, backend) =- self.anthropic_backend()- .ok_or_else(|| LlmClientError::Configuration {- message: "count_tokens is anthropic-only; this client has no anthropic backend"- .to_string(),- })?;- // Same encode-and-forward path as the completion call, only to the- // `count_tokens` URL; the response is the raw `{"input_tokens": N}` JSON.- let Request {- llm_request,- metadata,- ..- } = request;- let http_response = self- .send_encoded(- backend,- WireFormat::AnthropicMessages,- llm_request,- metadata.as_ref(),- model,- UpstreamEndpoint::CountTokens,- )- .await?;- let body = match http_response {- EncodedResponse::Buffered { body, .. } => body,- EncodedResponse::Streaming(_) => {- return Err(LlmClientError::InvalidRequest {- message: "count_tokens does not support streaming requests".to_string(),- });- }- };- serde_json::from_slice(&body).map_err(|error| LlmClientError::InvalidResponse {- source: Box::new(error),- })- }}#[derive(Clone, Copy)]diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex d7d2381a1..8d1314a56 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -30,7 +30,7 @@ use crate::core::algorithm::{self, Algorithm, Driver, LlmTarget, LlmTargetSet, Suse crate::core::classifier::{Classification, Classifier, Score};use crate::core::processor::{Event, Processor};use crate::{LibsyError, Result};-use switchyard_protocol::{Context, Decision, Request, Response, RoutedLlmClient};+use switchyard_protocol::{Context, Decision, Request, Response};struct SessionState<S> {state: Arc<AsyncMutex<S>>,@@ -420,10 +420,6 @@ where&self.name}- fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {- self.targets.count_tokens_client()- }-async fn create_run_task(self: Arc<Self>,ctx: Context,@@ -441,8 +437,8 @@ mod tests {use crate::{SystemPromptProcessor, TargetPrompts};use switchyard_protocol::{- LlmClientError, LlmRequest, LlmResponse, Message, Metadata, Role, completion_text,- text_request, text_response,+ LlmClientError, LlmRequest, LlmResponse, Message, Metadata, Role, RoutedLlmClient,+ completion_text, text_request, text_response,};#[derive(Debug, thiserror::Error)]diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex 4c627b09f..c2cce86e5 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -26,7 +26,7 @@ use crate::core::classifier::{Classification, Classifier, Score};use crate::core::state::{State, StateValue};use crate::{LibsyError, Result};use switchyard_protocol::{- AggLlmResponse, Context, LlmClientError, LlmResponse, Request, Response, RoutedLlmClient,+ AggLlmResponse, Context, LlmClientError, LlmResponse, Request, Response,};const PROMPT_TEMPLATE: &str = include_str!("../prompts/capability-classifier/prompt.md");@@ -936,10 +936,6 @@ impl Algorithm for LlmTaskClassifier {"llm_task_classifier"}- fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {- self.route.count_tokens_client()- }-async fn create_run_task(self: Arc<Self>,ctx: Context,diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rsindex 2a922f28d..9643fc4b4 100644--- a/crates/libsy/src/algorithms/passthrough.rs+++ b/crates/libsy/src/algorithms/passthrough.rs@@ -9,7 +9,7 @@ use switchyard_protocol::{Request, Response};use crate::Result;use crate::core::algorithm::{Algorithm, Driver, LlmTarget};-use switchyard_protocol::{Context, Decision, RoutedLlmClient};+use switchyard_protocol::{Context, Decision};/// Routing algorithm that always calls one configured target.pub struct Passthrough {@@ -47,14 +47,6 @@ impl Algorithm for Passthrough {"passthrough"}- fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {- self.target- .llm_client- .as_ref()- .filter(|client| client.supports_count_tokens())- .cloned()- }-async fn create_run_task(self: Arc<Self>,ctx: Context,diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rsindex 2eb93f024..3f41500c5 100644--- a/crates/libsy/src/algorithms/rand.rs+++ b/crates/libsy/src/algorithms/rand.rs@@ -19,7 +19,7 @@ use crate::algorithms::fall_through::{FallThrough, FallThroughDecision};use crate::core::algorithm::{Algorithm, Driver, LlmTargetSet};use crate::core::classifier::{Classification, Classifier, Score};use crate::{LibsyError, Result};-use switchyard_protocol::{Context, Request, Response, RoutedLlmClient};+use switchyard_protocol::{Context, Request, Response};/// Compatibility name for the decision produced by [`Random`].pub type RandomDecision = FallThroughDecision;@@ -162,10 +162,6 @@ impl Algorithm for Random {"random"}- fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {- self.inner.count_tokens_client()- }-async fn create_run_task(self: Arc<Self>,ctx: Context,diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rsindex a2612d69e..c82577c5b 100644--- a/crates/libsy/src/algorithms/stage.rs+++ b/crates/libsy/src/algorithms/stage.rs@@ -29,7 +29,7 @@ use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};use crate::core::classifier::{Classification, Classifier};use crate::core::state::State;use crate::{LibsyError, Result};-use switchyard_protocol::{Context, Request, Response, RoutedLlmClient};+use switchyard_protocol::{Context, Request, Response};/// Telemetry name for a router this module assembles.const STAGE_ROUTER: &str = "stage_router";@@ -142,10 +142,6 @@ impl Algorithm for StageRouter {STAGE_ROUTER}- fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {- self.route.count_tokens_client()- }-async fn create_run_task(self: Arc<Self>,ctx: Context,diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 676328bc9..d63294ee7 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -405,19 +405,6 @@ impl LlmTargetSet {.cloned().ok_or(LibsyError::AllTargetsExcluded)}-- /// The first target's client that can serve `count_tokens` (an Anthropic- /// upstream), or `None` when no target has one. Used by an algorithm's- /// [`count_tokens_client`](crate::Algorithm::count_tokens_client).- pub fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {- self.targets.iter().find_map(|target| {- target- .llm_client- .as_ref()- .filter(|client| client.supports_count_tokens())- .cloned()- })- }}/// Bounds process-local overflow history. Dropping a live session's entry costs one@@ -590,39 +577,6 @@ pub trait Algorithm: Send + Sync + 'static {Ok(())}- /// The client [`count_tokens`](Self::count_tokens) forwards to: the first of- /// this algorithm's targets whose client can count tokens (an Anthropic- /// upstream). The default is `None` — an algorithm with no Anthropic target- /// does not support token counting.- ///- /// CAVEAT: this picks the *first* Anthropic target, not a routed one —- /// count_tokens is a direct passthrough, so it does not run the routing- /// cascade. For a route with several Anthropic tiers "first" is arbitrary;- /// choosing which tier count_tokens should reflect is deferred.- fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {- None- }-- /// Count the tokens `request` would use — a **direct passthrough** to this- /// algorithm's Anthropic target (via- /// [`count_tokens_client`](Self::count_tokens_client)), **not** a routed- /// call. Token counting is a pre-flight estimate with no routing decision,- /// so it deliberately bypasses the classifier cascade (which runs only for- /// completions via [`run`](Self::run)). Returns the upstream's JSON- /// verbatim. Errors when the algorithm has no Anthropic target.- async fn count_tokens(&self, request: Request) -> Result<serde_json::Value> {- let client = self- .count_tokens_client()- .ok_or_else(|| LibsyError::AlgorithmError {- message: "no target supports count_tokens (needs an Anthropic upstream)"- .to_string(),- })?;- client- .count_tokens(request)- .await- .map_err(|source| LibsyError::client_call("count_tokens", source))- }-/// Process a request to completion, returning a stream of [`Step`]s.////// The consumer must fulfill every [`Step::CallLlm`] before the algorithm candiff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rsindex 233ddb38c..05124509b 100644--- a/crates/protocol/src/client.rs+++ b/crates/protocol/src/client.rs@@ -180,24 +180,4 @@ pub trait RoutedLlmClient: Send + Sync {request: Request,decision: Arc<dyn Decision>,) -> Result<Response, LlmClientError>;-- /// Whether this client can serve [`count_tokens`](Self::count_tokens) — i.e.- /// it has an Anthropic upstream. The default is `false`.- fn supports_count_tokens(&self) -> bool {- false- }-- /// Count the tokens `request` would use — a **direct passthrough**, not a- /// routed call. Forwards `request` to this client's Anthropic- /// `/v1/messages/count_tokens` endpoint (model restamped to the upstream- /// target id) and returns the JSON verbatim. Token counting is a pre-flight- /// estimate with no routing decision, so unlike [`call`](Self::call) it- /// takes no [`Decision`]. The default errors; only an Anthropic-backed- /// client overrides it.- async fn count_tokens(&self, request: Request) -> Result<serde_json::Value, LlmClientError> {- let _ = request;- Err(LlmClientError::Configuration {- message: "count_tokens is not supported by this client".to_string(),- })- }}diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex edbf5a258..afab8f986 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -119,7 +119,7 @@ documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_| `POST` | `/v1/chat/completions` | OpenAI Chat Completions || `POST` | `/v1/messages` | Anthropic Messages || `POST` | `/v1/responses` | OpenAI Responses |-| `POST` | `/v1/messages/count_tokens` | Anthropic token count |+| `POST` | `/v1/messages/count_tokens` | Token count from a route's Anthropic target || `GET` | `/v1/models` | Routes served by this deployment || `GET` | `/v1/stats` | Per-model request, token, and cost totals || `POST` | `/v1/stats/reset` | Clear accumulated stats |@@ -130,6 +130,9 @@ Requests name a route by its `id`, so `POST /v1/chat/completions` with `"model":routes through the `[routes.general]` entry above. Any of the three request formats can address anyroute, and the server translates between them.+Token counting selects an Anthropic-format completion target, preferring target names or model IDs+containing `opus`, `sonnet`, then `haiku`. Other ties preserve the route's target order.+## Metrics`GET /metrics` exposes Prometheus text from the server's process-wide OpenTelemetry provider.diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rsindex 95cc57b60..18ad3f7fe 100644--- a/crates/switchyard-server/src/config.rs+++ b/crates/switchyard-server/src/config.rs@@ -21,7 +21,7 @@ use switchyard_llm_client::{};use switchyard_protocol::RoutedLlmClient;-use crate::{ModelCapabilities, ServerError, ServerResult, ServerState};+use crate::{CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState};const SUPPORTED_SCHEMA_VERSION: u32 = 1;const MAX_CONFIGURED_RETRIES: u32 = 10;@@ -98,12 +98,18 @@ impl ServerConfig {)));}let algorithm = build_algorithm(route_name, config, &targets)?;- routes.push((config.id().to_string(), algorithm, capabilities));+ let count_tokens_target = self.build_count_tokens_target(config, &clients);+ routes.push((+ config.id().to_string(),+ algorithm,+ capabilities,+ count_tokens_target,+ ));}ServerState::new_with_capabilities(routes)}- fn build_clients(&self) -> ServerResult<BTreeMap<String, Arc<dyn RoutedLlmClient>>> {+ fn build_clients(&self) -> ServerResult<BTreeMap<String, Arc<TranslatingLlmClient>>> {let mut models_by_client = self.llm_clients.keys()@@ -132,7 +138,7 @@ impl ServerConfig {let mut clients = BTreeMap::new();for (name, model_configs) in models_by_client {- let client: Arc<dyn RoutedLlmClient> = Arc::new(+ let client = Arc::new(TranslatingLlmClient::new(&model_configs).map_err(|error| ServerError::new(error.to_string()))?,);@@ -143,7 +149,7 @@ impl ServerConfig {fn build_targets(&self,- clients: &BTreeMap<String, Arc<dyn RoutedLlmClient>>,+ clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,) -> ServerResult<BTreeMap<String, LlmTarget>> {self.targets.iter()@@ -151,16 +157,53 @@ impl ServerConfig {let client = clients.get(&config.llm_client).ok_or_else(|| {ServerError::new(format!("target {name} has no constructed llm client"))})?;+ let client: Arc<dyn RoutedLlmClient> = client.clone();Ok((name.clone(),LlmTarget {semantic_name: config.id.clone(),- llm_client: Some(Arc::clone(client)),+ llm_client: Some(client),},))}).collect()}++ fn build_count_tokens_target(+ &self,+ route_config: &RouteConfig,+ clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,+ ) -> Option<CountTokensTarget> {+ route_config+ .routing_target_names()+ .into_iter()+ .enumerate()+ .filter_map(|(index, name)| {+ let target = self.targets.get(name)?;+ let client = clients.get(&target.llm_client)?;+ client.supports_count_tokens(&target.id).then_some((+ count_tokens_priority(name, &target.id),+ index,+ target,+ client,+ ))+ })+ .min_by_key(|(priority, index, _, _)| (*priority, *index))+ .map(|(_, _, target, client)| CountTokensTarget {+ model: target.id.clone(),+ client: client.clone(),+ })+ }+}++// Prefer known Claude families, then preserve the route's target order.+fn count_tokens_priority(target_name: &str, model_id: &str) -> usize {+ let target_name = target_name.to_ascii_lowercase();+ let model_id = model_id.to_ascii_lowercase();+ ["opus", "sonnet", "haiku"]+ .iter()+ .position(|hint| target_name.contains(hint) || model_id.contains(hint))+ .unwrap_or(3)}#[derive(Debug, Deserialize)]@@ -399,6 +442,44 @@ impl RouteConfig {}}+ // Completion targets in algorithm order; judge-only targets are excluded.+ fn routing_target_names(&self) -> Vec<&str> {+ match self {+ Self::Noop { .. } => Vec::new(),+ Self::Random { targets, .. } => targets.iter().map(String::as_str).collect(),+ Self::Passthrough { target, .. } => vec![target],+ Self::LlmClassifier {+ mode,+ strong_target,+ weak_target,+ escalation,+ targets,+ ..+ } => match mode.unwrap_or(if escalation.is_some() {+ ClassifierMode::Escalation+ } else {+ ClassifierMode::Capability+ }) {+ ClassifierMode::Capability => weak_target+ .iter()+ .chain(strong_target)+ .map(String::as_str)+ .collect(),+ ClassifierMode::Escalation => strong_target+ .iter()+ .chain(weak_target)+ .map(String::as_str)+ .collect(),+ ClassifierMode::Custom => targets.iter().flatten().map(String::as_str).collect(),+ },+ Self::StageRouter {+ capable_target,+ efficient_target,+ ..+ } => vec![capable_target, efficient_target],+ }+ }+fn capabilities(&self) -> ModelCapabilities {use RouteConfig::*;match self {diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex edcdac377..8e39620e3 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -36,6 +36,7 @@ use libsy::{Algorithm, LibsyError, RunObservation, RunObserver};use parking_lot::Mutex;use serde::Deserialize;use serde_json::{Value, json};+use switchyard_llm_client::TranslatingLlmClient;use switchyard_protocol::{Context, Decision, LlmClientError, Metadata, Request, Usage};use tokio::net::{TcpListener, TcpSocket};use tokio::task;@@ -97,12 +98,24 @@ struct ModelCapabilities {tool_calling: Option<bool>,}-/// A registered route: the libsy algorithm that serves it and the capabilities-/// advertised for it on `GET /v1/models`. One entry owns both so the routing-/// runtime and the model listing can never drift apart.+/// A registered algorithm route and its server-owned endpoint metadata.struct RouteEntry {algorithm: Arc<dyn Algorithm>,capabilities: ModelCapabilities,+ count_tokens_target: Option<CountTokensTarget>,+}++/// Exact upstream model used by the server's Anthropic token-count endpoint.+#[derive(Clone)]+struct CountTokensTarget {+ model: String,+ client: Arc<TranslatingLlmClient>,+}++impl CountTokensTarget {+ async fn count_tokens(&self, request: Request) -> Result<Value, LlmClientError> {+ self.client.count_tokens(&self.model, request).await+ }}/// Shared server state used by all endpoint handlers.@@ -157,15 +170,22 @@ impl ServerState {Self::new_with_capabilities(routes.into_iter()- .map(|(model, algorithm)| (model, algorithm, ModelCapabilities::default())),+ .map(|(model, algorithm)| (model, algorithm, ModelCapabilities::default(), None)),)}fn new_with_capabilities(- routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>, ModelCapabilities)>,+ routes: impl IntoIterator<+ Item = (+ String,+ Arc<dyn Algorithm>,+ ModelCapabilities,+ Option<CountTokensTarget>,+ ),+ >,) -> ServerResult<Self> {let mut entries = BTreeMap::new();- for (model, algorithm, capabilities) in routes {+ for (model, algorithm, capabilities, count_tokens_target) in routes {let model = model.trim();if model.is_empty() {return Err(ServerError::new("route model must not be empty"));@@ -173,6 +193,7 @@ impl ServerState {let entry = RouteEntry {algorithm,capabilities,+ count_tokens_target,};if entries.insert(model.to_string(), entry).is_some() {return Err(ServerError::new(format!("duplicate route model {model}")));@@ -202,10 +223,8 @@ impl ServerState {self.routes.keys().map(String::as_str)}- fn algorithm_for_model(&self, model: &str) -> Option<Arc<dyn Algorithm>> {- self.routes- .get(model)- .map(|entry| Arc::clone(&entry.algorithm))+ fn route_for_model(&self, model: &str) -> Option<&RouteEntry> {+ self.routes.get(model)}}@@ -463,11 +482,7 @@ async fn openai_responses(handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await}-/// Anthropic token counting. Resolves the route named by `model`, then does a-/// **direct passthrough** via [`Algorithm::count_tokens`] to that route's-/// Anthropic target — it does *not* run the routing cascade (count_tokens is a-/// pre-flight estimate with no routing decision). Unknown route → 404; a route-/// with no Anthropic target → 400.+/// Anthropic token counting against the route's explicitly configured target.async fn anthropic_count_tokens(State(state): State<ServerState>,headers: HeaderMap,@@ -477,7 +492,7 @@ async fn anthropic_count_tokens(Ok(body) => body,Err(message) => return anthropic_error_response(invalid_body_error(message)),};- let (algorithm, request) = match resolve_route(+ let (route, request) = match resolve_route(&state,metadata_from_headers(headers),body,@@ -486,27 +501,23 @@ async fn anthropic_count_tokens(Ok(resolved) => resolved,Err(response) => return anthropic_error_response(response),};- anthropic_error_response(match algorithm.count_tokens(request).await {+ let Some(target) = route.count_tokens_target.as_ref() else {+ return anthropic_error_response(error_response(+ StatusCode::BAD_REQUEST,+ "route has no Anthropic target for token counting",+ "invalid_request_error",+ "count_tokens_unsupported",+ ));+ };+ anthropic_error_response(match target.count_tokens(request).await {Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),Err(error) => count_tokens_error(error),})}-/// Map a [`count_tokens`](Algorithm::count_tokens) failure to an HTTP response:-/// the route has no Anthropic target → 400, an upstream HTTP error → its own-/// status, anything else → 502.-fn count_tokens_error(error: LibsyError) -> Response {- // The one count_tokens-specific case is "no Anthropic target in the route";- // every upstream/client failure gets the same mapping completions use.- match &error {- LibsyError::AlgorithmError { message } => error_response(- StatusCode::BAD_REQUEST,- message.clone(),- "invalid_request_error",- "count_tokens_unsupported",- ),- _ => algorithm_error(error),- }+/// Maps a token-count client failure with the same policy as a routed client call.+fn count_tokens_error(error: LlmClientError) -> Response {+ client_error(&error)}async fn handle_endpoint(@@ -585,7 +596,7 @@ fn llm_json_body(/// Decode `body`, resolve the route named by its `model`, and build the/// [`Request`]. Shared by the completion and `count_tokens` handlers. Returns-/// the resolved algorithm and the built request — or an error [`Response`]+/// the resolved route and the built request — or an error [`Response`]/// (invalid body, empty `model` → 400, unknown route → 404).// Both callers immediately return the `Err(Response)` as the HTTP response, so// the large error type is intentional, not propagated up a call stack.@@ -595,7 +606,7 @@ fn resolve_route(metadata: Metadata,body: Value,wire_format: WireFormat,-) -> std::result::Result<(Arc<dyn Algorithm>, Request), Response> {+) -> std::result::Result<(&RouteEntry, Request), Response> {let llm_request = decode_request(wire_format, &body).map_err(|error| invalid_body_error(error.to_string()))?;let requested_model = llm_request@@ -610,7 +621,7 @@ fn resolve_route("invalid_request_error",)})?;- let algorithm = state.algorithm_for_model(&requested_model).ok_or_else(|| {+ let route = state.route_for_model(&requested_model).ok_or_else(|| {error_response(StatusCode::NOT_FOUND,format!("No route registered for model {requested_model}"),@@ -623,7 +634,7 @@ fn resolve_route(raw_request: Some(body),metadata: Some(metadata),};- Ok((algorithm, request))+ Ok((route, request))}async fn handle_llm_request(@@ -635,10 +646,11 @@ async fn handle_llm_request(routing_log_context: Option<routing_log::RoutingLogContext>,) -> Response {let cache_probe = state.track_cache_eligibility.then(|| prefix_probe(&body));- let (algorithm, request) = match resolve_route(&state, metadata, body, wire_format) {+ let (route, request) = match resolve_route(&state, metadata, body, wire_format) {Ok(resolved) => resolved,Err(response) => return response,};+ let algorithm = Arc::clone(&route.algorithm);let observer = stats_observer(state.stats.clone());let (trace, response) = match algorithm.run_observed(Context::default(), request, Some(observer))@@ -784,7 +796,11 @@ fn algorithm_error(error: LibsyError) -> Response {let LibsyError::ClientCall { source, .. } = &error else {return server_error(error.to_string());};- match source {+ client_error(source)+}++fn client_error(error: &LlmClientError) -> Response {+ match error {LlmClientError::InvalidRequest { message }| LlmClientError::RequestTranslation(message) => error_response(StatusCode::BAD_REQUEST,diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 42aac1665..3b7b1100a 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -980,10 +980,14 @@ base_url = "{base_url}"id = "real/opus"llm_client = "claude"+[targets.other]+id = "real/sonnet"+llm_client = "claude"+[routes.random]id = "switchyard/random"type = "random"-targets = ["strong"]+targets = ["other", "strong"]"#,base_url = upstream.base_url))?;@@ -1044,15 +1048,14 @@ targets = ["weak"]).await?;assert_eq!(response.status, StatusCode::BAD_REQUEST);- // The route's picked target is OpenAI, so count_tokens (Anthropic-only) is- // unsupported for it.+ // This route has no Anthropic-format target.assert_eq!(response.json()?,json!({"type": "error","error": {"type": "invalid_request_error",- "message": "no target supports count_tokens (needs an Anthropic upstream)"+ "message": "route has no Anthropic target for token counting"}}));