Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(server): /v1/messages/count_tokens forwarding on the Rust server (#138)
sabhatinas82d5fbb36dcb
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rsindex c907f290d..f847249e2 100644--- a/crates/libsy-llm-client/src/backend.rs+++ b/crates/libsy-llm-client/src/backend.rs@@ -124,6 +124,19 @@ impl Backend {&self.config().extra_headers}+ /// Whether this backend speaks the Anthropic Messages wire format — the only+ /// one with a `count_tokens` endpoint.+ pub fn is_anthropic(&self) -> bool {+ matches!(self, Backend::Anthropic(_))+ }++ /// The upstream `/v1/messages/count_tokens` URL, derived from the same base+ /// URL join as [`url`](Self::url).+ pub fn count_tokens_url(&self) -> String {+ let base_url = self.config().base_url.trim_end_matches('/');+ format!("{}/count_tokens", anthropic_url(base_url))+ }+/// Whether an upstream 400 `body` looks like a context-window overflow for/// this backend's provider.pub(crate) fn is_context_overflow(&self, body: &str) -> bool {@@ -218,6 +231,34 @@ mod tests {);}+ #[test]+ fn count_tokens_url_joins_every_base_url_shape() {+ assert_eq!(+ Backend::Anthropic(config("https://host")).count_tokens_url(),+ "https://host/v1/messages/count_tokens"+ );+ assert_eq!(+ Backend::Anthropic(config("https://host/v1")).count_tokens_url(),+ "https://host/v1/messages/count_tokens"+ );+ assert_eq!(+ Backend::Anthropic(config("https://host/v1/messages")).count_tokens_url(),+ "https://host/v1/messages/count_tokens"+ );+ // Trailing slash is trimmed before the join.+ assert_eq!(+ Backend::Anthropic(config("https://host/v1/")).count_tokens_url(),+ "https://host/v1/messages/count_tokens"+ );+ }++ #[test]+ fn only_anthropic_backend_is_anthropic() {+ assert!(Backend::Anthropic(config("x")).is_anthropic());+ assert!(!Backend::OpenAiChat(config("x")).is_anthropic());+ assert!(!Backend::OpenAiResponses(config("x")).is_anthropic());+ }+#[test]fn wire_format_matches_variant() {assert_eq!(diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex cc1c2b88a..5ca18cc16 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -11,7 +11,7 @@ use futures_util::StreamExt;use reqwest::RequestBuilder;use serde_json::Value;use switchyard_protocol::{- Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient,+ Context, Decision, LlmRequest, LlmResponse, Metadata, Request, Response, RoutedLlmClient,};use switchyard_translation::{decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,@@ -101,6 +101,23 @@ impl TranslatingLlmClient {})}+ /// A configured Anthropic backend to forward a direct `count_tokens` call to,+ /// paired with its upstream model id (the id the inbound route name is+ /// restamped to). `None` when this client has no Anthropic backend;+ /// `count_tokens` is Anthropic-only.+ fn anthropic_backend(&self) -> Option<(&str, &Backend)> {+ self.model_to_config.values().find_map(|config| {+ if config.default_backend.is_anthropic() {+ return Some((config.model_name.as_str(), &config.default_backend));+ }+ config+ .other_backends+ .as_ref()+ .and_then(|backends| backends.iter().find(|backend| backend.is_anthropic()))+ .map(|backend| (config.model_name.as_str(), backend))+ })+ }+/// The backend serving `model` over `format` — the default backend when its/// format matches, otherwise a matching entry in `other_backends`; `None` when/// the model is unknown or has no backend for `format`.@@ -117,6 +134,66 @@ impl TranslatingLlmClient {})}+ /// Encode `llm_request` (its model restamped to `model`) for `wire_format`,+ /// POST it to `url` with the request's forwarded headers plus the backend's+ /// static headers and auth, and return the successful upstream response and+ /// whether the request asked for a streamed body. A non-success status maps+ /// to a typed error — a 400 is classified as a context-window overflow via+ /// the backend's provider rules. Shared by+ /// [`call_rewrite_model`](Self::call_rewrite_model) (which POSTs to the+ /// backend's completion URL and decodes a response) and+ /// [`count_tokens`](RoutedLlmClient::count_tokens) (which POSTs to the+ /// `count_tokens` URL and returns the raw JSON).+ async fn send_encoded(+ &self,+ url: String,+ backend: &Backend,+ wire_format: WireFormat,+ mut llm_request: LlmRequest,+ metadata: Option<&Metadata>,+ model: &str,+ ) -> Result<(reqwest::Response, bool)> {+ // The resolved name is the upstream model id (per the crate contract).+ llm_request.model = Some(model.to_string());+ let mut body = encode_request(&llm_request, wire_format)+ .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?;+ // `encode_request` round-trips a preserved same-format body verbatim,+ // which keeps the caller's original `model`; force the resolved model so+ // the upstream always sees the target id.+ set_json_model(&mut body, model);+ let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false);++ let builder = self.client.post(url).json(&body);+ let builder = forward_metadata_headers(builder, metadata);+ let builder = apply_extra_headers(builder, backend);+ let builder = backend.apply_auth(builder);++ let http_response = builder.send().await.map_err(convert_reqwest_error)?;+ let status = http_response.status();+ if !status.is_success() {+ let body = match http_response.text().await {+ Ok(body) => body,+ Err(error) if error.is_timeout() => {+ return Err(LlmClientError::Timeout {+ source: Box::new(error),+ })+ }+ Err(error) => format!("<failed to read error body: {error}>"),+ };+ if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {+ return Err(LlmClientError::ContextWindowExceeded {+ model: model.to_string(),+ message: body,+ });+ }+ return Err(LlmClientError::UpstreamHttp {+ status: status.as_u16(),+ body,+ });+ }+ Ok((http_response, streaming))+ }+/// Calls the backend for `model_name` (or the request's own model), over the/// wire format the request pins in its metadata (else the model's default/// backend), and returns the neutral response.@@ -134,7 +211,7 @@ impl TranslatingLlmClient {// Own the request's parts so the model can be set without a `mut` param// and without cloning the messages. `raw_request` is unused here.let Request {- mut llm_request,+ llm_request,metadata,..} = request;@@ -161,45 +238,16 @@ impl TranslatingLlmClient {message: format!("model {model:?} has no backend for format {wire_format}"),})?;- // The resolved name is the upstream model id (per the crate contract).- llm_request.model = Some(model.clone());-- let mut body = encode_request(&llm_request, wire_format)- .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?;- // `encode_request` round-trips a preserved same-format body verbatim,- // which keeps the caller's original `model`; force the resolved model so- // the upstream always sees the target id.- set_json_model(&mut body, &model);- let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false);-- let builder = self.client.post(backend.url()).json(&body);- let builder = forward_metadata_headers(builder, metadata.as_ref());- let builder = apply_extra_headers(builder, backend);- let builder = backend.apply_auth(builder);-- let http_response = builder.send().await.map_err(convert_reqwest_error)?;- let status = http_response.status();- if !status.is_success() {- let body = match http_response.text().await {- Ok(body) => body,- Err(error) if error.is_timeout() => {- return Err(LlmClientError::Timeout {- source: Box::new(error),- })- }- Err(error) => format!("<failed to read error body: {error}>"),- };- if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {- return Err(LlmClientError::ContextWindowExceeded {- model,- message: body,- });- }- return Err(LlmClientError::UpstreamHttp {- status: status.as_u16(),- body,- });- }+ let (http_response, streaming) = self+ .send_encoded(+ backend.url(),+ backend,+ wire_format,+ llm_request,+ metadata.as_ref(),+ &model,+ )+ .await?;let llm_response = if streaming {// Adapt the reqwest body stream to plain bytes; the SSE-decode itself is@@ -304,6 +352,42 @@ impl RoutedLlmClient for TranslatingLlmClient {let model_name = Some(decision.selected_model());self.call_rewrite_model(ctx, request, model_name).await}++ fn supports_count_tokens(&self) -> bool {+ self.anthropic_backend().is_some()+ }++ async fn count_tokens(&self, request: Request) -> Result<Value> {+ // Direct passthrough: forward straight to this client's Anthropic+ // backend's count_tokens endpoint. Not routed — no decision.+ let (model, backend) =+ self.anthropic_backend()+ .ok_or_else(|| LlmClientError::Configuration {+ message: "count_tokens is anthropic-only; this client has no anthropic backend"+ .to_string(),+ })?;+ // Same encode-and-forward path as the completion call, only to the+ // `count_tokens` URL; the response is the raw `{"input_tokens": N}` JSON.+ let Request {+ llm_request,+ metadata,+ ..+ } = request;+ let (http_response, _) = self+ .send_encoded(+ backend.count_tokens_url(),+ backend,+ WireFormat::AnthropicMessages,+ llm_request,+ metadata.as_ref(),+ model,+ )+ .await?;+ let text = http_response.text().await.map_err(convert_reqwest_error)?;+ serde_json::from_str(&text).map_err(|error| LlmClientError::InvalidResponse {+ source: Box::new(error),+ })+ }}fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex a87e7edfd..b95f7b578 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -22,6 +22,7 @@ use async_trait::async_trait;use crate::core::{Classifier, Event, Processor, Score, SharedState};use crate::{Algorithm, Context, Decision, Driver, LibsyError, LlmTargetSet, Request, Response, Result,+ RoutedLlmClient,};/// The decision a fall-through run produces: the selected model plus a human-readable reason.@@ -100,6 +101,10 @@ impl Algorithm<SharedState> for FallThrough {"fall_through"}+ fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {+ self.targets.count_tokens_client()+ }+async fn create_run_task(self: Arc<Self>,ctx: Context<SharedState>,diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rsindex e2b71c100..4037cde68 100644--- a/crates/libsy/src/algorithms/passthrough.rs+++ b/crates/libsy/src/algorithms/passthrough.rs@@ -9,7 +9,7 @@ use std::sync::Arc;use switchyard_protocol::{Request, Response};-use crate::{Algorithm, Context, Decision, Driver, LlmTarget, Result};+use crate::{Algorithm, Context, Decision, Driver, LlmTarget, Result, RoutedLlmClient};/// See module commentpub struct Passthrough {@@ -48,6 +48,14 @@ where"passthrough"}+ fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {+ self.target+ .llm_client+ .as_ref()+ .filter(|client| client.supports_count_tokens())+ .cloned()+ }+async fn create_run_task(self: Arc<Self>,ctx: Context<S>,diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rsindex c91f7cfcf..40f1abcec 100644--- a/crates/libsy/src/algorithms/rand.rs+++ b/crates/libsy/src/algorithms/rand.rs@@ -16,7 +16,7 @@ use rand::SeedableRng;use crate::{Algorithm, Context, Decision, Driver, LibsyError, LlmTarget, LlmTargetSet, Request, Response,- Result,+ Result, RoutedLlmClient,};/// Decision produced by [`Random`]: which target was chosen and why.@@ -138,6 +138,10 @@ where"random"}+ fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {+ self.target_set.count_tokens_client()+ }+async fn create_run_task(self: Arc<Self>,ctx: Context<S>,diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex c12678b53..b3e71bc4e 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -307,6 +307,19 @@ impl LlmTargetSet {target: name.to_string(),})}++ /// The first target's client that can serve `count_tokens` (an Anthropic+ /// upstream), or `None` when no target has one. Used by an algorithm's+ /// [`count_tokens_client`](crate::Algorithm::count_tokens_client).+ pub fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {+ self.targets.iter().find_map(|target| {+ target+ .llm_client+ .as_ref()+ .filter(|client| client.supports_count_tokens())+ .cloned()+ })+ }}/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);@@ -352,6 +365,39 @@ whereOk(())}+ /// The client [`count_tokens`](Self::count_tokens) forwards to: the first of+ /// this algorithm's targets whose client can count tokens (an Anthropic+ /// upstream). The default is `None` — an algorithm with no Anthropic target+ /// does not support token counting.+ ///+ /// CAVEAT: this picks the *first* Anthropic target, not a routed one —+ /// count_tokens is a direct passthrough, so it does not run the routing+ /// cascade. For a route with several Anthropic tiers "first" is arbitrary;+ /// choosing which tier count_tokens should reflect is deferred.+ fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {+ None+ }++ /// Count the tokens `request` would use — a **direct passthrough** to this+ /// algorithm's Anthropic target (via+ /// [`count_tokens_client`](Self::count_tokens_client)), **not** a routed+ /// call. Token counting is a pre-flight estimate with no routing decision,+ /// so it deliberately bypasses the classifier cascade (which runs only for+ /// completions via [`run`](Self::run)). Returns the upstream's JSON+ /// verbatim. Errors when the algorithm has no Anthropic target.+ async fn count_tokens(&self, request: Request) -> Result<serde_json::Value> {+ let client = self+ .count_tokens_client()+ .ok_or_else(|| LibsyError::AlgorithmError {+ message: "no target supports count_tokens (needs an Anthropic upstream)"+ .to_string(),+ })?;+ client+ .count_tokens(request)+ .await+ .map_err(|source| LibsyError::client_call("count_tokens", source))+ }+/// Process a request to completion, returning a stream of [`Step`]s./// Each [`Step::CallLlm`] is an offloaded model call the consumer must serve./// The stream ends with a [`Step::ReturnToAgent`] on success, or an `Err` item on failure.diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rsindex 6e546054f..7130397d8 100644--- a/crates/protocol/src/client.rs+++ b/crates/protocol/src/client.rs@@ -145,4 +145,24 @@ pub trait RoutedLlmClient: Send + Sync {request: Request,decision: Arc<dyn Decision>,) -> Result<Response, LlmClientError>;++ /// Whether this client can serve [`count_tokens`](Self::count_tokens) — i.e.+ /// it has an Anthropic upstream. The default is `false`.+ fn supports_count_tokens(&self) -> bool {+ false+ }++ /// Count the tokens `request` would use — a **direct passthrough**, not a+ /// routed call. Forwards `request` to this client's Anthropic+ /// `/v1/messages/count_tokens` endpoint (model restamped to the upstream+ /// target id) and returns the JSON verbatim. Token counting is a pre-flight+ /// estimate with no routing decision, so unlike [`call`](Self::call) it+ /// takes no [`Decision`]. The default errors; only an Anthropic-backed+ /// client overrides it.+ async fn count_tokens(&self, request: Request) -> Result<serde_json::Value, LlmClientError> {+ let _ = request;+ Err(LlmClientError::Configuration {+ message: "count_tokens is not supported by this client".to_string(),+ })+ }}diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 5221b3358..f5d7e7d36 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -198,6 +198,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router {Router::new().route("/v1/chat/completions", post(openai_chat_completions)).route("/v1/messages", post(anthropic_messages))+ .route("/v1/messages/count_tokens", post(anthropic_count_tokens)).route("/v1/responses", post(openai_responses)).route("/v1/models", get(models)).route("/metrics", get(prometheus_metrics))@@ -257,6 +258,52 @@ async fn openai_responses(handle_endpoint(state, headers, body, WireFormat::OpenAiResponses).await}+/// Anthropic token counting. Resolves the route named by `model`, then does a+/// **direct passthrough** via [`Algorithm::count_tokens`] to that route's+/// Anthropic target — it does *not* run the routing cascade (count_tokens is a+/// pre-flight estimate with no routing decision). Unknown route → 404; a route+/// with no Anthropic target → 400.+async fn anthropic_count_tokens(+ State(state): State<ServerState>,+ headers: HeaderMap,+ body: std::result::Result<Json<Value>, JsonRejection>,+) -> Response {+ let body = match llm_json_body(body) {+ Ok(body) => body,+ Err(message) => return invalid_body_error(message),+ };+ let (algorithm, request, _) = match resolve_route(+ &state,+ metadata_from_headers(&headers),+ body,+ WireFormat::AnthropicMessages,+ ) {+ Ok(resolved) => resolved,+ Err(response) => return response,+ };+ match algorithm.count_tokens(request).await {+ Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),+ Err(error) => count_tokens_error(error),+ }+}++/// Map a [`count_tokens`](Algorithm::count_tokens) failure to an HTTP response:+/// the route has no Anthropic target → 400, an upstream HTTP error → its own+/// status, anything else → 502.+fn count_tokens_error(error: LibsyError) -> Response {+ // The one count_tokens-specific case is "no Anthropic target in the route";+ // every upstream/client failure gets the same mapping completions use.+ match &error {+ LibsyError::AlgorithmError { message } => error_response(+ StatusCode::BAD_REQUEST,+ message.clone(),+ "invalid_request_error",+ "count_tokens_unsupported",+ ),+ _ => algorithm_error(error),+ }+}+async fn handle_endpoint(state: ServerState,headers: HeaderMap,@@ -301,42 +348,61 @@ fn llm_json_body(}}-async fn handle_llm_request(- state: ServerState,+/// Decode `body`, resolve the route named by its `model`, and build the+/// [`Request`]. Shared by the completion and `count_tokens` handlers. Returns+/// the resolved algorithm, the built request, and the requested route model —+/// or an error [`Response`] (invalid body, empty `model` → 400, unknown route →+/// 404).+// Both callers immediately return the `Err(Response)` as the HTTP response, so+// the large error type is intentional, not propagated up a call stack.+#[allow(clippy::type_complexity, clippy::result_large_err)]+fn resolve_route(+ state: &ServerState,metadata: Metadata,body: Value,wire_format: WireFormat,-) -> Response {- let llm_request = match decode_request(wire_format, &body) {- Ok(request) => request,- Err(error) => return invalid_body_error(error.to_string()),- };- let Some(requested_model) = llm_request+) -> std::result::Result<(Arc<dyn Algorithm<SharedState>>, Request, String), Response> {+ let llm_request = decode_request(wire_format, &body)+ .map_err(|error| invalid_body_error(error.to_string()))?;+ let requested_model = llm_request.model.clone().filter(|model| !model.trim().is_empty())- else {- return error_response(- StatusCode::BAD_REQUEST,- "request body must include a non-empty string `model`",- "invalid_request_error",- "invalid_request_error",- );- };- let Some(algorithm) = state.algorithm_for_model(&requested_model) else {- return error_response(+ .ok_or_else(|| {+ error_response(+ StatusCode::BAD_REQUEST,+ "request body must include a non-empty string `model`",+ "invalid_request_error",+ "invalid_request_error",+ )+ })?;+ let algorithm = state.algorithm_for_model(&requested_model).ok_or_else(|| {+ error_response(StatusCode::NOT_FOUND,format!("No route registered for model {requested_model}"),"model_not_found","model_not_found",- );- };-+ )+ })?;let request = Request {llm_request,raw_request: Some(body),metadata: Some(metadata),};+ Ok((algorithm, request, requested_model))+}++async fn handle_llm_request(+ state: ServerState,+ metadata: Metadata,+ body: Value,+ wire_format: WireFormat,+) -> Response {+ let (algorithm, request, requested_model) =+ match resolve_route(&state, metadata, body, wire_format) {+ Ok(resolved) => resolved,+ Err(response) => return response,+ };let (trace, response) = match algorithm.run(Context::<SharedState>::default(), request).awaitdiff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 24dac9ef9..c8bb644d1 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -17,7 +17,8 @@ use axum::response::{IntoResponse, Response as HttpResponse};use axum::routing::post;use axum::{Json, Router};use http_body_util::BodyExt;-use libsy::algorithms::Random;+use libsy::algorithms::{FallThrough, Random};+use libsy::stage_router::{PickerMode, StageClassifier};use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, SharedState};use serde_json::{json, Value};use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};@@ -45,6 +46,7 @@ impl MockUpstream {let calls = Arc::new(Mutex::new(Vec::new()));let app = Router::new().route("/v1/chat/completions", post(upstream_chat))+ .route("/v1/messages/count_tokens", post(upstream_count_tokens)).with_state(Arc::clone(&calls));let listener = TcpListener::bind("127.0.0.1:0").await?;let addr = listener.local_addr()?;@@ -120,6 +122,14 @@ async fn upstream_chat(.into_response()}+async fn upstream_count_tokens(+ State(calls): State<Arc<Mutex<Vec<Value>>>>,+ Json(body): Json<Value>,+) -> HttpResponse {+ calls.lock().await.push(body.clone());+ Json(json!({"input_tokens": 7})).into_response()+}+fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<ServerState> {let backend = Backend::OpenAiChat(HttpBackendConfig {base_url: base_url.to_string(),@@ -335,6 +345,162 @@ target = "weak"Ok(())}+#[tokio::test]+async fn count_tokens_forwards_to_configured_anthropic_target() -> TestResult {+ let upstream = MockUpstream::start().await?;+ let state = load_test_config(&format!(+ r#"+schema_version = 1++[llm_clients.claude]+format = "anthropic_messages"+base_url = "{base_url}"++[targets.strong]+id = "real/opus"+llm_client = "claude"++[routes.random]+id = "switchyard/random"+type = "random"+targets = ["strong"]+"#,+ base_url = upstream.base_url+ ))?;+ let app = build_switchyard_router(state);++ let response = send(+ &app,+ "POST",+ "/v1/messages/count_tokens",+ Some(json!({+ "model": "switchyard/random",+ "messages": [{"role": "user", "content": "hi"}]+ })),+ )+ .await?;+ assert_eq!(response.status, StatusCode::OK);+ assert_eq!(response.json()?["input_tokens"], 7);++ let calls = upstream.calls.lock().await;+ assert_eq!(calls.len(), 1);+ // The inbound route name is rewritten to the real upstream model.+ assert_eq!(calls[0]["model"], "real/opus");+ Ok(())+}++#[tokio::test]+async fn count_tokens_without_anthropic_target_returns_bad_request() -> TestResult {+ let upstream = MockUpstream::start().await?;+ let state = load_test_config(&format!(+ r#"+schema_version = 1++[llm_clients.upstream]+format = "openai_chat"+base_url = "{base_url}"++[targets.weak]+id = "model/weak"+llm_client = "upstream"++[routes.random]+id = "switchyard/random"+type = "random"+targets = ["weak"]+"#,+ base_url = upstream.base_url+ ))?;+ let app = build_switchyard_router(state);++ let response = send(+ &app,+ "POST",+ "/v1/messages/count_tokens",+ Some(json!({+ "model": "switchyard/random",+ "messages": [{"role": "user", "content": "hi"}]+ })),+ )+ .await?;+ assert_eq!(response.status, StatusCode::BAD_REQUEST);+ // The route's picked target is OpenAI, so count_tokens (Anthropic-only) is+ // unsupported for it.+ assert_eq!(+ response.json()?["error"]["code"],+ "count_tokens_unsupported"+ );+ Ok(())+}++// Build a stage_router (FallThrough) with an Anthropic `strong` tier and an+// OpenAI `weak` tier, both pointed at the mock upstream, and register it.+fn stage_router_state(upstream: &MockUpstream, mode: PickerMode) -> TestResult<ServerState> {+ let cfg = |url: &str| HttpBackendConfig {+ base_url: url.to_string(),+ api_key: Some("k".to_string()),+ extra_headers: BTreeMap::new(),+ };+ let strong: Arc<dyn RoutedLlmClient> =+ Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(+ "strong",+ Backend::Anthropic(cfg(&upstream.base_url)),+ None,+ )])?);+ let weak: Arc<dyn RoutedLlmClient> = Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(+ "weak",+ Backend::OpenAiChat(cfg(&upstream.base_url)),+ None,+ )])?);+ let targets = LlmTargetSet::new(vec![+ LlmTarget {+ semantic_name: "strong".to_string(),+ llm_client: Some(strong),+ },+ LlmTarget {+ semantic_name: "weak".to_string(),+ llm_client: Some(weak),+ },+ ]);+ let stage: Arc<dyn Algorithm<SharedState>> = Arc::new(+ FallThrough::new(targets).with_classifier(Arc::new(StageClassifier::new(mode, 0.5))),+ );+ Ok(ServerState::new([("switchyard/stage".to_string(), stage)])?)+}++/// `count_tokens` is a **direct passthrough**, not a routed call, so on a stage+/// router it goes straight to the Anthropic (`strong`) tier — bypassing the+/// classifier cascade. This is exactly what makes it work where a completion+/// can't: a signal-less request (as `count_tokens` always is) gives the+/// `StageClassifier` nothing to score, so a *completion* on this bare stage+/// router abstains — but `count_tokens` still succeeds.+#[tokio::test]+async fn count_tokens_on_a_stage_router_passes_through_to_the_anthropic_tier() -> TestResult {+ let upstream = MockUpstream::start().await?;+ let app = build_switchyard_router(stage_router_state(&upstream, PickerMode::EfficientFirst)?);+ let body = json!({+ "model": "switchyard/stage",+ "messages": [{"role": "user", "content": "hi"}]+ });++ // A completion routes → the bare StageClassifier abstains on a signal-less+ // request → error.+ let completion = send(&app, "POST", "/v1/messages", Some(body.clone())).await?;+ assert!(completion.text()?.contains("abstained"));++ // count_tokens does NOT route — it passes through to the strong (Anthropic)+ // tier and succeeds.+ let count = send(&app, "POST", "/v1/messages/count_tokens", Some(body)).await?;+ assert_eq!(count.status, StatusCode::OK);+ assert_eq!(count.json()?["input_tokens"], 7);++ // The forwarded call went to count_tokens with the strong tier's model id.+ let calls = upstream.calls.lock().await;+ assert_eq!(calls.len(), 1);+ assert_eq!(calls[0]["model"], "strong");+ Ok(())+}+#[tokio::test]async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult {let (upstream, app) = test_app(&[