Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(server): /v1/messages/count_tokens forwarding on the Rust server (#138)

sabhatinas82d5fbb36dcb
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs
index c907f290d..f847249e2 100644
--- a/crates/libsy-llm-client/src/backend.rs
+++ b/crates/libsy-llm-client/src/backend.rs
@@ -124,6 +124,19 @@ impl Backend {
&self.config().extra_headers
}
+ /// Whether this backend speaks the Anthropic Messages wire format — the only
+ /// one with a `count_tokens` endpoint.
+ pub fn is_anthropic(&self) -> bool {
+ matches!(self, Backend::Anthropic(_))
+ }
+
+ /// The upstream `/v1/messages/count_tokens` URL, derived from the same base
+ /// URL join as [`url`](Self::url).
+ pub fn count_tokens_url(&self) -> String {
+ let base_url = self.config().base_url.trim_end_matches('/');
+ format!("{}/count_tokens", anthropic_url(base_url))
+ }
+
/// Whether an upstream 400 `body` looks like a context-window overflow for
/// this backend's provider.
pub(crate) fn is_context_overflow(&self, body: &str) -> bool {
@@ -218,6 +231,34 @@ mod tests {
);
}
+ #[test]
+ fn count_tokens_url_joins_every_base_url_shape() {
+ assert_eq!(
+ Backend::Anthropic(config("https://host")).count_tokens_url(),
+ "https://host/v1/messages/count_tokens"
+ );
+ assert_eq!(
+ Backend::Anthropic(config("https://host/v1")).count_tokens_url(),
+ "https://host/v1/messages/count_tokens"
+ );
+ assert_eq!(
+ Backend::Anthropic(config("https://host/v1/messages")).count_tokens_url(),
+ "https://host/v1/messages/count_tokens"
+ );
+ // Trailing slash is trimmed before the join.
+ assert_eq!(
+ Backend::Anthropic(config("https://host/v1/")).count_tokens_url(),
+ "https://host/v1/messages/count_tokens"
+ );
+ }
+
+ #[test]
+ fn only_anthropic_backend_is_anthropic() {
+ assert!(Backend::Anthropic(config("x")).is_anthropic());
+ assert!(!Backend::OpenAiChat(config("x")).is_anthropic());
+ assert!(!Backend::OpenAiResponses(config("x")).is_anthropic());
+ }
+
#[test]
fn wire_format_matches_variant() {
assert_eq!(
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index cc1c2b88a..5ca18cc16 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -11,7 +11,7 @@ use futures_util::StreamExt;
use reqwest::RequestBuilder;
use serde_json::Value;
use switchyard_protocol::{
- Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient,
+ Context, Decision, LlmRequest, LlmResponse, Metadata, Request, Response, RoutedLlmClient,
};
use switchyard_translation::{
decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,
@@ -101,6 +101,23 @@ impl TranslatingLlmClient {
})
}
+ /// A configured Anthropic backend to forward a direct `count_tokens` call to,
+ /// paired with its upstream model id (the id the inbound route name is
+ /// restamped to). `None` when this client has no Anthropic backend;
+ /// `count_tokens` is Anthropic-only.
+ fn anthropic_backend(&self) -> Option<(&str, &Backend)> {
+ self.model_to_config.values().find_map(|config| {
+ if config.default_backend.is_anthropic() {
+ return Some((config.model_name.as_str(), &config.default_backend));
+ }
+ config
+ .other_backends
+ .as_ref()
+ .and_then(|backends| backends.iter().find(|backend| backend.is_anthropic()))
+ .map(|backend| (config.model_name.as_str(), backend))
+ })
+ }
+
/// The backend serving `model` over `format` — the default backend when its
/// format matches, otherwise a matching entry in `other_backends`; `None` when
/// the model is unknown or has no backend for `format`.
@@ -117,6 +134,66 @@ impl TranslatingLlmClient {
})
}
+ /// Encode `llm_request` (its model restamped to `model`) for `wire_format`,
+ /// POST it to `url` with the request's forwarded headers plus the backend's
+ /// static headers and auth, and return the successful upstream response and
+ /// whether the request asked for a streamed body. A non-success status maps
+ /// to a typed error — a 400 is classified as a context-window overflow via
+ /// the backend's provider rules. Shared by
+ /// [`call_rewrite_model`](Self::call_rewrite_model) (which POSTs to the
+ /// backend's completion URL and decodes a response) and
+ /// [`count_tokens`](RoutedLlmClient::count_tokens) (which POSTs to the
+ /// `count_tokens` URL and returns the raw JSON).
+ async fn send_encoded(
+ &self,
+ url: String,
+ backend: &Backend,
+ wire_format: WireFormat,
+ mut llm_request: LlmRequest,
+ metadata: Option<&Metadata>,
+ model: &str,
+ ) -> Result<(reqwest::Response, bool)> {
+ // The resolved name is the upstream model id (per the crate contract).
+ llm_request.model = Some(model.to_string());
+ let mut body = encode_request(&llm_request, wire_format)
+ .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?;
+ // `encode_request` round-trips a preserved same-format body verbatim,
+ // which keeps the caller's original `model`; force the resolved model so
+ // the upstream always sees the target id.
+ set_json_model(&mut body, model);
+ let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false);
+
+ let builder = self.client.post(url).json(&body);
+ let builder = forward_metadata_headers(builder, metadata);
+ let builder = apply_extra_headers(builder, backend);
+ let builder = backend.apply_auth(builder);
+
+ let http_response = builder.send().await.map_err(convert_reqwest_error)?;
+ let status = http_response.status();
+ if !status.is_success() {
+ let body = match http_response.text().await {
+ Ok(body) => body,
+ Err(error) if error.is_timeout() => {
+ return Err(LlmClientError::Timeout {
+ source: Box::new(error),
+ })
+ }
+ Err(error) => format!("<failed to read error body: {error}>"),
+ };
+ if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {
+ return Err(LlmClientError::ContextWindowExceeded {
+ model: model.to_string(),
+ message: body,
+ });
+ }
+ return Err(LlmClientError::UpstreamHttp {
+ status: status.as_u16(),
+ body,
+ });
+ }
+ Ok((http_response, streaming))
+ }
+
/// Calls the backend for `model_name` (or the request's own model), over the
/// wire format the request pins in its metadata (else the model's default
/// backend), and returns the neutral response.
@@ -134,7 +211,7 @@ impl TranslatingLlmClient {
// Own the request's parts so the model can be set without a `mut` param
// and without cloning the messages. `raw_request` is unused here.
let Request {
- mut llm_request,
+ llm_request,
metadata,
..
} = request;
@@ -161,45 +238,16 @@ impl TranslatingLlmClient {
message: format!("model {model:?} has no backend for format {wire_format}"),
})?;
- // The resolved name is the upstream model id (per the crate contract).
- llm_request.model = Some(model.clone());
-
- let mut body = encode_request(&llm_request, wire_format)
- .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?;
- // `encode_request` round-trips a preserved same-format body verbatim,
- // which keeps the caller's original `model`; force the resolved model so
- // the upstream always sees the target id.
- set_json_model(&mut body, &model);
- let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false);
-
- let builder = self.client.post(backend.url()).json(&body);
- let builder = forward_metadata_headers(builder, metadata.as_ref());
- let builder = apply_extra_headers(builder, backend);
- let builder = backend.apply_auth(builder);
-
- let http_response = builder.send().await.map_err(convert_reqwest_error)?;
- let status = http_response.status();
- if !status.is_success() {
- let body = match http_response.text().await {
- Ok(body) => body,
- Err(error) if error.is_timeout() => {
- return Err(LlmClientError::Timeout {
- source: Box::new(error),
- })
- }
- Err(error) => format!("<failed to read error body: {error}>"),
- };
- if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {
- return Err(LlmClientError::ContextWindowExceeded {
- model,
- message: body,
- });
- }
- return Err(LlmClientError::UpstreamHttp {
- status: status.as_u16(),
- body,
- });
- }
+ let (http_response, streaming) = self
+ .send_encoded(
+ backend.url(),
+ backend,
+ wire_format,
+ llm_request,
+ metadata.as_ref(),
+ &model,
+ )
+ .await?;
let llm_response = if streaming {
// Adapt the reqwest body stream to plain bytes; the SSE-decode itself is
@@ -304,6 +352,42 @@ impl RoutedLlmClient for TranslatingLlmClient {
let model_name = Some(decision.selected_model());
self.call_rewrite_model(ctx, request, model_name).await
}
+
+ fn supports_count_tokens(&self) -> bool {
+ self.anthropic_backend().is_some()
+ }
+
+ async fn count_tokens(&self, request: Request) -> Result<Value> {
+ // Direct passthrough: forward straight to this client's Anthropic
+ // backend's count_tokens endpoint. Not routed — no decision.
+ let (model, backend) =
+ self.anthropic_backend()
+ .ok_or_else(|| LlmClientError::Configuration {
+ message: "count_tokens is anthropic-only; this client has no anthropic backend"
+ .to_string(),
+ })?;
+ // Same encode-and-forward path as the completion call, only to the
+ // `count_tokens` URL; the response is the raw `{"input_tokens": N}` JSON.
+ let Request {
+ llm_request,
+ metadata,
+ ..
+ } = request;
+ let (http_response, _) = self
+ .send_encoded(
+ backend.count_tokens_url(),
+ backend,
+ WireFormat::AnthropicMessages,
+ llm_request,
+ metadata.as_ref(),
+ model,
+ )
+ .await?;
+ let text = http_response.text().await.map_err(convert_reqwest_error)?;
+ serde_json::from_str(&text).map_err(|error| LlmClientError::InvalidResponse {
+ source: Box::new(error),
+ })
+ }
}
fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index a87e7edfd..b95f7b578 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -22,6 +22,7 @@ use async_trait::async_trait;
use crate::core::{Classifier, Event, Processor, Score, SharedState};
use crate::{
Algorithm, Context, Decision, Driver, LibsyError, LlmTargetSet, Request, Response, Result,
+ RoutedLlmClient,
};
/// The decision a fall-through run produces: the selected model plus a human-readable reason.
@@ -100,6 +101,10 @@ impl Algorithm<SharedState> for FallThrough {
"fall_through"
}
+ fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {
+ self.targets.count_tokens_client()
+ }
+
async fn create_run_task(
self: Arc<Self>,
ctx: Context<SharedState>,
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index e2b71c100..4037cde68 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -9,7 +9,7 @@ use std::sync::Arc;
use switchyard_protocol::{Request, Response};
-use crate::{Algorithm, Context, Decision, Driver, LlmTarget, Result};
+use crate::{Algorithm, Context, Decision, Driver, LlmTarget, Result, RoutedLlmClient};
/// See module comment
pub struct Passthrough {
@@ -48,6 +48,14 @@ where
"passthrough"
}
+ fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {
+ self.target
+ .llm_client
+ .as_ref()
+ .filter(|client| client.supports_count_tokens())
+ .cloned()
+ }
+
async fn create_run_task(
self: Arc<Self>,
ctx: Context<S>,
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index c91f7cfcf..40f1abcec 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -16,7 +16,7 @@ use rand::SeedableRng;
use crate::{
Algorithm, Context, Decision, Driver, LibsyError, LlmTarget, LlmTargetSet, Request, Response,
- Result,
+ Result, RoutedLlmClient,
};
/// Decision produced by [`Random`]: which target was chosen and why.
@@ -138,6 +138,10 @@ where
"random"
}
+ fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {
+ self.target_set.count_tokens_client()
+ }
+
async fn create_run_task(
self: Arc<Self>,
ctx: Context<S>,
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index c12678b53..b3e71bc4e 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -307,6 +307,19 @@ impl LlmTargetSet {
target: name.to_string(),
})
}
+
+ /// The first target's client that can serve `count_tokens` (an Anthropic
+ /// upstream), or `None` when no target has one. Used by an algorithm's
+ /// [`count_tokens_client`](crate::Algorithm::count_tokens_client).
+ pub fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {
+ self.targets.iter().find_map(|target| {
+ target
+ .llm_client
+ .as_ref()
+ .filter(|client| client.supports_count_tokens())
+ .cloned()
+ })
+ }
}
/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);
@@ -352,6 +365,39 @@ where
Ok(())
}
+ /// The client [`count_tokens`](Self::count_tokens) forwards to: the first of
+ /// this algorithm's targets whose client can count tokens (an Anthropic
+ /// upstream). The default is `None` — an algorithm with no Anthropic target
+ /// does not support token counting.
+ ///
+ /// CAVEAT: this picks the *first* Anthropic target, not a routed one —
+ /// count_tokens is a direct passthrough, so it does not run the routing
+ /// cascade. For a route with several Anthropic tiers "first" is arbitrary;
+ /// choosing which tier count_tokens should reflect is deferred.
+ fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {
+ None
+ }
+
+ /// Count the tokens `request` would use — a **direct passthrough** to this
+ /// algorithm's Anthropic target (via
+ /// [`count_tokens_client`](Self::count_tokens_client)), **not** a routed
+ /// call. Token counting is a pre-flight estimate with no routing decision,
+ /// so it deliberately bypasses the classifier cascade (which runs only for
+ /// completions via [`run`](Self::run)). Returns the upstream's JSON
+ /// verbatim. Errors when the algorithm has no Anthropic target.
+ async fn count_tokens(&self, request: Request) -> Result<serde_json::Value> {
+ let client = self
+ .count_tokens_client()
+ .ok_or_else(|| LibsyError::AlgorithmError {
+ message: "no target supports count_tokens (needs an Anthropic upstream)"
+ .to_string(),
+ })?;
+ client
+ .count_tokens(request)
+ .await
+ .map_err(|source| LibsyError::client_call("count_tokens", source))
+ }
+
/// Process a request to completion, returning a stream of [`Step`]s.
/// Each [`Step::CallLlm`] is an offloaded model call the consumer must serve.
/// The stream ends with a [`Step::ReturnToAgent`] on success, or an `Err` item on failure.
diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs
index 6e546054f..7130397d8 100644
--- a/crates/protocol/src/client.rs
+++ b/crates/protocol/src/client.rs
@@ -145,4 +145,24 @@ pub trait RoutedLlmClient: Send + Sync {
request: Request,
decision: Arc<dyn Decision>,
) -> Result<Response, LlmClientError>;
+
+ /// Whether this client can serve [`count_tokens`](Self::count_tokens) — i.e.
+ /// it has an Anthropic upstream. The default is `false`.
+ fn supports_count_tokens(&self) -> bool {
+ false
+ }
+
+ /// Count the tokens `request` would use — a **direct passthrough**, not a
+ /// routed call. Forwards `request` to this client's Anthropic
+ /// `/v1/messages/count_tokens` endpoint (model restamped to the upstream
+ /// target id) and returns the JSON verbatim. Token counting is a pre-flight
+ /// estimate with no routing decision, so unlike [`call`](Self::call) it
+ /// takes no [`Decision`]. The default errors; only an Anthropic-backed
+ /// client overrides it.
+ async fn count_tokens(&self, request: Request) -> Result<serde_json::Value, LlmClientError> {
+ let _ = request;
+ Err(LlmClientError::Configuration {
+ message: "count_tokens is not supported by this client".to_string(),
+ })
+ }
}
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 5221b3358..f5d7e7d36 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -198,6 +198,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router {
Router::new()
.route("/v1/chat/completions", post(openai_chat_completions))
.route("/v1/messages", post(anthropic_messages))
+ .route("/v1/messages/count_tokens", post(anthropic_count_tokens))
.route("/v1/responses", post(openai_responses))
.route("/v1/models", get(models))
.route("/metrics", get(prometheus_metrics))
@@ -257,6 +258,52 @@ async fn openai_responses(
handle_endpoint(state, headers, body, WireFormat::OpenAiResponses).await
}
+/// Anthropic token counting. Resolves the route named by `model`, then does a
+/// **direct passthrough** via [`Algorithm::count_tokens`] to that route's
+/// Anthropic target — it does *not* run the routing cascade (count_tokens is a
+/// pre-flight estimate with no routing decision). Unknown route → 404; a route
+/// with no Anthropic target → 400.
+async fn anthropic_count_tokens(
+ State(state): State<ServerState>,
+ headers: HeaderMap,
+ body: std::result::Result<Json<Value>, JsonRejection>,
+) -> Response {
+ let body = match llm_json_body(body) {
+ Ok(body) => body,
+ Err(message) => return invalid_body_error(message),
+ };
+ let (algorithm, request, _) = match resolve_route(
+ &state,
+ metadata_from_headers(&headers),
+ body,
+ WireFormat::AnthropicMessages,
+ ) {
+ Ok(resolved) => resolved,
+ Err(response) => return response,
+ };
+ match algorithm.count_tokens(request).await {
+ Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),
+ Err(error) => count_tokens_error(error),
+ }
+}
+
+/// Map a [`count_tokens`](Algorithm::count_tokens) failure to an HTTP response:
+/// the route has no Anthropic target → 400, an upstream HTTP error → its own
+/// status, anything else → 502.
+fn count_tokens_error(error: LibsyError) -> Response {
+ // The one count_tokens-specific case is "no Anthropic target in the route";
+ // every upstream/client failure gets the same mapping completions use.
+ match &error {
+ LibsyError::AlgorithmError { message } => error_response(
+ StatusCode::BAD_REQUEST,
+ message.clone(),
+ "invalid_request_error",
+ "count_tokens_unsupported",
+ ),
+ _ => algorithm_error(error),
+ }
+}
+
async fn handle_endpoint(
state: ServerState,
headers: HeaderMap,
@@ -301,42 +348,61 @@ fn llm_json_body(
}
}
-async fn handle_llm_request(
- state: ServerState,
+/// Decode `body`, resolve the route named by its `model`, and build the
+/// [`Request`]. Shared by the completion and `count_tokens` handlers. Returns
+/// the resolved algorithm, the built request, and the requested route model —
+/// or an error [`Response`] (invalid body, empty `model` → 400, unknown route →
+/// 404).
+// Both callers immediately return the `Err(Response)` as the HTTP response, so
+// the large error type is intentional, not propagated up a call stack.
+#[allow(clippy::type_complexity, clippy::result_large_err)]
+fn resolve_route(
+ state: &ServerState,
metadata: Metadata,
body: Value,
wire_format: WireFormat,
-) -> Response {
- let llm_request = match decode_request(wire_format, &body) {
- Ok(request) => request,
- Err(error) => return invalid_body_error(error.to_string()),
- };
- let Some(requested_model) = llm_request
+) -> std::result::Result<(Arc<dyn Algorithm<SharedState>>, Request, String), Response> {
+ let llm_request = decode_request(wire_format, &body)
+ .map_err(|error| invalid_body_error(error.to_string()))?;
+ let requested_model = llm_request
.model
.clone()
.filter(|model| !model.trim().is_empty())
- else {
- return error_response(
- StatusCode::BAD_REQUEST,
- "request body must include a non-empty string `model`",
- "invalid_request_error",
- "invalid_request_error",
- );
- };
- let Some(algorithm) = state.algorithm_for_model(&requested_model) else {
- return error_response(
+ .ok_or_else(|| {
+ error_response(
+ StatusCode::BAD_REQUEST,
+ "request body must include a non-empty string `model`",
+ "invalid_request_error",
+ "invalid_request_error",
+ )
+ })?;
+ let algorithm = state.algorithm_for_model(&requested_model).ok_or_else(|| {
+ error_response(
StatusCode::NOT_FOUND,
format!("No route registered for model {requested_model}"),
"model_not_found",
"model_not_found",
- );
- };
-
+ )
+ })?;
let request = Request {
llm_request,
raw_request: Some(body),
metadata: Some(metadata),
};
+ Ok((algorithm, request, requested_model))
+}
+
+async fn handle_llm_request(
+ state: ServerState,
+ metadata: Metadata,
+ body: Value,
+ wire_format: WireFormat,
+) -> Response {
+ let (algorithm, request, requested_model) =
+ match resolve_route(&state, metadata, body, wire_format) {
+ Ok(resolved) => resolved,
+ Err(response) => return response,
+ };
let (trace, response) = match algorithm
.run(Context::<SharedState>::default(), request)
.await
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 24dac9ef9..c8bb644d1 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -17,7 +17,8 @@ use axum::response::{IntoResponse, Response as HttpResponse};
use axum::routing::post;
use axum::{Json, Router};
use http_body_util::BodyExt;
-use libsy::algorithms::Random;
+use libsy::algorithms::{FallThrough, Random};
+use libsy::stage_router::{PickerMode, StageClassifier};
use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, SharedState};
use serde_json::{json, Value};
use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
@@ -45,6 +46,7 @@ impl MockUpstream {
let calls = Arc::new(Mutex::new(Vec::new()));
let app = Router::new()
.route("/v1/chat/completions", post(upstream_chat))
+ .route("/v1/messages/count_tokens", post(upstream_count_tokens))
.with_state(Arc::clone(&calls));
let listener = TcpListener::bind("127.0.0.1:0").await?;
let addr = listener.local_addr()?;
@@ -120,6 +122,14 @@ async fn upstream_chat(
.into_response()
}
+async fn upstream_count_tokens(
+ State(calls): State<Arc<Mutex<Vec<Value>>>>,
+ Json(body): Json<Value>,
+) -> HttpResponse {
+ calls.lock().await.push(body.clone());
+ Json(json!({"input_tokens": 7})).into_response()
+}
+
fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<ServerState> {
let backend = Backend::OpenAiChat(HttpBackendConfig {
base_url: base_url.to_string(),
@@ -335,6 +345,162 @@ target = "weak"
Ok(())
}
+#[tokio::test]
+async fn count_tokens_forwards_to_configured_anthropic_target() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let state = load_test_config(&format!(
+ r#"
+schema_version = 1
+
+[llm_clients.claude]
+format = "anthropic_messages"
+base_url = "{base_url}"
+
+[targets.strong]
+id = "real/opus"
+llm_client = "claude"
+
+[routes.random]
+id = "switchyard/random"
+type = "random"
+targets = ["strong"]
+"#,
+ base_url = upstream.base_url
+ ))?;
+ let app = build_switchyard_router(state);
+
+ let response = send(
+ &app,
+ "POST",
+ "/v1/messages/count_tokens",
+ Some(json!({
+ "model": "switchyard/random",
+ "messages": [{"role": "user", "content": "hi"}]
+ })),
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK);
+ assert_eq!(response.json()?["input_tokens"], 7);
+
+ let calls = upstream.calls.lock().await;
+ assert_eq!(calls.len(), 1);
+ // The inbound route name is rewritten to the real upstream model.
+ assert_eq!(calls[0]["model"], "real/opus");
+ Ok(())
+}
+
+#[tokio::test]
+async fn count_tokens_without_anthropic_target_returns_bad_request() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let state = load_test_config(&format!(
+ r#"
+schema_version = 1
+
+[llm_clients.upstream]
+format = "openai_chat"
+base_url = "{base_url}"
+
+[targets.weak]
+id = "model/weak"
+llm_client = "upstream"
+
+[routes.random]
+id = "switchyard/random"
+type = "random"
+targets = ["weak"]
+"#,
+ base_url = upstream.base_url
+ ))?;
+ let app = build_switchyard_router(state);
+
+ let response = send(
+ &app,
+ "POST",
+ "/v1/messages/count_tokens",
+ Some(json!({
+ "model": "switchyard/random",
+ "messages": [{"role": "user", "content": "hi"}]
+ })),
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::BAD_REQUEST);
+ // The route's picked target is OpenAI, so count_tokens (Anthropic-only) is
+ // unsupported for it.
+ assert_eq!(
+ response.json()?["error"]["code"],
+ "count_tokens_unsupported"
+ );
+ Ok(())
+}
+
+// Build a stage_router (FallThrough) with an Anthropic `strong` tier and an
+// OpenAI `weak` tier, both pointed at the mock upstream, and register it.
+fn stage_router_state(upstream: &MockUpstream, mode: PickerMode) -> TestResult<ServerState> {
+ let cfg = |url: &str| HttpBackendConfig {
+ base_url: url.to_string(),
+ api_key: Some("k".to_string()),
+ extra_headers: BTreeMap::new(),
+ };
+ let strong: Arc<dyn RoutedLlmClient> =
+ Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(
+ "strong",
+ Backend::Anthropic(cfg(&upstream.base_url)),
+ None,
+ )])?);
+ let weak: Arc<dyn RoutedLlmClient> = Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(
+ "weak",
+ Backend::OpenAiChat(cfg(&upstream.base_url)),
+ None,
+ )])?);
+ let targets = LlmTargetSet::new(vec![
+ LlmTarget {
+ semantic_name: "strong".to_string(),
+ llm_client: Some(strong),
+ },
+ LlmTarget {
+ semantic_name: "weak".to_string(),
+ llm_client: Some(weak),
+ },
+ ]);
+ let stage: Arc<dyn Algorithm<SharedState>> = Arc::new(
+ FallThrough::new(targets).with_classifier(Arc::new(StageClassifier::new(mode, 0.5))),
+ );
+ Ok(ServerState::new([("switchyard/stage".to_string(), stage)])?)
+}
+
+/// `count_tokens` is a **direct passthrough**, not a routed call, so on a stage
+/// router it goes straight to the Anthropic (`strong`) tier — bypassing the
+/// classifier cascade. This is exactly what makes it work where a completion
+/// can't: a signal-less request (as `count_tokens` always is) gives the
+/// `StageClassifier` nothing to score, so a *completion* on this bare stage
+/// router abstains — but `count_tokens` still succeeds.
+#[tokio::test]
+async fn count_tokens_on_a_stage_router_passes_through_to_the_anthropic_tier() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let app = build_switchyard_router(stage_router_state(&upstream, PickerMode::EfficientFirst)?);
+ let body = json!({
+ "model": "switchyard/stage",
+ "messages": [{"role": "user", "content": "hi"}]
+ });
+
+ // A completion routes → the bare StageClassifier abstains on a signal-less
+ // request → error.
+ let completion = send(&app, "POST", "/v1/messages", Some(body.clone())).await?;
+ assert!(completion.text()?.contains("abstained"));
+
+ // count_tokens does NOT route — it passes through to the strong (Anthropic)
+ // tier and succeeds.
+ let count = send(&app, "POST", "/v1/messages/count_tokens", Some(body)).await?;
+ assert_eq!(count.status, StatusCode::OK);
+ assert_eq!(count.json()?["input_tokens"], 7);
+
+ // The forwarded call went to count_tokens with the strong tier's model id.
+ let calls = upstream.calls.lock().await;
+ assert_eq!(calls.len(), 1);
+ assert_eq!(calls[0]["model"], "strong");
+ Ok(())
+}
+
#[tokio::test]
async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult {
let (upstream, app) = test_app(&[