Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat: Introduce ModelId type (#373)

Graham King48b3b71d3cc6
diff --git a/AGENTS.md b/AGENTS.md
index 463ba90c9..ee8b8d4a0 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -140,7 +140,7 @@ current source and CI workflows for implementation details.
The supported serving path is native Rust:
```
-HTTP request → switchyard-server → libsy Algorithm → LlmTarget/RoutedLlmClient
+HTTP request → switchyard-server → libsy Algorithm → RoutedLlmClient
→ switchyard-translation → upstream model
```
diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md
index ed9d99188..30fef3a4a 100644
--- a/crates/libsy-llm-client/README.md
+++ b/crates/libsy-llm-client/README.md
@@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0
# libsy-llm-client
An HTTP client that speaks Switchyard's neutral IR directly. You hand it a
-[`switchyard_protocol::Request`] and a model name; it looks up the configured backend,
+[`switchyard_protocol::Request`] and a [`ModelId`]; it looks up the configured backend,
encodes the request to that backend's wire format, adds auth and forwards your
headers, makes the call with a shared `reqwest::Client`, and decodes the reply
back into a [`switchyard_protocol::Response`] — buffered or streamed.
@@ -20,16 +20,20 @@ It depends on `switchyard-libsy`, `switchyard-protocol`, and
## Concepts
-- **Model configs.** A client is built from [`ModelConfig`] values. Each model has a
- default [`Backend`] and can have additional backends for other wire formats.
- [`TranslatingLlmClient::call_rewrite_model`] uses the request's metadata wire format
- when set, otherwise the model's default backend.
+- **Model configs.** A client is built from [`ModelConfig`] values, each keyed by a
+ [`ModelId`]. Each model has a default [`Backend`] and can have additional backends
+ for other wire formats. [`TranslatingLlmClient::call_rewrite_model`] uses the
+ request's metadata wire format when set, otherwise the model's default backend.
+- **Model ids.** A [`ModelId`] is a newtype over `String` that behaves like one — it
+ derefs to `str`, compares against string literals, and converts from `&str` or
+ `String` with `.into()`. Anywhere below that takes `impl Into<ModelId>` accepts a
+ bare literal; the borrowed positions need a `ModelId` value to reference.
- **Backends.** A [`Backend`] is one of `OpenAiChat`, `OpenAiResponses`, or
`Anthropic`, each wrapping an [`HttpBackendConfig`] (`base_url`, `api_key`,
static `extra_headers`, default `extra_body` fields, and `max_retries`). The
variant fixes the URL path and auth scheme (Bearer vs `x-api-key` +
`anthropic-version`).
-- **Model rewrite.** The resolved model name is both the map key and the model id
+- **Model rewrite.** The resolved [`ModelId`] is both the map key and the model id
sent upstream — it overwrites whatever `model` the request arrived with.
- **Streaming is chosen by the request.** If the encoded body has `stream: true`
(i.e. `request.llm_request.stream`), you get `LlmResponse::Stream`; otherwise
@@ -80,7 +84,7 @@ fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {
```rust
use switchyard_llm_client::{LlmClientError, TranslatingLlmClient};
-use switchyard_protocol::{completion_text, text_request, LlmResponse, Request};
+use switchyard_protocol::{completion_text, text_request, LlmResponse, ModelId, Request};
async fn ask(client: &TranslatingLlmClient) -> switchyard_llm_client::Result<String> {
let request = Request {
@@ -90,9 +94,8 @@ async fn ask(client: &TranslatingLlmClient) -> switchyard_llm_client::Result<Str
};
// model_name wins over request.llm_request.model; it is also sent upstream.
- let response = client
- .call_rewrite_model(request, Some("gpt-4o-mini"))
- .await?;
+ let model = ModelId::from("gpt-4o-mini");
+ let response = client.call_rewrite_model(request, Some(&model)).await?;
match response.llm_response {
LlmResponse::Agg(agg) => Ok(completion_text(&agg)),
@@ -110,7 +113,7 @@ Set `stream` on the IR request and drive the returned chunk stream:
```rust
use futures_util::StreamExt;
use switchyard_llm_client::TranslatingLlmClient;
-use switchyard_protocol::{text_request, LlmResponse, LlmResponseChunk, Request};
+use switchyard_protocol::{text_request, LlmResponse, LlmResponseChunk, ModelId, Request};
async fn stream(
client: &TranslatingLlmClient,
@@ -119,16 +122,17 @@ async fn stream(
llm_request.stream = true;
let request = Request { llm_request, raw_request: None, metadata: None };
- let response = client
- .call_rewrite_model(request, Some("gpt-4o-mini"))
- .await?;
-
- if let LlmResponse::Stream(mut chunks) = response.llm_response {
- while let Some(item) = chunks.next().await {
- match item {
- Ok(LlmResponseChunk::TextDelta { text, .. }) => print!("{text}"),
- Ok(_) => {} // usage, tool-call deltas, message start/stop
- Err(error) => return Err(error),
+ let model = ModelId::from("gpt-4o-mini");
+ let response = client.call_rewrite_model(request, Some(&model)).await?;
+
+ if let LlmResponse::Stream(mut events) = response.llm_response {
+ while let Some(item) = events.next().await {
+ // Each event carries zero or more provider-neutral chunks; the ones
+ // ignored here are usage, tool-call deltas, and message start/stop.
+ for chunk in item?.normalized() {
+ if let LlmResponseChunk::TextDelta { text, .. } = chunk {
+ print!("{text}");
+ }
}
}
}
@@ -165,10 +169,27 @@ async fn route(
```
When targets are served by different clients — a judge on one provider and the serving
-models on another, say — build the router from `model name -> client` with
+models on another, say — build the router from `ModelId -> client` with
[`ClientRouter::new`] instead. A model missing from that map fails with
`LlmClientError::Configuration` rather than silently going to another provider.
+```rust
+use std::collections::HashMap;
+use std::sync::Arc;
+use switchyard_llm_client::ClientRouter;
+use switchyard_protocol::{ModelId, RoutedLlmClient};
+
+fn split_router(
+ judge: Arc<dyn RoutedLlmClient>,
+ serving: Arc<dyn RoutedLlmClient>,
+) -> ClientRouter {
+ ClientRouter::new(HashMap::from([
+ (ModelId::from("gpt-4o-mini"), judge),
+ (ModelId::from("claude-sonnet-4-5"), serving),
+ ]))
+}
+```
+
A router is not itself a client: [`ClientRouter::route`] hands back a
[`switchyard_protocol::RoutedLlmClient`] and the caller makes the call.
@@ -248,5 +269,6 @@ and the retry budget plus capped `Retry-After` delays determines total latency.
[`Backend`]: src/backend.rs
[`HttpBackendConfig`]: src/backend.rs
[`ModelConfig`]: src/client.rs
+[`ModelId`]: ../protocol/src/model_id.rs
[`TranslatingLlmClient::call_rewrite_model`]: src/client.rs
[`LlmClientError`]: ../protocol/src/client.rs
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index d6ca1191d..8928808e9 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -13,7 +13,7 @@ use reqwest::RequestBuilder;
use reqwest::header::{HeaderMap, RETRY_AFTER};
use serde_json::{Map, Value};
use switchyard_protocol::{
- Decision, LlmRequest, LlmResponse, Metadata, Request, Response, RoutedLlmClient,
+ Decision, LlmRequest, LlmResponse, Metadata, ModelId, Request, Response, RoutedLlmClient,
};
use switchyard_translation::{
WireFormat, decode_aggregated_response, decode_request, decode_stream,
@@ -56,7 +56,7 @@ const MAX_RETRY_AFTER: Duration = Duration::from_secs(60);
/// pin a wire format, plus any `other_backends` reachable over additional formats.
#[derive(Clone, Debug)]
pub struct ModelConfig {
- model_name: String,
+ model_name: ModelId,
default_backend: Backend,
other_backends: Option<Vec<Backend>>,
}
@@ -65,7 +65,7 @@ impl ModelConfig {
/// A model named `model_name` served by `default_backend`, optionally reachable
/// over additional wire formats via `other_backends`.
pub fn new(
- model_name: impl Into<String>,
+ model_name: impl Into<ModelId>,
default_backend: Backend,
other_backends: Option<Vec<Backend>>,
) -> Self {
@@ -86,7 +86,7 @@ impl ModelConfig {
/// [`reqwest::Client`], and decodes the response back to the neutral IR (buffered
/// or streamed).
pub struct TranslatingLlmClient {
- model_to_config: HashMap<String, ModelConfig>,
+ model_to_config: HashMap<ModelId, ModelConfig>,
client: reqwest::Client,
}
@@ -114,7 +114,7 @@ impl TranslatingLlmClient {
/// The backend serving `model` over `format` — the default backend when its
/// format matches, otherwise a matching entry in `other_backends`; `None` when
/// the model is unknown or has no backend for `format`.
- pub fn backend_for(&self, model: &str, format: WireFormat) -> Option<&Backend> {
+ pub fn backend_for(&self, model: &ModelId, format: WireFormat) -> Option<&Backend> {
self.model_to_config.get(model).and_then(|config| {
if config.default_backend.wire_format() == format {
Some(&config.default_backend)
@@ -128,7 +128,7 @@ impl TranslatingLlmClient {
}
/// Whether `model` has an Anthropic backend that supports token counting.
- pub fn supports_count_tokens(&self, model: &str) -> bool {
+ pub fn supports_count_tokens(&self, model: &ModelId) -> bool {
self.backend_for(model, WireFormat::AnthropicMessages)
.is_some()
}
@@ -137,7 +137,7 @@ impl TranslatingLlmClient {
///
/// Returns an error when the model has no Anthropic backend or the upstream
/// request fails or returns invalid JSON.
- pub async fn count_tokens(&self, model: &str, request: Request) -> Result<Value> {
+ pub async fn count_tokens(&self, model: &ModelId, request: Request) -> Result<Value> {
let backend = self
.backend_for(model, WireFormat::AnthropicMessages)
.ok_or_else(|| LlmClientError::Configuration {
@@ -188,7 +188,7 @@ impl TranslatingLlmClient {
wire_format: WireFormat,
mut llm_request: LlmRequest,
metadata: Option<&Metadata>,
- model: &str,
+ model: &ModelId,
endpoint: UpstreamEndpoint,
) -> Result<EncodedResponse> {
// The resolved name is the upstream model id (per the crate contract).
@@ -224,7 +224,7 @@ impl TranslatingLlmClient {
let span = tracing::debug_span!(
target: "libsy",
"libsy.upstream_attempt",
- model,
+ model = %model,
wire_format = %wire_format,
attempt = attempt + 1,
max_attempts,
@@ -276,7 +276,7 @@ impl TranslatingLlmClient {
backend: &Backend,
body: &Value,
metadata: Option<&Metadata>,
- model: &str,
+ model: &ModelId,
streaming: bool,
) -> std::result::Result<EncodedResponse, AttemptFailure> {
let builder = self.client.post(url).json(body);
@@ -336,7 +336,7 @@ impl TranslatingLlmClient {
let error =
if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {
LlmClientError::ContextWindowExceeded {
- model: model.to_string(),
+ model: model.clone(),
message: body,
}
} else {
@@ -363,7 +363,7 @@ impl TranslatingLlmClient {
pub async fn call_rewrite_model(
&self,
request: Request,
- model_name: Option<&str>,
+ model_name: Option<&ModelId>,
) -> Result<Response> {
// Own the request's parts so the model can be set without a `mut` param
// and without cloning the messages. `raw_request` is unused here.
@@ -374,8 +374,8 @@ impl TranslatingLlmClient {
} = request;
let model = model_name
- .map(str::to_string)
- .or_else(|| llm_request.model.clone())
+ .cloned()
+ .or_else(|| llm_request.model.clone().map(ModelId::from))
.ok_or_else(|| LlmClientError::InvalidRequest {
message: "no model given".to_string(),
})?;
@@ -460,7 +460,7 @@ impl TranslatingLlmClient {
&self,
raw_http_request: Value,
http_headers: Option<http::HeaderMap>,
- model: Option<&str>,
+ model: Option<&ModelId>,
wire_format: WireFormat,
) -> Result<RawResponse> {
let llm_request = decode_request(wire_format, &raw_http_request)
@@ -469,7 +469,7 @@ impl TranslatingLlmClient {
// one, else the request's own model. Mirrors `call_rewrite_model`'s own
// resolution so the response names whoever answered.
let served_model = model
- .map(str::to_string)
+ .map(ModelId::to_string)
.or_else(|| llm_request.model.clone());
let request = Request {
@@ -1019,15 +1019,19 @@ mod tests {
-> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?;
// "gpt" is served over OpenAI Chat only; other formats and models miss.
- assert!(client.backend_for("gpt", WireFormat::OpenAiChat).is_some());
assert!(
client
- .backend_for("gpt", WireFormat::AnthropicMessages)
+ .backend_for(&ModelId::from("gpt"), WireFormat::OpenAiChat)
+ .is_some()
+ );
+ assert!(
+ client
+ .backend_for(&ModelId::from("gpt"), WireFormat::AnthropicMessages)
.is_none()
);
assert!(
client
- .backend_for("missing", WireFormat::OpenAiChat)
+ .backend_for(&ModelId::from("missing"), WireFormat::OpenAiChat)
.is_none()
);
Ok(())
@@ -1039,7 +1043,7 @@ mod tests {
let client = TranslatingLlmClient::new(&[])?;
// Arg "b" is looked up (and reported), not the request's "a".
let Err(error) = client
- .call_rewrite_model(request_for(Some("a"), false), Some("b"))
+ .call_rewrite_model(request_for(Some("a"), false), Some(&ModelId::from("b")))
.await
else {
panic!("expected an error");
@@ -1213,7 +1217,10 @@ mod tests {
let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
// Inbound model differs from the map key / resolved model.
client
- .call_rewrite_model(request_for(Some("switchyard"), false), Some("gpt"))
+ .call_rewrite_model(
+ request_for(Some("switchyard"), false),
+ Some(&ModelId::from("gpt")),
+ )
.await?;
// The body_partial_json matcher asserts the upstream saw model "gpt".
Ok(())
@@ -1258,7 +1265,12 @@ mod tests {
});
client
- .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat)
+ .call_rewrite_model_raw(
+ raw,
+ None,
+ Some(&ModelId::from("gpt")),
+ WireFormat::OpenAiChat,
+ )
.await?;
Ok(())
}
@@ -1327,7 +1339,12 @@ mod tests {
});
client
- .call_rewrite_model_raw(raw, None, Some("claude"), WireFormat::AnthropicMessages)
+ .call_rewrite_model_raw(
+ raw,
+ None,
+ Some(&ModelId::from("claude")),
+ WireFormat::AnthropicMessages,
+ )
.await?;
Ok(())
}
@@ -1369,7 +1386,12 @@ mod tests {
});
client
- .call_rewrite_model_raw(raw, None, Some("claude"), WireFormat::AnthropicMessages)
+ .call_rewrite_model_raw(
+ raw,
+ None,
+ Some(&ModelId::from("claude")),
+ WireFormat::AnthropicMessages,
+ )
.await?;
Ok(())
}
@@ -1431,7 +1453,12 @@ mod tests {
});
let response = client
- .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat)
+ .call_rewrite_model_raw(
+ raw,
+ None,
+ Some(&ModelId::from("gpt")),
+ WireFormat::OpenAiChat,
+ )
.await?;
assert!(matches!(response, RawResponse::Stream(_)));
Ok(())
@@ -1641,7 +1668,7 @@ mod tests {
let context_window = AttemptFailure {
error: LlmClientError::ContextWindowExceeded {
- model: "gpt".to_string(),
+ model: ModelId::from("gpt"),
message: "too long".to_string(),
},
status: Some(400),
@@ -1821,7 +1848,12 @@ mod tests {
-> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
let client = TranslatingLlmClient::new(&[])?;
let Err(error) = client
- .call_rewrite_model_raw(json!("invalid"), None, Some("gpt"), WireFormat::OpenAiChat)
+ .call_rewrite_model_raw(
+ json!("invalid"),
+ None,
+ Some(&ModelId::from("gpt")),
+ WireFormat::OpenAiChat,
+ )
.await
else {
panic!("expected request translation to fail");
@@ -1861,7 +1893,12 @@ mod tests {
"messages": [{"role": "user", "content": "hi"}]
});
let RawResponse::Buffered(body) = client
- .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat)
+ .call_rewrite_model_raw(
+ raw,
+ None,
+ Some(&ModelId::from("gpt")),
+ WireFormat::OpenAiChat,
+ )
.await?
else {
panic!("expected a buffered response");
@@ -1897,7 +1934,12 @@ mod tests {
"stream": true
});
let RawResponse::Stream(stream) = client
- .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat)
+ .call_rewrite_model_raw(
+ raw,
+ None,
+ Some(&ModelId::from("gpt")),
+ WireFormat::OpenAiChat,
+ )
.await?
else {
panic!("expected a streamed response");
@@ -1945,7 +1987,12 @@ mod tests {
// Matchers assert the forwarded x-request-id survives and reserved
// authorization is the backend's, not the client's.
client
- .call_rewrite_model_raw(raw, Some(headers), Some("gpt"), WireFormat::OpenAiChat)
+ .call_rewrite_model_raw(
+ raw,
+ Some(headers),
+ Some(&ModelId::from("gpt")),
+ WireFormat::OpenAiChat,
+ )
.await?;
Ok(())
}
diff --git a/crates/libsy-llm-client/src/observation.rs b/crates/libsy-llm-client/src/observation.rs
index 06352e25f..cff08fff4 100644
--- a/crates/libsy-llm-client/src/observation.rs
+++ b/crates/libsy-llm-client/src/observation.rs
@@ -6,13 +6,13 @@
use std::sync::Arc;
use std::time::Duration;
-use switchyard_protocol::Usage;
+use switchyard_protocol::{ModelId, Usage};
/// One completed model call observed at the algorithm offload boundary.
#[derive(Clone, Debug)]
pub struct LlmCallObservation {
/// Model selected for the completed call.
- pub selected_model: String,
+ pub selected_model: ModelId,
/// Whether this call generated an answer rather than a routing verdict.
pub is_answer_call: bool,
/// Whether the call completed successfully.
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index 4427fdf3a..0324035a8 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -17,7 +17,7 @@ use std::time::{Duration, Instant};
use parking_lot::Mutex;
use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, drive};
-use switchyard_protocol::{Decision, LlmClientError, Request, Response, RoutedLlmClient};
+use switchyard_protocol::{Decision, LlmClientError, ModelId, Request, Response, RoutedLlmClient};
use crate::observation::{LlmCallObservation, RunObservation, RunObserver};
use crate::{metrics, observability};
@@ -109,12 +109,12 @@ impl RoutedCallWindows {
fields(
algorithm = call.algorithm,
switchyard.algorithm = call.algorithm,
- selected_model = call.decision.selected_model_id(),
+ selected_model = %call.decision.selected_model_id(),
otel.kind = "client",
otel.name = %format_args!("chat {}", call.decision.selected_model_id()),
openinference.span.kind = "LLM",
gen_ai.operation.name = "chat",
- gen_ai.request.model = call.decision.selected_model_id(),
+ gen_ai.request.model = %call.decision.selected_model_id(),
gen_ai.request.stream = tracing::field::Empty,
gen_ai.request.temperature = tracing::field::Empty,
gen_ai.request.top_p = tracing::field::Empty,
@@ -157,7 +157,7 @@ async fn serve(
}
let request = call.request.clone();
let decision = call.decision.clone();
- let target = decision.selected_model_id().to_string();
+ let target = decision.selected_model_id().clone();
let is_answer_call = decision.is_answer_call();
// Resolved before the clock starts: picking the client is Switchyard's work, not
// the provider's, so it belongs in the routing overhead.
@@ -174,7 +174,7 @@ async fn serve(
let result = observability::observe_client_call(result);
if let Some(observer) = observer {
observer(RunObservation::LlmCall(LlmCallObservation {
- selected_model: call.decision.selected_model_id().to_string(),
+ selected_model: call.decision.selected_model_id().clone(),
is_answer_call,
is_success: result.is_ok(),
duration,
@@ -209,12 +209,12 @@ enum Routing {
/// One client serves every model.
Single(Arc<dyn RoutedLlmClient>),
/// Each model is served by the client configured for it.
- ByModel(HashMap<String, Arc<dyn RoutedLlmClient>>),
+ ByModel(HashMap<ModelId, Arc<dyn RoutedLlmClient>>),
}
impl ClientRouter {
/// Build a router over `model name -> client`, for targets spread across providers.
- pub fn new(by_model: HashMap<String, Arc<dyn RoutedLlmClient>>) -> Self {
+ pub fn new(by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>) -> Self {
Self {
routing: Arc::new(Routing::ByModel(by_model)),
}
@@ -237,7 +237,7 @@ impl ClientRouter {
/// entry for this one, rather than silently sending the call to another provider.
pub fn route(
&self,
- model: &str,
+ model: &ModelId,
) -> std::result::Result<&Arc<dyn RoutedLlmClient>, LlmClientError> {
match self.routing.as_ref() {
Routing::Single(client) => Ok(client),
@@ -252,8 +252,8 @@ impl ClientRouter {
}
}
-impl FromIterator<(String, Arc<dyn RoutedLlmClient>)> for ClientRouter {
- fn from_iter<I: IntoIterator<Item = (String, Arc<dyn RoutedLlmClient>)>>(iter: I) -> Self {
+impl FromIterator<(ModelId, Arc<dyn RoutedLlmClient>)> for ClientRouter {
+ fn from_iter<I: IntoIterator<Item = (ModelId, Arc<dyn RoutedLlmClient>)>>(iter: I) -> Self {
Self::new(iter.into_iter().collect())
}
}
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index daa84dbfa..d6f34be9c 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -33,11 +33,11 @@ use tracing_subscriber::layer::{Context as LayerContext, SubscriberExt};
use tracing_subscriber::registry::LookupSpan;
use switchyard_libsy::{
- AffinityRouter, Algorithm, Classifier, Driver, LibsyError, LlmClassifierConfig, LlmTarget,
- LlmTargetSet, LlmTaskClassifier, PickerMode, StageRouter, StageRouterConfig, Step,
- TaskClassifierConfig,
+ AffinityRouter, Algorithm, Classifier, Driver, LibsyError, LlmClassifierConfig,
+ LlmTaskClassifier, PickerMode, StageRouter, StageRouterConfig, Step, TaskClassifierConfig,
};
use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver};
+use switchyard_protocol::ModelId;
use switchyard_protocol::{
ContentBlock, Decision, LlmRequest, LlmResponse, Message, Metadata, Request, Response, Role,
RoutedLlmClient, ToolCall, ToolResult, Usage, WireFormat,
@@ -429,7 +429,7 @@ impl RoutedLlmClient for UsageClient {
/// algorithm exercising both instrumented driver paths.
struct SingleCallAlgo {
name: String,
- target_set: LlmTargetSet,
+ target_set: Vec<String>,
}
#[async_trait]
@@ -445,15 +445,10 @@ impl Algorithm for SingleCallAlgo {
) -> switchyard_libsy::Result<Response> {
let target = self
.target_set
- .targets()
.first()
.ok_or(LibsyError::NoTargets)?
.clone();
- let decision = Decision::new(
- target.semantic_name.clone(),
- Some(format!("picked '{}'", target.semantic_name)),
- true,
- );
+ let decision = Decision::new(target.clone(), Some(format!("picked '{target}'")), true);
driver.decide(decision.clone()).await?;
driver.call_model(request, decision).await
}
@@ -478,9 +473,7 @@ fn request_with_metadata(session_id: &str, correlation_id: &str) -> Request {
fn algo(name: &str, model: &str) -> Arc<dyn Algorithm> {
Arc::new(SingleCallAlgo {
name: name.to_string(),
- target_set: LlmTargetSet::new(vec![LlmTarget {
- semantic_name: model.to_string(),
- }]),
+ target_set: vec![model.to_string()],
})
}
@@ -499,15 +492,11 @@ fn classifier_router(
efficient_model: &str,
capable_model: &str,
) -> switchyard_libsy::Result<Arc<dyn Algorithm>> {
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- };
- let targets = LlmTargetSet::new(vec![target(efficient_model), target(capable_model)]);
Ok(Arc::new(LlmTaskClassifier::new(
LlmClassifierConfig::Capability {
- judge_target: target(judge_model),
- efficient_target: targets.get_target(efficient_model)?,
- capable_target: targets.get_target(capable_model)?,
+ judge_target: ModelId::from(judge_model),
+ efficient_target: ModelId::from(efficient_model),
+ capable_target: ModelId::from(capable_model),
config: TaskClassifierConfig {
base_threshold: 0.5,
..TaskClassifierConfig::default()
@@ -862,12 +851,10 @@ async fn stage_router_records_algorithm_owned_metrics() -> switchyard_libsy::Res
let (_, exporter, provider, _, _) = telemetry();
const STRONG: &str = "obs-stage-strong";
const WEAK: &str = "obs-stage-weak";
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- };
+ let target = |name: &str| name.to_string();
let algorithm = Arc::new(StageRouter::new(
- target(STRONG),
- target(WEAK),
+ target(STRONG).into(),
+ target(WEAK).into(),
StageRouterConfig::new(PickerMode::EfficientFirst, 0.5),
)?) as Arc<dyn Algorithm>;
let request = Request {
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index 5a2ea5638..895c69f82 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -30,7 +30,7 @@ tokio = { version = "1", features = ["macros", "rt"] }
## How it fits together
-[`LlmTarget`] names a routing destination. An [`Algorithm`] selects targets and
+A target is a bare model id naming a routing destination. An [`Algorithm`] selects targets and
records [`Decision`](switchyard_protocol::Decision)s, offloading every model call
to its caller: [`Algorithm::run_stream`] yields a [`Step`] stream whose
[`Step::CallModel`] items the host serves over its own transport. libsy makes no
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index ec9d48fb2..391a24f34 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -28,13 +28,11 @@ use async_trait::async_trait;
use parking_lot::Mutex;
use tokio::sync::Mutex as AsyncMutex;
-use crate::core::algorithm::{
- self, Algorithm, Driver, LlmTarget, LlmTargetSet, RoutingIdentity, SessionEvictions,
-};
+use crate::core::algorithm::{self, Algorithm, Driver, RoutingIdentity, SessionEvictions};
use crate::core::classifier::{Classification, Classifier, Score};
use crate::core::processor::{Event, Processor};
use crate::{LibsyError, Result};
-use switchyard_protocol::{Decision, Request, Response, RoutingFallbackReason};
+use switchyard_protocol::{Decision, ModelId, Request, Response, RoutingFallbackReason};
struct SessionState<S> {
state: Arc<AsyncMutex<S>>,
@@ -59,12 +57,12 @@ const SESSION_CLEANUP_INTERVAL: Duration = Duration::from_secs(60 * 60);
/// decider that never does. Which target that is belongs to whoever assembles the
/// cascade, not to the classifiers in it.
pub struct DefaultTarget {
- target: String,
+ target: ModelId,
}
impl DefaultTarget {
/// Close a cascade with `target`.
- pub fn new(target: impl Into<String>) -> Self {
+ pub fn new(target: impl Into<ModelId>) -> Self {
Self {
target: target.into(),
}
@@ -98,7 +96,7 @@ pub struct FallThrough<S = ()> {
decision_reason: fn(&str, &Score) -> String,
processors: Vec<Arc<dyn Processor<S>>>,
classifiers: Vec<Arc<dyn Classifier<S>>>,
- targets: LlmTargetSet,
+ targets: Vec<ModelId>,
session_states: Option<Arc<SessionStates<S>>>,
cleanup_started: Once,
session_evictions: SessionEvictions,
@@ -106,7 +104,7 @@ pub struct FallThrough<S = ()> {
impl FallThrough<()> {
/// Creates an empty stateless router.
- pub fn new(targets: LlmTargetSet) -> Self {
+ pub fn new(targets: Vec<ModelId>) -> Self {
Self {
name: "fall_through".to_string(),
decision_reason: default_decision_reason,
@@ -125,7 +123,7 @@ where
S: Default + Send + 'static,
{
/// Creates a router that retains one private `S` per session.
- pub fn new_with_state(targets: LlmTargetSet) -> Self {
+ pub fn new_with_state(targets: Vec<ModelId>) -> Self {
Self {
name: "fall_through".to_string(),
decision_reason: default_decision_reason,
@@ -258,8 +256,8 @@ where
fn fallback_decision(
&self,
deciding: &dyn Classifier<S>,
- from: &LlmTarget,
- to: &LlmTarget,
+ from: &ModelId,
+ to: &ModelId,
reason: RoutingFallbackReason,
) -> Decision {
let failure = match reason {
@@ -267,15 +265,13 @@ where
RoutingFallbackReason::Unavailable => "was unavailable",
};
Decision::new(
- to.semantic_name.clone(),
+ to.clone(),
Some(with_routing_tier(
format!(
- "{} {failure}; fell back to {} (fallback reason: {})",
- from.semantic_name,
- to.semantic_name,
+ "{from} {failure}; fell back to {to} (fallback reason: {})",
reason.as_str(),
),
- deciding.routing_tier(&to.semantic_name),
+ deciding.routing_tier(to),
)),
true,
)
@@ -298,15 +294,10 @@ where
async fn route(
&self,
state: &mut S,
- excluded: &HashSet<String>,
+ excluded: &HashSet<ModelId>,
driver: &Driver,
request: &mut Request,
- ) -> Result<(
- LlmTarget,
- Decision,
- Option<Response>,
- Arc<dyn Classifier<S>>,
- )> {
+ ) -> Result<(ModelId, Decision, Option<Response>, Arc<dyn Classifier<S>>)> {
// 1. Processor chain accumulates request-side facts into the composition's state.
for processor in &self.processors {
processor.process(state, Event::Request(request)).await?;
@@ -332,21 +323,18 @@ where
// 3. Resolve the target and publish the decision. When an excluded target sends
// the request elsewhere, the reasoning describes where it actually went.
- let target = self.targets.resolve_target(&score.target, excluded)?;
- let reasoning = if target.semantic_name == score.target {
+ let target = algorithm::select_eligible_model(&self.targets, &score.target, excluded)?;
+ let reasoning = if target == score.target {
(self.decision_reason)(&self.name, &score)
} else {
format!(
"{} exceeded its context window; fell back to {}",
- score.target, target.semantic_name
+ score.target, target
)
};
let decision: Decision = Decision::new(
- target.semantic_name.clone(),
- Some(with_routing_tier(
- reasoning,
- deciding.routing_tier(&target.semantic_name),
- )),
+ target.clone(),
+ Some(with_routing_tier(reasoning, deciding.routing_tier(&target))),
true,
);
driver.decide(decision.clone()).await?;
@@ -512,15 +500,8 @@ mod tests {
}
}
- fn target_set(names: &[&str]) -> LlmTargetSet {
- LlmTargetSet::new(
- names
- .iter()
- .map(|name| LlmTarget {
- semantic_name: name.to_string(),
- })
- .collect(),
- )
+ fn target_set(names: &[&str]) -> Vec<ModelId> {
+ names.iter().map(|name| ModelId::from(*name)).collect()
}
fn target_prompts() -> TargetPrompts {
@@ -587,7 +568,7 @@ mod tests {
fn score(target: &str, confidence: f64) -> Score {
Score {
confidence,
- target: target.to_string(),
+ target: ModelId::from(target),
}
}
@@ -659,8 +640,8 @@ mod tests {
move |decision: Decision, _request: Request| {
let calls = Arc::clone(&calls);
async move {
- let model = decision.selected_model_id().to_string();
- calls.lock().push(model.clone());
+ let model = decision.selected_model_id().clone();
+ calls.lock().push(model.to_string());
if overflowing.contains(&model.as_str()) {
return Err(LlmClientError::ContextWindowExceeded {
model,
@@ -872,8 +853,11 @@ mod tests {
#[async_trait]
impl Classifier for TieredClassifier {
- fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
- match selected_model_id {
+ fn routing_tier(
+ &self,
+ selected_model_id: &switchyard_protocol::ModelId,
+ ) -> Option<&'static str> {
+ match selected_model_id.as_str() {
"weak" => Some("weak"),
"strong" => Some("strong"),
_ => None,
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 80c173a87..ec5be5957 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -9,7 +9,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use serde::{Deserialize, Deserializer};
use serde_json::Value;
-use switchyard_protocol::{ContentBlock, Decision, Message, Role};
+use switchyard_protocol::{ContentBlock, Decision, Message, ModelId, Role};
use super::fall_through::{DefaultTarget, FallThrough};
use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS;
@@ -21,7 +21,7 @@ use super::util::llm_judge::{
SerdeDecoder, StructuredJudge,
};
use super::util::target_selector::TargetSelectorPolicy;
-use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};
+use crate::core::algorithm::{self, Algorithm, Driver};
use crate::core::classifier::{Classification, Classifier, Score};
use crate::core::state::{State, StateValue};
use crate::{LibsyError, Result};
@@ -165,16 +165,16 @@ impl ClassifierInput for TaskInput {
type CapabilityJudge = StructuredJudge<TaskInput, SerdeDecoder<TaskClassifierVerdict>>;
struct TaskClassifierPolicy {
- efficient_target: String,
- capable_target: String,
+ efficient_target: ModelId,
+ capable_target: ModelId,
base_threshold: f64,
threshold_step: f64,
}
impl TaskClassifierPolicy {
fn new(
- efficient_target: impl Into<String>,
- capable_target: impl Into<String>,
+ efficient_target: impl Into<ModelId>,
+ capable_target: impl Into<ModelId>,
config: &TaskClassifierConfig,
) -> Self {
Self {
@@ -433,8 +433,8 @@ impl JudgePolicy for CustomPolicyRuntime {
struct TaskClassifier {
classifier: JudgeClassifier<CapabilityJudge, TaskClassifierPolicy>,
- efficient_target: String,
- capable_target: String,
+ efficient_target: ModelId,
+ capable_target: ModelId,
}
// ── Escalation classifier ──────────────────────────────────────────────────
@@ -449,9 +449,9 @@ fn streak(state: &State) -> u32 {
}
}
-fn decisive(target: &str) -> Classification {
+fn decisive(target: &ModelId) -> Classification {
Classification::Scores(vec![Score {
- target: target.to_string(),
+ target: target.clone(),
confidence: 1.0,
}])
}
@@ -471,20 +471,20 @@ fn assistant_message(response: &AggLlmResponse) -> Message {
/// not pay for a second model call.
struct EscalationClassifier {
judge: JudgeClassifier<EscalationJudge, EscalationPolicy>,
- capable: LlmTarget,
- efficient: LlmTarget,
+ capable: ModelId,
+ efficient: ModelId,
/// Consecutive escalate verdicts required to latch.
confirmations: u32,
}
#[async_trait]
impl Classifier<State> for EscalationClassifier {
- fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
- if self.capable.semantic_name == self.efficient.semantic_name {
+ fn routing_tier(&self, selected_model_id: &ModelId) -> Option<&'static str> {
+ if self.capable == self.efficient {
None
- } else if selected_model_id == self.capable.semantic_name {
+ } else if *selected_model_id == self.capable {
Some("strong")
- } else if selected_model_id == self.efficient.semantic_name {
+ } else if *selected_model_id == self.efficient {
Some("weak")
} else {
None
@@ -505,7 +505,7 @@ impl Classifier<State> for EscalationClassifier {
// A confirmed session stays capable without a judge call.
if streak(state) >= self.confirmations {
- return Ok((decisive(&self.capable.semantic_name), None));
+ return Ok((decisive(&self.capable), None));
}
// Call efficient model and buffer the response so the judge can read it.
@@ -517,7 +517,7 @@ impl Classifier<State> for EscalationClassifier {
.call_model(
request.clone(),
Decision::new(
- self.efficient.semantic_name.clone(),
+ self.efficient.clone(),
Some("escalation classifier: efficient tier".into()),
true,
),
@@ -528,7 +528,7 @@ impl Classifier<State> for EscalationClassifier {
Err(LibsyError::ClientCall {
source: LlmClientError::ContextWindowExceeded { .. },
..
- }) => return Ok((decisive(&self.capable.semantic_name), None)),
+ }) => return Ok((decisive(&self.capable), None)),
Err(e) => return Err(e),
};
let agg = efficient_response
@@ -561,7 +561,7 @@ impl Classifier<State> for EscalationClassifier {
let held = streak(state);
let best = classification.argmax(false)?;
let (escalate, pending) = match &best {
- Some(score) if score.target == self.capable.semantic_name => (true, held + 1),
+ Some(score) if score.target == self.capable => (true, held + 1),
Some(_) => (false, 0),
None => (false, held),
};
@@ -571,13 +571,10 @@ impl Classifier<State> for EscalationClassifier {
if escalate && pending >= self.confirmations {
// Streak confirmed: drop the efficient response, caller will serve capable.
- return Ok((decisive(&self.capable.semantic_name), None));
+ return Ok((decisive(&self.capable), None));
}
- Ok((
- decisive(&self.efficient.semantic_name),
- Some(efficient_response),
- ))
+ Ok((decisive(&self.efficient), Some(efficient_response)))
}
}
@@ -589,7 +586,7 @@ pub struct LlmTaskClassifier {
}
struct ClassifierRouteConfig {
- default_target: String,
+ default_target: ModelId,
session_affinity: bool,
message_hash_fallback: bool,
}
@@ -600,22 +597,22 @@ pub enum LlmClassifierConfig {
/// Routes between efficient and capable targets from a task-level verdict.
Capability {
/// Target that produces classifier verdicts.
- judge_target: LlmTarget,
+ judge_target: ModelId,
/// Target used when the efficient tier can handle the task.
- efficient_target: LlmTarget,
+ efficient_target: ModelId,
/// Target used when the task needs the capable tier.
- capable_target: LlmTarget,
+ capable_target: ModelId,
/// Capability classifier settings.
config: TaskClassifierConfig,
},
/// Judges efficient responses and escalates after a confirmed streak.
Escalation {
/// Target that produces escalation verdicts.
- judge_target: LlmTarget,
+ judge_target: ModelId,
/// Target called before each escalation decision.
- efficient_target: LlmTarget,
+ efficient_target: ModelId,
/// Target used after escalation is confirmed.
- capable_target: LlmTarget,
+ capable_target: ModelId,
/// Prompt and verdict contract settings for the escalation judge.
contract: ClassifierContractConfig,
/// Escalation policy settings.
@@ -626,9 +623,9 @@ pub enum LlmClassifierConfig {
/// Routes among named targets using a user-supplied schema and policy.
Custom {
/// Target that produces classifier verdicts.
- judge_target: LlmTarget,
+ judge_target: ModelId,
/// User-facing labels paired with their resolved routing targets.
- targets: Vec<(String, LlmTarget)>,
+ targets: Vec<(String, ModelId)>,
/// Label selected when the judge does not produce a usable verdict.
default_target: String,
/// Custom classifier settings.
@@ -676,14 +673,14 @@ impl LlmTaskClassifier {
}
fn build_capability(
- judge_target: LlmTarget,
- efficient_target: LlmTarget,
- capable_target: LlmTarget,
+ judge_target: ModelId,
+ efficient_target: ModelId,
+ capable_target: ModelId,
config: TaskClassifierConfig,
) -> Result<Self> {
config.validate()?;
let contract = Self::load_capability_contract(&config.contract)?;
- let targets = LlmTargetSet::new(vec![efficient_target.clone(), capable_target.clone()]);
+ let targets = vec![efficient_target.clone(), capable_target.clone()];
let session_affinity = config.session_affinity;
let message_hash_fallback = config.message_hash_fallback;
let classifier = Arc::new(TaskClassifier {
@@ -698,13 +695,13 @@ impl LlmTaskClassifier {
),
judge_target.clone(),
TaskClassifierPolicy::new(
- efficient_target.semantic_name.clone(),
- capable_target.semantic_name.clone(),
+ efficient_target.clone(),
+ capable_target.clone(),
&config,
),
),
- efficient_target: efficient_target.semantic_name.clone(),
- capable_target: capable_target.semantic_name.clone(),
+ efficient_target: efficient_target.clone(),
+ capable_target: capable_target.clone(),
});
let inner: Arc<dyn Classifier<State>> = classifier.clone();
Self::from_classifier(
@@ -719,8 +716,8 @@ impl LlmTaskClassifier {
}
fn build_custom(
- judge_target: LlmTarget,
- targets: Vec<(String, LlmTarget)>,
+ judge_target: ModelId,
+ targets: Vec<(String, ModelId)>,
default_target: String,
config: CustomClassifierConfig,
) -> Result<Self> {
@@ -732,7 +729,7 @@ impl LlmTaskClassifier {
}
let mut labels = BTreeSet::new();
- let mut semantic_names = BTreeSet::new();
+ let mut resolved_names = BTreeSet::new();
let mut target_map = BTreeMap::new();
let mut resolved_targets = Vec::with_capacity(targets.len());
for (label, target) in targets {
@@ -747,18 +744,15 @@ impl LlmTaskClassifier {
message: format!("custom classifier target label {label:?} is duplicated"),
});
}
- if !semantic_names.insert(target.semantic_name.clone()) {
+ if !resolved_names.insert(target.clone()) {
return Err(LibsyError::AlgorithmError {
- message: format!(
- "custom classifier resolved target {:?} is duplicated",
- target.semantic_name
- ),
+ message: format!("custom classifier resolved target {target:?} is duplicated"),
});
}
- target_map.insert(label, target.semantic_name.clone());
+ target_map.insert(label, target.clone());
resolved_targets.push(target);
}
- let default_semantic_name =
+ let default_name =
target_map
.get(&default_target)
.cloned()
@@ -797,10 +791,10 @@ impl LlmTaskClassifier {
));
Self::from_classifier(
- LlmTargetSet::new(resolved_targets),
+ resolved_targets,
classifier,
ClassifierRouteConfig {
- default_target: default_semantic_name,
+ default_target: default_name,
session_affinity,
message_hash_fallback,
},
@@ -808,15 +802,15 @@ impl LlmTaskClassifier {
}
fn build_escalation(
- judge_target: LlmTarget,
- efficient_target: LlmTarget,
- capable_target: LlmTarget,
+ judge_target: ModelId,
+ efficient_target: ModelId,
+ capable_target: ModelId,
contract_config: ClassifierContractConfig,
config: EscalationJudgeConfig,
max_output_tokens: u64,
) -> Result<Self> {
- let capable_name = capable_target.semantic_name.clone();
- let efficient_name = efficient_target.semantic_name.clone();
+ let capable_name = capable_target.clone();
+ let efficient_name = efficient_target.clone();
let confirmations = config.confirmations;
let esc = Arc::new(EscalationClassifier {
judge: escalation::build_judge(
@@ -832,7 +826,7 @@ impl LlmTaskClassifier {
confirmations,
});
let inner: Arc<dyn Classifier<State>> = esc.clone();
- let targets = LlmTargetSet::new(vec![capable_target, efficient_target]);
+ let targets = vec![capable_target, efficient_target];
Ok(Self {
route: FallThrough::<State>::new_with_state(targets)
.with_name(ALGORITHM_NAME)
@@ -848,11 +842,11 @@ impl LlmTaskClassifier {
/// Keeps affinity and fallback ordering identical across judge-backed modes.
fn from_classifier(
- targets: LlmTargetSet,
+ targets: Vec<ModelId>,
inner: Arc<dyn Classifier<State>>,
config: ClassifierRouteConfig,
) -> Result<Self> {
- targets.get_target(&config.default_target)?;
+ algorithm::ensure_model_is_target(&targets, &config.default_target)?;
if config.message_hash_fallback && !config.session_affinity {
return Err(LibsyError::AlgorithmError {
message: "message_hash_fallback requires session_affinity".to_string(),
@@ -886,12 +880,12 @@ impl LlmTaskClassifier {
#[async_trait]
impl Classifier<State> for TaskClassifier {
- fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
+ fn routing_tier(&self, selected_model_id: &ModelId) -> Option<&'static str> {
if self.efficient_target == self.capable_target {
None
- } else if selected_model_id == self.efficient_target {
+ } else if *selected_model_id == self.efficient_target {
Some("weak")
- } else if selected_model_id == self.capable_target {
+ } else if *selected_model_id == self.capable_target {
Some("strong")
} else {
None
@@ -910,7 +904,7 @@ impl Classifier<State> for TaskClassifier {
#[async_trait]
impl Classifier<State> for LlmTaskClassifier {
- fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
+ fn routing_tier(&self, selected_model_id: &ModelId) -> Option<&'static str> {
self.inner.routing_tier(selected_model_id)
}
@@ -981,7 +975,7 @@ mod tests {
fn selected(
policy: &TaskClassifierPolicy,
verdict: Option<&TaskClassifierVerdict>,
- ) -> Result<String> {
+ ) -> Result<ModelId> {
policy
.to_classification(verdict)
.argmax(false)?
@@ -1079,14 +1073,11 @@ mod tests {
}
fn router() -> Result<Arc<LlmTaskClassifier>> {
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- };
Ok(Arc::new(LlmTaskClassifier::new(
LlmClassifierConfig::Capability {
- judge_target: target("judge"),
- efficient_target: target("efficient"),
- capable_target: target("capable"),
+ judge_target: ModelId::from("judge"),
+ efficient_target: ModelId::from("efficient"),
+ capable_target: ModelId::from("capable"),
config: test_config(TEST_THRESHOLD),
},
)?))
@@ -1129,7 +1120,10 @@ mod tests {
let (trace, response) = test_drive(router, classify_request(), unreachable_judge()).await?;
- assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
+ assert_eq!(
+ trace.last().map(|d| d.selected_model_id().as_str()),
+ Some("capable")
+ );
assert_eq!(
response.llm_response.as_agg().map(completion_text),
Some("answer from capable".to_string())
@@ -1165,13 +1159,10 @@ mod tests {
#[tokio::test]
async fn classifier_config_sets_the_judge_completion_cap() -> Result<()> {
let recorder = Arc::new(Recorder::default());
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- };
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
- judge_target: target("judge"),
- efficient_target: target("efficient"),
- capable_target: target("capable"),
+ judge_target: ModelId::from("judge"),
+ efficient_target: ModelId::from("efficient"),
+ capable_target: ModelId::from("capable"),
config: TaskClassifierConfig {
max_output_tokens: 512,
..test_config(TEST_THRESHOLD)
@@ -1187,13 +1178,10 @@ mod tests {
#[tokio::test]
async fn classifier_config_overrides_the_packaged_prompt() -> Result<()> {
let recorder = Arc::new(Recorder::default());
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- };
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
- judge_target: target("judge"),
- efficient_target: target("efficient"),
- capable_target: target("capable"),
+ judge_target: ModelId::from("judge"),
+ efficient_target: ModelId::from("efficient"),
+ capable_target: ModelId::from("capable"),
config: TaskClassifierConfig {
contract: ClassifierContractConfig::default()
.with_prompt("Custom capability rubric."),
@@ -1212,13 +1200,10 @@ mod tests {
#[tokio::test]
async fn classifier_config_enables_session_affinity() -> Result<()> {
let recorder = Arc::new(Recorder::default());
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- };
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
- judge_target: target("judge"),
- efficient_target: target("efficient"),
- capable_target: target("capable"),
+ judge_target: ModelId::from("judge"),
+ efficient_target: ModelId::from("efficient"),
+ capable_target: ModelId::from("capable"),
config: TaskClassifierConfig {
session_affinity: true,
..test_config(TEST_THRESHOLD)
@@ -1236,13 +1221,10 @@ mod tests {
#[tokio::test]
async fn classifier_config_reuses_message_hash_affinity_for_a_follow_up() -> Result<()> {
let recorder = Arc::new(Recorder::default());
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- };
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
- judge_target: target("judge"),
- efficient_target: target("efficient"),
- capable_target: target("capable"),
+ judge_target: ModelId::from("judge"),
+ efficient_target: ModelId::from("efficient"),
+ capable_target: ModelId::from("capable"),
config: TaskClassifierConfig {
session_affinity: true,
message_hash_fallback: true,
@@ -1301,15 +1283,12 @@ mod tests {
#[test]
fn invalid_classifier_config_is_rejected() -> Result<()> {
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- };
for bad in [1.5, -0.1, f64::NAN, f64::INFINITY] {
assert!(
LlmTaskClassifier::new(LlmClassifierConfig::Capability {
- judge_target: target("judge"),
- efficient_target: target("e"),
- capable_target: target("c"),
+ judge_target: ModelId::from("judge"),
+ efficient_target: ModelId::from("e"),
+ capable_target: ModelId::from("c"),
config: test_config(bad),
})
.is_err(),
@@ -1340,9 +1319,9 @@ mod tests {
] {
assert!(
LlmTaskClassifier::new(LlmClassifierConfig::Capability {
- judge_target: target("judge"),
- efficient_target: target("e"),
- capable_target: target("c"),
+ judge_target: ModelId::from("judge"),
+ efficient_target: ModelId::from("e"),
+ capable_target: ModelId::from("c"),
config,
})
.is_err()
@@ -1350,9 +1329,9 @@ mod tests {
}
for base_threshold in [0.0, 1.0] {
LlmTaskClassifier::new(LlmClassifierConfig::Capability {
- judge_target: target("judge"),
- efficient_target: target("e"),
- capable_target: target("c"),
+ judge_target: ModelId::from("judge"),
+ efficient_target: ModelId::from("e"),
+ capable_target: ModelId::from("c"),
config: test_config(base_threshold),
})?;
}
@@ -1794,14 +1773,11 @@ mod tests {
/// Builds a router with escalation enabled (`confirmations=1` latches on the first verdict).
fn escalation_router() -> Result<Arc<LlmTaskClassifier>> {
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- };
Ok(Arc::new(LlmTaskClassifier::new(
LlmClassifierConfig::Escalation {
- judge_target: target("judge"),
- efficient_target: target("efficient"),
- capable_target: target("capable"),
+ judge_target: ModelId::from("judge"),
+ efficient_target: ModelId::from("efficient"),
+ capable_target: ModelId::from("capable"),
contract: ClassifierContractConfig::default(),
config: EscalationJudgeConfig {
confirmations: 1,
@@ -1824,7 +1800,7 @@ mod tests {
// The efficient model is the serving target, and the response comes from its call.
assert_eq!(
- trace.last().map(|d| d.selected_model_id()),
+ trace.last().map(|d| d.selected_model_id().as_str()),
Some("efficient")
);
assert!(
@@ -1843,13 +1819,10 @@ mod tests {
#[tokio::test]
async fn escalation_config_overrides_the_packaged_prompt() -> Result<()> {
let recorder = Arc::new(Recorder::default());
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- };
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation {
- judge_target: target("judge"),
- efficient_target: target("efficient"),
- capable_target: target("capable"),
+ judge_target: ModelId::from("judge"),
+ efficient_target: ModelId::from("efficient"),
+ capable_target: ModelId::from("capable"),
contract: ClassifierContractConfig::default().with_prompt("Custom trajectory rubric."),
config: EscalationJudgeConfig {
confirmations: 1,
@@ -1877,7 +1850,10 @@ mod tests {
let (trace, response) =
test_drive(router, classify_request(), queued(model, judge)).await?;
- assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
+ assert_eq!(
+ trace.last().map(|d| d.selected_model_id().as_str()),
+ Some("capable")
+ );
assert!(
trace
.last()
@@ -1908,7 +1884,10 @@ mod tests {
.await?;
let (trace, _) = test_drive(router.clone(), session_request, queued(model, judge)).await?;
- assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
+ assert_eq!(
+ trace.last().map(|d| d.selected_model_id().as_str()),
+ Some("capable")
+ );
Ok(())
}
@@ -1921,9 +1900,9 @@ mod tests {
// Efficient overflows, capable answers, and the judge must never be called.
let serve = |decision: Decision, _request: Request| async move {
- match decision.selected_model_id() {
+ match decision.selected_model_id().as_str() {
"efficient" => Err(LlmClientError::ContextWindowExceeded {
- model: decision.selected_model_id().to_string(),
+ model: decision.selected_model_id().clone(),
message: "prompt is too long".to_string(),
}),
"judge" => panic!("the judge must not be consulted when efficient overflows"),
@@ -1933,7 +1912,10 @@ mod tests {
let (trace, response) = test_drive(router, classify_request(), serve).await?;
- assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable"));
+ assert_eq!(
+ trace.last().map(|d| d.selected_model_id().as_str()),
+ Some("capable")
+ );
assert_eq!(
response.llm_response.as_agg().map(completion_text),
Some("capable answer".to_string())
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index d80b16f17..0727175b3 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -5,21 +5,23 @@
use std::sync::Arc;
-use switchyard_protocol::{Request, Response};
+use switchyard_protocol::{ModelId, Request, Response};
use crate::Result;
-use crate::core::algorithm::{Algorithm, Driver, LlmTarget};
+use crate::core::algorithm::{Algorithm, Driver};
use switchyard_protocol::Decision;
/// Routing algorithm that always calls one configured target.
pub struct Passthrough {
- target: LlmTarget,
+ target: ModelId,
}
impl Passthrough {
/// Creates an algorithm that always calls `target`.
- pub fn new(target: LlmTarget) -> Self {
- Passthrough { target }
+ pub fn new(target: impl Into<ModelId>) -> Self {
+ Passthrough {
+ target: target.into(),
+ }
}
}
@@ -31,11 +33,8 @@ impl Algorithm for Passthrough {
async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
let decision: Decision = Decision::new(
- self.target.semantic_name.clone(),
- Some(format!(
- "passthrough selected target '{}'",
- self.target.semantic_name
- )),
+ self.target.clone(),
+ Some(format!("passthrough selected target '{}'", self.target)),
true,
);
driver.decide(decision.clone()).await?;
@@ -48,7 +47,7 @@ mod tests {
use std::sync::Arc;
use super::Passthrough;
- use crate::core::algorithm::{Algorithm, LlmTarget};
+ use crate::core::algorithm::Algorithm;
use crate::core::testing::{echo, test_drive};
use switchyard_protocol::{Request, completion_text, text_request};
@@ -60,9 +59,7 @@ mod tests {
raw_request: None,
metadata: None,
};
- let algorithm: Arc<dyn Algorithm> = Arc::new(Passthrough::new(LlmTarget {
- semantic_name: MODEL_ID.to_string(),
- }));
+ let algorithm: Arc<dyn Algorithm> = Arc::new(Passthrough::new(MODEL_ID));
let (trace, response) = test_drive(algorithm, request, echo()).await?;
assert_eq!(
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index 2a7b61c84..bb52aa331 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -16,14 +16,14 @@ use rand::distr::{Distribution, weighted::WeightedIndex};
use rand::rngs::StdRng;
use crate::algorithms::fall_through::FallThrough;
-use crate::core::algorithm::{Algorithm, Driver, LlmTargetSet};
+use crate::core::algorithm::{Algorithm, Driver};
use crate::core::classifier::{Classification, Classifier, Score};
use crate::{LibsyError, Result};
-use switchyard_protocol::{Request, Response};
+use switchyard_protocol::{ModelId, Request, Response};
/// Stateless weighted classifier used by random fall-through routing.
pub struct RandomClassifier {
- targets: Vec<String>,
+ targets: Vec<ModelId>,
distribution: WeightedIndex<f64>,
rng: Mutex<StdRng>,
}
@@ -40,12 +40,16 @@ impl RandomClassifier {
/// Returns an error when targets are empty or duplicated, or when explicit
/// weights have the wrong length, are negative or non-finite, or contain no
/// positive value.
- pub fn new(targets: Vec<String>, weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Self> {
+ pub fn new(
+ targets: Vec<ModelId>,
+ weights: Option<Vec<f64>>,
+ seed: Option<u64>,
+ ) -> Result<Self> {
let target_count = targets.len();
if target_count == 0 {
return Err(LibsyError::NoTargets);
}
- let unique_targets = targets.iter().map(String::as_str).collect::<BTreeSet<_>>();
+ let unique_targets = targets.iter().map(ModelId::as_str).collect::<BTreeSet<_>>();
if unique_targets.len() != target_count {
return Err(LibsyError::AlgorithmError {
message: "random targets must be unique".to_string(),
@@ -85,7 +89,7 @@ impl RandomClassifier {
})
}
- fn select_target(&self) -> String {
+ fn select_target(&self) -> ModelId {
let mut rng = self.rng.lock();
let index = self.distribution.sample(&mut *rng);
self.targets[index].clone()
@@ -125,23 +129,18 @@ pub struct Random {
}
impl Random {
- /// Creates a router over `target_set`.
+ /// Creates a router over `targets`.
///
/// # Errors
///
/// Returns an error when targets or weights are invalid for [`RandomClassifier`].
pub fn new(
- target_set: LlmTargetSet,
+ targets: Vec<ModelId>,
weights: Option<Vec<f64>>,
seed: Option<u64>,
) -> Result<Self> {
- let target_names = target_set
- .targets()
- .iter()
- .map(|target| target.semantic_name.clone())
- .collect();
- let classifier = Arc::new(RandomClassifier::new(target_names, weights, seed)?);
- let inner = FallThrough::<()>::new(target_set)
+ let classifier = Arc::new(RandomClassifier::new(targets.clone(), weights, seed)?);
+ let inner = FallThrough::<()>::new(targets)
.with_name("random")
.with_decision_reason(random_decision_reason)
.with_classifier(classifier);
@@ -172,7 +171,6 @@ mod tests {
use switchyard_protocol::{Metadata, completion_text, text_request};
use crate::algorithms::util::affinity::AffinityRouter;
- use crate::core::algorithm::LlmTarget;
use crate::core::testing::{echo, test_drive};
use switchyard_protocol::Request;
@@ -194,14 +192,8 @@ mod tests {
}
}
- fn target_set(names: &[&str]) -> LlmTargetSet {
- let targets = names
- .iter()
- .map(|name| LlmTarget {
- semantic_name: (*name).to_string(),
- })
- .collect();
- LlmTargetSet::new(targets)
+ fn target_set(names: &[&str]) -> Vec<ModelId> {
+ names.iter().map(|name| ModelId::from(*name)).collect()
}
fn algorithm(names: &[&str], weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Random> {
@@ -324,7 +316,7 @@ mod tests {
let names = ["a/model", "b/model"];
let affinity = Arc::new(AffinityRouter::new());
let random = Arc::new(RandomClassifier::new(
- names.iter().map(|name| (*name).to_string()).collect(),
+ names.iter().map(|name| ModelId::from(*name)).collect(),
None,
Some(42),
)?);
@@ -353,7 +345,7 @@ mod tests {
.argmax(false)?;
assert_eq!(
retained.map(|score| score.target),
- Some(selected.to_string())
+ Some(ModelId::from(selected.clone()))
);
let (_, second) = test_drive(algorithm, request_for_session("session-1"), echo()).await?;
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index c2ee63c39..6512d7636 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -25,11 +25,11 @@ use super::util::stage::{
record_decision_source, record_routing_decision,
};
use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor};
-use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};
+use crate::core::algorithm::{Algorithm, Driver};
use crate::core::classifier::{Classification, Classifier};
use crate::core::state::State;
use crate::{LibsyError, Result};
-use switchyard_protocol::{Request, Response};
+use switchyard_protocol::{ModelId, Request, Response};
/// Telemetry name for a router this module assembles.
const STAGE_ROUTER: &str = "stage_router";
@@ -45,7 +45,7 @@ struct SourceStamp {
#[async_trait]
impl Classifier<State> for SourceStamp {
- fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
+ fn routing_tier(&self, selected_model_id: &ModelId) -> Option<&'static str> {
self.inner.routing_tier(selected_model_id)
}
@@ -69,7 +69,7 @@ impl Classifier<State> for SourceStamp {
pub struct LlmFallback {
/// Target the judge model is called through. It is not a routing
/// destination, so it does not belong in the router's target set.
- pub judge_target: LlmTarget,
+ pub judge_target: ModelId,
/// Judge configuration. `recent_turn_window` is worth setting to this router's
/// `recent_window` so the judge reads the same span the signal scorer scored.
/// Note: `session_affinity` and `message_hash_fallback` have no effect here —
@@ -126,11 +126,7 @@ impl StageRouter {
/// routing destination.
///
/// Errors if either threshold in `config` is outside `[0.0, 1.0]`.
- pub fn new(
- capable: LlmTarget,
- efficient: LlmTarget,
- config: StageRouterConfig,
- ) -> Result<Self> {
+ pub fn new(capable: ModelId, efficient: ModelId, config: StageRouterConfig) -> Result<Self> {
Ok(Self {
route: build_route(capable, efficient, config)?,
})
@@ -150,8 +146,8 @@ impl Algorithm for StageRouter {
/// Wires the cascade the wrapper drives.
fn build_route(
- capable: LlmTarget,
- efficient: LlmTarget,
+ capable: ModelId,
+ efficient: ModelId,
config: StageRouterConfig,
) -> Result<FallThrough<State>> {
if !(0.0..=1.0).contains(&config.confidence_threshold) {
@@ -164,10 +160,7 @@ fn build_route(
}
// The tiers are a fixed pair; their targets are whatever the deployment calls
// them, and the classifier scores onto those names.
- let targets = StageTargets::new(
- capable.semantic_name.clone(),
- efficient.semantic_name.clone(),
- );
+ let targets = StageTargets::new(capable.clone(), efficient.clone());
// The picker's mode fixes the fallback tier up front, so the terminal
// classifier is a constant rather than a per-turn lookup.
let fall_open = targets.name(config.mode.default_tier()).to_string();
@@ -179,7 +172,7 @@ fn build_route(
let signals = ToolSignalProcessor {
recent_window: config.recent_window.unwrap_or(DEFAULT_RECENT_WINDOW),
};
- let target_set = LlmTargetSet::new(vec![capable.clone(), efficient.clone()]);
+ let target_set = vec![capable.clone(), efficient.clone()];
let mut router = FallThrough::<State>::new_with_state(target_set)
.with_name(STAGE_ROUTER)
.with_processor(Arc::new(signals))
@@ -223,18 +216,11 @@ mod tests {
use super::*;
use crate::algorithms::util::stage::DECISION_SOURCE_KEY;
- use crate::core::algorithm::LlmTarget;
use crate::core::classifier::Score;
use crate::core::state::StateValue;
use crate::core::testing::{Serve, reply, test_drive};
use switchyard_protocol::{Decision, Metadata, Response};
- fn tier_target(name: &str) -> LlmTarget {
- LlmTarget {
- semantic_name: name.to_string(),
- }
- }
-
/// A classifier that always picks `target`, standing in for a cascade member.
struct Fixed(&'static str);
@@ -248,7 +234,7 @@ mod tests {
) -> Result<(Classification, Option<Response>)> {
Ok((
Classification::Scores(vec![Score {
- target: self.0.to_string(),
+ target: ModelId::from(self.0),
confidence: 1.0,
}]),
None,
@@ -311,7 +297,7 @@ mod tests {
let mut config = config();
config.confidence_threshold = 1.5;
assert!(matches!(
- StageRouter::new(tier_target("strong"), tier_target("weak"), config),
+ StageRouter::new(ModelId::from("strong"), ModelId::from("weak"), config),
Err(LibsyError::AlgorithmError { .. })
));
}
@@ -320,23 +306,21 @@ mod tests {
fn rejects_an_out_of_range_judge_threshold() {
let mut config = config();
config.llm_fallback = Some(LlmFallback {
- judge_target: LlmTarget {
- semantic_name: "judge".to_string(),
- },
+ judge_target: ModelId::from("judge"),
config: TaskClassifierConfig {
base_threshold: -0.1,
..Default::default()
},
});
assert!(matches!(
- StageRouter::new(tier_target("strong"), tier_target("weak"), config),
+ StageRouter::new(ModelId::from("strong"), ModelId::from("weak"), config),
Err(LibsyError::AlgorithmError { .. })
));
}
#[test]
fn builds_over_both_tiers() -> Result<()> {
- let router = StageRouter::new(tier_target("strong"), tier_target("weak"), config())?;
+ let router = StageRouter::new(ModelId::from("strong"), ModelId::from("weak"), config())?;
assert_eq!(router.name(), STAGE_ROUTER);
Ok(())
}
@@ -402,16 +386,10 @@ mod tests {
}
}
- fn recording_target(name: &str) -> LlmTarget {
- LlmTarget {
- semantic_name: name.to_string(),
- }
- }
-
fn recording_router(config: StageRouterConfig) -> Result<Arc<StageRouter>> {
Ok(Arc::new(StageRouter::new(
- recording_target("strong"),
- recording_target("weak"),
+ ModelId::from("strong"),
+ ModelId::from("weak"),
config,
)?))
}
@@ -426,7 +404,7 @@ mod tests {
*recorder.judge_p_solve.lock() = p_solve;
let mut c = config();
c.llm_fallback = Some(LlmFallback {
- judge_target: recording_target(JUDGE),
+ judge_target: ModelId::from(JUDGE),
config: TaskClassifierConfig {
base_threshold: 0.5,
recent_turn_window: Some(3),
diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
index 02b808934..4a390351c 100644
--- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs
+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
@@ -15,11 +15,11 @@ use super::fall_through::FallThrough;
use super::util::affinity::AffinityRouter;
use super::util::subagent::SubagentOverride;
use crate::Result;
-use crate::core::algorithm::{Driver, LlmTarget, LlmTargetSet};
+use crate::core::algorithm::Driver;
use crate::core::classifier::{Classification, Classifier, Score};
use crate::core::testing::{echo, test_drive};
use switchyard_protocol::{
- Metadata, Request, Response, completion_text, slice_to_header_map, text_request,
+ Metadata, ModelId, Request, Response, completion_text, slice_to_header_map, text_request,
};
/// The cascade's terminal classifier: always picks the orchestrator.
@@ -36,22 +36,18 @@ impl Classifier for AlwaysOrchestrator {
Ok((
Classification::Scores(vec![Score {
confidence: 0.5,
- target: "orchestrator".to_string(),
+ target: ModelId::from("orchestrator"),
}]),
None,
))
}
}
-fn targets() -> LlmTargetSet {
- LlmTargetSet::new(
- ["orchestrator", "worker", "reviewer"]
- .iter()
- .map(|name| LlmTarget {
- semantic_name: (*name).to_string(),
- })
- .collect(),
- )
+fn targets() -> Vec<ModelId> {
+ ["orchestrator", "worker", "reviewer"]
+ .iter()
+ .map(|name| ModelId::from(*name))
+ .collect()
}
fn request(headers: &[(&str, &str)]) -> Request {
diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs
index bef22d17d..5ffc869cc 100644
--- a/crates/libsy/src/algorithms/util/affinity.rs
+++ b/crates/libsy/src/algorithms/util/affinity.rs
@@ -23,7 +23,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
use async_trait::async_trait;
use parking_lot::Mutex;
-use switchyard_protocol::{Request, Role};
+use switchyard_protocol::{ModelId, Request, Role};
use crate::core::algorithm::{Driver, RoutingIdentity};
use crate::core::classifier::{Classification, Classifier, Score};
@@ -46,7 +46,7 @@ const MAX_ASSIGNMENTS: usize = 4096;
#[derive(Default)]
pub struct AffinityRouter {
/// When set, only these models are retained; a decision for any other model is not latched.
- latch_only: Option<HashSet<String>>,
+ latch_only: Option<HashSet<ModelId>>,
/// Whether root-session requests should abstain instead of being retained.
subagents_only: bool,
/// In absence of headers, use the message hash based fallback key to do task based routing
@@ -55,7 +55,7 @@ pub struct AffinityRouter {
///
/// Held on the instance so the two roles share one process-local map through a
/// single registered [`Arc`](std::sync::Arc); bounded by [`MAX_ASSIGNMENTS`].
- assignments: Mutex<HashMap<RoutingIdentity, String>>,
+ assignments: Mutex<HashMap<RoutingIdentity, ModelId>>,
/// Whether the "no identity to key on" warning has already been emitted.
unkeyed_warning_emitted: AtomicBool,
}
@@ -84,7 +84,7 @@ impl AffinityRouter {
/// Restricts latching to `models`; a decision for any other model routes but is not
/// retained.
- pub fn with_latch_only(mut self, models: impl IntoIterator<Item = impl Into<String>>) -> Self {
+ pub fn with_latch_only(mut self, models: impl IntoIterator<Item = impl Into<ModelId>>) -> Self {
self.latch_only = Some(models.into_iter().map(Into::into).collect());
self
}
@@ -153,7 +153,7 @@ where
let mut assignments = self.assignments.lock();
if self.should_latch(model) && !assignments.contains_key(&key) {
evict_if_full(&mut assignments);
- assignments.insert(key, model.to_string());
+ assignments.insert(key, model.clone());
}
}
Ok(())
@@ -179,7 +179,7 @@ impl<S> Classifier<S> for AffinityRouter
where
S: Send + 'static,
{
- fn target_unavailable(&self, request: &Request, target: &str) {
+ fn target_unavailable(&self, request: &Request, target: &ModelId) {
let Some(key) = self.affinity_key(request) else {
return;
};
@@ -216,7 +216,7 @@ where
}
/// Evicts one arbitrary assignment when the map has reached [`MAX_ASSIGNMENTS`].
-fn evict_if_full(assignments: &mut HashMap<RoutingIdentity, String>) {
+fn evict_if_full(assignments: &mut HashMap<RoutingIdentity, ModelId>) {
if assignments.len() >= MAX_ASSIGNMENTS
&& let Some(evicted) = assignments.keys().next().cloned()
{
diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs
index b8387f625..54ded690c 100644
--- a/crates/libsy/src/algorithms/util/escalation.rs
+++ b/crates/libsy/src/algorithms/util/escalation.rs
@@ -8,14 +8,13 @@
//! lives with the assembled algorithm in [`crate::algorithms::escalation`].
use serde::Deserialize;
-use switchyard_protocol::{ContentBlock, Message, Role};
+use switchyard_protocol::{ContentBlock, Message, ModelId, Role};
use super::classifier_contract::{ClassifierContract, ClassifierContractConfig};
use super::llm_judge::{
ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder,
StructuredJudge,
};
-use crate::core::algorithm::LlmTarget;
use crate::core::classifier::{Classification, Score};
use crate::core::state::State;
use crate::{LibsyError, Result};
@@ -118,8 +117,8 @@ pub(crate) type EscalationJudge = StructuredJudge<EscalationInput, SerdeDecoder<
/// decline and an outage stay efficient, but only a decline is evidence, so only a decline
/// clears the streak.
pub(crate) struct EscalationPolicy {
- capable: String,
- efficient: String,
+ capable: ModelId,
+ efficient: ModelId,
}
impl JudgePolicy for EscalationPolicy {
@@ -145,9 +144,9 @@ impl JudgePolicy for EscalationPolicy {
/// Loads the packaged prompt and schema, so an unusable asset or an unusable `config` value
/// fails here rather than on the first request.
pub(crate) fn build_judge(
- judge_target: LlmTarget,
- capable: String,
- efficient: String,
+ judge_target: ModelId,
+ capable: ModelId,
+ efficient: ModelId,
contract_config: &ClassifierContractConfig,
config: EscalationJudgeConfig,
max_output_tokens: u64,
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index 464544e24..bb271ac5a 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -13,11 +13,12 @@ use async_trait::async_trait;
use serde::de::DeserializeOwned;
use serde_json::Value;
use switchyard_protocol::{
- AggLlmResponse, InstructionBlock, LlmRequest, Message, OutputParams, Role, completion_text,
+ AggLlmResponse, InstructionBlock, LlmRequest, Message, ModelId, OutputParams, Role,
+ completion_text,
};
use super::classifier_contract::ClassifierContract;
-use crate::core::algorithm::{Driver, LlmTarget};
+use crate::core::algorithm::Driver;
use crate::core::classifier::{Classification, Classifier};
use crate::core::state::State;
use crate::{LibsyError, Result};
@@ -192,7 +193,7 @@ pub trait JudgePolicy: Send + Sync {
/// A classifier that calls one judge target and routes through its verdict policy.
pub struct JudgeClassifier<J, P> {
judge: J,
- target: LlmTarget,
+ target: ModelId,
policy: P,
}
@@ -202,7 +203,7 @@ where
P: JudgePolicy<Verdict = J::Verdict>,
{
/// Combines a judge target with a verdict policy.
- pub fn new(judge: J, target: LlmTarget, policy: P) -> Self {
+ pub fn new(judge: J, target: ModelId, policy: P) -> Self {
Self {
judge,
target,
@@ -223,13 +224,13 @@ where
request: &Request,
driver: &Driver,
) -> Option<J::Verdict> {
- let judge_model = self.target.semantic_name.as_str();
+ let judge_model = self.target.as_str();
let response = driver
.call_model(
self.judge.build_request(state, request),
Decision::new(
- self.target.semantic_name.to_string(),
+ self.target.to_string(),
Some("llm judge consultation".to_string()),
false,
),
@@ -303,7 +304,7 @@ where
return Err(LibsyError::AlgorithmError {
message: format!(
"judge classifier for target {:?} requires a driver to call it",
- self.target.semantic_name
+ self.target
),
});
};
@@ -375,20 +376,14 @@ mod tests {
"no-verdict"
};
Classification::Scores(vec![Score {
- target: target.to_string(),
+ target: ModelId::from(target),
confidence: 1.0,
}])
}
}
fn classifier() -> JudgeClassifier<TestJudge, TestPolicy> {
- JudgeClassifier::new(
- TestJudge,
- LlmTarget {
- semantic_name: "judge".to_string(),
- },
- TestPolicy,
- )
+ JudgeClassifier::new(TestJudge, ModelId::from("judge"), TestPolicy)
}
fn request() -> Request {
@@ -450,7 +445,7 @@ mod tests {
}
}
- fn selected(classification: Classification) -> Result<String> {
+ fn selected(classification: Classification) -> Result<ModelId> {
classification
.argmax(false)?
.map(|score| score.target)
@@ -460,7 +455,7 @@ mod tests {
}
/// Serves the single offloaded judge call with `reply` through a standalone step receiver.
- async fn score_served_with(reply: Result<Response>) -> Result<String> {
+ async fn score_served_with(reply: Result<Response>) -> Result<ModelId> {
let (driver, step_rx) = Driver::new("test");
let mut steps = tokio_stream::wrappers::ReceiverStream::new(step_rx);
let classifier = classifier();
diff --git a/crates/libsy/src/algorithms/util/prompts.rs b/crates/libsy/src/algorithms/util/prompts.rs
index 9d45e65db..e6fd6c673 100644
--- a/crates/libsy/src/algorithms/util/prompts.rs
+++ b/crates/libsy/src/algorithms/util/prompts.rs
@@ -26,7 +26,7 @@
use std::collections::BTreeMap;
use async_trait::async_trait;
-use switchyard_protocol::{ContentBlock, InstructionBlock, Message, Request, Role};
+use switchyard_protocol::{ContentBlock, InstructionBlock, Message, ModelId, Request, Role};
use crate::Result;
use crate::core::processor::{Event, Processor};
@@ -74,18 +74,18 @@ fn drop_exact_replay(request: &mut Request) {
/// untouched.
#[derive(Clone, Debug, Default)]
pub struct TargetPrompts {
- by_target: BTreeMap<String, String>,
+ by_target: BTreeMap<ModelId, String>,
}
impl TargetPrompts {
/// Hand `target` this prompt on every turn it serves.
- pub fn with(mut self, target: impl Into<String>, prompt: impl Into<String>) -> Self {
+ pub fn with(mut self, target: impl Into<ModelId>, prompt: impl Into<String>) -> Self {
self.by_target.insert(target.into(), prompt.into());
self
}
/// The prompt configured for `target`, if any.
- pub fn get(&self, target: &str) -> Option<&str> {
+ pub fn get(&self, target: &ModelId) -> Option<&str> {
self.by_target.get(target).map(String::as_str)
}
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index 1976f6b94..3f16334fc 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -28,6 +28,7 @@ use crate::core::algorithm::Driver;
use crate::core::classifier::{Classification, Classifier, Score};
use crate::core::state::{State, StateValue};
use crate::observability::meter;
+use switchyard_protocol::ModelId;
use switchyard_protocol::Request;
/// Turn depth below which stall signals stay quiet — early no-write turns are
@@ -96,13 +97,13 @@ impl Tier {
/// reaches the right model.
#[derive(Clone, Debug)]
pub struct StageTargets {
- capable: String,
- efficient: String,
+ capable: ModelId,
+ efficient: ModelId,
}
impl StageTargets {
/// Name the targets the two tiers route to.
- pub fn new(capable: impl Into<String>, efficient: impl Into<String>) -> Self {
+ pub fn new(capable: impl Into<ModelId>, efficient: impl Into<ModelId>) -> Self {
Self {
capable: capable.into(),
efficient: efficient.into(),
@@ -110,7 +111,7 @@ impl StageTargets {
}
/// The target `tier` routes to.
- pub fn name(&self, tier: Tier) -> &str {
+ pub fn name(&self, tier: Tier) -> &ModelId {
match tier {
Tier::Capable => &self.capable,
Tier::Efficient => &self.efficient,
@@ -118,10 +119,10 @@ impl StageTargets {
}
/// The tier label for a routed target, or `None` for one outside the pair.
- pub fn label_for(&self, target: &str) -> Option<&'static str> {
- if target == self.capable {
+ pub fn label_for(&self, target: &ModelId) -> Option<&'static str> {
+ if *target == self.capable {
Some(Tier::Capable.label())
- } else if target == self.efficient {
+ } else if *target == self.efficient {
Some(Tier::Efficient.label())
} else {
None
@@ -542,7 +543,7 @@ impl StageClassifier {
#[async_trait]
impl Classifier<State> for StageClassifier {
- fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> {
+ fn routing_tier(&self, selected_model_id: &ModelId) -> Option<&'static str> {
self.targets.label_for(selected_model_id)
}
@@ -578,7 +579,7 @@ impl Classifier<State> for StageClassifier {
let conf = score.abs();
Ok((
Classification::Scores(vec![Score {
- target: target.to_string(),
+ target: target.clone(),
confidence: conf,
}]),
None,
diff --git a/crates/libsy/src/algorithms/util/subagent.rs b/crates/libsy/src/algorithms/util/subagent.rs
index 829d1cb29..c2077ad12 100644
--- a/crates/libsy/src/algorithms/util/subagent.rs
+++ b/crates/libsy/src/algorithms/util/subagent.rs
@@ -20,12 +20,13 @@ use async_trait::async_trait;
use crate::Result;
use crate::core::algorithm::Driver;
use crate::core::classifier::{Classification, Classifier, Score};
+use switchyard_protocol::ModelId;
use switchyard_protocol::{Metadata, Request, Response};
/// Scores a fixed worker target for delegated sub-agent work; abstains otherwise.
pub struct SubagentOverride {
/// Name of the worker target, resolved by the cascade against its target set.
- worker: String,
+ worker: ModelId,
}
impl SubagentOverride {
@@ -33,7 +34,7 @@ impl SubagentOverride {
///
/// `worker` must name a target in the cascade's set, or routing a sub-agent request
/// fails with [`LibsyError::TargetNotFound`](crate::LibsyError::TargetNotFound).
- pub fn new(worker: impl Into<String>) -> Self {
+ pub fn new(worker: impl Into<ModelId>) -> Self {
Self {
worker: worker.into(),
}
@@ -80,14 +81,14 @@ mod tests {
let metadata =
(!headers.is_empty()).then(|| Metadata::from_headers(&slice_to_header_map(headers)));
Request {
- llm_request: text_request(Some("auto".to_string()), "hi"),
+ llm_request: text_request(Some(ModelId::from("auto").to_string()), "hi"),
raw_request: None,
metadata,
}
}
/// Scores `headers` through the override, returning the winning target if it scored.
- async fn selected(headers: &[(&str, &str)]) -> Result<Option<String>> {
+ async fn selected(headers: &[(&str, &str)]) -> Result<Option<ModelId>> {
let mut state = ();
let classification = SubagentOverride::new("worker")
.score(&mut state, &mut request(headers), None)
@@ -108,16 +109,16 @@ mod tests {
("x-claude-code-session-id", "root"),
("x-claude-code-agent-id", "child-1"),
];
- assert_eq!(selected(claude).await?, Some("worker".to_string()));
+ assert_eq!(selected(claude).await?, Some(ModelId::from("worker")));
// Codex delegated-work kinds.
assert_eq!(
selected(&[("x-openai-subagent", "review")]).await?,
- Some("worker".to_string())
+ Some(ModelId::from("worker"))
);
assert_eq!(
selected(&[("x-openai-subagent", "collab_spawn")]).await?,
- Some("worker".to_string())
+ Some(ModelId::from("worker"))
);
Ok(())
}
diff --git a/crates/libsy/src/algorithms/util/target_selector.rs b/crates/libsy/src/algorithms/util/target_selector.rs
index da6c46597..4e36b648a 100644
--- a/crates/libsy/src/algorithms/util/target_selector.rs
+++ b/crates/libsy/src/algorithms/util/target_selector.rs
@@ -11,18 +11,19 @@ use serde_json::Value;
use super::llm_judge::JudgePolicy;
use crate::core::classifier::{Classification, Score};
use crate::{LibsyError, Result};
+use switchyard_protocol::ModelId;
/// Maps one string field in a validated verdict to a configured routing target.
pub(crate) struct TargetSelectorPolicy {
selector: PointerBuf,
- targets: BTreeMap<String, String>,
+ targets: BTreeMap<String, ModelId>,
}
impl TargetSelectorPolicy {
/// Parses a JSON Pointer used to read validated verdicts.
pub(crate) fn new(
selector: impl Into<String>,
- targets: BTreeMap<String, String>,
+ targets: BTreeMap<String, ModelId>,
) -> Result<Self> {
let selector =
PointerBuf::parse(selector.into()).map_err(|error| LibsyError::AlgorithmError {
@@ -67,8 +68,8 @@ mod tests {
let policy = TargetSelectorPolicy::new(
"/decision/target",
BTreeMap::from([
- ("opus".to_string(), "model/opus".to_string()),
- ("sonnet".to_string(), "model/sonnet".to_string()),
+ ("opus".to_string(), ModelId::from("model/opus")),
+ ("sonnet".to_string(), ModelId::from("model/sonnet")),
]),
)?;
let classification = policy.to_classification(Some(&json!({
@@ -77,7 +78,7 @@ mod tests {
assert_eq!(
classification.argmax(false)?.map(|score| score.target),
- Some("model/sonnet".to_string())
+ Some(ModelId::from("model/sonnet"))
);
Ok(())
}
@@ -86,7 +87,7 @@ mod tests {
fn a_missing_or_unknown_target_abstains() -> Result<()> {
let policy = TargetSelectorPolicy::new(
"/target",
- BTreeMap::from([("sonnet".to_string(), "model/sonnet".to_string())]),
+ BTreeMap::from([("sonnet".to_string(), ModelId::from("model/sonnet"))]),
)?;
assert_eq!(
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index f63d6e107..6df5c98eb 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -28,7 +28,9 @@ use tracing::Instrument;
/// [`switchyard_protocol::LlmResponseStreamEvent`] is its host/algorithm envelope; and
/// [`switchyard_protocol::LlmResponse`] carries either a live
/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.
-use switchyard_protocol::{Decision, LlmClientError, Request, Response, RoutingFallbackReason};
+use switchyard_protocol::{
+ Decision, LlmClientError, ModelId, Request, Response, RoutingFallbackReason,
+};
use crate::{DriverError, LibsyError, Result, observability};
@@ -139,7 +141,7 @@ impl Driver {
skip_all,
fields(
algorithm = self.algorithm,
- selected_model = decision.selected_model_id(),
+ selected_model = %decision.selected_model_id(),
openinference.span.kind = "CHAIN",
outcome = tracing::field::Empty,
error = tracing::field::Empty,
@@ -290,60 +292,36 @@ impl Drop for AbortOnDrop {
}
}
-/// A named routing target an algorithm routes by. Serving its calls is the stream
-/// consumer's concern: the selected identifier reaches the consumer as
-/// `decision.selected_model_id()` on the offloaded [`CallModel`].
-#[derive(Clone)]
-pub struct LlmTarget {
- /// The routing label an algorithm selects this target by — a logical tier like
- /// `"strong"`, or the model id when they coincide. Mapping it to a provider model
- /// id is the consumer's concern, never the algorithm's.
- pub semantic_name: String,
-}
-
-/// The set of targets an algorithm may route among. An algorithm is constructed
-/// with one and picks targets by position ([`targets`](Self::targets)) or by name
-/// ([`get_target`](Self::get_target)).
-#[derive(Clone)]
-pub struct LlmTargetSet {
- targets: Vec<LlmTarget>,
+/// Errors unless `targets` contains `name`.
+///
+/// Config target names must be resolved before an algorithm is built. This list contains
+/// model IDs, not target names.
+pub(crate) fn ensure_model_is_target(targets: &[ModelId], name: &ModelId) -> Result<()> {
+ targets
+ .iter()
+ .any(|target| target == name)
+ .then_some(())
+ .ok_or_else(|| LibsyError::TargetNotFound {
+ target: name.clone(),
+ })
}
-impl LlmTargetSet {
- /// Build a target set from a list of targets.
- pub fn new(targets: Vec<LlmTarget>) -> Self {
- Self { targets }
- }
-
- /// All targets in the set — e.g. for an algorithm to select among.
- pub fn targets(&self) -> &[LlmTarget] {
- &self.targets
- }
-
- /// Look up a target by name; errors if no target has that name.
- pub fn get_target(&self, name: &str) -> Result<LlmTarget> {
- self.targets
- .iter()
- .find(|t| t.semantic_name == name)
- .cloned()
- .ok_or_else(|| LibsyError::TargetNotFound {
- target: name.to_string(),
- })
- }
-
- /// The named target, or the first one not in `excluded` when it has been barred.
- /// Errors if every target is excluded.
- pub fn resolve_target(&self, name: &str, excluded: &HashSet<String>) -> Result<LlmTarget> {
- let target = self.get_target(name)?;
- if !excluded.contains(&target.semantic_name) {
- return Ok(target);
- }
- self.targets
- .iter()
- .find(|t| !excluded.contains(&t.semantic_name))
- .cloned()
- .ok_or(LibsyError::AllTargetsExcluded)
+/// `name` itself, or the first target not in `excluded` when `name` has been barred.
+/// Errors if `name` is unknown, or if every target is excluded.
+pub(crate) fn select_eligible_model(
+ targets: &[ModelId],
+ name: &ModelId,
+ excluded: &HashSet<ModelId>,
+) -> Result<ModelId> {
+ ensure_model_is_target(targets, name)?;
+ if !excluded.contains(name) {
+ return Ok(name.clone());
}
+ targets
+ .iter()
+ .find(|target| !excluded.contains(*target))
+ .cloned()
+ .ok_or(LibsyError::AllTargetsExcluded)
}
/// Key for overflow history: a root request by its session, a child request by its session
@@ -395,7 +373,7 @@ const MAX_EVICTION_IDENTITIES: usize = 1_024;
/// without a routing identity are not tracked — there is nothing to remember them by.
#[derive(Default)]
pub(crate) struct SessionEvictions {
- by_identity: Mutex<HashMap<RoutingIdentity, HashSet<String>>>,
+ by_identity: Mutex<HashMap<RoutingIdentity, HashSet<ModelId>>>,
}
impl SessionEvictions {
@@ -407,7 +385,7 @@ impl SessionEvictions {
}
/// The targets `identity` has already overflowed; empty for an untracked request.
- fn evicted_for(&self, identity: Option<&RoutingIdentity>) -> Vec<String> {
+ fn evicted_for(&self, identity: Option<&RoutingIdentity>) -> Vec<ModelId> {
let Some(identity) = identity else {
return Vec::new();
};
@@ -420,7 +398,7 @@ impl SessionEvictions {
/// Remembers that `target` overflowed for `identity`, tracking at most
/// [`MAX_EVICTION_IDENTITIES`] identities.
- fn record(&self, identity: Option<&RoutingIdentity>, target: &str) {
+ fn record(&self, identity: Option<&RoutingIdentity>, target: &ModelId) {
let Some(identity) = identity else { return };
let mut histories = self.by_identity.lock();
if histories.len() >= MAX_EVICTION_IDENTITIES
@@ -432,24 +410,23 @@ impl SessionEvictions {
histories
.entry(identity.clone())
.or_default()
- .insert(target.to_string());
+ .insert(target.clone());
}
}
/// How many of `targets` this request is still allowed to reach.
-fn eligible_targets(targets: &LlmTargetSet, excluded: &HashSet<String>) -> usize {
+fn eligible_targets(targets: &[ModelId], excluded: &HashSet<ModelId>) -> usize {
targets
- .targets()
.iter()
- .filter(|t| !excluded.contains(&t.semantic_name))
+ .filter(|target| !excluded.contains(*target))
.count()
}
/// Bars the targets `identity` has already overflowed from this request, so routing does
/// not select one that is certain to fail again.
pub(crate) fn exclude_evicted(
- excluded: &mut HashSet<String>,
- targets: &LlmTargetSet,
+ excluded: &mut HashSet<ModelId>,
+ targets: &[ModelId],
evictions: &SessionEvictions,
identity: Option<&RoutingIdentity>,
) {
@@ -464,7 +441,7 @@ pub(crate) fn exclude_evicted(
}
/// Returns the failed target and routing fallback policy for a terminal client error.
-fn classify_fallback(error: &LibsyError) -> Option<(&str, RoutingFallbackReason)> {
+fn classify_fallback(error: &LibsyError) -> Option<(&ModelId, RoutingFallbackReason)> {
let LibsyError::ClientCall { target, source } = error else {
return None;
};
@@ -492,16 +469,16 @@ fn classify_fallback(error: &LibsyError) -> Option<(&str, RoutingFallbackReason)
/// overflows are recorded for `identity`; unavailable targets remain request-local.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn call_model_with_fallback(
- excluded: &mut HashSet<String>,
+ excluded: &mut HashSet<ModelId>,
driver: &Driver,
- targets: &LlmTargetSet,
- mut target: LlmTarget,
+ targets: &[ModelId],
+ mut target: ModelId,
mut decision: Decision,
request: Request,
identity: Option<&RoutingIdentity>,
evictions: &SessionEvictions,
- target_unavailable: impl Fn(&Request, &str),
- fallback_decision: impl Fn(&LlmTarget, &LlmTarget, RoutingFallbackReason) -> Decision,
+ target_unavailable: impl Fn(&Request, &ModelId),
+ fallback_decision: impl Fn(&ModelId, &ModelId, RoutingFallbackReason) -> Decision,
) -> Result<Response> {
loop {
let result = driver.call_model(request.clone(), decision.clone()).await;
@@ -511,14 +488,14 @@ pub(crate) async fn call_model_with_fallback(
};
// A target already excluded means the pool is spent; surface the client error
// so the caller still sees the concrete upstream failure.
- if !excluded.insert(failed.to_string()) {
+ if !excluded.insert(failed.clone()) {
return Err(error);
}
match reason {
RoutingFallbackReason::ContextWindow => evictions.record(identity, failed),
RoutingFallbackReason::Unavailable => target_unavailable(&request, failed),
}
- let Ok(next) = targets.resolve_target(&target.semantic_name, excluded) else {
+ let Ok(next) = select_eligible_model(targets, &target, excluded) else {
return Err(error);
};
decision = fallback_decision(&target, &next, reason);
@@ -627,7 +604,7 @@ mod tests {
fn route_fallback_only_accepts_context_and_unavailable_failures() {
assert_eq!(
classified_client_error(LlmClientError::ContextWindowExceeded {
- model: "target".to_string(),
+ model: ModelId::from("target"),
message: "too long".to_string(),
}),
Some(RoutingFallbackReason::ContextWindow)
@@ -675,14 +652,14 @@ mod tests {
}
/// Build a routed decision for orchestration tests.
- fn test_decision(selected_model_id: String) -> Decision {
+ fn test_decision(selected_model_id: ModelId) -> Decision {
Decision::new(selected_model_id, None, true)
}
/// Trivial algo used only to exercise the orchestrator: calls the first target
/// and returns its response with a one-item trace.
struct TestAlgo {
- target_set: LlmTargetSet,
+ target_set: Vec<ModelId>,
}
#[async_trait]
@@ -694,18 +671,17 @@ mod tests {
async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
let target = self
.target_set
- .targets()
.first()
.ok_or(LibsyError::NoTargets)?
.clone();
- let decision = test_decision(target.semantic_name.clone());
+ let decision = test_decision(target.clone());
driver.decide(decision.clone()).await?;
driver.call_model(request, decision).await
}
}
/// Build a shared `TestAlgo` over the given target set.
- fn orch(target_set: LlmTargetSet) -> Arc<dyn Algorithm> {
+ fn orch(target_set: Vec<ModelId>) -> Arc<dyn Algorithm> {
Arc::new(TestAlgo { target_set })
}
@@ -717,14 +693,8 @@ mod tests {
}
}
- fn target_set(names: &[&str]) -> LlmTargetSet {
- let targets = names
- .iter()
- .map(|name| LlmTarget {
- semantic_name: name.to_string(),
- })
- .collect();
- LlmTargetSet::new(targets)
+ fn target_set(names: &[&str]) -> Vec<ModelId> {
+ names.iter().map(|name| ModelId::from(*name)).collect()
}
#[tokio::test]
@@ -736,12 +706,12 @@ mod tests {
let first_driver = driver.clone();
let mut first = tokio::spawn(async move {
first_driver
- .call_model(request(), test_decision("first".to_string()))
+ .call_model(request(), test_decision(ModelId::from("first")))
.await
});
let second = tokio::spawn(async move {
driver
- .call_model(request(), test_decision("second".to_string()))
+ .call_model(request(), test_decision(ModelId::from("second")))
.await
});
@@ -787,7 +757,7 @@ mod tests {
let (driver, mut step_rx) = Driver::new("test");
let producer = tokio::spawn(async move {
driver
- .call_model(request(), test_decision("dropped".to_string()))
+ .call_model(request(), test_decision(ModelId::from("dropped")))
.await
});
let step = step_rx.recv().await.ok_or(DriverError::StreamClosed)??;
@@ -806,7 +776,7 @@ mod tests {
// A standalone driver reports the typed step receiver disappearing at its next send.
let (driver, step_rx) = Driver::new("test");
drop(step_rx);
- let decision = test_decision("closed".to_string());
+ let decision = test_decision(ModelId::from("closed"));
let result = driver.decide(decision).await;
assert!(matches!(
result,
@@ -825,7 +795,7 @@ mod tests {
#[tokio::test]
async fn into_parts_yields_the_call_without_answering_it() -> Result<()> {
let (driver, mut step_rx) = Driver::new("test");
- let decision = test_decision("answer/model".to_string());
+ let decision = test_decision(ModelId::from("answer/model"));
let producer = tokio::spawn({
let decision = decision.clone();
async move { driver.call_model(request(), decision).await }
@@ -864,7 +834,7 @@ mod tests {
#[test]
fn target_lookup_returns_the_missing_target() {
- let error = target_set(&[]).get_target("missing").err();
+ let error = ensure_model_is_target(&target_set(&[]), &ModelId::from("missing")).err();
assert!(matches!(
error,
Some(LibsyError::TargetNotFound { target }) if target == "missing"
@@ -1307,8 +1277,8 @@ mod tests {
/// Offloads two targets concurrently and returns the first to resolve, dropping the
/// loser's call (first-wins hedging).
struct Hedge {
- winner: LlmTarget,
- loser: LlmTarget,
+ winner: String,
+ loser: String,
}
#[async_trait]
@@ -1318,8 +1288,8 @@ mod tests {
}
async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
- let dec_w = test_decision(self.winner.semantic_name.clone());
- let dec_l = test_decision(self.loser.semantic_name.clone());
+ let dec_w = test_decision(self.winner.clone().into());
+ let dec_l = test_decision(self.loser.clone().into());
let win = driver.call_model(request.clone(), dec_w);
let lose = driver.call_model(request, dec_l);
// First to resolve wins; `select!` drops the losing future (and its promise).
@@ -1336,12 +1306,8 @@ mod tests {
fn hedge(loser_delay: Option<std::time::Duration>) -> (Arc<dyn Algorithm>, impl Serve) {
let started = Arc::new(tokio::sync::Notify::new());
let algo = Arc::new(Hedge {
- winner: LlmTarget {
- semantic_name: "winner".to_string(),
- },
- loser: LlmTarget {
- semantic_name: "loser".to_string(),
- },
+ winner: "winner".to_string(),
+ loser: "loser".to_string(),
});
let serve = move |decision: Decision, _request: Request| {
let started = started.clone();
@@ -1421,7 +1387,7 @@ mod tests {
async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
let offloads = futures::future::join_all((0..self.n).map(|i| {
- let decision = test_decision(format!("m{i}"));
+ let decision = test_decision(format!("m{i}").into());
driver.call_model(request.clone(), decision)
}));
tokio::select! {
diff --git a/crates/libsy/src/core/classifier.rs b/crates/libsy/src/core/classifier.rs
index f277b293f..f6212f679 100644
--- a/crates/libsy/src/core/classifier.rs
+++ b/crates/libsy/src/core/classifier.rs
@@ -4,7 +4,7 @@
use crate::core::algorithm::Driver;
use crate::{LibsyError, Result};
use async_trait::async_trait;
-use switchyard_protocol::{Request, Response};
+use switchyard_protocol::{ModelId, Request, Response};
/// One classifier's recommendation of a routing `target`, with a `[0.0, 1.0]` confidence.
#[derive(Debug, Clone, PartialEq)]
@@ -12,7 +12,7 @@ pub struct Score {
/// `[0.0, 1.0]` confidence in `target`.
pub confidence: f64,
/// The target (model / tier) being recommended.
- pub target: String,
+ pub target: ModelId,
}
/// A classifier's verdict for a request: a set of target [`Score`]s, flagged by how
@@ -72,14 +72,14 @@ fn argmax(scores: &[Score]) -> Result<Option<Score>> {
#[async_trait]
pub trait Classifier<S = ()>: Send + Sync {
/// Stable tier represented by `selected_model_id`, when this classifier defines one.
- fn routing_tier(&self, _selected_model_id: &str) -> Option<&'static str> {
+ fn routing_tier(&self, _selected_model_id: &ModelId) -> Option<&'static str> {
None
}
/// Drops retained routing state when `target` was unavailable for `request`.
///
/// Stateless classifiers do not need to implement this hook.
- fn target_unavailable(&self, _request: &Request, _target: &str) {}
+ fn target_unavailable(&self, _request: &Request, _target: &ModelId) {}
/// Score the classifier's targets given the current state and request.
///
@@ -107,7 +107,7 @@ mod tests {
/// Terse `Score` builder for the assertions below.
fn score(target: &str, confidence: f64) -> Score {
Score {
- target: target.to_string(),
+ target: ModelId::from(target),
confidence,
}
}
@@ -125,7 +125,7 @@ mod tests {
// Equal confidence: the earlier target in cascade order wins the tie.
let scores = vec![score("first", 0.7), score("second", 0.7)];
let best = Classification::Scores(scores).argmax(false)?;
- assert_eq!(best.map(|s| s.target), Some("first".to_string()));
+ assert_eq!(best.map(|s| s.target), Some(ModelId::from("first")));
Ok(())
}
@@ -196,7 +196,7 @@ mod tests {
let target = request.requested_model().unwrap_or("auto").to_string();
Ok((
Classification::Scores(vec![Score {
- target,
+ target: target.into(),
confidence: 1.0,
}]),
None,
@@ -218,7 +218,7 @@ mod tests {
.await?;
assert_eq!(
classification.argmax(false)?.map(|s| s.target),
- Some("strong".to_string())
+ Some(ModelId::from("strong"))
);
assert!(state);
Ok(())
@@ -238,7 +238,7 @@ mod tests {
request.llm_request.model = Some("rewritten".to_string());
Ok((
Classification::Scores(vec![Score {
- target: "rewritten".to_string(),
+ target: ModelId::from("rewritten"),
confidence: 1.0,
}]),
None,
diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rs
index 4703cb445..aa5c63b4a 100644
--- a/crates/libsy/src/error.rs
+++ b/crates/libsy/src/error.rs
@@ -5,7 +5,7 @@
use std::error::Error as StdError;
-use switchyard_protocol::LlmClientError;
+use switchyard_protocol::{LlmClientError, ModelId};
use thiserror::Error;
/// Result type returned by libsy APIs.
@@ -17,8 +17,8 @@ pub enum LibsyError {
/// A named target was not present in the configured target set.
#[error("target {target:?} was not found")]
TargetNotFound {
- /// Missing semantic target name.
- target: String,
+ /// Missing target model id.
+ target: ModelId,
},
/// Routing was attempted without any configured targets.
@@ -44,7 +44,7 @@ pub enum LibsyError {
#[error("client call to target {target:?} failed: {source}")]
ClientCall {
/// Target whose client failed.
- target: String,
+ target: ModelId,
/// Typed error supplied by the protocol-owned client trait.
#[source]
source: LlmClientError,
@@ -67,7 +67,7 @@ pub enum LibsyError {
impl LibsyError {
/// Wrap an error returned by the protocol-owned client trait.
- pub fn client_call(target: impl Into<String>, source: LlmClientError) -> Self {
+ pub fn client_call(target: impl Into<ModelId>, source: LlmClientError) -> Self {
Self::ClientCall {
target: target.into(),
source,
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 3a48885aa..3b1ede9ac 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -5,9 +5,7 @@
#![doc = include_str!("../README.md")]
mod core;
-pub use core::algorithm::{
- Algorithm, CallModel, Driver, LlmTarget, LlmTargetSet, Step, StepStream, drive,
-};
+pub use core::algorithm::{Algorithm, CallModel, Driver, Step, StepStream, drive};
pub use core::classifier::{Classification, Classifier, Score};
pub use core::processor::{Event, Processor};
pub use core::state::{State, StateValue};
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index ca093dae5..2435cb026 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -292,7 +292,7 @@ pub(crate) fn record_decision(algorithm: &str, decision: &Decision) {
tracing::debug!(
target: TRACING_TARGET,
algorithm,
- selected_model,
+ selected_model = %selected_model,
reasoning = decision.reasoning().unwrap_or(""),
"routing decision"
);
diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs
index cf137e84f..21f2ac2e6 100644
--- a/crates/protocol/src/client.rs
+++ b/crates/protocol/src/client.rs
@@ -13,7 +13,7 @@
use async_trait::async_trait;
use thiserror::Error;
-use crate::{Request, Response};
+use crate::{ModelId, Request, Response};
/// A boxed client-specific error preserved as the source of a routed call failure.
pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
@@ -72,7 +72,7 @@ pub enum LlmClientError {
#[error("context window exceeded for model {model}: {message}")]
ContextWindowExceeded {
/// Model whose context window was exceeded.
- model: String,
+ model: ModelId,
/// Upstream error message.
message: String,
},
@@ -131,7 +131,7 @@ impl RoutingFallbackReason {
#[derive(Clone, Debug)]
pub struct Decision {
/// The model identifier selected for the call.
- selected_model_id: String,
+ selected_model_id: ModelId,
/// Why, for logs and traces.
reasoning: Option<String>,
/// True for an answer-generating call. False for classifier and judge calls.
@@ -141,7 +141,7 @@ pub struct Decision {
impl Decision {
/// Creates a decision and records whether its call produces the answer.
pub fn new(
- selected_model_id: impl Into<String>,
+ selected_model_id: impl Into<ModelId>,
reasoning: Option<String>,
is_answer_call: bool,
) -> Self {
@@ -153,8 +153,8 @@ impl Decision {
}
/// The model identifier selected for the call.
- pub fn selected_model_id(&self) -> &str {
- self.selected_model_id.as_str()
+ pub fn selected_model_id(&self) -> &ModelId {
+ &self.selected_model_id
}
/// Why this decision was made.
diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs
index 2747a12e7..15352b44f 100644
--- a/crates/protocol/src/lib.rs
+++ b/crates/protocol/src/lib.rs
@@ -9,6 +9,7 @@ pub mod envelope;
pub mod format;
pub mod llm;
pub mod metadata;
+pub mod model_id;
pub mod stream;
pub use client::*;
@@ -16,6 +17,7 @@ pub use envelope::*;
pub use format::*;
pub use llm::*;
pub use metadata::*;
+pub use model_id::*;
pub use stream::*;
/// Builds a single-turn request: one user message carrying `prompt`, for `model`.
diff --git a/crates/protocol/src/model_id.rs b/crates/protocol/src/model_id.rs
new file mode 100644
index 000000000..854e9d77c
--- /dev/null
+++ b/crates/protocol/src/model_id.rs
@@ -0,0 +1,188 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Model identifier used by Switchyard. See ModelId docs for details.
+
+use std::borrow::Borrow;
+use std::fmt;
+use std::ops::Deref;
+
+use serde::{Deserialize, Serialize};
+
+/// A model name used in a request or routing decision.
+///
+/// It can name a provider model, such as `"openai/gpt-oss-20b"`. It can also name a
+/// Switchyard route, such as `"switchyard/random"`.
+///
+/// A target name from server config, such as `"capable"`, is not a model ID. The server
+/// resolves target names to model IDs before routing.
+///
+/// This type acts like a string. It can be printed, compared, and saved as a JSON string.
+#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct ModelId(String);
+
+/// Forwards to the wrapped string rather than deriving, so `{:?}` renders `"gpt-4"`
+/// instead of `ModelId("gpt-4")`.
+impl fmt::Debug for ModelId {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ fmt::Debug::fmt(&self.0, formatter)
+ }
+}
+
+impl ModelId {
+ /// Wraps a model identifier.
+ pub fn new(id: impl Into<String>) -> Self {
+ Self(id.into())
+ }
+
+ /// The identifier as a string slice.
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+
+ /// Unwraps to the owned identifier.
+ pub fn into_string(self) -> String {
+ self.0
+ }
+}
+
+impl fmt::Display for ModelId {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str(&self.0)
+ }
+}
+
+impl Deref for ModelId {
+ type Target = str;
+
+ fn deref(&self) -> &str {
+ &self.0
+ }
+}
+
+impl AsRef<str> for ModelId {
+ fn as_ref(&self) -> &str {
+ &self.0
+ }
+}
+
+/// Lets a `HashMap<ModelId, _>` or `HashSet<ModelId>` be looked up by `&str`, so
+/// callers holding a borrowed id do not have to allocate one to query.
+impl Borrow<str> for ModelId {
+ fn borrow(&self) -> &str {
+ &self.0
+ }
+}
+
+impl From<String> for ModelId {
+ fn from(id: String) -> Self {
+ Self(id)
+ }
+}
+
+impl From<&str> for ModelId {
+ fn from(id: &str) -> Self {
+ Self(id.to_string())
+ }
+}
+
+/// Mirrors `From<&str> for String`, so a borrowed id reaches an owning
+/// `impl Into<ModelId>` parameter without an explicit clone.
+impl From<&ModelId> for ModelId {
+ fn from(id: &ModelId) -> Self {
+ id.clone()
+ }
+}
+
+impl From<ModelId> for String {
+ fn from(id: ModelId) -> Self {
+ id.0
+ }
+}
+
+impl From<&ModelId> for String {
+ fn from(id: &ModelId) -> Self {
+ id.0.clone()
+ }
+}
+
+// Comparison against bare strings, in both directions, so an id can be checked
+// against a literal or a configured name without wrapping either side.
+// This is likely excessive. We can reduce when things settle.
+
+impl PartialEq<str> for ModelId {
+ fn eq(&self, other: &str) -> bool {
+ self.0 == other
+ }
+}
+
+impl PartialEq<&str> for ModelId {
+ fn eq(&self, other: &&str) -> bool {
+ self.0 == *other
+ }
+}
+
+impl PartialEq<String> for ModelId {
+ fn eq(&self, other: &String) -> bool {
+ &self.0 == other
+ }
+}
+
+impl PartialEq<ModelId> for str {
+ fn eq(&self, other: &ModelId) -> bool {
+ self == other.0
+ }
+}
+
+impl PartialEq<ModelId> for &str {
+ fn eq(&self, other: &ModelId) -> bool {
+ *self == other.0
+ }
+}
+
+impl PartialEq<ModelId> for String {
+ fn eq(&self, other: &ModelId) -> bool {
+ self == &other.0
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::HashMap;
+
+ use super::*;
+
+ #[test]
+ fn it_behaves_like_the_string_it_wraps() {
+ let id = ModelId::new("openai/gpt-oss-20b");
+
+ assert_eq!(id.to_string(), "openai/gpt-oss-20b");
+ assert_eq!(id, "openai/gpt-oss-20b");
+ assert_eq!("openai/gpt-oss-20b", id);
+ assert!(id.starts_with("openai/"));
+ assert_eq!(takes_str(&id), "openai/gpt-oss-20b");
+ }
+
+ /// Deref coercion means an `&ModelId` reaches a `&str` parameter unchanged.
+ fn takes_str(model: &str) -> &str {
+ model
+ }
+
+ #[test]
+ fn a_map_of_ids_is_queryable_by_str() {
+ let by_model = HashMap::from([(ModelId::new("aws/anthropic/claude-opus-4-5"), 1)]);
+
+ assert_eq!(by_model.get("aws/anthropic/claude-opus-4-5"), Some(&1));
+ }
+
+ #[test]
+ fn it_serializes_as_a_bare_string() -> serde_json::Result<()> {
+ let id = ModelId::new("openai/gpt-oss-20b");
+
+ let json = serde_json::to_string(&id)?;
+ assert_eq!(json, "\"openai/gpt-oss-20b\"");
+ assert_eq!(serde_json::from_str::<ModelId>(&json)?, id);
+ Ok(())
+ }
+}
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 8be1bede0..2bed6b405 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -13,12 +13,12 @@ use pyo3::prelude::*;
use serde_json::{Value, json};
use switchyard_libsy::{
Algorithm, ClassifierContractConfig, HandoffNoteConfig, LibsyError as RustLibsyError,
- LlmClassifierConfig, LlmFallback, LlmTarget, LlmTargetSet, LlmTaskClassifier, Noop, PickerMode,
- Random, StageRouter, StageRouterConfig, TaskClassifierConfig,
+ LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, StageRouter,
+ StageRouterConfig, TaskClassifierConfig,
};
use switchyard_llm_client::ClientRouter;
use switchyard_protocol::{
- AggLlmResponse, Decision, LlmClientError, LlmResponse, Metadata, Request, Response,
+ AggLlmResponse, Decision, LlmClientError, LlmResponse, Metadata, ModelId, Request, Response,
RoutedLlmClient,
};
@@ -78,10 +78,9 @@ struct PyLlmTarget {
}
impl PyLlmTarget {
- fn clone_core(&self, _py: Python<'_>) -> LlmTarget {
- LlmTarget {
- semantic_name: self.name.clone(),
- }
+ /// The bare model id libsy routes by; the client behind it stays with the bindings.
+ fn clone_core(&self, _py: Python<'_>) -> ModelId {
+ ModelId::new(self.name.clone())
}
/// The `selected_model -> client` entry this target contributes to the algorithm's
@@ -89,7 +88,7 @@ impl PyLlmTarget {
/// mapping and serve the calls themselves.
fn client_entry(&self, py: Python<'_>) -> ClientEntry {
(
- self.name.clone(),
+ ModelId::new(self.name.clone()),
Arc::new(PythonLlmClient {
inner: self.client.clone_ref(py),
}),
@@ -229,7 +228,7 @@ struct PyAlgorithm {
}
/// One target's `selected_model -> client` mapping for an algorithm's router.
-type ClientEntry = (String, Arc<dyn RoutedLlmClient>);
+type ClientEntry = (ModelId, Arc<dyn RoutedLlmClient>);
impl PyAlgorithm {
fn new(inner: Arc<dyn Algorithm>, clients: impl IntoIterator<Item = ClientEntry>) -> Self {
@@ -310,13 +309,10 @@ fn random_algorithm(
seed: Option<u64>,
) -> PyResult<PyAlgorithm> {
let (cores, clients) = target_cores(py, &targets)?;
- let algorithm =
- Random::new(LlmTargetSet::new(cores), weights, seed).map_err(|error| match error {
- RustLibsyError::NoTargets => {
- PyValueError::new_err("random requires at least one target")
- }
- other => PyValueError::new_err(other.to_string()),
- })?;
+ let algorithm = Random::new(cores, weights, seed).map_err(|error| match error {
+ RustLibsyError::NoTargets => PyValueError::new_err("random requires at least one target"),
+ other => PyValueError::new_err(other.to_string()),
+ })?;
Ok(PyAlgorithm::new(Arc::new(algorithm), clients))
}
@@ -325,7 +321,7 @@ fn random_algorithm(
fn target_cores(
py: Python<'_>,
targets: &[Py<PyLlmTarget>],
-) -> PyResult<(Vec<LlmTarget>, Vec<ClientEntry>)> {
+) -> PyResult<(Vec<ModelId>, Vec<ClientEntry>)> {
let mut cores = Vec::with_capacity(targets.len());
let mut clients = Vec::with_capacity(targets.len());
for target in targets {
@@ -429,14 +425,10 @@ fn stage_router_algorithm(
(None, None) => None,
};
if let Some(prompt) = capable_system_prompt {
- config.tier_prompts = config
- .tier_prompts
- .with(capable.semantic_name.clone(), prompt);
+ config.tier_prompts = config.tier_prompts.with(capable.clone(), prompt);
}
if let Some(prompt) = efficient_system_prompt {
- config.tier_prompts = config
- .tier_prompts
- .with(efficient.semantic_name.clone(), prompt);
+ config.tier_prompts = config.tier_prompts.with(efficient.clone(), prompt);
}
// The judge is only reachable through the optional classifier fallback, so its client
// joins the router only when a fallback is configured.
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index f7be8c646..83b4f3d99 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -10,9 +10,9 @@ use std::sync::Arc;
use libsy::{
Algorithm, ClassifierContractConfig, CustomClassifierConfig, CustomClassifierPolicy,
- EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTarget,
- LlmTargetSet, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter,
- StageRouterConfig, TargetPrompts, TaskClassifierConfig,
+ EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier,
+ Noop, Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, TargetPrompts,
+ TaskClassifierConfig,
};
use serde::Deserialize;
use serde_json::Value;
@@ -20,7 +20,7 @@ use switchyard_llm_client::{
Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,
TranslatingLlmClient,
};
-use switchyard_protocol::RoutedLlmClient;
+use switchyard_protocol::{ModelId, RoutedLlmClient};
use crate::{CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState};
@@ -102,7 +102,7 @@ impl ServerConfig {
let client = self.build_client_router(config, &clients)?;
let count_tokens_target = self.build_count_tokens_target(config, &clients);
routes.push((
- config.id().to_string(),
+ config.id().clone(),
algorithm,
client,
capabilities,
@@ -133,7 +133,7 @@ impl ServerConfig {
.get_mut(&target.llm_client)
.ok_or_else(|| ServerError::new("validated llm client was not initialized"))?;
model_configs.push(ModelConfig::new(
- &target.id,
+ target.id.clone(),
build_backend(&target.llm_client, client_config, &target.extra_body)?,
None,
));
@@ -150,18 +150,11 @@ impl ServerConfig {
Ok(clients)
}
- fn build_targets(&self) -> ServerResult<BTreeMap<String, LlmTarget>> {
+ fn build_targets(&self) -> ServerResult<BTreeMap<String, ModelId>> {
Ok(self
.targets
.iter()
- .map(|(name, config)| {
- (
- name.clone(),
- LlmTarget {
- semantic_name: config.id.clone(),
- },
- )
- })
+ .map(|(name, config)| (name.clone(), config.id.clone()))
.collect())
}
@@ -225,7 +218,7 @@ impl ServerConfig {
}
// Prefer known Claude families, then preserve the route's target order.
-fn count_tokens_priority(target_name: &str, model_id: &str) -> usize {
+fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize {
let target_name = target_name.to_ascii_lowercase();
let model_id = model_id.to_ascii_lowercase();
["opus", "sonnet", "haiku"]
@@ -249,7 +242,7 @@ struct LlmClientConfig {
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct TargetConfig {
- id: String,
+ id: ModelId,
llm_client: String,
#[serde(default)]
extra_body: BTreeMap<String, Value>,
@@ -333,7 +326,7 @@ struct CustomClassifierRouteConfig {
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
enum RouteConfig {
Noop {
- id: String,
+ id: ModelId,
#[serde(default)]
context_window: Option<u32>,
#[serde(default)]
@@ -342,7 +335,7 @@ enum RouteConfig {
reasoning: Option<bool>,
},
Random {
- id: String,
+ id: ModelId,
#[serde(default)]
context_window: Option<u32>,
#[serde(default)]
@@ -354,7 +347,7 @@ enum RouteConfig {
seed: Option<u64>,
},
Passthrough {
- id: String,
+ id: ModelId,
#[serde(default)]
context_window: Option<u32>,
#[serde(default)]
@@ -364,7 +357,7 @@ enum RouteConfig {
target: String,
},
LlmClassifier {
- id: String,
+ id: ModelId,
#[serde(default)]
context_window: Option<u32>,
#[serde(default)]
@@ -404,7 +397,7 @@ enum RouteConfig {
policy: Option<ClassifierPolicyConfig>,
},
StageRouter {
- id: String,
+ id: ModelId,
#[serde(default)]
context_window: Option<u32>,
#[serde(default)]
@@ -469,7 +462,7 @@ impl StageClassifierConfig {
}
impl RouteConfig {
- fn id(&self) -> &str {
+ fn id(&self) -> &ModelId {
use RouteConfig::*;
match self {
Noop { id, .. }
@@ -821,7 +814,7 @@ const fn default_max_retries() -> u32 {
fn build_algorithm(
route_name: &str,
config: &RouteConfig,
- targets: &BTreeMap<String, LlmTarget>,
+ targets: &BTreeMap<String, ModelId>,
) -> ServerResult<Arc<dyn Algorithm>> {
match config {
RouteConfig::Noop { .. } => Ok(Arc::new(Noop {})),
@@ -838,18 +831,19 @@ fn build_algorithm(
Ok(Arc::new(algorithm))
}
RouteConfig::Passthrough { target, .. } => {
- let target = resolve_target(route_name, target, targets)?;
+ let target = resolve_target_model_id(route_name, target, targets)?;
Ok(Arc::new(Passthrough::new(target)))
}
RouteConfig::LlmClassifier {
classifier_target, ..
} => {
- let classifier = resolve_target(route_name, classifier_target, targets)?;
+ let classifier = resolve_target_model_id(route_name, classifier_target, targets)?;
let mode = config.classifier_mode(route_name)?;
let algorithm = match mode {
LlmClassifierModeConfig::Capability(config) => {
- let strong = resolve_target(route_name, &config.strong_target, targets)?;
- let weak = resolve_target(route_name, &config.weak_target, targets)?;
+ let strong =
+ resolve_target_model_id(route_name, &config.strong_target, targets)?;
+ let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?;
let classifier_config = TaskClassifierConfig {
base_threshold: config.base_threshold,
threshold_step: config.threshold_step,
@@ -867,8 +861,9 @@ fn build_algorithm(
})
}
LlmClassifierModeConfig::Escalation(config) => {
- let strong = resolve_target(route_name, &config.strong_target, targets)?;
- let weak = resolve_target(route_name, &config.weak_target, targets)?;
+ let strong =
+ resolve_target_model_id(route_name, &config.strong_target, targets)?;
+ let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?;
LlmTaskClassifier::new(LlmClassifierConfig::Escalation {
judge_target: classifier,
efficient_target: weak,
@@ -883,7 +878,7 @@ fn build_algorithm(
.targets
.iter()
.map(|name| {
- resolve_target(route_name, name, targets)
+ resolve_target_model_id(route_name, name, targets)
.map(|target| (name.clone(), target))
})
.collect::<ServerResult<Vec<_>>>()?;
@@ -933,15 +928,15 @@ fn build_algorithm(
"stage_router route {route_name} uses picker \"capable_first\", which is experimental: published thresholds and routing results all come from \"efficient_first\", so there is no calibrated confidence_threshold for it and no measured accuracy or cost. Use \"efficient_first\" unless you are running your own calibration."
);
}
- let capable = resolve_target(route_name, capable_target, targets)?;
- let efficient = resolve_target(route_name, efficient_target, targets)?;
+ let capable = resolve_target_model_id(route_name, capable_target, targets)?;
+ let efficient = resolve_target_model_id(route_name, efficient_target, targets)?;
let mut config = StageRouterConfig::new(*picker, *confidence_threshold);
config.recent_window = *recent_turn_window;
config.handoff_notes = handoff_notes.clone();
config.tier_prompts = tier_prompts(
- &capable.semantic_name,
+ &capable,
capable_system_prompt.as_deref(),
- &efficient.semantic_name,
+ &efficient,
efficient_system_prompt.as_deref(),
);
// The judge is called through its own target, so it is not a routing
@@ -949,12 +944,12 @@ fn build_algorithm(
config.llm_fallback = classifier
.as_ref()
.map(|classifier| {
- resolve_target(route_name, &classifier.target, targets).map(|judge_target| {
- LlmFallback {
+ resolve_target_model_id(route_name, &classifier.target, targets).map(
+ |judge_target| LlmFallback {
judge_target,
config: classifier.task_classifier_config(),
- }
- })
+ },
+ )
})
.transpose()?;
let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| {
@@ -995,20 +990,19 @@ fn tier_prompts(
fn resolve_targets<'a>(
route_name: &str,
names: impl IntoIterator<Item = &'a str>,
- targets: &BTreeMap<String, LlmTarget>,
-) -> ServerResult<LlmTargetSet> {
- let resolved = names
+ targets: &BTreeMap<String, ModelId>,
+) -> ServerResult<Vec<ModelId>> {
+ names
.into_iter()
- .map(|name| resolve_target(route_name, name, targets))
- .collect::<ServerResult<Vec<_>>>()?;
- Ok(LlmTargetSet::new(resolved))
+ .map(|name| resolve_target_model_id(route_name, name, targets))
+ .collect()
}
-fn resolve_target(
+fn resolve_target_model_id(
route_name: &str,
name: &str,
- targets: &BTreeMap<String, LlmTarget>,
-) -> ServerResult<LlmTarget> {
+ targets: &BTreeMap<String, ModelId>,
+) -> ServerResult<ModelId> {
targets.get(name).cloned().ok_or_else(|| {
ServerError::new(format!(
"route {route_name} references unknown target {name}"
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 7ad32d81b..9c538ea0d 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -37,7 +37,7 @@ use parking_lot::Mutex;
use serde::Deserialize;
use serde_json::{Value, json};
use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient};
-use switchyard_protocol::{Decision, LlmClientError, Metadata, Request, Usage};
+use switchyard_protocol::{Decision, LlmClientError, Metadata, ModelId, Request, Usage};
use tokio::net::{TcpListener, TcpSocket};
use tokio::task;
use tracing::{Instrument, Level};
@@ -117,7 +117,7 @@ struct RouteEntry {
/// Exact upstream model used by the server's Anthropic token-count endpoint.
#[derive(Clone)]
struct CountTokensTarget {
- model: String,
+ model: ModelId,
client: Arc<TranslatingLlmClient>,
}
@@ -130,7 +130,7 @@ impl CountTokensTarget {
/// Shared server state used by all endpoint handlers.
#[derive(Clone)]
pub struct ServerState {
- routes: Arc<BTreeMap<String, RouteEntry>>,
+ routes: Arc<BTreeMap<ModelId, RouteEntry>>,
metrics: prometheus::Registry,
stats: StatsAccumulator,
routing_log: Option<SharedRoutingLog>,
@@ -175,7 +175,7 @@ impl ServerState {
/// Creates server state from route model IDs, their libsy algorithms, and the
/// per-target client routing each route's calls are resolved through.
pub fn new(
- routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>, ClientRouter)>,
+ routes: impl IntoIterator<Item = (ModelId, Arc<dyn Algorithm>, ClientRouter)>,
) -> ServerResult<Self> {
Self::new_with_capabilities(routes.into_iter().map(|(model, algorithm, clients)| {
(
@@ -191,7 +191,7 @@ impl ServerState {
fn new_with_capabilities(
routes: impl IntoIterator<
Item = (
- String,
+ ModelId,
Arc<dyn Algorithm>,
ClientRouter,
ModelCapabilities,
@@ -201,7 +201,7 @@ impl ServerState {
) -> ServerResult<Self> {
let mut entries = BTreeMap::new();
for (model, algorithm, target_clients, capabilities, count_tokens_target) in routes {
- let model = model.trim();
+ let model = ModelId::from(model.trim());
if model.is_empty() {
return Err(ServerError::new("route model must not be empty"));
}
@@ -211,7 +211,7 @@ impl ServerState {
capabilities,
count_tokens_target,
};
- if entries.insert(model.to_string(), entry).is_some() {
+ if entries.insert(model.clone(), entry).is_some() {
return Err(ServerError::new(format!("duplicate route model {model}")));
}
}
@@ -240,7 +240,7 @@ impl ServerState {
/// Returns the route model IDs served by the configured algorithms.
pub fn models(&self) -> impl Iterator<Item = &str> {
- self.routes.keys().map(String::as_str)
+ self.routes.keys().map(ModelId::as_str)
}
fn route_for_model(&self, model: &str) -> Option<&RouteEntry> {
@@ -1327,7 +1327,7 @@ mod tests {
let call = |model: &str, is_answer_call: bool| {
RunObservation::LlmCall(LlmCallObservation {
- selected_model: model.to_string(),
+ selected_model: ModelId::from(model),
is_answer_call,
is_success: true,
duration: Duration::from_millis(3),
diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs
index 9060887e5..8d44f46f9 100644
--- a/crates/switchyard-server/src/routing_log.rs
+++ b/crates/switchyard-server/src/routing_log.rs
@@ -12,7 +12,7 @@ use std::time::SystemTime;
use humantime::format_rfc3339_millis;
use serde::{Deserialize, Serialize};
-use switchyard_protocol::Usage;
+use switchyard_protocol::{ModelId, Usage};
use crate::usage_metrics::token_usage;
use crate::{ServerError, ServerResult};
@@ -145,7 +145,7 @@ pub(crate) struct SessionStatsSnapshot {
total_cached_tokens: u64,
total_cache_creation_tokens: u64,
total_completion_tokens: u64,
- models: BTreeMap<String, SessionModelStats>,
+ models: BTreeMap<ModelId, SessionModelStats>,
}
#[derive(Default, Serialize)]
@@ -178,7 +178,7 @@ impl SessionStatsSnapshot {
"" => "unknown",
model => model,
};
- let stats = self.models.entry(model.to_string()).or_default();
+ let stats = self.models.entry(ModelId::from(model)).or_default();
stats.calls = stats.calls.saturating_add(1);
stats.prompt_tokens = stats.prompt_tokens.saturating_add(record.prompt_tokens);
stats.cached_tokens = stats.cached_tokens.saturating_add(record.cached_tokens);
diff --git a/crates/switchyard-server/src/stats/accumulator.rs b/crates/switchyard-server/src/stats/accumulator.rs
index 56d2df552..7ddfb06e5 100644
--- a/crates/switchyard-server/src/stats/accumulator.rs
+++ b/crates/switchyard-server/src/stats/accumulator.rs
@@ -12,6 +12,7 @@ use serde::Serialize;
use super::algorithms::{AlgorithmStats, AlgorithmStatsSnapshot};
use super::cache_eligibility::PrefixProbe;
+use switchyard_protocol::ModelId;
const MAX_LATENCY_SAMPLES: usize = 10_000;
@@ -50,7 +51,7 @@ impl StatsAccumulator {
}
/// Records one successful routed backend call.
- pub(crate) fn record_success(&self, model: impl Into<String>, backend_latency_ms: f64) {
+ pub(crate) fn record_success(&self, model: impl Into<ModelId>, backend_latency_ms: f64) {
let mut inner = self.lock();
inner.total_requests = inner.total_requests.saturating_add(1);
let stats = inner.model_stats_mut(model.into());
@@ -59,7 +60,7 @@ impl StatsAccumulator {
}
/// Records one failed routed backend call.
- pub(crate) fn record_error(&self, model: impl Into<String>) {
+ pub(crate) fn record_error(&self, model: impl Into<ModelId>) {
let mut inner = self.lock();
inner.total_requests = inner.total_requests.saturating_add(1);
inner.total_errors = inner.total_errors.saturating_add(1);
@@ -68,7 +69,7 @@ impl StatsAccumulator {
}
/// Records a stream failure after its routed call was already counted.
- pub(crate) fn record_stream_error(&self, model: impl Into<String>) {
+ pub(crate) fn record_stream_error(&self, model: impl Into<ModelId>) {
let mut inner = self.lock();
inner.total_errors = inner.total_errors.saturating_add(1);
let stats = inner.model_stats_mut(model.into());
@@ -78,7 +79,7 @@ impl StatsAccumulator {
/// Records usage and terminal latency after a successful routed call.
pub(crate) fn record_usage(
&self,
- model: impl Into<String>,
+ model: impl Into<ModelId>,
usage: TokenUsage,
total_latency_ms: f64,
) {
@@ -96,7 +97,7 @@ impl StatsAccumulator {
/// Records one successful classifier or judge call.
pub(crate) fn record_classifier_success(
&self,
- model: impl Into<String>,
+ model: impl Into<ModelId>,
usage: Option<TokenUsage>,
latency_ms: f64,
) {
@@ -112,7 +113,7 @@ impl StatsAccumulator {
}
/// Records one failed classifier or judge call.
- pub(crate) fn record_classifier_error(&self, model: impl Into<String>) {
+ pub(crate) fn record_classifier_error(&self, model: impl Into<ModelId>) {
let mut inner = self.lock();
inner.classifier_requests = inner.classifier_requests.saturating_add(1);
inner.classifier_errors = inner.classifier_errors.saturating_add(1);
@@ -121,9 +122,9 @@ impl StatsAccumulator {
}
/// Returns the cache-eligible fraction for `model` and records the prefix as seen.
- pub(crate) fn prefix_eligibility(&self, model: &str, probe: &PrefixProbe) -> f64 {
+ pub(crate) fn prefix_eligibility(&self, model: &ModelId, probe: &PrefixProbe) -> f64 {
let mut inner = self.lock();
- let stats = inner.model_stats_mut(model.to_string());
+ let stats = inner.model_stats_mut(model.clone());
let fraction = probe.eligible_fraction(&stats.seen_prefixes);
if let Some(hash) = probe.full_hash() {
stats.seen_prefixes.insert(hash);
@@ -147,12 +148,12 @@ impl StatsAccumulator {
}
struct StatsAccumulatorInner {
- by_model: BTreeMap<String, ModelStats>,
+ by_model: BTreeMap<ModelId, ModelStats>,
total_requests: u64,
total_errors: u64,
routing_overhead: LatencyHistogram,
routing_fallbacks: RoutingFallbackStats,
- by_classifier: BTreeMap<String, ModelStats>,
+ by_classifier: BTreeMap<ModelId, ModelStats>,
classifier_requests: u64,
classifier_errors: u64,
algorithm_stats: AlgorithmStats,
@@ -173,11 +174,11 @@ impl StatsAccumulatorInner {
}
}
- fn model_stats_mut(&mut self, model: String) -> &mut ModelStats {
+ fn model_stats_mut(&mut self, model: ModelId) -> &mut ModelStats {
self.by_model.entry(model).or_default()
}
- fn classifier_stats_mut(&mut self, model: String) -> &mut ModelStats {
+ fn classifier_stats_mut(&mut self, model: ModelId) -> &mut ModelStats {
self.by_classifier.entry(model).or_default()
}
@@ -315,7 +316,7 @@ pub(crate) struct StatsSnapshot {
pub total_requests: u64,
pub total_errors: u64,
pub total_tokens: TokenTotals,
- pub models: BTreeMap<String, ModelStatsSnapshot>,
+ pub models: BTreeMap<ModelId, ModelStatsSnapshot>,
pub routing_overhead: LatencyHistogramSnapshot,
pub routing_fallbacks: RoutingFallbackStats,
pub classifier: ClassifierStatsSnapshot,
@@ -336,7 +337,7 @@ pub(crate) struct ClassifierStatsSnapshot {
pub total_requests: u64,
pub total_errors: u64,
pub total_tokens: TokenTotals,
- pub models: BTreeMap<String, ModelStatsSnapshot>,
+ pub models: BTreeMap<ModelId, ModelStatsSnapshot>,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)]
@@ -382,9 +383,9 @@ pub(crate) struct LatencyHistogramSnapshot {
}
fn build_model_snapshots(
- by_model: &BTreeMap<String, ModelStats>,
+ by_model: &BTreeMap<ModelId, ModelStats>,
total_requests: u64,
-) -> (BTreeMap<String, ModelStatsSnapshot>, TokenTotals) {
+) -> (BTreeMap<ModelId, ModelStatsSnapshot>, TokenTotals) {
let mut totals = TokenTotals::default();
for stats in by_model.values() {
totals.prompt = totals.prompt.saturating_add(stats.prompt_tokens);
@@ -430,7 +431,7 @@ fn build_model_snapshots(
}
fn build_classifier_snapshot(
- models: &BTreeMap<String, ModelStats>,
+ models: &BTreeMap<ModelId, ModelStats>,
total_requests: u64,
total_errors: u64,
) -> ClassifierStatsSnapshot {
@@ -536,12 +537,15 @@ mod tests {
let probe = prefix_probe(&json!({
"messages": [{"role": "user", "content": "repeat me"}],
}));
- stats.prefix_eligibility("model/a", &probe);
+ stats.prefix_eligibility(&ModelId::from("model/a"), &probe);
stats.reset();
assert_eq!(stats.snapshot(), StatsSnapshot::default());
- assert_eq!(stats.prefix_eligibility("model/a", &probe), 0.0);
+ assert_eq!(
+ stats.prefix_eligibility(&ModelId::from("model/a"), &probe),
+ 0.0
+ );
}
#[test]
@@ -550,7 +554,7 @@ mod tests {
let first = prefix_probe(&json!({
"messages": [{"role": "user", "content": "aaaa"}],
}));
- let first_eligible = stats.prefix_eligibility("model/a", &first);
+ let first_eligible = stats.prefix_eligibility(&ModelId::from("model/a"), &first);
stats.record_usage(
"model/a",
TokenUsage {
@@ -566,7 +570,7 @@ mod tests {
{"role": "user", "content": "bbbb"},
],
}));
- let second_eligible = stats.prefix_eligibility("model/a", &second);
+ let second_eligible = stats.prefix_eligibility(&ModelId::from("model/a"), &second);
stats.record_usage(
"model/a",
TokenUsage {
@@ -582,6 +586,9 @@ mod tests {
stats.snapshot().models["model/a"].theoretical_cache_hit_rate,
0.25
);
- assert_eq!(stats.prefix_eligibility("model/b", &second), 0.0);
+ assert_eq!(
+ stats.prefix_eligibility(&ModelId::from("model/b"), &second),
+ 0.0
+ );
}
}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 65bd5ab79..a1241c59d 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -17,11 +17,12 @@ use axum::response::{IntoResponse, Response as HttpResponse};
use axum::routing::post;
use axum::{Json, Router};
use http_body_util::BodyExt;
-use libsy::{Algorithm, LlmTarget, LlmTargetSet, Random};
+use libsy::{Algorithm, Random};
use serde_json::{Value, json};
use switchyard_llm_client::{
Backend, ClientRouter, HttpBackendConfig, ModelConfig, TranslatingLlmClient,
};
+use switchyard_protocol::ModelId;
use switchyard_protocol::RoutedLlmClient;
use switchyard_server::config::load_server_state;
use switchyard_server::{ServerState, build_switchyard_router};
@@ -227,17 +228,10 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Server
let entries = routes
.iter()
.map(|(route_model, targets)| {
- let target_set = LlmTargetSet::new(
- targets
- .iter()
- .map(|model| LlmTarget {
- semantic_name: (*model).to_string(),
- })
- .collect(),
- );
+ let target_set = targets.iter().map(|model| ModelId::from(*model)).collect();
let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set, None, None)?);
Ok((
- (*route_model).to_string(),
+ ModelId::from(*route_model),
algorithm,
ClientRouter::single(Arc::clone(&client)),
))