Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(client): serve System One decision calls (#907)
nachiketb-nvidia1cac9637b6f0
diff --git a/Cargo.lock b/Cargo.lockindex aeeeaf37b..48b15ca63 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2358,6 +2358,7 @@ dependencies = ["opentelemetry_sdk","parking_lot","reqwest",+ "serde","serde_json","switchyard-libsy","switchyard-protocol",diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.tomlindex cb8ea4f93..ded0ecab8 100644--- a/crates/libsy-llm-client/Cargo.toml+++ b/crates/libsy-llm-client/Cargo.toml@@ -29,6 +29,7 @@ parking_lot.workspace = truehttp.workspace = truehttpdate.workspace = trueserde_json.workspace = true+serde.workspace = truetokio.workspace = truetracing.workspace = truetracing-opentelemetry.workspace = truediff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex 7348012f7..e1976de29 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -1,7 +1,7 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! [`TranslatingLlmClient`] — the crate's single public entry point: encode a neutral+//! [`TranslatingLlmClient`]: encode a neutral//! request, call the configured backend over HTTP, decode the neutral response.use std::collections::{BTreeMap, BTreeSet, HashMap};@@ -897,7 +897,7 @@ fn record_gen_ai_request(url: &str, model: &str, streaming: bool) {}}-fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {+pub(crate) fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {// Reqwest labels truncated or otherwise unreadable response bodies as decode// errors, so distinguish them from serde JSON failures at the call site.let error = error.without_url();diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rsindex d5f579c7d..ee1d9d616 100644--- a/crates/libsy-llm-client/src/lib.rs+++ b/crates/libsy-llm-client/src/lib.rs@@ -11,6 +11,10 @@//! back to a [`switchyard_protocol::Response`] — supporting both buffered and//! streamed responses.//!+//! [`SystemOneClient`] serves typed decision requests. Register decision targets+//! with [`ClientRouter::new_with_decision_clients`] or+//! [`ClientRouter::single_with_decision_clients`] to serve them alongside LLM calls.+//!//! [`run()`] pairs the client with a libsy algorithm: it drives//! [`switchyard_libsy::Algorithm::run_stream`], serves routing-time calls, and makes the terminal//! answer call from the routing outcome when needed. A host that just wants the answer does not@@ -24,14 +28,16 @@ mod observability;mod observation;pub mod raw;pub mod run;+mod system_one;pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig};pub use client::{AuxiliaryOperation, ModelConfig, TranslatingLlmClient};pub use error::{LlmClientError, Result};-pub use observation::{LlmCallObservation, RunObservation, RunObserver};+pub use observation::{ModelCallObservation, RunObservation, RunObserver};pub use raw::RawResponse;pub use run::{ClientRouter, decide, run};pub use switchyard_translation::RawEventStream;+pub use system_one::SystemOneClient;/// Registers process-wide compatibility gauges with the global meter provider.pub fn initialize_metrics() {diff --git a/crates/libsy-llm-client/src/observation.rs b/crates/libsy-llm-client/src/observation.rsindex 308ad4b28..53e660f14 100644--- a/crates/libsy-llm-client/src/observation.rs+++ b/crates/libsy-llm-client/src/observation.rs@@ -11,7 +11,7 @@ use switchyard_protocol::{ModelId, Usage};/// One completed model call observed while serving an algorithm run.#[derive(Clone, Debug)]-pub struct LlmCallObservation {+pub struct ModelCallObservation {/// Model selected for the completed call.pub selected_model: ModelId,/// Whether the call completed successfully.@@ -22,15 +22,17 @@ pub struct LlmCallObservation {pub usage: Option<Usage>,}-/// Events emitted inline while [`crate::run`] serves a routing request.+/// Events emitted inline while [`crate::run()`] serves a routing request.#[derive(Clone, Debug)]pub enum RunObservation {/// Metadata attached to the completed routing outcome.Outcome(OutcomeMetadata),/// A completed model call requested by the algorithm for routing work.- LlmCall(LlmCallObservation),+ LlmCall(ModelCallObservation),+ /// A completed decision call requested by the algorithm for routing work.+ DecisionCall(ModelCallObservation),/// A completed terminal model call made from the routing outcome.- AnswerCall(LlmCallObservation),+ AnswerCall(ModelCallObservation),/// Routing time recorded by the `switchyard.routing_overhead_ms` metric.RoutingOverhead(Duration),}diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex a3dfc339a..dcb4a6f7d 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -32,12 +32,12 @@ use switchyard_libsy::{};use switchyard_protocol::{AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, Message,- ModelId, Request, Response, ResponseAccumulator, RoutedLlmClient, RoutingFallbackReason,- WireFormat,+ ModelId, Request, Response, ResponseAccumulator, RoutedDecisionClient, RoutedLlmClient,+ RoutingFallbackReason, WireFormat,};use switchyard_translation::prepare_request_for_target;-use crate::observation::{LlmCallObservation, RunObservation, RunObserver};+use crate::observation::{ModelCallObservation, RunObservation, RunObserver};use crate::{metrics, observability};/// Run one request to completion, serving every offloaded model call with `client`.@@ -152,18 +152,38 @@ pub async fn decide(Ok(outcome)}-async fn unsupported_decision(call: CallDecision) -> Result<()> {- let model = call.model.clone();- call.respond(Err(LibsyError::client_call(- model,- LlmClientError::General("decision calls are not supported by this client".to_string()),- )))+#[tracing::instrument(+ name = "libsy.decision_client_call",+ skip_all,+ fields(algorithm = %call.algorithm, selected_model = %call.model, outcome = tracing::field::Empty),+)]+async fn serve_decision(+ clients: &ClientRouter,+ call: CallDecision,+ observations: &Option<Arc<Mutex<Vec<RunObservation>>>>,+) -> Result<()> {+ let client = clients.route_decision(&call.model);+ let started = Instant::now();+ let result = async { client?.call(call.request.clone()).await }.await;+ tracing::Span::current().record("outcome", if result.is_ok() { "ok" } else { "error" });+ if let Some(observations) = observations {+ observations+ .lock()+ .push(RunObservation::DecisionCall(ModelCallObservation {+ selected_model: call.model.clone(),+ is_success: result.is_ok(),+ duration: started.elapsed(),+ usage: result.as_ref().ok().map(|response| response.usage.clone()),+ }));+ }+ let result = result.map_err(|error| LibsyError::client_call(call.model.clone(), error));+ call.respond(result)}/// Emits completed routing calls after the outcome reveals whether one response became the answer.fn emit_routing_observations(observer: &Option<RunObserver>,- observations: &Option<Arc<Mutex<Vec<LlmCallObservation>>>>,+ observations: &Option<Arc<Mutex<Vec<RunObservation>>>>,answered_model: Option<&ModelId>,) {let (Some(observer), Some(observations)) = (observer, observations) else {@@ -171,31 +191,36 @@ fn emit_routing_observations(};let mut answer_observed = false;for observation in observations.lock().drain(..) {- if !answer_observed && answered_model == Some(&observation.selected_model) {- answer_observed = true;- observer(RunObservation::AnswerCall(observation));- } else {- observer(RunObservation::LlmCall(observation));- }+ observer(match observation {+ RunObservation::LlmCall(call)+ if !answer_observed && answered_model == Some(&call.selected_model) =>+ {+ answer_observed = true;+ RunObservation::AnswerCall(call)+ }+ observation => observation,+ });}}/// Serve one offloaded call and fulfill its promise.///-/// LLM failures stop the run unless the call enables recovery. Unsupported decisions-/// return an error to the algorithm.+/// LLM failures stop the run unless the call enables recovery. Decision results,+/// including client failures, go back to the algorithm for its routing policy.async fn serve(clients: ClientRouter,call: Call,- observations: Option<Arc<Mutex<Vec<LlmCallObservation>>>>,+ observations: Option<Arc<Mutex<Vec<RunObservation>>>>,) -> Result<()> {let call = match call {Call::Model(call) => *call,- Call::Decision(call) => return unsupported_decision(*call).await,+ Call::Decision(call) => return serve_decision(&clients, *call, &observations).await,};let observe = |observation| {if let Some(observations) = &observations {- observations.lock().push(observation);+ observations+ .lock()+ .push(RunObservation::LlmCall(observation));}};let target = call.models.first().ok_or(LibsyError::NoTargets)?;@@ -224,7 +249,7 @@ async fn call_first_available(algorithm: &str,request: &Request,models: &[ModelId],- observe: &(dyn Fn(LlmCallObservation) + Send + Sync),+ observe: &(dyn Fn(ModelCallObservation) + Send + Sync),) -> Result<Response> {for (index, target) in models.iter().enumerate() {let request = clients.prepare_completion_request(request.clone(), target);@@ -301,7 +326,7 @@ async fn call_one(model_id: &ModelId,request: Request,algorithm: &str,- observe: &(dyn Fn(LlmCallObservation) + Send + Sync),+ observe: &(dyn Fn(ModelCallObservation) + Send + Sync),// index is for span logindex: usize,// count is for span log@@ -342,7 +367,7 @@ async fn call_one(} else {result};- observe(LlmCallObservation {+ observe(ModelCallObservation {selected_model: model_id.clone(),is_success: result.is_ok(),duration,@@ -557,8 +582,7 @@ fn conversation_id(fields: &serde_json::Map<String, Value>) -> Option<&str> {////// An algorithm routes among named targets; which provider each target lives on is the/// host's concern, and two targets in one run may sit on different providers. A router owns-/// that mapping. It is *not* itself a client: it hands back a [`RoutedLlmClient`] and the-/// caller makes the call.+/// that mapping and resolves the client for each kind of call.////// Cloning is cheap — the mapping is shared, so one router can serve every request.#[derive(Clone)]@@ -568,6 +592,7 @@ pub struct ClientRouter {struct ClientRouting {routing: Routing,+ decision_clients: HashMap<ModelId, Arc<dyn RoutedDecisionClient>>,target_prompts: HashMap<ModelId, String>,routing_answer_target: Option<ModelId>,/// Provider-owned routing pins and materialized cross-format history.@@ -613,7 +638,7 @@ impl ClientRouter {/// Build a router with an explicit list of models that can answer requests.///- /// `by_model` contains every callable model, including classifiers. Only the model IDs in+ /// `by_model` contains callable LLMs, including LLM classifiers. Only the model IDs in/// `completion_targets` determine whether native Responses state needs a routing pin./// Cross-format state is recorded lazily when a Responses request uses a Chat or Anthropic/// target. `target_prompts` and `routing_answer_target` have the same meaning as in@@ -623,6 +648,24 @@ impl ClientRouter {target_prompts: HashMap<ModelId, String>,routing_answer_target: Option<ModelId>,completion_targets: &[ModelId],+ ) -> Self {+ Self::new_with_decision_clients(+ by_model,+ HashMap::new(),+ target_prompts,+ routing_answer_target,+ completion_targets,+ )+ }++ /// Build a router with separate LLM and decision mappings, shared across clones.+ /// LLM prompt and completion settings follow [`Self::new_with_completion_targets`].+ pub fn new_with_decision_clients(+ by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,+ decision_clients: HashMap<ModelId, Arc<dyn RoutedDecisionClient>>,+ target_prompts: HashMap<ModelId, String>,+ routing_answer_target: Option<ModelId>,+ completion_targets: &[ModelId],) -> Self {let mut clients = completion_targets.iter()@@ -633,6 +676,7 @@ impl ClientRouter {Self {inner: Arc::new(ClientRouting {routing: Routing::ByModel(by_model),+ decision_clients,target_prompts,routing_answer_target,state_owners: Mutex::default(),@@ -647,9 +691,18 @@ impl ClientRouter {/// backends internally and rejects ones it does not know, so enumerating them here would/// only duplicate that.pub fn single(client: Arc<dyn RoutedLlmClient>) -> Self {+ Self::single_with_decision_clients(client, HashMap::new())+ }++ /// Use one client for all LLM targets and explicit clients for decision targets.+ pub fn single_with_decision_clients(+ client: Arc<dyn RoutedLlmClient>,+ decision_clients: HashMap<ModelId, Arc<dyn RoutedDecisionClient>>,+ ) -> Self {Self {inner: Arc::new(ClientRouting {routing: Routing::Single(client),+ decision_clients,target_prompts: HashMap::new(),routing_answer_target: None,state_owners: Mutex::default(),@@ -678,6 +731,19 @@ impl ClientRouter {}}+ /// Resolve a decision target without falling back to an LLM client.+ fn route_decision(+ &self,+ model: &ModelId,+ ) -> std::result::Result<&Arc<dyn RoutedDecisionClient>, LlmClientError> {+ self.inner+ .decision_clients+ .get(model)+ .ok_or_else(|| LlmClientError::Configuration {+ message: format!("no decision client is configured for model {model:?}"),+ })+ }+/// Return the recorded model for the requested response or conversation ID.fn stored_state_owner(&self, request: &Request) -> Option<StateOwner> {let fields = &request.llm_request.extensions.fields;@@ -1072,6 +1138,239 @@ mod tests {}}+ #[tokio::test]+ async fn system_one_serves_decisions_and_returns_failures_to_the_classifier()+ -> std::result::Result<(), Box<dyn std::error::Error>> {+ use std::time::Duration;+ use switchyard_libsy::{+ CapabilityJudgeConfig, DecisionJudgeConfig, LlmClassifierConfig, LlmTaskClassifier,+ TaskClassifierConfig,+ };+ use switchyard_protocol::{+ BooleanEstimate, DecisionRequest, DecisionValue, Probability, ScoreValue,+ };+ use wiremock::matchers::{body_partial_json, header, path};++ let server = MockServer::start().await;+ let client = Arc::new(crate::SystemOneClient::new(+ format!("{}/v1/systemone", server.uri()).parse()?,+ "test-key".into(),+ Duration::from_secs(2),+ )?);+ assert!(matches!(+ crate::SystemOneClient::new(+ format!("{}/v1/systemone", server.uri()).parse()?,+ "invalid\nkey".into(),+ Duration::from_secs(2),+ ),+ Err(LlmClientError::Configuration { .. })+ ));+ let mixed: DecisionRequest = serde_json::from_value(json!({+ "model": "judge", "context": {"task": "A simple task"},+ "questions": {+ "boolean": {"instructions": "Is it simple?", "kind": {"type": "boolean", "data": {+ "true_description": ["Simple"], "false_description": null+ }}},+ "route": {"instructions": {"task": "Choose"}, "kind": {"type": "choice", "data": {+ "options": [{"id": "advantage", "description": "A wins"}, {"id": "no_advantage"}]+ }}},+ "score": {"instructions": "Rate difficulty", "kind": {"type": "score", "data": {+ "levels": ["Easy", {"description": "Hard"}]+ }}}+ }+ }))?;+ let mut body = json!({+ "model": "jev-1.13.0", "usage": {"input_tokens": 42, "output_tokens": 3},+ "answers": {+ "boolean": {"type": "noul", "noul": 0.9},+ "route": {"type": "choice", "choice": "no_advantage", "confidence": 0.7,+ "probabilities": {"advantage": 0.2, "no_advantage": 0.8}},+ "score": {"type": "score", "score": 0.6, "confidence": 0.3,+ "legend": {"0": "Easy", "1": {"description": "Hard"}},+ "probabilities": {"1": 0.6, "0": 0.4}}+ }+ });+ Mock::given(method("POST"))+ .and(path("/v1/systemone"))+ .and(header("authorization", "Bearer test-key"))+ .and(body_partial_json(json!({+ "model": "judge", "state": mixed.context,+ "questions": {+ "boolean": {"type": "noul", "criteria": {"true": ["Simple"]}},+ "route": {"type": "choice", "instructions": {"task": "Choose"},+ "criteria": {"advantage": "A wins", "no_advantage": null}},+ "score": {"type": "score", "criteria": ["Easy", {"description": "Hard"}]}+ }+ })))+ .respond_with(ResponseTemplate::new(200).set_body_json(&body))+ .expect(1)+ .mount(&server)+ .await;+ let response = client.call(mixed.clone()).await?;+ assert_eq!(response.model.as_deref(), Some("jev-1.13.0"));+ assert_eq!(response.id, None);+ assert_eq!(response.usage.input_tokens, Some(42));+ assert_eq!(response.usage.total_tokens, None);+ assert_eq!(+ response.answers["boolean"].value,+ DecisionValue::Boolean(BooleanEstimate::ProbabilityTrue(Probability(0.9)))+ );+ assert_eq!(+ response.answers["score"].value,+ DecisionValue::Score {+ value: ScoreValue(0.6),+ probabilities: Some(vec![Probability(0.4), Probability(0.6)]),+ }+ );+ assert_eq!(+ response.answers["route"].provider_confidence.map(|c| c.0),+ Some(0.7)+ );+ server.verify().await;++ let mut partial = body.clone();+ partial["answers"]+ .as_object_mut()+ .unwrap()+ .remove("boolean");+ let mut invalid_responses = vec![json!({"answers": {}}), partial];+ for (id, answer) in [+ ("unknown-question", json!({"type": "noul", "noul": 0.9})),+ ("boolean", json!({"type": "score", "score": 0.0})),+ (+ "route",+ json!({"type": "choice", "choice": "unknown-option",+ "probabilities": {"advantage": 0.2, "no_advantage": 0.8}}),+ ),+ ("score", json!({"type": "score", "score": -0.1})),+ (+ "score",+ json!({"type": "score", "score": 1.1,+ "probabilities": {"0": 0.4, "1": 0.6}}),+ ),+ ] {+ let mut invalid = body.clone();+ invalid["answers"][id] = answer;+ if id == "unknown-question" {+ invalid["answers"]+ .as_object_mut()+ .unwrap()+ .remove("boolean");+ }+ invalid_responses.push(invalid);+ }+ for template in invalid_responses+ .into_iter()+ .map(|body| ResponseTemplate::new(200).set_body_json(body))+ .chain([ResponseTemplate::new(200).set_body_string("invalid JSON")])+ {+ server.reset().await;+ Mock::given(method("POST"))+ .respond_with(template)+ .expect(1)+ .mount(&server)+ .await;+ assert!(matches!(+ client.call(mixed.clone()).await,+ Err(LlmClientError::ResponseTranslation(_) | LlmClientError::InvalidResponse { .. })+ ));+ server.verify().await;+ }++ body["answers"].as_object_mut().unwrap().remove("boolean");+ body["answers"].as_object_mut().unwrap().remove("score");+ let models = Arc::new(RuntimeModels::new(HashMap::from([+ (Category::Judge, vec!["judge".into()]),+ (Category::Capable, vec!["capable".into()]),+ (Category::Efficient, vec!["efficient".into()]),+ (Category::Any, vec!["efficient".into(), "capable".into()]),+ ])));+ let algorithm = || -> Result<Arc<dyn Algorithm>> {+ Ok(Arc::new(LlmTaskClassifier::new(+ LlmClassifierConfig::Capability {+ config: TaskClassifierConfig {+ judge: CapabilityJudgeConfig::Decision(DecisionJudgeConfig {+ cutoff: 0.4,+ instructions: None,+ candidates: BTreeMap::from([+ ("a".into(), "capable".into()),+ ("b".into(), "efficient".into()),+ ]),+ evidence: json!({}),+ }),+ ..TaskClassifierConfig::default()+ },+ },+ )?))+ };+ for (template, expected, is_success) in [+ (+ ResponseTemplate::new(200).set_body_json(&body),+ "efficient",+ true,+ ),+ (+ ResponseTemplate::new(503).set_body_string("unavailable"),+ "capable",+ false,+ ),+ ] {+ server.reset().await;+ Mock::given(method("POST"))+ .and(path("/v1/systemone"))+ .respond_with(template)+ .expect(2)+ .mount(&server)+ .await;+ let llm = Arc::new(CandidateClient {+ calls: Mutex::default(),+ requests: Mutex::default(),+ first: FirstOutcome::Unauthorized,+ });+ let clients = ClientRouter::single_with_decision_clients(+ llm.clone(),+ HashMap::from([(+ ModelId::from("judge"),+ client.clone() as Arc<dyn RoutedDecisionClient>,+ )]),+ );+ let outcome = decide(algorithm()?, clients.clone(), request(), models.clone()).await?;+ assert_eq!(outcome.selected_model_id()?, expected);+ assert!(llm.calls.lock().is_empty());+ let events = Arc::new(Mutex::new(Vec::new()));+ let captured = events.clone();+ let (selected, _) = run(+ algorithm()?,+ clients,+ request(),+ models.clone(),+ Some(Arc::new(move |event| captured.lock().push(event))),+ )+ .await?;+ assert_eq!(selected, expected);+ assert_eq!(&*llm.calls.lock(), &[ModelId::from(expected)]);+ {+ let events = events.lock();+ let RunObservation::DecisionCall(call) = &events[0] else {+ panic!("missing decision observation")+ };+ assert_eq!(call.selected_model, "judge");+ assert_eq!(call.is_success, is_success);+ assert_eq!(+ call.usage.as_ref().and_then(|u| u.input_tokens),+ is_success.then_some(42)+ );+ assert!(+ events+ .iter()+ .any(|event| matches!(event, RunObservation::AnswerCall(_)))+ );+ }+ server.verify().await;+ }+ Ok(())+ }+fn instruction_text(request: &Request) -> Vec<&str> {request.llm_request@@ -1159,12 +1458,13 @@ mod tests {fn answer_observation_keeps_call_order() {let pending = Some(Arc::new(Mutex::new(["answer", "judge"]- .map(|model| LlmCallObservation {+ .map(|model| ModelCallObservation {selected_model: model.into(),is_success: true,duration: std::time::Duration::ZERO,usage: None,})+ .map(RunObservation::LlmCall).into(),)));let emitted = Arc::new(Mutex::new(Vec::new()));diff --git a/crates/libsy-llm-client/src/system_one.rs b/crates/libsy-llm-client/src/system_one.rsnew file mode 100644index 000000000..611832ca8--- /dev/null+++ b/crates/libsy-llm-client/src/system_one.rs@@ -0,0 +1,243 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Buffered System One calls using the provider-neutral decision IR.++use std::collections::BTreeMap;+use std::time::Duration;++use async_trait::async_trait;+use reqwest::{Url, header::HeaderValue};+use serde::Deserialize;+use serde_json::{Value, json};+use switchyard_protocol::{+ BooleanEstimate, DecisionAnswer, DecisionKind, DecisionRequest, DecisionResponse,+ DecisionValue, ModelId, Probability, ProviderConfidence, RoutedDecisionClient, ScoreValue,+ Usage,+};++use crate::client::convert_reqwest_error;+use crate::{LlmClientError, Result, metrics};++/// Serves decision requests through a System One endpoint, such as TypeSafe's Jev API.+pub struct SystemOneClient {+ client: reqwest::Client,+ endpoint: Url,+ api_key: String,+}++impl SystemOneClient {+ /// `endpoint` is the full URL, including `/v1/systemone`. Each call makes one+ /// attempt; `timeout` covers sending the request and reading the response body.+ pub fn new(endpoint: Url, api_key: String, timeout: Duration) -> Result<Self> {+ if HeaderValue::try_from(format!("Bearer {api_key}")).is_err() {+ return Err(LlmClientError::Configuration {+ message: "System One API key cannot be encoded as an HTTP header".into(),+ });+ }+ let client = reqwest::Client::builder()+ .timeout(timeout)+ .redirect(reqwest::redirect::Policy::none())+ .build()+ .map_err(convert_reqwest_error)?;+ Ok(Self {+ client,+ endpoint,+ api_key,+ })+ }+}++#[async_trait]+impl RoutedDecisionClient for SystemOneClient {+ async fn call(&self, request: DecisionRequest) -> Result<DecisionResponse> {+ let response = self+ .client+ .post(self.endpoint.clone())+ .bearer_auth(&self.api_key)+ .json(&encode(&request)?)+ .send()+ .await+ .inspect_err(|_| metrics::record_upstream_attempt(None))+ .map_err(convert_reqwest_error)?;+ let status = response.status();+ let body = response+ .bytes()+ .await+ .inspect_err(|_| metrics::record_upstream_attempt(None))+ .map_err(convert_reqwest_error)?;+ metrics::record_upstream_attempt(Some(status.as_u16()));+ if !status.is_success() {+ return Err(LlmClientError::UpstreamHttp {+ status,+ body: String::from_utf8_lossy(&body).replace(&self.api_key, "[REDACTED]"),+ });+ }+ let response: WireResponse =+ serde_json::from_slice(&body).map_err(|source| LlmClientError::InvalidResponse {+ source: Box::new(source),+ })?;+ if !response.answers.keys().eq(request.questions.keys()) {+ return Err(LlmClientError::ResponseTranslation(+ "System One answer keys do not match the requested question keys".into(),+ ));+ }+ let answers = response+ .answers+ .into_iter()+ .map(|(id, answer)| {+ let value = match (request.questions.get(&id).map(|q| &q.kind), answer.value) {+ (Some(DecisionKind::Boolean { .. }), WireValue::Noul { noul }) => {+ DecisionValue::Boolean(BooleanEstimate::ProbabilityTrue(noul))+ }+ (+ Some(DecisionKind::Choice { options }),+ WireValue::Choice {+ choice,+ probabilities,+ },+ ) if options.iter().any(|option| option.id == choice) => {+ DecisionValue::Choice {+ selected: choice,+ probabilities,+ }+ }+ (+ Some(DecisionKind::Score { levels }),+ WireValue::Score {+ score,+ probabilities,+ },+ ) if !levels.is_empty()+ && (0.0..=(levels.len() - 1) as f64).contains(&score.0) =>+ {+ let probabilities = probabilities+ .map(|mut probabilities| {+ let invalid_rubric = || {+ LlmClientError::ResponseTranslation(format!(+ "score probabilities for {id:?} do not match its rubric"+ ))+ };+ if probabilities.len() != levels.len() {+ return Err(invalid_rubric());+ }+ // JSON keys are strings; order probabilities by the request's rubric.+ (0..levels.len())+ .map(|index| {+ probabilities+ .remove(&index.to_string())+ .ok_or_else(invalid_rubric)+ })+ .collect()+ })+ .transpose()?;+ DecisionValue::Score {+ value: score,+ probabilities,+ }+ }+ _ => {+ return Err(LlmClientError::ResponseTranslation(format!(+ "answer {id:?} has an unexpected question ID, kind, or value"+ )));+ }+ };+ Ok((+ id,+ DecisionAnswer {+ value,+ provider_confidence: answer.confidence,+ },+ ))+ })+ .collect::<Result<_>>()?;+ Ok(DecisionResponse {+ id: response.id,+ model: response.model,+ answers,+ usage: response.usage,+ })+ }+}++fn encode(request: &DecisionRequest) -> Result<Value> {+ let model = request+ .model+ .as_ref()+ .ok_or_else(|| LlmClientError::InvalidRequest {+ message: "System One requires a selected model".into(),+ })?;+ let mut questions = serde_json::Map::new();+ for (id, question) in &request.questions {+ let (kind, criteria) = match &question.kind {+ DecisionKind::Boolean {+ true_description,+ false_description,+ } => {+ let mut criteria = serde_json::Map::new();+ for (key, description) in [("true", true_description), ("false", false_description)]+ {+ if let Some(description) = description {+ criteria.insert(key.into(), description.clone());+ }+ }+ ("noul", Value::Object(criteria))+ }+ DecisionKind::Choice { options } => {+ let mut criteria = serde_json::Map::new();+ for option in options {+ let description = option.description.as_ref().unwrap_or(&Value::Null);+ if criteria+ .insert(option.id.clone(), description.clone())+ .is_some()+ {+ return Err(LlmClientError::InvalidRequest {+ message: format!("duplicate option {:?} in questions.{id}", option.id),+ });+ }+ }+ ("choice", Value::Object(criteria))+ }+ DecisionKind::Score { levels } => ("score", json!(levels)),+ };+ questions.insert(+ id.clone(),+ json!({+ "type": kind, "instructions": question.instructions, "criteria": criteria,+ }),+ );+ }+ Ok(json!({"model": model, "state": request.context, "questions": questions}))+}++#[derive(Deserialize)]+struct WireResponse {+ id: Option<String>,+ model: Option<ModelId>,+ answers: BTreeMap<String, WireAnswer>,+ #[serde(default)]+ usage: Usage,+}++#[derive(Deserialize)]+struct WireAnswer {+ #[serde(flatten)]+ value: WireValue,+ confidence: Option<ProviderConfidence>,+}++#[derive(Deserialize)]+#[serde(tag = "type", rename_all = "snake_case")]+enum WireValue {+ Noul {+ noul: Probability,+ },+ Choice {+ choice: String,+ probabilities: Option<BTreeMap<String, Probability>>,+ },+ Score {+ score: ScoreValue,+ probabilities: Option<BTreeMap<String, Probability>>,+ },+}diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rsindex 8d312e72d..fa37f01c1 100644--- a/crates/protocol/src/client.rs+++ b/crates/protocol/src/client.rs@@ -1,17 +1,15 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! The routed-call server trait and its shared error types.+//! Routed-call client contracts and shared error types.//!-//! [`RoutedLlmClient`] is the one piece of I/O the protocol does not own: a host-//! implements it to actually perform a model call. It lives here — rather than in-//! libsy's orchestration crate — so a client crate that depends only on the protocol-//! can serve routed calls without pulling in the orchestrator.+//! Hosts implement these traits to perform model calls. Keeping the contracts here+//! lets clients depend on the protocol without pulling in libsy's orchestration.use async_trait::async_trait;use thiserror::Error;-use crate::{ModelId, Request, Response};+use crate::{DecisionRequest, DecisionResponse, ModelId, Request, Response};/// A boxed client-specific error preserved as the source of a routed call failure.pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;@@ -157,3 +155,11 @@ pub trait RoutedLlmClient: Send + Sync {/// Make a requestasync fn call(&self, request: Request) -> Result<Response, LlmClientError>;}++/// Serves typed decision calls over a host-owned transport.+/// Implementations may be shared across targets and called concurrently.+#[async_trait]+pub trait RoutedDecisionClient: Send + Sync {+ /// Evaluate the request using its selected model.+ async fn call(&self, request: DecisionRequest) -> Result<DecisionResponse, LlmClientError>;+}diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rsindex 95794f0d8..4c1b7ab05 100644--- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs+++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs@@ -11,7 +11,7 @@ use nemo_relay_plugin::{MetricValueType, PluginRuntime,};use serde_json::{Map, json};-use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver};+use switchyard_llm_client::{ModelCallObservation, RunObservation, RunObserver};use switchyard_protocol::{LlmClientError, LlmResponse, LlmResponseChunk, LlmStreamError, Metadata, ProviderExtensions,Request, Response, Usage, WireFormat,@@ -336,7 +336,23 @@ impl SwitchyardRuntime {}RunObservation::LlmCall(call) => {call_index += 1;- self.routing_call_events(events, call, call_index, metadata);+ self.routing_call_events(+ events,+ call,+ call_index,+ metadata,+ "switchyard.routing.llm_call",+ );+ }+ RunObservation::DecisionCall(call) => {+ call_index += 1;+ self.routing_call_events(+ events,+ call,+ call_index,+ metadata,+ "switchyard.routing.decision_call",+ );}RunObservation::RoutingOverhead(duration) => {let latency_ms = duration.as_secs_f64() * 1_000.0;@@ -374,15 +390,16 @@ impl SwitchyardRuntime {fn routing_call_events(&self,events: &mut Vec<RoutingEvent>,- call: LlmCallObservation,+ call: ModelCallObservation,call_index: usize,metadata: &Json,+ mark_name: &str,) {let outcome = if call.is_success { "ok" } else { "error" };let latency_ms = call.duration.as_secs_f64() * 1_000.0;let token_metrics = token_usage_metrics("routing", &call, metadata);events.push(RoutingEvent::Mark(RoutingMark {- name: "switchyard.routing.llm_call".into(),+ name: mark_name.into(),data: json!({"call_index": call_index,"selected_model": call.selected_model.as_str(),@@ -717,7 +734,7 @@ fn routing_overhead_metric(latency_ms: f64, metadata: Json) -> RoutingEvent {fn token_usage_metrics(call_role: &str,- call: &LlmCallObservation,+ call: &ModelCallObservation,metadata: &Json,) -> Vec<RoutingEvent> {let Some(usage) = call.usage.as_ref() else {@@ -1299,7 +1316,7 @@ mod tests {runtime.emit_observations(&mut events,vec![- RunObservation::LlmCall(LlmCallObservation {+ RunObservation::LlmCall(ModelCallObservation {selected_model: ModelId::from("routing-model"),is_success: false,duration: std::time::Duration::from_millis(12),@@ -1390,7 +1407,7 @@ mod tests {#[test]fn token_usage_metrics_distinguish_routing_and_answer_targets() {- let call = LlmCallObservation {+ let call = ModelCallObservation {selected_model: ModelId::from("judge-model"),is_success: true,duration: std::time::Duration::from_millis(1),@@ -1452,13 +1469,13 @@ mod tests {runtime.emit_observations(&mut events,vec![- RunObservation::AnswerCall(LlmCallObservation {+ RunObservation::AnswerCall(ModelCallObservation {selected_model: ModelId::from("weak-target"),is_success: false,duration: std::time::Duration::from_millis(2),usage: None,}),- RunObservation::AnswerCall(LlmCallObservation {+ RunObservation::AnswerCall(ModelCallObservation {selected_model: ModelId::from("strong-target"),is_success: true,duration: std::time::Duration::from_millis(3),diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 5f44eef52..9ee670071 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -447,7 +447,7 @@ fn stats_observer(stats.record_error(&call.selected_model);}}- RunObservation::LlmCall(call) => {+ RunObservation::LlmCall(call) | RunObservation::DecisionCall(call) => {let latency_ms = call.duration.as_secs_f64() * 1_000.0;if call.is_success {if let (Some((log, context)), Some(usage)) =@@ -1687,7 +1687,7 @@ fn endpoint_listing(has_routing_log: bool) -> String {#[cfg(test)]mod tests {- use switchyard_llm_client::LlmCallObservation;+ use switchyard_llm_client::ModelCallObservation;use tokio::io::{AsyncReadExt, AsyncWriteExt};use tokio::sync::{Notify, oneshot};@@ -1708,7 +1708,7 @@ mod tests {let observer = stats_observer(StatsAccumulator::default(), Some((log.clone(), context)));let call = |model: &str, answer: bool| {- let observation = LlmCallObservation {+ let observation = ModelCallObservation {selected_model: ModelId::from(model),is_success: true,duration: Duration::from_millis(3),