Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(runner): support OpenAI Decisions (#1070)
nachiketb-nvidia7558a1d8a5f2
diff --git a/Cargo.lock b/Cargo.lockindex 8773c11b3..93d7f5e3f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3848,6 +3848,7 @@ dependencies = ["tokio","toml 1.1.3+spec-1.1.0","tracing",+ "wiremock",][[package]]diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rsindex 5c9fd1f4f..414a32167 100644--- a/crates/libsy-llm-client/src/lib.rs+++ b/crates/libsy-llm-client/src/lib.rs@@ -27,6 +27,7 @@ pub mod metrics;mod observability;mod observation;mod observed_translation;+mod openai_decisions;pub mod raw;pub mod run;mod system_one;@@ -35,6 +36,7 @@ pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig};pub use client::{AuxiliaryOperation, ModelConfig, TranslatingLlmClient};pub use error::{LlmClientError, Result};pub use observation::{ModelCallObservation, RunObservation, RunObserver};+pub use openai_decisions::OpenAiDecisionClient;pub use raw::RawResponse;pub use run::{ClientRouter, decide, run};pub use switchyard_translation::RawEventStream;diff --git a/crates/libsy-llm-client/src/openai_decisions.rs b/crates/libsy-llm-client/src/openai_decisions.rsnew file mode 100644index 000000000..231240bf4--- /dev/null+++ b/crates/libsy-llm-client/src/openai_decisions.rs@@ -0,0 +1,312 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! OpenAI Decisions calls using the provider-neutral decision IR.++use std::collections::BTreeMap;+use std::time::Duration;++use async_trait::async_trait;+use reqwest::{Url, header::HeaderValue};+use serde::Deserialize;+use serde_json::{Value, json};+use switchyard_protocol::{+ BooleanEstimate, DecisionAnswer, DecisionKind, DecisionRequest, DecisionResponse,+ DecisionValue, ModelId, Probability, ProviderConfidence, RoutedDecisionClient, ScoreValue,+ Usage,+};++use crate::client::convert_reqwest_error;+use crate::{LlmClientError, Result, metrics};++/// Serves typed questions through OpenAI's Decisions API.+pub struct OpenAiDecisionClient {+ client: reqwest::Client,+ endpoint: Url,+ api_key: String,+}++impl OpenAiDecisionClient {+ /// `endpoint` includes `/v1/decisions`. Each call makes one attempt, with+ /// `timeout` covering both the request and response body.+ pub fn new(endpoint: Url, api_key: String, timeout: Duration) -> Result<Self> {+ if HeaderValue::try_from(format!("Bearer {api_key}")).is_err() {+ return Err(LlmClientError::Configuration {+ message: "OpenAI Decisions API key cannot be encoded as an HTTP header".into(),+ });+ }+ let client = reqwest::Client::builder()+ .timeout(timeout)+ .redirect(reqwest::redirect::Policy::none())+ .build()+ .map_err(convert_reqwest_error)?;+ Ok(Self {+ client,+ endpoint,+ api_key,+ })+ }+}++#[async_trait]+impl RoutedDecisionClient for OpenAiDecisionClient {+ async fn call(&self, request: DecisionRequest) -> Result<DecisionResponse> {+ let response = self+ .client+ .post(self.endpoint.clone())+ .bearer_auth(&self.api_key)+ .json(&encode(&request)?)+ .send()+ .await+ .inspect_err(|_| metrics::record_upstream_attempt(None))+ .map_err(convert_reqwest_error)?;+ let status = response.status();+ let body = response+ .bytes()+ .await+ .inspect_err(|_| metrics::record_upstream_attempt(None))+ .map_err(convert_reqwest_error)?;+ metrics::record_upstream_attempt(Some(status.as_u16()));+ if !status.is_success() {+ return Err(LlmClientError::UpstreamHttp {+ status,+ body: String::from_utf8_lossy(&body).replace(&self.api_key, "[REDACTED]"),+ });+ }+ let response: WireResponse =+ serde_json::from_slice(&body).map_err(|source| LlmClientError::InvalidResponse {+ source: Box::new(source),+ })?;+ let mut questions = request.questions;+ let mut answers = BTreeMap::new();+ for answer in response.answers {+ let invalid = || {+ LlmClientError::ResponseTranslation(format!(+ "OpenAI Decisions answer {:?} has an unexpected question name, kind, or value",+ answer.name,+ ))+ };+ let question = questions.remove(&answer.name).ok_or_else(invalid)?;+ let value = match (&question.kind, answer.value) {+ (DecisionKind::Boolean { .. }, WireValue::Predicate { probability }) => {+ DecisionValue::Boolean(BooleanEstimate::ProbabilityTrue(probability))+ }+ (+ DecisionKind::Choice { options },+ WireValue::Choice {+ choice,+ probabilities,+ },+ ) => {+ let count = probabilities.len();+ let probabilities: BTreeMap<_, _> = probabilities+ .into_iter()+ .map(|p| (p.value, p.probability))+ .collect();+ if count != options.len()+ || probabilities.len() != options.len()+ || !options.iter().any(|option| option.id == choice)+ || options+ .iter()+ .any(|option| !probabilities.contains_key(&option.id))+ {+ return Err(invalid());+ }+ DecisionValue::Choice {+ selected: choice,+ probabilities: Some(probabilities),+ }+ }+ (+ DecisionKind::Score { levels },+ WireValue::Score {+ score,+ probabilities,+ },+ ) => {+ let mut probabilities = probabilities;+ probabilities.sort_by_key(|p| p.value);+ if probabilities.len() != levels.len()+ || probabilities+ .iter()+ .enumerate()+ .any(|(index, p)| p.value != index)+ {+ return Err(invalid());+ }+ DecisionValue::Score {+ value: score,+ probabilities: Some(+ probabilities.into_iter().map(|p| p.probability).collect(),+ ),+ }+ }+ (_, WireValue::Refusal) => {+ return Err(LlmClientError::ResponseTranslation(format!(+ "OpenAI Decisions refused question {:?}",+ answer.name,+ )));+ }+ _ => return Err(invalid()),+ };+ answers.insert(+ answer.name,+ DecisionAnswer {+ value,+ provider_confidence: answer.confidence,+ },+ );+ }+ if !questions.is_empty() {+ return Err(LlmClientError::ResponseTranslation(+ "OpenAI Decisions answer names do not match the requested question keys".into(),+ ));+ }+ let usage = response.usage;+ let cached = usage.input_tokens_details.cached_tokens;+ let created = usage.input_tokens_details.cache_write_tokens;+ Ok(DecisionResponse {+ id: None,+ model: Some(response.model),+ answers,+ usage: Usage {+ input_tokens: Some(+ usage+ .input_tokens+ .saturating_sub(cached)+ .saturating_sub(created),+ ),+ cache: Usage::cache_details(Some(cached), Some(created)),+ output_tokens: Some(usage.output_tokens),+ total_tokens: Some(usage.total_tokens),+ reasoning_tokens: Some(usage.output_tokens_details.reasoning_tokens),+ ..Usage::default()+ },+ })+ }+}++fn encode(request: &DecisionRequest) -> Result<Value> {+ let model = request+ .model+ .as_ref()+ .ok_or_else(|| LlmClientError::InvalidRequest {+ message: "OpenAI Decisions requires a selected model".into(),+ })?;+ let questions: Vec<_> = request+ .questions+ .iter()+ .map(|(name, question)| {+ let mut wire = json!({"name": name, "instructions": text(&question.instructions)});+ match &question.kind {+ DecisionKind::Boolean {+ true_description,+ false_description,+ } => {+ wire["type"] = json!("predicate");+ if true_description.is_some() || false_description.is_some() {+ wire["instructions"] = json!(+ json!({+ "instructions": question.instructions,+ "true": true_description, "false": false_description,+ })+ .to_string()+ );+ }+ }+ DecisionKind::Choice { options } => {+ wire["type"] = json!("choice");+ wire["choices"] = Value::Array(+ options+ .iter()+ .map(|option| {+ let mut choice = json!({"value": option.id});+ if let Some(description) = &option.description {+ choice["description"] = json!(text(description));+ }+ choice+ })+ .collect(),+ );+ }+ DecisionKind::Score { levels } => {+ wire["type"] = json!("score");+ wire["levels"] = Value::Array(+ levels+ .iter()+ .map(|level| json!({"label": text(level)}))+ .collect(),+ );+ }+ }+ wire+ })+ .collect();+ Ok(json!({"model": model, "input": text(&request.context), "questions": questions}))+}++// OpenAI requires text; retain structured IR content as JSON rather than dropping it.+fn text(value: &Value) -> String {+ value+ .as_str()+ .map(str::to_owned)+ .unwrap_or_else(|| value.to_string())+}++#[derive(Deserialize)]+struct WireResponse {+ model: ModelId,+ answers: Vec<WireAnswer>,+ usage: WireUsage,+}++#[derive(Deserialize)]+struct WireAnswer {+ name: String,+ #[serde(flatten)]+ value: WireValue,+ confidence: Option<ProviderConfidence>,+}++#[derive(Deserialize)]+#[serde(tag = "type", rename_all = "snake_case")]+enum WireValue {+ Predicate {+ probability: Probability,+ },+ Choice {+ choice: String,+ probabilities: Vec<WireProbability<String>>,+ },+ Score {+ score: ScoreValue,+ probabilities: Vec<WireProbability<usize>>,+ },+ Refusal,+}++#[derive(Deserialize)]+struct WireProbability<T> {+ value: T,+ probability: Probability,+}++#[derive(Deserialize)]+struct WireUsage {+ input_tokens: u64,+ output_tokens: u64,+ total_tokens: u64,+ input_tokens_details: InputTokenDetails,+ output_tokens_details: OutputTokenDetails,+}++#[derive(Deserialize)]+struct InputTokenDetails {+ cached_tokens: u64,+ cache_write_tokens: u64,+}++#[derive(Deserialize)]+struct OutputTokenDetails {+ reasoning_tokens: u64,+}diff --git a/crates/switchyard-runner/Cargo.toml b/crates/switchyard-runner/Cargo.tomlindex 3a80b61b3..d55e60097 100644--- a/crates/switchyard-runner/Cargo.toml+++ b/crates/switchyard-runner/Cargo.toml@@ -33,3 +33,4 @@ tracing.workspace = trueasync-trait.workspace = truefutures-util.workspace = truetokio.workspace = true+wiremock = "0.6"diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex ddd3812ab..7e3999c43 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -14,7 +14,7 @@ use serde::{Deserialize, Deserializer};use serde_json::Value;use switchyard_llm_client::{AuxiliaryOperation, Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,- SystemOneClient, TranslatingLlmClient,+ OpenAiDecisionClient, SystemOneClient, TranslatingLlmClient,};use switchyard_protocol::{ModelId, RoutedDecisionClient, RoutedLlmClient, WireFormat};@@ -46,8 +46,7 @@ pub fn api_key_variables(source: &str) -> Result<BTreeSet<String>, RunnerError>.filter_map(|client| client.api_key_env.clone()).collect();for client in config.decision_clients.values() {- let DecisionClientConfig::SystemOne { api_key_env, .. } = client;- variables.insert(api_key_env.clone());+ variables.insert(client.api_key_env.clone());}if variables.iter()@@ -421,7 +420,8 @@ impl DeploymentConfig {.iter().map(|(name, config)| {validate_value("decision client name", name)?;- let DecisionClientConfig::SystemOne {+ let DecisionClientConfig {+ format,endpoint,api_key_env,timeout_ms,@@ -432,17 +432,20 @@ impl DeploymentConfig {)));}let api_key = read_api_key(&format!("decision client {name}"), api_key_env)?;- let client = SystemOneClient::new(- endpoint.0.clone(),- api_key.clone(),- Duration::from_millis(*timeout_ms),- )+ let timeout = Duration::from_millis(*timeout_ms);+ let client = match format {+ DecisionFormat::SystemOne => {+ SystemOneClient::new(endpoint.0.clone(), api_key.clone(), timeout)+ .map(|client| Arc::new(client) as Arc<dyn RoutedDecisionClient>)+ }+ DecisionFormat::OpenaiDecisions => {+ OpenAiDecisionClient::new(endpoint.0.clone(), api_key.clone(), timeout)+ .map(|client| Arc::new(client) as Arc<dyn RoutedDecisionClient>)+ }+ }.map_err(|error| RunnerError::configuration(error.to_string()))?;provider_api_keys.push(api_key);- Ok((- name.clone(),- Arc::new(client) as Arc<dyn RoutedDecisionClient>,- ))+ Ok((name.clone(), client))}).collect()}@@ -766,13 +769,19 @@ enum ContextDiscovery {}#[derive(Debug, Deserialize)]-#[serde(tag = "format", rename_all = "snake_case", deny_unknown_fields)]-enum DecisionClientConfig {- SystemOne {- endpoint: HttpBaseUrl,- api_key_env: String,- timeout_ms: u64,- },+#[serde(deny_unknown_fields)]+struct DecisionClientConfig {+ format: DecisionFormat,+ endpoint: HttpBaseUrl,+ api_key_env: String,+ timeout_ms: u64,+}++#[derive(Debug, Deserialize)]+#[serde(rename_all = "snake_case")]+enum DecisionFormat {+ SystemOne,+ OpenaiDecisions,}#[derive(Debug, Deserialize)]diff --git a/crates/switchyard-runner/tests/openai_decisions.rs b/crates/switchyard-runner/tests/openai_decisions.rsnew file mode 100644index 000000000..6447e212e--- /dev/null+++ b/crates/switchyard-runner/tests/openai_decisions.rs@@ -0,0 +1,119 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use std::sync::{Arc, mpsc};++use serde_json::{Value, json};+use switchyard_llm_client::RunObservation;+use switchyard_protocol::{Request, Usage, text_request};+use switchyard_runner::Runner;+use wiremock::matchers::{body_partial_json, header, method, path};+use wiremock::{Mock, MockServer, ResponseTemplate};++#[tokio::test]+async fn openai_decisions_routes_from_toml() -> Result<(), Box<dyn std::error::Error>> {+ let server = MockServer::start().await;+ let endpoint = server.uri();+ let source = format!(+ r#"+schema_version = 1+[llm_clients.answers]+format = "openai_chat"+base_url = "{endpoint}/v1"+[targets]+strong = {{ id = "strong/model", llm_client = "answers" }}+weak = {{ id = "weak/model", llm_client = "answers" }}+[decision_clients.judge]+format = "openai_decisions"+endpoint = "{endpoint}/v1/decisions"+api_key_env = "SWITCHYARD_OPENAI_DECISIONS_TEST_KEY"+timeout_ms = 2000+[decision_targets.judge]+id = "gpt-6-luna"+decision_client = "judge"+[routes.classifier]+id = "switchyard/classifier"+type = "llm_classifier"+classifier_target = "judge"+strong_target = "strong"+weak_target = "weak"+fail_open = false+[routes.classifier.decision]+cutoff = 0.7+candidates = {{ a = "strong", b = "weak" }}+instructions = {{ task = "Choose advantage only when a succeeds and b fails." }}+evidence = {{ description = "Compare a and b" }}+"#+ );+ // This unique variable is only read while constructing this client.+ unsafe {+ std::env::set_var("SWITCHYARD_OPENAI_DECISIONS_TEST_KEY", "test-key");+ }+ let runner = Runner::from_toml(&source);+ unsafe {+ std::env::remove_var("SWITCHYARD_OPENAI_DECISIONS_TEST_KEY");+ }+ let runner = runner?;+ let question = json!({+ "name": "route", "type": "choice",+ "instructions": json!({"task": "Choose advantage only when a succeeds and b fails."}).to_string(),+ "choices": [{"value": "advantage"}, {"value": "no_advantage"}]+ });+ Mock::given(method("POST")).and(path("/v1/decisions"))+ .and(header("authorization", "Bearer test-key"))+ .and(body_partial_json(json!({"model": "gpt-6-luna", "questions": [question]})))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "model": "gpt-6-luna", "answers": [{+ "type": "choice", "name": "route", "choice": "advantage", "confidence": 0.9,+ "probabilities": [{"value": "advantage", "probability": 0.6},+ {"value": "no_advantage", "probability": 0.4}]+ }],+ "usage": {"input_tokens": 42, "input_tokens_details": {"cached_tokens": 10, "cache_write_tokens": 2},+ "output_tokens": 3, "output_tokens_details": {"reasoning_tokens": 1}, "total_tokens": 45}+ }))).expect(1).mount(&server).await;+ Mock::given(method("POST")).and(path("/v1/chat/completions"))+ .and(body_partial_json(json!({"model": "weak/model"})))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "id": "answer-1", "object": "chat.completion", "model": "weak/model",+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}]+ }))).expect(1).mount(&server).await;+ let (sender, observations) = mpsc::channel();+ let output = runner+ .route("switchyard/classifier")+ .unwrap()+ .execute(+ Request {+ llm_request: text_request(None, "Route this request"),+ ..Request::default()+ },+ Some(Arc::new(move |event| {+ sender.send(event).unwrap();+ })),+ )+ .await?;+ assert_eq!(output.selected_model, "weak/model");+ let usage = observations.try_iter().find_map(|event| match event {+ RunObservation::DecisionCall(call) => call.usage,+ _ => None,+ });+ assert_eq!(+ usage,+ Some(Usage {+ input_tokens: Some(30),+ cache: Usage::cache_details(Some(10), Some(2)),+ output_tokens: Some(3),+ total_tokens: Some(45),+ reasoning_tokens: Some(1),+ ..Usage::default()+ })+ );+ let requests = server.received_requests().await.unwrap();+ let decision: Value = serde_json::from_slice(&requests[0].body)?;+ let context: Value = serde_json::from_str(decision["input"].as_str().unwrap())?;+ assert_eq!(+ context["evidence"],+ json!({"description": "Compare a and b"})+ );+ server.verify().await;+ Ok(())+}diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.mdindex fc1d6c09e..f5f0b8e54 100644--- a/docs/reference/toml_schema.md+++ b/docs/reference/toml_schema.md@@ -175,6 +175,34 @@ model ID as that route's judge or reviewer because those calls would otherwise bat the client boundary; Switchyard rejects that configuration when it loads.Token-count requests do not apply target system prompts.+## `[decision_clients.<name>]`++Decision clients serve typed judgments for routing classifiers.++| Key | Required | Meaning |+|---|:---:|---|+| `format` | Yes | `system_one` or `openai_decisions`. |+| `endpoint` | Yes | Full API URL, including the endpoint path. |+| `api_key_env` | Yes | Environment variable holding a non-empty API key. |+| `timeout_ms` | Yes | Request and response deadline in milliseconds; must be at least `1`. |++For [OpenAI Decisions](https://developers.openai.com/api/docs/guides/decisions):++```toml+[decision_clients.decisions]+format = "openai_decisions"+endpoint = "https://api.openai.com/v1/decisions"+api_key_env = "OPENAI_API_KEY"+timeout_ms = 5000++[decision_targets.judge]+id = "gpt-6-luna"+decision_client = "decisions"+```++Each `decision_targets` entry pairs an upstream model `id` with a `decision_client`.+OpenAI receives structured context, instructions, and descriptions as JSON text.+## `[routes.<name>]`Every route takes the common keys below, plus the keys for its type.diff --git a/docs/routing_algorithms/composite_routing.md b/docs/routing_algorithms/composite_routing.mdindex 0e77945c0..1e5d7ee09 100644--- a/docs/routing_algorithms/composite_routing.md+++ b/docs/routing_algorithms/composite_routing.md@@ -111,6 +111,9 @@ evidence = { description = "Add task-relevant evidence comparing a and b." }```Both candidates must be completion targets. The judge must be a decision target.+See the [decision client reference](../reference/toml_schema.md#decision_clientsname)+for OpenAI Decisions configuration.+The classifier selects capable when `probabilities.advantage` exceeds `cutoff`.Calibrate the cutoff for your tasks and models. Omit LLM-only settings such as`base_threshold`, `threshold_step` and `prompt` when using a decision model.