Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(runner): support OpenAI Decisions (#1070)

nachiketb-nvidia7558a1d8a5f2
diff --git a/Cargo.lock b/Cargo.lock
index 8773c11b3..93d7f5e3f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3848,6 +3848,7 @@ dependencies = [
"tokio",
"toml 1.1.3+spec-1.1.0",
"tracing",
+ "wiremock",
]
[[package]]
diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs
index 5c9fd1f4f..414a32167 100644
--- a/crates/libsy-llm-client/src/lib.rs
+++ b/crates/libsy-llm-client/src/lib.rs
@@ -27,6 +27,7 @@ pub mod metrics;
mod observability;
mod observation;
mod observed_translation;
+mod openai_decisions;
pub mod raw;
pub mod run;
mod system_one;
@@ -35,6 +36,7 @@ pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig};
pub use client::{AuxiliaryOperation, ModelConfig, TranslatingLlmClient};
pub use error::{LlmClientError, Result};
pub use observation::{ModelCallObservation, RunObservation, RunObserver};
+pub use openai_decisions::OpenAiDecisionClient;
pub use raw::RawResponse;
pub use run::{ClientRouter, decide, run};
pub use switchyard_translation::RawEventStream;
diff --git a/crates/libsy-llm-client/src/openai_decisions.rs b/crates/libsy-llm-client/src/openai_decisions.rs
new file mode 100644
index 000000000..231240bf4
--- /dev/null
+++ b/crates/libsy-llm-client/src/openai_decisions.rs
@@ -0,0 +1,312 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! OpenAI Decisions calls using the provider-neutral decision IR.
+
+use std::collections::BTreeMap;
+use std::time::Duration;
+
+use async_trait::async_trait;
+use reqwest::{Url, header::HeaderValue};
+use serde::Deserialize;
+use serde_json::{Value, json};
+use switchyard_protocol::{
+ BooleanEstimate, DecisionAnswer, DecisionKind, DecisionRequest, DecisionResponse,
+ DecisionValue, ModelId, Probability, ProviderConfidence, RoutedDecisionClient, ScoreValue,
+ Usage,
+};
+
+use crate::client::convert_reqwest_error;
+use crate::{LlmClientError, Result, metrics};
+
+/// Serves typed questions through OpenAI's Decisions API.
+pub struct OpenAiDecisionClient {
+ client: reqwest::Client,
+ endpoint: Url,
+ api_key: String,
+}
+
+impl OpenAiDecisionClient {
+ /// `endpoint` includes `/v1/decisions`. Each call makes one attempt, with
+ /// `timeout` covering both the request and response body.
+ pub fn new(endpoint: Url, api_key: String, timeout: Duration) -> Result<Self> {
+ if HeaderValue::try_from(format!("Bearer {api_key}")).is_err() {
+ return Err(LlmClientError::Configuration {
+ message: "OpenAI Decisions API key cannot be encoded as an HTTP header".into(),
+ });
+ }
+ let client = reqwest::Client::builder()
+ .timeout(timeout)
+ .redirect(reqwest::redirect::Policy::none())
+ .build()
+ .map_err(convert_reqwest_error)?;
+ Ok(Self {
+ client,
+ endpoint,
+ api_key,
+ })
+ }
+}
+
+#[async_trait]
+impl RoutedDecisionClient for OpenAiDecisionClient {
+ async fn call(&self, request: DecisionRequest) -> Result<DecisionResponse> {
+ let response = self
+ .client
+ .post(self.endpoint.clone())
+ .bearer_auth(&self.api_key)
+ .json(&encode(&request)?)
+ .send()
+ .await
+ .inspect_err(|_| metrics::record_upstream_attempt(None))
+ .map_err(convert_reqwest_error)?;
+ let status = response.status();
+ let body = response
+ .bytes()
+ .await
+ .inspect_err(|_| metrics::record_upstream_attempt(None))
+ .map_err(convert_reqwest_error)?;
+ metrics::record_upstream_attempt(Some(status.as_u16()));
+ if !status.is_success() {
+ return Err(LlmClientError::UpstreamHttp {
+ status,
+ body: String::from_utf8_lossy(&body).replace(&self.api_key, "[REDACTED]"),
+ });
+ }
+ let response: WireResponse =
+ serde_json::from_slice(&body).map_err(|source| LlmClientError::InvalidResponse {
+ source: Box::new(source),
+ })?;
+ let mut questions = request.questions;
+ let mut answers = BTreeMap::new();
+ for answer in response.answers {
+ let invalid = || {
+ LlmClientError::ResponseTranslation(format!(
+ "OpenAI Decisions answer {:?} has an unexpected question name, kind, or value",
+ answer.name,
+ ))
+ };
+ let question = questions.remove(&answer.name).ok_or_else(invalid)?;
+ let value = match (&question.kind, answer.value) {
+ (DecisionKind::Boolean { .. }, WireValue::Predicate { probability }) => {
+ DecisionValue::Boolean(BooleanEstimate::ProbabilityTrue(probability))
+ }
+ (
+ DecisionKind::Choice { options },
+ WireValue::Choice {
+ choice,
+ probabilities,
+ },
+ ) => {
+ let count = probabilities.len();
+ let probabilities: BTreeMap<_, _> = probabilities
+ .into_iter()
+ .map(|p| (p.value, p.probability))
+ .collect();
+ if count != options.len()
+ || probabilities.len() != options.len()
+ || !options.iter().any(|option| option.id == choice)
+ || options
+ .iter()
+ .any(|option| !probabilities.contains_key(&option.id))
+ {
+ return Err(invalid());
+ }
+ DecisionValue::Choice {
+ selected: choice,
+ probabilities: Some(probabilities),
+ }
+ }
+ (
+ DecisionKind::Score { levels },
+ WireValue::Score {
+ score,
+ probabilities,
+ },
+ ) => {
+ let mut probabilities = probabilities;
+ probabilities.sort_by_key(|p| p.value);
+ if probabilities.len() != levels.len()
+ || probabilities
+ .iter()
+ .enumerate()
+ .any(|(index, p)| p.value != index)
+ {
+ return Err(invalid());
+ }
+ DecisionValue::Score {
+ value: score,
+ probabilities: Some(
+ probabilities.into_iter().map(|p| p.probability).collect(),
+ ),
+ }
+ }
+ (_, WireValue::Refusal) => {
+ return Err(LlmClientError::ResponseTranslation(format!(
+ "OpenAI Decisions refused question {:?}",
+ answer.name,
+ )));
+ }
+ _ => return Err(invalid()),
+ };
+ answers.insert(
+ answer.name,
+ DecisionAnswer {
+ value,
+ provider_confidence: answer.confidence,
+ },
+ );
+ }
+ if !questions.is_empty() {
+ return Err(LlmClientError::ResponseTranslation(
+ "OpenAI Decisions answer names do not match the requested question keys".into(),
+ ));
+ }
+ let usage = response.usage;
+ let cached = usage.input_tokens_details.cached_tokens;
+ let created = usage.input_tokens_details.cache_write_tokens;
+ Ok(DecisionResponse {
+ id: None,
+ model: Some(response.model),
+ answers,
+ usage: Usage {
+ input_tokens: Some(
+ usage
+ .input_tokens
+ .saturating_sub(cached)
+ .saturating_sub(created),
+ ),
+ cache: Usage::cache_details(Some(cached), Some(created)),
+ output_tokens: Some(usage.output_tokens),
+ total_tokens: Some(usage.total_tokens),
+ reasoning_tokens: Some(usage.output_tokens_details.reasoning_tokens),
+ ..Usage::default()
+ },
+ })
+ }
+}
+
+fn encode(request: &DecisionRequest) -> Result<Value> {
+ let model = request
+ .model
+ .as_ref()
+ .ok_or_else(|| LlmClientError::InvalidRequest {
+ message: "OpenAI Decisions requires a selected model".into(),
+ })?;
+ let questions: Vec<_> = request
+ .questions
+ .iter()
+ .map(|(name, question)| {
+ let mut wire = json!({"name": name, "instructions": text(&question.instructions)});
+ match &question.kind {
+ DecisionKind::Boolean {
+ true_description,
+ false_description,
+ } => {
+ wire["type"] = json!("predicate");
+ if true_description.is_some() || false_description.is_some() {
+ wire["instructions"] = json!(
+ json!({
+ "instructions": question.instructions,
+ "true": true_description, "false": false_description,
+ })
+ .to_string()
+ );
+ }
+ }
+ DecisionKind::Choice { options } => {
+ wire["type"] = json!("choice");
+ wire["choices"] = Value::Array(
+ options
+ .iter()
+ .map(|option| {
+ let mut choice = json!({"value": option.id});
+ if let Some(description) = &option.description {
+ choice["description"] = json!(text(description));
+ }
+ choice
+ })
+ .collect(),
+ );
+ }
+ DecisionKind::Score { levels } => {
+ wire["type"] = json!("score");
+ wire["levels"] = Value::Array(
+ levels
+ .iter()
+ .map(|level| json!({"label": text(level)}))
+ .collect(),
+ );
+ }
+ }
+ wire
+ })
+ .collect();
+ Ok(json!({"model": model, "input": text(&request.context), "questions": questions}))
+}
+
+// OpenAI requires text; retain structured IR content as JSON rather than dropping it.
+fn text(value: &Value) -> String {
+ value
+ .as_str()
+ .map(str::to_owned)
+ .unwrap_or_else(|| value.to_string())
+}
+
+#[derive(Deserialize)]
+struct WireResponse {
+ model: ModelId,
+ answers: Vec<WireAnswer>,
+ usage: WireUsage,
+}
+
+#[derive(Deserialize)]
+struct WireAnswer {
+ name: String,
+ #[serde(flatten)]
+ value: WireValue,
+ confidence: Option<ProviderConfidence>,
+}
+
+#[derive(Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+enum WireValue {
+ Predicate {
+ probability: Probability,
+ },
+ Choice {
+ choice: String,
+ probabilities: Vec<WireProbability<String>>,
+ },
+ Score {
+ score: ScoreValue,
+ probabilities: Vec<WireProbability<usize>>,
+ },
+ Refusal,
+}
+
+#[derive(Deserialize)]
+struct WireProbability<T> {
+ value: T,
+ probability: Probability,
+}
+
+#[derive(Deserialize)]
+struct WireUsage {
+ input_tokens: u64,
+ output_tokens: u64,
+ total_tokens: u64,
+ input_tokens_details: InputTokenDetails,
+ output_tokens_details: OutputTokenDetails,
+}
+
+#[derive(Deserialize)]
+struct InputTokenDetails {
+ cached_tokens: u64,
+ cache_write_tokens: u64,
+}
+
+#[derive(Deserialize)]
+struct OutputTokenDetails {
+ reasoning_tokens: u64,
+}
diff --git a/crates/switchyard-runner/Cargo.toml b/crates/switchyard-runner/Cargo.toml
index 3a80b61b3..d55e60097 100644
--- a/crates/switchyard-runner/Cargo.toml
+++ b/crates/switchyard-runner/Cargo.toml
@@ -33,3 +33,4 @@ tracing.workspace = true
async-trait.workspace = true
futures-util.workspace = true
tokio.workspace = true
+wiremock = "0.6"
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index ddd3812ab..7e3999c43 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -14,7 +14,7 @@ use serde::{Deserialize, Deserializer};
use serde_json::Value;
use switchyard_llm_client::{
AuxiliaryOperation, Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,
- SystemOneClient, TranslatingLlmClient,
+ OpenAiDecisionClient, SystemOneClient, TranslatingLlmClient,
};
use switchyard_protocol::{ModelId, RoutedDecisionClient, RoutedLlmClient, WireFormat};
@@ -46,8 +46,7 @@ pub fn api_key_variables(source: &str) -> Result<BTreeSet<String>, RunnerError>
.filter_map(|client| client.api_key_env.clone())
.collect();
for client in config.decision_clients.values() {
- let DecisionClientConfig::SystemOne { api_key_env, .. } = client;
- variables.insert(api_key_env.clone());
+ variables.insert(client.api_key_env.clone());
}
if variables
.iter()
@@ -421,7 +420,8 @@ impl DeploymentConfig {
.iter()
.map(|(name, config)| {
validate_value("decision client name", name)?;
- let DecisionClientConfig::SystemOne {
+ let DecisionClientConfig {
+ format,
endpoint,
api_key_env,
timeout_ms,
@@ -432,17 +432,20 @@ impl DeploymentConfig {
)));
}
let api_key = read_api_key(&format!("decision client {name}"), api_key_env)?;
- let client = SystemOneClient::new(
- endpoint.0.clone(),
- api_key.clone(),
- Duration::from_millis(*timeout_ms),
- )
+ let timeout = Duration::from_millis(*timeout_ms);
+ let client = match format {
+ DecisionFormat::SystemOne => {
+ SystemOneClient::new(endpoint.0.clone(), api_key.clone(), timeout)
+ .map(|client| Arc::new(client) as Arc<dyn RoutedDecisionClient>)
+ }
+ DecisionFormat::OpenaiDecisions => {
+ OpenAiDecisionClient::new(endpoint.0.clone(), api_key.clone(), timeout)
+ .map(|client| Arc::new(client) as Arc<dyn RoutedDecisionClient>)
+ }
+ }
.map_err(|error| RunnerError::configuration(error.to_string()))?;
provider_api_keys.push(api_key);
- Ok((
- name.clone(),
- Arc::new(client) as Arc<dyn RoutedDecisionClient>,
- ))
+ Ok((name.clone(), client))
})
.collect()
}
@@ -766,13 +769,19 @@ enum ContextDiscovery {
}
#[derive(Debug, Deserialize)]
-#[serde(tag = "format", rename_all = "snake_case", deny_unknown_fields)]
-enum DecisionClientConfig {
- SystemOne {
- endpoint: HttpBaseUrl,
- api_key_env: String,
- timeout_ms: u64,
- },
+#[serde(deny_unknown_fields)]
+struct DecisionClientConfig {
+ format: DecisionFormat,
+ endpoint: HttpBaseUrl,
+ api_key_env: String,
+ timeout_ms: u64,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "snake_case")]
+enum DecisionFormat {
+ SystemOne,
+ OpenaiDecisions,
}
#[derive(Debug, Deserialize)]
diff --git a/crates/switchyard-runner/tests/openai_decisions.rs b/crates/switchyard-runner/tests/openai_decisions.rs
new file mode 100644
index 000000000..6447e212e
--- /dev/null
+++ b/crates/switchyard-runner/tests/openai_decisions.rs
@@ -0,0 +1,119 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::sync::{Arc, mpsc};
+
+use serde_json::{Value, json};
+use switchyard_llm_client::RunObservation;
+use switchyard_protocol::{Request, Usage, text_request};
+use switchyard_runner::Runner;
+use wiremock::matchers::{body_partial_json, header, method, path};
+use wiremock::{Mock, MockServer, ResponseTemplate};
+
+#[tokio::test]
+async fn openai_decisions_routes_from_toml() -> Result<(), Box<dyn std::error::Error>> {
+ let server = MockServer::start().await;
+ let endpoint = server.uri();
+ let source = format!(
+ r#"
+schema_version = 1
+[llm_clients.answers]
+format = "openai_chat"
+base_url = "{endpoint}/v1"
+[targets]
+strong = {{ id = "strong/model", llm_client = "answers" }}
+weak = {{ id = "weak/model", llm_client = "answers" }}
+[decision_clients.judge]
+format = "openai_decisions"
+endpoint = "{endpoint}/v1/decisions"
+api_key_env = "SWITCHYARD_OPENAI_DECISIONS_TEST_KEY"
+timeout_ms = 2000
+[decision_targets.judge]
+id = "gpt-6-luna"
+decision_client = "judge"
+[routes.classifier]
+id = "switchyard/classifier"
+type = "llm_classifier"
+classifier_target = "judge"
+strong_target = "strong"
+weak_target = "weak"
+fail_open = false
+[routes.classifier.decision]
+cutoff = 0.7
+candidates = {{ a = "strong", b = "weak" }}
+instructions = {{ task = "Choose advantage only when a succeeds and b fails." }}
+evidence = {{ description = "Compare a and b" }}
+"#
+ );
+ // This unique variable is only read while constructing this client.
+ unsafe {
+ std::env::set_var("SWITCHYARD_OPENAI_DECISIONS_TEST_KEY", "test-key");
+ }
+ let runner = Runner::from_toml(&source);
+ unsafe {
+ std::env::remove_var("SWITCHYARD_OPENAI_DECISIONS_TEST_KEY");
+ }
+ let runner = runner?;
+ let question = json!({
+ "name": "route", "type": "choice",
+ "instructions": json!({"task": "Choose advantage only when a succeeds and b fails."}).to_string(),
+ "choices": [{"value": "advantage"}, {"value": "no_advantage"}]
+ });
+ Mock::given(method("POST")).and(path("/v1/decisions"))
+ .and(header("authorization", "Bearer test-key"))
+ .and(body_partial_json(json!({"model": "gpt-6-luna", "questions": [question]})))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "model": "gpt-6-luna", "answers": [{
+ "type": "choice", "name": "route", "choice": "advantage", "confidence": 0.9,
+ "probabilities": [{"value": "advantage", "probability": 0.6},
+ {"value": "no_advantage", "probability": 0.4}]
+ }],
+ "usage": {"input_tokens": 42, "input_tokens_details": {"cached_tokens": 10, "cache_write_tokens": 2},
+ "output_tokens": 3, "output_tokens_details": {"reasoning_tokens": 1}, "total_tokens": 45}
+ }))).expect(1).mount(&server).await;
+ Mock::given(method("POST")).and(path("/v1/chat/completions"))
+ .and(body_partial_json(json!({"model": "weak/model"})))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "answer-1", "object": "chat.completion", "model": "weak/model",
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "Answer"}, "finish_reason": "stop"}]
+ }))).expect(1).mount(&server).await;
+ let (sender, observations) = mpsc::channel();
+ let output = runner
+ .route("switchyard/classifier")
+ .unwrap()
+ .execute(
+ Request {
+ llm_request: text_request(None, "Route this request"),
+ ..Request::default()
+ },
+ Some(Arc::new(move |event| {
+ sender.send(event).unwrap();
+ })),
+ )
+ .await?;
+ assert_eq!(output.selected_model, "weak/model");
+ let usage = observations.try_iter().find_map(|event| match event {
+ RunObservation::DecisionCall(call) => call.usage,
+ _ => None,
+ });
+ assert_eq!(
+ usage,
+ Some(Usage {
+ input_tokens: Some(30),
+ cache: Usage::cache_details(Some(10), Some(2)),
+ output_tokens: Some(3),
+ total_tokens: Some(45),
+ reasoning_tokens: Some(1),
+ ..Usage::default()
+ })
+ );
+ let requests = server.received_requests().await.unwrap();
+ let decision: Value = serde_json::from_slice(&requests[0].body)?;
+ let context: Value = serde_json::from_str(decision["input"].as_str().unwrap())?;
+ assert_eq!(
+ context["evidence"],
+ json!({"description": "Compare a and b"})
+ );
+ server.verify().await;
+ Ok(())
+}
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index fc1d6c09e..f5f0b8e54 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -175,6 +175,34 @@ model ID as that route's judge or reviewer because those calls would otherwise b
at the client boundary; Switchyard rejects that configuration when it loads.
Token-count requests do not apply target system prompts.
+## `[decision_clients.<name>]`
+
+Decision clients serve typed judgments for routing classifiers.
+
+| Key | Required | Meaning |
+|---|:---:|---|
+| `format` | Yes | `system_one` or `openai_decisions`. |
+| `endpoint` | Yes | Full API URL, including the endpoint path. |
+| `api_key_env` | Yes | Environment variable holding a non-empty API key. |
+| `timeout_ms` | Yes | Request and response deadline in milliseconds; must be at least `1`. |
+
+For [OpenAI Decisions](https://developers.openai.com/api/docs/guides/decisions):
+
+```toml
+[decision_clients.decisions]
+format = "openai_decisions"
+endpoint = "https://api.openai.com/v1/decisions"
+api_key_env = "OPENAI_API_KEY"
+timeout_ms = 5000
+
+[decision_targets.judge]
+id = "gpt-6-luna"
+decision_client = "decisions"
+```
+
+Each `decision_targets` entry pairs an upstream model `id` with a `decision_client`.
+OpenAI receives structured context, instructions, and descriptions as JSON text.
+
## `[routes.<name>]`
Every route takes the common keys below, plus the keys for its type.
diff --git a/docs/routing_algorithms/composite_routing.md b/docs/routing_algorithms/composite_routing.md
index 0e77945c0..1e5d7ee09 100644
--- a/docs/routing_algorithms/composite_routing.md
+++ b/docs/routing_algorithms/composite_routing.md
@@ -111,6 +111,9 @@ evidence = { description = "Add task-relevant evidence comparing a and b." }
```
Both candidates must be completion targets. The judge must be a decision target.
+See the [decision client reference](../reference/toml_schema.md#decision_clientsname)
+for OpenAI Decisions configuration.
+
The classifier selects capable when `probabilities.advantage` exceeds `cutoff`.
Calibrate the cutoff for your tasks and models. Omit LLM-only settings such as
`base_threshold`, `threshold_step` and `prompt` when using a decision model.