Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(libsy): use conversation IR request and response types (#58)

nachiketb-nvidia64d02dba674b
diff --git a/Cargo.lock b/Cargo.lock
index 0beaae069..565426fb2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -855,6 +855,7 @@ version = "0.1.0"
dependencies = [
"async-trait",
"futures",
+ "libsy-protocol",
"serde_json",
"tokio",
"tokio-stream",
@@ -872,6 +873,14 @@ dependencies = [
"tokio-stream",
]
+[[package]]
+name = "libsy-protocol"
+version = "0.1.0"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "libyaml-rs"
version = "0.3.0"
@@ -1684,6 +1693,7 @@ dependencies = [
name = "switchyard-translation"
version = "0.1.0"
dependencies = [
+ "libsy-protocol",
"pretty_assertions",
"serde",
"serde_json",
diff --git a/Cargo.toml b/Cargo.toml
index 3295507e0..582ea8dfe 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,6 +11,7 @@ members = [
"crates/switchyard-components-v2-macros",
"crates/switchyard-core",
"crates/switchyard-py",
+ "crates/libsy-protocol",
"crates/switchyard-server",
"crates/switchyard-skill-distillation",
"crates/switchyard-translation",
diff --git a/crates/libsy-examples/examples/research_agent.rs b/crates/libsy-examples/examples/research_agent.rs
index 8038ca383..0ea5804ca 100644
--- a/crates/libsy-examples/examples/research_agent.rs
+++ b/crates/libsy-examples/examples/research_agent.rs
@@ -16,8 +16,8 @@ use std::sync::Arc;
use async_trait::async_trait;
use libsy::{
- Algorithm, Context, LlmClient, LlmRequest, LlmResponse, LlmTarget, LlmTargetSet, Request,
- Response, RoutedRequest,
+ Algorithm, ContentBlock, Context, LlmClient, LlmRequest, LlmResponse, LlmTarget, LlmTargetSet,
+ Message, Request, Response, ResponseOutput, Role, RoutedRequest,
};
use libsy_examples::llm_class::LlmClassifierOrchAlgo;
@@ -42,8 +42,12 @@ impl LlmClient for StubClient {
};
Ok(Response {
llm_response: LlmResponse {
- completion,
- raw_response: None,
+ outputs: vec![ResponseOutput {
+ role: Role::Assistant,
+ content: vec![ContentBlock::Text { text: completion }],
+ stop_reason: None,
+ }],
+ ..LlmResponse::default()
},
metadata: None,
})
@@ -74,15 +78,31 @@ impl ResearchAgent {
for step in self.plan(question) {
let request = Request {
llm_request: LlmRequest {
- inbound_model_name: "auto".to_string(),
- prompt: step,
+ model: Some("auto".to_string()),
+ messages: vec![Message::text(Role::User, step)],
+ ..LlmRequest::default()
},
raw_request: None,
metadata: None,
};
let (_trace, response) = self.algo.clone().run(Context::default(), request).await?;
- notes.push(response.llm_response.completion);
+ notes.push(
+ response
+ .llm_response
+ .outputs
+ .iter()
+ .flat_map(|output| output.content.iter())
+ .filter_map(|block| match block {
+ ContentBlock::Text { text }
+ | ContentBlock::Refusal { text }
+ | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
+ ContentBlock::Unknown { raw, .. } => raw.as_str(),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n"),
+ );
}
Ok(notes.join("\n"))
}
diff --git a/crates/libsy-examples/examples/research_agent_core.rs b/crates/libsy-examples/examples/research_agent_core.rs
index 73fbfcc60..e6b3bfed6 100644
--- a/crates/libsy-examples/examples/research_agent_core.rs
+++ b/crates/libsy-examples/examples/research_agent_core.rs
@@ -13,8 +13,8 @@ use std::error::Error;
use std::sync::Arc;
use libsy::{
- Algorithm, Context, Decision, LlmRequest, LlmResponse, LlmTarget, LlmTargetSet, Request,
- Response, Step,
+ Algorithm, ContentBlock, Context, Decision, LlmRequest, LlmResponse, LlmTarget, LlmTargetSet,
+ Message, Request, Response, ResponseOutput, Role, Step,
};
use libsy_examples::llm_class::LlmClassifierOrchAlgo;
use tokio_stream::StreamExt;
@@ -35,8 +35,12 @@ async fn call_model(model: &str) -> Response {
};
Response {
llm_response: LlmResponse {
- completion,
- raw_response: None,
+ outputs: vec![ResponseOutput {
+ role: Role::Assistant,
+ content: vec![ContentBlock::Text { text: completion }],
+ stop_reason: None,
+ }],
+ ..LlmResponse::default()
},
metadata: None,
}
@@ -66,8 +70,9 @@ impl ResearchAgent {
for step in self.plan(question) {
let request = Request {
llm_request: LlmRequest {
- inbound_model_name: "auto".to_string(),
- prompt: step,
+ model: Some("auto".to_string()),
+ messages: vec![Message::text(Role::User, step)],
+ ..LlmRequest::default()
},
raw_request: None,
metadata: None,
@@ -84,7 +89,22 @@ impl ResearchAgent {
// Decisions stream in as the algorithm makes them.
Step::Decision(decision) => print_decision(decision.as_ref()),
Step::ReturnToAgent(response) => {
- notes.push(response.llm_response.completion);
+ notes.push(
+ response
+ .llm_response
+ .outputs
+ .iter()
+ .flat_map(|output| output.content.iter())
+ .filter_map(|block| match block {
+ ContentBlock::Text { text }
+ | ContentBlock::Refusal { text }
+ | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
+ ContentBlock::Unknown { raw, .. } => raw.as_str(),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n"),
+ );
}
}
}
diff --git a/crates/libsy-examples/src/ensemble.rs b/crates/libsy-examples/src/ensemble.rs
index 2ce088202..e558ac098 100644
--- a/crates/libsy-examples/src/ensemble.rs
+++ b/crates/libsy-examples/src/ensemble.rs
@@ -23,7 +23,8 @@ use std::sync::{Arc, Mutex};
use async_trait::async_trait;
use libsy::{
- Algorithm, Context, Decision, Driver, LlmRequest, LlmTargetSet, Request, Response, Signals,
+ Algorithm, ContentBlock, Context, Decision, Driver, LlmRequest, LlmTargetSet, Message, Request,
+ Response, Role, Signals,
};
/// Which step of the ensemble flow produced a decision.
@@ -184,12 +185,7 @@ impl EnsembleOrchAlgo {
phase: EnsemblePhase::Committed,
});
let routed = Request {
- llm_request: LlmRequest {
- // The agent's inbound name rides through; the committed model is on
- // the decision, not stamped onto the request.
- inbound_model_name: request.llm_request.inbound_model_name,
- prompt: request.llm_request.prompt,
- },
+ llm_request: request.llm_request,
raw_request: request.raw_request,
metadata: request.metadata,
};
@@ -206,10 +202,27 @@ impl EnsembleOrchAlgo {
driver: &Driver,
request: Request,
) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {
- let user_prompt = request.llm_request.prompt.clone();
- // The agent's inbound name rides through every sub-call unchanged; the model
- // each call hits is carried by its decision, not stamped onto the request.
- let inbound = request.llm_request.inbound_model_name.clone();
+ let user_prompt = request
+ .llm_request
+ .instructions
+ .iter()
+ .flat_map(|instruction| instruction.content.iter())
+ .chain(
+ request
+ .llm_request
+ .messages
+ .iter()
+ .flat_map(|message| message.content.iter()),
+ )
+ .filter_map(|block| match block {
+ ContentBlock::Text { text }
+ | ContentBlock::Refusal { text }
+ | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
+ ContentBlock::Unknown { raw, .. } => raw.as_str(),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n");
// Fan out to all candidates concurrently with the same user prompt. Each
// call is annotated with its own candidate decision so the caller can see
@@ -225,10 +238,7 @@ impl EnsembleOrchAlgo {
});
candidate_decisions.push(decision.clone());
let call_request = Request {
- llm_request: LlmRequest {
- inbound_model_name: inbound.clone(),
- prompt: user_prompt.clone(),
- },
+ llm_request: request.llm_request.clone(),
raw_request: request.raw_request.clone(),
metadata: request.metadata.clone(),
};
@@ -273,17 +283,32 @@ impl EnsembleOrchAlgo {
});
let judge_request = Request {
llm_request: LlmRequest {
- inbound_model_name: inbound.clone(),
- prompt: judge_prompt,
+ model: request.llm_request.model.clone(),
+ messages: vec![Message::text(Role::User, judge_prompt)],
+ ..LlmRequest::default()
},
- raw_request: request.raw_request.clone(),
+ raw_request: None,
metadata: request.metadata.clone(),
};
let judge_response = driver
.call_llm_target(&judge_target, judge_request, judge_decision.clone())
.await?;
// Fail open: an unparseable pick falls back to the first response.
- let choice = parse_choice(&judge_response.llm_response.completion, survivors.len());
+ let judge_text = judge_response
+ .llm_response
+ .outputs
+ .iter()
+ .flat_map(|output| output.content.iter())
+ .filter_map(|block| match block {
+ ContentBlock::Text { text }
+ | ContentBlock::Refusal { text }
+ | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
+ ContentBlock::Unknown { raw, .. } => raw.as_str(),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n");
+ let choice = parse_choice(&judge_text, survivors.len());
let (model, response) = survivors
.into_iter()
.nth(choice)
@@ -328,11 +353,21 @@ fn build_judge_prompt(user_prompt: &str, survivors: &[(String, Response)]) -> St
prompt.push_str(user_prompt);
prompt.push_str("\n\n");
for (i, (_model, response)) in survivors.iter().enumerate() {
- prompt.push_str(&format!(
- "Response {}:\n{}\n\n",
- i + 1,
- response.llm_response.completion
- ));
+ let candidate_text = response
+ .llm_response
+ .outputs
+ .iter()
+ .flat_map(|output| output.content.iter())
+ .filter_map(|block| match block {
+ ContentBlock::Text { text }
+ | ContentBlock::Refusal { text }
+ | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
+ ContentBlock::Unknown { raw, .. } => raw.as_str(),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n");
+ prompt.push_str(&format!("Response {}:\n{}\n\n", i + 1, candidate_text));
}
prompt.push_str(&format!(
"Reply with only the number (1-{}) of the best response.",
@@ -395,7 +430,7 @@ impl Algorithm for EnsembleOrchAlgo {
#[cfg(test)]
mod tests {
use super::*;
- use libsy::{LlmClient, LlmRequest, LlmResponse, LlmTarget, Response, RoutedRequest};
+ use libsy::{LlmClient, LlmResponse, LlmTarget, Response, ResponseOutput, RoutedRequest};
use std::sync::Mutex as StdMutex;
/// Mock client that answers candidate calls with `answer from {model}` and,
@@ -420,14 +455,21 @@ mod tests {
.map_err(|_| "lock poisoned")?
.push(name.clone());
let completion = if name == self.judge_model {
- judge_pick(&routed.request.llm_request.prompt, &self.prefer)
+ let prompt = routed.request.llm_request.messages[0]
+ .text_content("\n")
+ .unwrap_or_default();
+ judge_pick(&prompt, &self.prefer)
} else {
format!("answer from {name}")
};
Ok(Response {
llm_response: LlmResponse {
- completion,
- raw_response: None,
+ outputs: vec![ResponseOutput {
+ role: Role::Assistant,
+ content: vec![ContentBlock::Text { text: completion }],
+ stop_reason: None,
+ }],
+ ..LlmResponse::default()
},
metadata: None,
})
@@ -506,8 +548,9 @@ mod tests {
fn request(prompt: &str) -> Request {
Request {
llm_request: LlmRequest {
- inbound_model_name: "auto".to_string(),
- prompt: prompt.to_string(),
+ model: Some("auto".to_string()),
+ messages: vec![Message::text(Role::User, prompt)],
+ ..LlmRequest::default()
},
raw_request: None,
metadata: None,
@@ -536,7 +579,12 @@ mod tests {
let (trace, response) = orch(algo)
.run(Context::default(), request("solve it"))
.await?;
- assert_eq!(response.llm_response.completion, "answer from b/model");
+ assert_eq!(
+ response.llm_response.outputs[0].content,
+ vec![ContentBlock::Text {
+ text: "answer from b/model".to_string()
+ }]
+ );
// Both candidates and the judge were called.
let calls = calls.lock().map_err(|_| "lock poisoned")?;
@@ -576,7 +624,12 @@ mod tests {
// Third request: committed fast path — routes straight to b/model with no
// fan-out to a/model and no judge call.
let (trace, response) = orch.clone().run(Context::default(), request("t3")).await?;
- assert_eq!(response.llm_response.completion, "answer from b/model");
+ assert_eq!(
+ response.llm_response.outputs[0].content,
+ vec![ContentBlock::Text {
+ text: "answer from b/model".to_string()
+ }]
+ );
assert_eq!(trace.len(), 1);
let decision = as_ensemble(&trace[0])?;
assert_eq!(decision.phase, EnsemblePhase::Committed);
@@ -594,7 +647,12 @@ mod tests {
async fn single_candidate_skips_the_judge() -> Result<(), Box<dyn Error + Send + Sync>> {
let (algo, calls) = algo(&["only/model"], "judge/haiku", "only/model", 100);
let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;
- assert_eq!(response.llm_response.completion, "answer from only/model");
+ assert_eq!(
+ response.llm_response.outputs[0].content,
+ vec![ContentBlock::Text {
+ text: "answer from only/model".to_string()
+ }]
+ );
// No judge call for a lone candidate.
assert!(!calls
.lock()
@@ -710,7 +768,10 @@ mod tests {
let name = routed.decision.selected_model().to_string();
let completion = if name == self.judge_model {
// Judge runs after the barrier releases; it must not wait.
- judge_pick(&routed.request.llm_request.prompt, &self.prefer)
+ let prompt = routed.request.llm_request.messages[0]
+ .text_content("\n")
+ .unwrap_or_default();
+ judge_pick(&prompt, &self.prefer)
} else {
// Hold every candidate call until all sessions have fanned out.
self.barrier.wait().await;
@@ -718,8 +779,12 @@ mod tests {
};
Ok(Response {
llm_response: LlmResponse {
- completion,
- raw_response: None,
+ outputs: vec![ResponseOutput {
+ role: Role::Assistant,
+ content: vec![ContentBlock::Text { text: completion }],
+ stop_reason: None,
+ }],
+ ..LlmResponse::default()
},
metadata: None,
})
@@ -755,7 +820,18 @@ mod tests {
session
.run(Context::default(), request(prompt))
.await
- .map(|(_, response)| response.llm_response.completion)
+ .map(|(_, response)| {
+ match response
+ .llm_response
+ .outputs
+ .into_iter()
+ .next()
+ .and_then(|output| output.content.into_iter().next())
+ {
+ Some(ContentBlock::Text { text }) => text,
+ _ => String::new(),
+ }
+ })
})
};
let handle_a = run(session_a, "from A");
@@ -801,10 +877,17 @@ mod tests {
.and_then(|d| d.as_any().downcast_ref::<EnsembleDecision>())
.map(|d| d.phase)
.ok_or("missing final decision")?;
- Ok::<(String, EnsemblePhase), Box<dyn Error + Send + Sync>>((
- response.llm_response.completion,
- phase,
- ))
+ let completion = match response
+ .llm_response
+ .outputs
+ .into_iter()
+ .next()
+ .and_then(|output| output.content.into_iter().next())
+ {
+ Some(ContentBlock::Text { text }) => text,
+ _ => String::new(),
+ };
+ Ok::<(String, EnsemblePhase), Box<dyn Error + Send + Sync>>((completion, phase))
})
};
// The two sessions run in parallel; each committed independently.
diff --git a/crates/libsy-examples/src/llm_class.rs b/crates/libsy-examples/src/llm_class.rs
index 94e252094..66e2008b5 100644
--- a/crates/libsy-examples/src/llm_class.rs
+++ b/crates/libsy-examples/src/llm_class.rs
@@ -16,7 +16,8 @@ use std::sync::Arc;
use async_trait::async_trait;
use libsy::{
- Algorithm, Context, Decision, Driver, LlmRequest, LlmTargetSet, Request, Response, Signals,
+ Algorithm, ContentBlock, Context, Decision, Driver, LlmRequest, LlmTargetSet, Message, Request,
+ Response, Role, Signals,
};
/// Preamble prepended to the user prompt when asking the classifier target for a
@@ -105,19 +106,40 @@ impl Algorithm for LlmClassifierOrchAlgo {
driver: Driver,
request: Request,
) -> Result<Response, Box<dyn Error + Send + Sync>> {
- let user_prompt = request.llm_request.prompt.clone();
- // The agent's inbound name rides through unchanged on every sub-call; the
- // model each sub-call actually hits is carried by its decision instead.
- let inbound = request.llm_request.inbound_model_name.clone();
+ let user_prompt = request
+ .llm_request
+ .instructions
+ .iter()
+ .flat_map(|instruction| instruction.content.iter())
+ .chain(
+ request
+ .llm_request
+ .messages
+ .iter()
+ .flat_map(|message| message.content.iter()),
+ )
+ .filter_map(|block| match block {
+ ContentBlock::Text { text }
+ | ContentBlock::Refusal { text }
+ | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
+ ContentBlock::Unknown { raw, .. } => raw.as_str(),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n");
// 1. Classify: call the classifier target with the score-eliciting prompt.
let classifier_target = self.target_set.get_target(&self.classifier_model)?;
let classify_request = Request {
llm_request: LlmRequest {
- inbound_model_name: inbound.clone(),
- prompt: format!("{CLASSIFIER_PROMPT_PREAMBLE}{user_prompt}"),
+ model: request.llm_request.model.clone(),
+ messages: vec![Message::text(
+ Role::User,
+ format!("{CLASSIFIER_PROMPT_PREAMBLE}{user_prompt}"),
+ )],
+ ..LlmRequest::default()
},
- raw_request: request.raw_request.clone(),
+ raw_request: None,
metadata: request.metadata.clone(),
};
let classify_decision: Arc<dyn Decision> = Arc::new(ClassifierDecision {
@@ -130,12 +152,21 @@ impl Algorithm for LlmClassifierOrchAlgo {
let classify_response = driver
.call_llm_target(&classifier_target, classify_request, classify_decision)
.await?;
- let score = classify_response
+ let score_text = classify_response
.llm_response
- .completion
- .trim()
- .parse::<f64>()
- .ok();
+ .outputs
+ .iter()
+ .flat_map(|output| output.content.iter())
+ .filter_map(|block| match block {
+ ContentBlock::Text { text }
+ | ContentBlock::Refusal { text }
+ | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
+ ContentBlock::Unknown { raw, .. } => raw.as_str(),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n");
+ let score = score_text.trim().parse::<f64>().ok();
// 2. Route: pick strong/weak. Fail open — an unparseable score routes strong.
let (tier, model) = match score {
@@ -155,10 +186,7 @@ impl Algorithm for LlmClassifierOrchAlgo {
tier: Some(tier),
});
let routed_request = Request {
- llm_request: LlmRequest {
- inbound_model_name: inbound,
- prompt: user_prompt,
- },
+ llm_request: request.llm_request,
raw_request: request.raw_request,
metadata: request.metadata,
};
@@ -180,7 +208,7 @@ impl Algorithm for LlmClassifierOrchAlgo {
#[cfg(test)]
mod tests {
use super::*;
- use libsy::{LlmClient, LlmRequest, LlmResponse, LlmTarget, Response, RoutedRequest};
+ use libsy::{LlmClient, LlmResponse, LlmTarget, Response, ResponseOutput, RoutedRequest};
use std::sync::Mutex;
/// Returns `score` for the classifier target, an answer tagged with the model
@@ -199,10 +227,16 @@ mod tests {
routed: RoutedRequest,
) -> Result<Response, Box<dyn Error + Send + Sync>> {
let name = routed.decision.selected_model().to_string();
- let completion = if name == self.classifier_model {
- self.score.clone()
+ let outputs = if name == self.classifier_model {
+ vec![ResponseOutput {
+ role: Role::Assistant,
+ content: vec![ContentBlock::Text {
+ text: self.score.clone(),
+ }],
+ stop_reason: None,
+ }]
} else {
- format!("answer from {name}")
+ Vec::new()
};
self.seen
.lock()
@@ -210,8 +244,9 @@ mod tests {
.push(routed.request);
Ok(Response {
llm_response: LlmResponse {
- completion,
- raw_response: None,
+ model: Some(name),
+ outputs,
+ ..LlmResponse::default()
},
metadata: None,
})
@@ -248,8 +283,9 @@ mod tests {
fn request(prompt: &str) -> Request {
Request {
llm_request: LlmRequest {
- inbound_model_name: "auto".to_string(),
- prompt: prompt.to_string(),
+ model: Some("auto".to_string()),
+ messages: vec![Message::text(Role::User, prompt)],
+ ..LlmRequest::default()
},
raw_request: None,
metadata: None,
@@ -278,8 +314,8 @@ mod tests {
.run(Context::default(), request("solve this proof"))
.await?;
assert_eq!(
- response.llm_response.completion,
- "answer from frontier/model"
+ response.llm_response.model.as_deref(),
+ Some("frontier/model")
);
// Trace: [classify, route].
assert_eq!(trace[0].selected_model(), "router/classifier");
@@ -296,7 +332,7 @@ mod tests {
let (trace, response) = orch(algo)
.run(Context::default(), request("say hello"))
.await?;
- assert_eq!(response.llm_response.completion, "answer from cheap/model");
+ assert_eq!(response.llm_response.model.as_deref(), Some("cheap/model"));
let routed = as_classifier(&trace[1])?;
assert_eq!(routed.tier, Some(ClassifierTier::Weak));
assert_eq!(routed.score, Some(0.2));
@@ -311,8 +347,8 @@ mod tests {
.run(Context::default(), request("borderline"))
.await?;
assert_eq!(
- response.llm_response.completion,
- "answer from frontier/model"
+ response.llm_response.model.as_deref(),
+ Some("frontier/model")
);
Ok(())
}
@@ -322,8 +358,8 @@ mod tests {
let (algo, _) = algo(0.5, "not-a-number");
let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;
assert_eq!(
- response.llm_response.completion,
- "answer from frontier/model"
+ response.llm_response.model.as_deref(),
+ Some("frontier/model")
);
let routed = as_classifier(&trace[1])?;
assert_eq!(routed.tier, Some(ClassifierTier::Strong));
@@ -341,9 +377,15 @@ mod tests {
let seen = seen.lock().map_err(|_| "lock poisoned")?;
// Two calls: the classifier (preamble + user text), then the routed model.
assert_eq!(seen.len(), 2);
- assert!(seen[0].llm_request.prompt.contains("prove it"));
- assert!(seen[0].llm_request.prompt.contains("frontier model"));
- assert_eq!(seen[1].llm_request.prompt, "prove it");
+ let classifier_prompt = seen[0].llm_request.messages[0]
+ .text_content("\n")
+ .ok_or("classifier prompt is not text")?;
+ assert!(classifier_prompt.contains("prove it"));
+ assert!(classifier_prompt.contains("frontier model"));
+ assert_eq!(
+ seen[1].llm_request.messages[0].text_content("\n"),
+ Some("prove it".to_string())
+ );
Ok(())
}
}
diff --git a/crates/libsy-examples/src/rand.rs b/crates/libsy-examples/src/rand.rs
index 9e78f72f8..bebaa3803 100644
--- a/crates/libsy-examples/src/rand.rs
+++ b/crates/libsy-examples/src/rand.rs
@@ -96,7 +96,9 @@ impl Algorithm for RandomOrchAlgo {
#[cfg(test)]
mod tests {
use super::*;
- use libsy::{LlmClient, LlmRequest, LlmResponse, LlmTarget, Response, RoutedRequest};
+ use libsy::{
+ LlmClient, LlmRequest, LlmResponse, LlmTarget, Message, Response, Role, RoutedRequest,
+ };
use std::collections::HashSet;
/// Echoes back the target name it was called with, so a test can tell which
@@ -111,8 +113,8 @@ mod tests {
) -> Result<Response, Box<dyn Error + Send + Sync>> {
Ok(Response {
llm_response: LlmResponse {
- completion: routed.decision.selected_model().to_string(),
- raw_response: None,
+ model: Some(routed.decision.selected_model().to_string()),
+ ..LlmResponse::default()
},
metadata: None,
})
@@ -122,8 +124,9 @@ mod tests {
fn request() -> Request {
Request {
llm_request: LlmRequest {
- inbound_model_name: "auto".to_string(),
- prompt: "hi".to_string(),
+ model: Some("auto".to_string()),
+ messages: vec![Message::text(Role::User, "hi")],
+ ..LlmRequest::default()
},
raw_request: None,
metadata: None,
@@ -151,7 +154,7 @@ mod tests {
) -> Result<(), Box<dyn Error + Send + Sync>> {
let orch = orch(&["only/model"]);
let (trace, response) = orch.clone().run(Context::default(), request()).await?;
- assert_eq!(response.llm_response.completion, "only/model");
+ assert_eq!(response.llm_response.model.as_deref(), Some("only/model"));
assert_eq!(trace.len(), 1);
assert_eq!(trace[0].selected_model(), "only/model");
Ok(())
@@ -164,7 +167,7 @@ mod tests {
let orch = orch(&names);
for _ in 0..50 {
let (trace, response) = orch.clone().run(Context::default(), request()).await?;
- let selected = response.llm_response.completion.clone();
+ let selected = response.llm_response.model.unwrap_or_default();
assert!(
names.contains(&selected.as_str()),
"selected {selected} not in target set"
@@ -182,7 +185,7 @@ mod tests {
let mut seen = HashSet::new();
for _ in 0..100 {
let (_, response) = orch.clone().run(Context::default(), request()).await?;
- seen.insert(response.llm_response.completion);
+ seen.insert(response.llm_response.model.unwrap_or_default());
}
// 100 uniform draws over two targets: both should appear (miss ~ 2^-99).
assert_eq!(
diff --git a/crates/libsy-protocol/Cargo.toml b/crates/libsy-protocol/Cargo.toml
new file mode 100644
index 000000000..9ce25154f
--- /dev/null
+++ b/crates/libsy-protocol/Cargo.toml
@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "libsy-protocol"
+version = "0.1.0"
+description = "Provider-neutral LLM protocol types for Switchyard"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
diff --git a/crates/switchyard-translation/src/format.rs b/crates/libsy-protocol/src/format.rs
similarity index 97%
rename from crates/switchyard-translation/src/format.rs
rename to crates/libsy-protocol/src/format.rs
index a2a941f08..5a24c9185 100644
--- a/crates/switchyard-translation/src/format.rs
+++ b/crates/libsy-protocol/src/format.rs
@@ -1,12 +1,13 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Wire-format identifiers understood by the translation engine.
+//! Wire-format identifiers carried by the shared protocol types.
-use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::fmt;
+use serde::{Deserialize, Serialize};
+
/// Built-in provider API formats.
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
pub enum WireFormat {
diff --git a/crates/libsy-protocol/src/lib.rs b/crates/libsy-protocol/src/lib.rs
new file mode 100644
index 000000000..c844a2d72
--- /dev/null
+++ b/crates/libsy-protocol/src/lib.rs
@@ -0,0 +1,10 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Provider-neutral LLM protocol types shared across Switchyard crates.
+
+pub mod format;
+pub mod llm;
+
+pub use format::*;
+pub use llm::*;
diff --git a/crates/switchyard-translation/src/ir.rs b/crates/libsy-protocol/src/llm.rs
similarity index 84%
rename from crates/switchyard-translation/src/ir.rs
rename to crates/libsy-protocol/src/llm.rs
index a4df4bd12..9cb3ef874 100644
--- a/crates/switchyard-translation/src/ir.rs
+++ b/crates/libsy-protocol/src/llm.rs
@@ -1,11 +1,11 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Neutral conversation IR used for loss-aware provider translation.
+//! Provider-neutral conversation types shared by routing, clients, and translation.
-use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
+use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use crate::format::FormatId;
@@ -20,20 +20,6 @@ pub enum Role {
Tool,
}
-/// Returns true when `name` is a message role recognized by at least one
-/// supported provider API (OpenAI Chat, OpenAI Responses, or Anthropic
-/// Messages). Codecs use this to distinguish a genuinely-unsupported role —
-/// which is rejected on request decode to preserve the provider contract —
-/// from a known role that a given codec maps to a default. `function` is
-/// included because it is a legacy OpenAI Chat role that older clients may
-/// still send and which has always been coerced to `user` here.
-pub(crate) fn is_known_role_name(name: &str) -> bool {
- matches!(
- name,
- "system" | "developer" | "user" | "assistant" | "tool" | "function"
- )
-}
-
/// Instruction content separated from normal conversation messages.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct InstructionBlock {
@@ -109,7 +95,7 @@ pub enum ContentBlock {
},
}
-/// Image payload forms supported by the neutral IR.
+/// Image payload forms supported by the conversation model.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum ImageSource {
Url {
@@ -123,7 +109,7 @@ pub enum ImageSource {
Raw(Value),
}
-/// File payload forms supported by the neutral IR.
+/// File payload forms supported by the conversation model.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum FileSource {
FileId(String),
@@ -134,7 +120,7 @@ pub enum FileSource {
Raw(Value),
}
-/// Audio and video payload forms supported by the neutral IR.
+/// Audio and video payload forms supported by the conversation model.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum MediaSource {
Url {
@@ -205,7 +191,7 @@ pub struct ReasoningParams {
pub raw: Option<Value>,
}
-/// Provider-specific fields that do not have first-class IR slots.
+/// Provider-specific fields that do not have first-class conversation fields.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct ProviderExtensions {
pub fields: Map<String, Value>,
@@ -218,9 +204,9 @@ pub struct PreservationMetadata {
pub responses: BTreeMap<FormatId, Value>,
}
-/// Normalized request representation used between decoder and encoder.
+/// Normalized request representation shared by Switchyard components.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
-pub struct ConversationRequest {
+pub struct LlmRequest {
pub model: Option<String>,
pub instructions: Vec<InstructionBlock>,
pub messages: Vec<Message>,
@@ -243,6 +229,36 @@ pub struct Usage {
pub reasoning_tokens: Option<u64>,
}
+/// Provider-neutral event used between stream decoders and encoders.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub enum LlmStreamEvent {
+ MessageStart {
+ id: Option<String>,
+ model: Option<String>,
+ },
+ TextDelta {
+ index: usize,
+ text: String,
+ },
+ ReasoningDelta {
+ index: usize,
+ text: String,
+ },
+ ToolCallDelta {
+ index: usize,
+ id: Option<String>,
+ name: Option<String>,
+ arguments_delta: Option<String>,
+ },
+ Usage(Usage),
+ MessageStop {
+ reason: Option<String>,
+ },
+ Error {
+ message: String,
+ },
+}
+
/// Normalized reason a model stopped producing output.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub enum StopReason {
@@ -262,9 +278,9 @@ pub struct ResponseOutput {
pub stop_reason: Option<StopReason>,
}
-/// Normalized response representation used between decoder and encoder.
+/// Normalized response representation shared by Switchyard components.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
-pub struct ConversationResponse {
+pub struct LlmResponse {
pub id: Option<String>,
pub model: Option<String>,
pub outputs: Vec<ResponseOutput>,
@@ -273,7 +289,7 @@ pub struct ConversationResponse {
pub preservation: PreservationMetadata,
}
-impl ConversationResponse {
+impl LlmResponse {
/// Returns the first output item when a response has any output.
pub fn first_output(&self) -> Option<&ResponseOutput> {
self.outputs.first()
diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml
index dfd6bda0c..609073a14 100644
--- a/crates/libsy/Cargo.toml
+++ b/crates/libsy/Cargo.toml
@@ -15,6 +15,7 @@ rust-version.workspace = true
async-trait = "0.1"
serde_json = "1"
futures = "0.3"
+libsy-protocol = { path = "../libsy-protocol" }
tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1"
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index 68e66808d..aee56ed26 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -11,8 +11,11 @@ so it drops into a proxy, gateway, or agent runtime.
Build a target set, pick an algorithm, run a request:
```rust
-use libsy::llm_class::LlmClassifierOrchAlgo;
-use libsy::{Algorithm, Context, LlmClient, LlmRequest, LlmTarget, LlmTargetSet, Request};
+use libsy::{
+ Algorithm, ContentBlock, Context, LlmRequest, LlmClient, LlmTarget, LlmTargetSet,
+ Message, Request, Role,
+};
+use libsy_examples::llm_class::LlmClassifierOrchAlgo;
use std::sync::Arc;
// Targets the algorithm routes among, each backed by your LlmClient (see below).
@@ -25,7 +28,11 @@ let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(
));
let req = Request {
- llm_request: LlmRequest { inbound_model_name: "auto".into(), prompt: "explain tail latency".into() },
+ llm_request: LlmRequest {
+ model: Some("auto".into()),
+ messages: vec![Message::text(Role::User, "explain tail latency")],
+ ..LlmRequest::default()
+ },
raw_request: None,
metadata: None,
};
@@ -39,17 +46,24 @@ Runnable: [`research_agent`](../libsy-examples/examples/research_agent.rs) (in t
```rust
pub struct Request {
- pub llm_request: LlmRequest, // normalized: { inbound_model_name, prompt }
- pub raw_request: Option<serde_json::Value>, // original provider body, forwarded verbatim if present
+ pub llm_request: LlmRequest,
+ pub raw_request: Option<serde_json::Value>, // optional original provider body for exact-fidelity hosts
pub metadata: Option<Metadata>, // correlation: session / agent / task / correlation_id / extra
}
pub struct Response {
- pub llm_response: LlmResponse, // normalized: { completion, raw_response? }
+ pub llm_response: LlmResponse,
pub metadata: Option<Metadata>,
}
```
+`libsy-protocol` owns `LlmRequest`, `LlmResponse`, `Message`,
+`ContentBlock`, `ResponseOutput`, and `Role`. `libsy` re-exports those canonical types
+unchanged. Construct and inspect the conversation model directly so tools, sampling
+parameters, reasoning, and provider extensions remain visible instead of being hidden
+behind a second convenience API. `raw_request` remains available when a host needs exact
+source-body fidelity.
+
## Targets and clients
An `LlmTarget` pairs a routing `semantic_name` with an optional `LlmClient`. Mapping that
@@ -64,9 +78,20 @@ impl LlmClient for MyClient {
async fn call(&self, routed: RoutedRequest)
-> Result<Response, Box<dyn std::error::Error + Send + Sync>> {
let model = routed.decision.selected_model(); // the routed target — map it to a provider id
- // routed.request.llm_request.inbound_model_name is the agent's original name (not a call target)
+ // routed.request.llm_request.model is the agent's original name (not a call target)
// ... POST to your endpoint, read the completion ...
- Ok(Response { llm_response: LlmResponse { completion, raw_response: None }, metadata: None })
+ let completion = String::from("provider response text");
+ Ok(Response {
+ llm_response: LlmResponse {
+ outputs: vec![ResponseOutput {
+ role: Role::Assistant,
+ content: vec![ContentBlock::Text { text: completion }],
+ stop_reason: None,
+ }],
+ ..LlmResponse::default()
+ },
+ metadata: None,
+ })
}
}
```
@@ -153,8 +178,16 @@ impl Algorithm for LlmClassifierOrchAlgo {
// 1. Classify: ask the classifier target for a score.
let classifier = self.target_set.get_target(&self.classifier_model)?;
driver.info(classify_decision.clone()).await?;
- let score = driver.call_llm_target(&classifier, classify_req, classify_decision).await?
- .llm_response.completion.trim().parse::<f64>().ok();
+ let classify_response =
+ driver.call_llm_target(&classifier, classify_req, classify_decision).await?;
+ // Abbreviated: this assumes a text score in the first output block. The full
+ // implementation scans all outputs and text-like blocks.
+ let score = classify_response.llm_response.first_output()
+ .and_then(|output| output.content.first())
+ .and_then(|block| match block {
+ ContentBlock::Text { text } => text.trim().parse::<f64>().ok(),
+ _ => None,
+ });
// 2. Route: strong if score >= threshold, else weak (fail open on None).
let model = if score.map_or(true, |s| s >= self.threshold) { &self.strong_model } else { &self.weak_model };
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 7cbae1ccb..d28d8a740 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -66,6 +66,8 @@ use futures::{Stream, StreamExt};
use crate::driver::{DriverRequest, DriverStep, TypeErasedDriver};
+pub use libsy_protocol::{ContentBlock, LlmRequest, LlmResponse, Message, ResponseOutput, Role};
+
/// Shorthand for the crate's boxed, thread-safe error type.
type BoxErr = Box<dyn Error + Send + Sync>;
@@ -93,27 +95,13 @@ pub struct Metadata {
pub extra_metadata: Option<std::collections::BTreeMap<String, String>>,
}
-/// The normalized model request an algorithm reasons over and hands to a target.
-///
-/// Deliberately minimal: a target model name and the user prompt. The full
-/// provider-shaped request (messages, params, tools) rides on
-/// [`Request::raw_request`] when a host needs to forward it losslessly.
-#[derive(Clone)]
-pub struct LlmRequest {
- /// The model to call. Algorithms rewrite this as they route.
- pub inbound_model_name: String,
- /// The user prompt an algorithm inspects (e.g. to classify) and sends.
- pub prompt: String,
-}
-
/// A request entering the orchestrator: the normalized [`LlmRequest`] plus the
/// original provider payload and correlation [`Metadata`].
#[derive(Clone)]
pub struct Request {
/// The normalized request an algorithm routes.
pub llm_request: LlmRequest,
- /// The original provider-shaped request body, if the host wants to forward it
- /// verbatim (e.g. a proxy preserving messages/params). libsy does not read it.
+ /// Optional source provider body, carried only for hosts that need exact fidelity.
pub raw_request: Option<serde_json::Value>,
/// Correlation metadata carried through the request.
pub metadata: Option<Metadata>,
@@ -127,17 +115,6 @@ pub struct Request {
#[derive(Clone)]
pub struct Signals {}
-/// The neutral model response returned by a target.
-#[derive(Clone)]
-pub struct LlmResponse {
- /// The model's completion text — what an algorithm inspects (e.g. a
- /// classifier score) or returns.
- pub completion: String,
- /// Optional raw provider response body, so a host (e.g. a proxy) can forward
- /// the upstream response losslessly instead of rebuilding it from `completion`.
- pub raw_response: Option<serde_json::Value>,
-}
-
/// A response leaving the orchestrator: the neutral [`LlmResponse`] plus optional
/// correlation [`Metadata`].
#[derive(Clone)]
@@ -169,12 +146,11 @@ pub trait Decision: Send + Sync {
///
/// The two model identifiers live in separate, unambiguous places: the model to
/// call is [`decision.selected_model()`](Decision::selected_model), while
-/// `request.llm_request.inbound_model_name` is the *inbound* name the agent asked
-/// for (libsy never overwrites it). A client maps `selected_model()` to the
-/// provider model id it hits.
+/// `request.llm_request.model` is the inbound name the agent asked for. A client
+/// maps `selected_model()` to the provider model id it hits.
#[derive(Clone)]
pub struct RoutedRequest {
- /// The request to serve; its `inbound_model_name` is the agent's original name.
+ /// The request to serve; its `model` is the agent's original inbound model.
pub request: Request,
/// The routing decision behind this call; `selected_model()` is the model to hit.
pub decision: Arc<dyn Decision>,
@@ -193,27 +169,22 @@ pub struct RoutedRequest {
/// [`Driver::call_llm`] on the other side.
pub struct CallLlmRequest {
inner: DriverRequest,
- routed: RoutedRequest,
}
impl CallLlmRequest {
- /// Wrap a driver request whose payload is a [`RoutedRequest`]. Caches an owned copy
- /// so the accessors are plain field reads.
+ /// Wrap a driver request whose payload is a [`RoutedRequest`].
fn new(inner: DriverRequest) -> Self {
- // The payload is always a `RoutedRequest` (set by `Driver::call_llm`); a
- // mismatch would be a libsy bug, not a runtime condition.
- let routed = match inner.request::<RoutedRequest>() {
- Ok(routed) => routed.clone(),
- Err(_) => unreachable!("CallLlmRequest payload is always a RoutedRequest"),
- };
- Self { inner, routed }
+ Self { inner }
}
/// The routed request the host should serve. Its
/// [`default_client`](RoutedRequest::default_client) serves the call by default,
/// and its `decision.selected_model()` names the model to hit.
pub fn get_routed(&self) -> &RoutedRequest {
- &self.routed
+ match self.inner.request::<RoutedRequest>() {
+ Ok(routed) => routed,
+ Err(_) => unreachable!("CallLlmRequest payload is always a RoutedRequest"),
+ }
}
/// The model request to perform (the [`Request`] inside the routed request).
@@ -266,7 +237,7 @@ impl Driver {
/// default client into a [`RoutedRequest`], then publish it (see
/// [`call_llm`](Self::call_llm)). The convenience most algorithms use;
/// `decision.selected_model()` names the model to hit, and `request`'s
- /// `inbound_model_name` is left untouched.
+ /// inbound `model` is left untouched.
pub async fn call_llm_target(
&self,
target: &LlmTarget,
@@ -308,7 +279,7 @@ impl Driver {
.map_err(|_| "driver: info payload was not a Decision".into()),
DriverStep::Done(payload) => payload
.downcast::<Response>()
- .map(|response| Step::ReturnToAgent(*response))
+ .map(Step::ReturnToAgent)
.map_err(|_| "driver: done payload was not a Response".into()),
})
}
@@ -343,7 +314,7 @@ pub enum Step {
/// happens (rather than collected into a trace returned at the end).
Decision(Arc<dyn Decision>),
/// The algorithm finished with its final response — the last step of a run.
- ReturnToAgent(Response),
+ ReturnToAgent(Box<Response>),
}
/// Performs the actual model call for a target. This is the one piece of I/O
@@ -355,8 +326,8 @@ pub trait LlmClient: Send + Sync {
/// Serve `routed`, returning the model's response. Call the model named by
/// [`routed.decision.selected_model()`](Decision::selected_model) — the target
/// the algorithm routed to — mapping it to whatever provider model id this
- /// client hits. `routed.request.llm_request.inbound_model_name` is the agent's
- /// original name, carried through for reference, not a call target.
+ /// client hits. `routed.request.llm_request.model` is the agent's original
+ /// inbound name, carried through for reference, not a call target.
async fn call(&self, request: RoutedRequest) -> Result<Response, Box<dyn Error + Send + Sync>>;
}
@@ -502,7 +473,7 @@ pub trait Algorithm: Send + Sync + 'static {
Step::CallLlm(call) => in_flight.push(serve(call)),
Step::Decision(decision) => trace.push(decision),
Step::ReturnToAgent(response) => {
- final_response = Some(response);
+ final_response = Some(*response);
stream_open = false;
}
},
@@ -534,8 +505,8 @@ mod tests {
// Echo back the model the algorithm routed to (the decision's selection).
Ok(Response {
llm_response: LlmResponse {
- completion: routed.decision.selected_model().to_string(),
- raw_response: None,
+ model: Some(routed.decision.selected_model().to_string()),
+ ..LlmResponse::default()
},
metadata: None,
})
@@ -601,8 +572,9 @@ mod tests {
fn request() -> Request {
Request {
llm_request: LlmRequest {
- inbound_model_name: "auto".to_string(),
- prompt: "hi".to_string(),
+ model: Some("auto".to_string()),
+ messages: vec![Message::text(Role::User, "hi")],
+ ..LlmRequest::default()
},
raw_request: None,
metadata: None,
@@ -641,8 +613,8 @@ mod tests {
// Fulfilling the promise is the "real" model call the caller makes.
call.respond(Ok(Response {
llm_response: LlmResponse {
- completion: "fulfilled".to_string(),
- raw_response: None,
+ model: Some("fulfilled".to_string()),
+ ..LlmResponse::default()
},
metadata: None,
}))?;
@@ -651,7 +623,7 @@ mod tests {
assert_eq!(decision.selected_model(), "offload/model");
}
Step::ReturnToAgent(response) => {
- final_completion = Some(response.llm_response.completion);
+ final_completion = response.llm_response.model.clone();
}
}
}
@@ -687,7 +659,7 @@ mod tests {
}
Step::Decision(_) => {}
Step::ReturnToAgent(response) => {
- final_completion = Some(response.llm_response.completion);
+ final_completion = response.llm_response.model.clone();
}
}
}
@@ -706,7 +678,7 @@ mod tests {
.run(Context::default(), request())
.await?;
// TestAlgo calls the first target; EchoClient echoes its name.
- assert_eq!(response.llm_response.completion, "direct/model");
+ assert_eq!(response.llm_response.model.as_deref(), Some("direct/model"));
assert_eq!(trace[0].selected_model(), "direct/model");
Ok(())
}
@@ -747,8 +719,8 @@ mod tests {
self.barrier.wait().await;
Ok(Response {
llm_response: LlmResponse {
- completion: routed.decision.selected_model().to_string(),
- raw_response: None,
+ model: Some(routed.decision.selected_model().to_string()),
+ ..LlmResponse::default()
},
metadata: None,
})
@@ -771,7 +743,7 @@ mod tests {
handles.push(tokio::spawn(async move {
algo.run(Context::default(), request())
.await
- .map(|(_, response)| response.llm_response.completion)
+ .map(|(_, response)| response.llm_response.model.unwrap_or_default())
}));
}
@@ -850,8 +822,14 @@ mod tests {
self.current.fetch_sub(1, Ordering::SeqCst);
Ok(Response {
llm_response: LlmResponse {
- completion: routed.decision.selected_model().to_string(),
- raw_response: None,
+ outputs: vec![ResponseOutput {
+ role: Role::Assistant,
+ content: vec![ContentBlock::Text {
+ text: routed.decision.selected_model().to_string(),
+ }],
+ stop_reason: None,
+ }],
+ ..LlmResponse::default()
},
metadata: None,
})
@@ -885,8 +863,14 @@ mod tests {
futures::future::join_all(calls).await;
Ok(Response {
llm_response: LlmResponse {
- completion: "done".to_string(),
- raw_response: None,
+ outputs: vec![ResponseOutput {
+ role: Role::Assistant,
+ content: vec![ContentBlock::Text {
+ text: "done".to_string(),
+ }],
+ stop_reason: None,
+ }],
+ ..LlmResponse::default()
},
metadata: None,
})
@@ -945,7 +929,15 @@ mod tests {
// Release the gate; every call completes and the run finishes.
gate.add_permits(TOTAL_CALLS);
let (_trace, response) = tokio::time::timeout(Duration::from_secs(5), handle).await???;
- assert_eq!(response.llm_response.completion, "done");
+ assert_eq!(
+ response
+ .llm_response
+ .first_output()
+ .and_then(|output| output.content.first()),
+ Some(&ContentBlock::Text {
+ text: "done".to_string()
+ })
+ );
assert_eq!(
max.load(Ordering::SeqCst),
CAP,
diff --git a/crates/switchyard-translation/Cargo.toml b/crates/switchyard-translation/Cargo.toml
index 4f7d7e9f2..cd3d5f061 100644
--- a/crates/switchyard-translation/Cargo.toml
+++ b/crates/switchyard-translation/Cargo.toml
@@ -14,6 +14,7 @@ rust-version.workspace = true
[dependencies]
serde = { version = "1", features = ["derive"] }
serde_json = "1"
+libsy-protocol = { path = "../libsy-protocol" }
thiserror = "2"
[dev-dependencies]
diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
index e21508648..453292ba5 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
@@ -5,7 +5,7 @@
use serde_json::{json, Map, Value};
-use crate::codecs::common::{provider_extensions, text_from_blocks};
+use crate::codecs::common::{is_known_role_name, provider_extensions, text_from_blocks};
use crate::codecs::openai_chat::{decode_file_source, decode_image_source};
use crate::codecs::{
DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec,
@@ -13,11 +13,10 @@ use crate::codecs::{
use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
-use crate::ir::{
- is_known_role_name, ContentBlock, ConversationRequest, ConversationResponse, FileSource,
- ImageSource, InstructionBlock, MediaSource, Message, OutputParams, ProviderExtensions,
- ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice,
- ToolDefinition, ToolResult, Usage,
+use crate::llm::{
+ ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, LlmResponse, MediaSource,
+ Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,
+ SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,
};
use crate::policy::{DeterministicIdPolicy, TranslationPolicy};
use crate::util::sanitize_anthropic_tool_use_id;
@@ -40,7 +39,7 @@ impl FormatCodec for AnthropicMessagesCodec {
fn decode_request(&self, body: &Value, policy: &TranslationPolicy) -> Result<DecodedRequest> {
let body = crate::util::object(body, "$")?;
let mut diagnostics = Vec::new();
- let mut request = ConversationRequest {
+ let mut request = LlmRequest {
model: body
.get("model")
.and_then(Value::as_str)
@@ -70,7 +69,7 @@ impl FormatCodec for AnthropicMessagesCodec {
&Value::Object(body.clone()),
policy,
),
- ..ConversationRequest::default()
+ ..LlmRequest::default()
};
if let Some(system) = body.get("system") {
if let Some(content) = decode_anthropic_system(system, &mut diagnostics, policy)? {
@@ -148,7 +147,7 @@ impl FormatCodec for AnthropicMessagesCodec {
fn encode_request(
&self,
- request: &ConversationRequest,
+ request: &LlmRequest,
policy: &TranslationPolicy,
) -> Result<EncodedRequest> {
if let Some(body) =
@@ -253,7 +252,7 @@ impl FormatCodec for AnthropicMessagesCodec {
text: String::new(),
});
}
- let response = ConversationResponse {
+ let response = LlmResponse {
id: body
.get("id")
.and_then(Value::as_str)
@@ -298,7 +297,7 @@ impl FormatCodec for AnthropicMessagesCodec {
fn encode_response(
&self,
- response: &ConversationResponse,
+ response: &LlmResponse,
_policy: &TranslationPolicy,
) -> Result<EncodedResponse> {
if let Some(body) = exact_preserved_response(
diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
index d18e9a371..215173b4b 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
@@ -7,9 +7,10 @@ use serde_json::{json, Map, Value};
use crate::codecs::stream::{
record_source_identity, target_message_id_or_source_message_id, target_model_or_source_model,
- ConversationStreamEvent, StreamCodec, StreamTranslationState,
+ StreamCodec, StreamTranslationState,
};
use crate::format::{FormatId, WireFormat};
+use crate::llm::LlmStreamEvent;
use crate::util::sanitize_anthropic_tool_use_id;
/// Stream codec for Anthropic Messages events.
@@ -24,14 +25,14 @@ impl StreamCodec for AnthropicMessagesStreamCodec {
&self,
state: &mut StreamTranslationState,
event: &Value,
- ) -> Vec<ConversationStreamEvent> {
+ ) -> Vec<LlmStreamEvent> {
decode_anthropic_stream(state, event)
}
fn encode_event(
&self,
state: &mut StreamTranslationState,
- event: ConversationStreamEvent,
+ event: LlmStreamEvent,
) -> Vec<Value> {
encode_anthropic_stream(state, event)
}
@@ -45,9 +46,9 @@ impl StreamCodec for AnthropicMessagesStreamCodec {
fn decode_anthropic_stream(
state: &mut StreamTranslationState,
event: &Value,
-) -> Vec<ConversationStreamEvent> {
+) -> Vec<LlmStreamEvent> {
let Some(object) = event.as_object() else {
- return vec![ConversationStreamEvent::Error {
+ return vec![LlmStreamEvent::Error {
message: "Anthropic stream event is not an object".to_string(),
}];
};
@@ -72,7 +73,7 @@ fn decode_anthropic_stream(
capture_anthropic_usage(state, usage);
}
}
- vec![ConversationStreamEvent::MessageStart {
+ vec![LlmStreamEvent::MessageStart {
id: state.message_id.clone(),
model: state.model.clone(),
}]
@@ -83,7 +84,7 @@ fn decode_anthropic_stream(
let mut out = Vec::new();
if let Some(usage) = object.get("usage") {
capture_anthropic_usage(state, usage);
- out.push(ConversationStreamEvent::Usage(state.usage.clone()));
+ out.push(LlmStreamEvent::Usage(state.usage.clone()));
}
if let Some(stop_reason) = object
.get("delta")
@@ -91,14 +92,14 @@ fn decode_anthropic_stream(
.and_then(|delta| delta.get("stop_reason"))
.and_then(Value::as_str)
{
- out.push(ConversationStreamEvent::MessageStop {
+ out.push(LlmStreamEvent::MessageStop {
reason: Some(stop_reason.to_string()),
});
}
out
}
- Some("message_stop") => vec![ConversationStreamEvent::MessageStop { reason: None }],
- Some("error") => vec![ConversationStreamEvent::Error {
+ Some("message_stop") => vec![LlmStreamEvent::MessageStop { reason: None }],
+ Some("error") => vec![LlmStreamEvent::Error {
message: object
.get("error")
.and_then(Value::as_object)
@@ -114,10 +115,10 @@ fn decode_anthropic_stream(
// Encodes neutral streaming events into Anthropic Messages events.
fn encode_anthropic_stream(
state: &mut StreamTranslationState,
- event: ConversationStreamEvent,
+ event: LlmStreamEvent,
) -> Vec<Value> {
match event {
- ConversationStreamEvent::MessageStart { id, model } => {
+ LlmStreamEvent::MessageStart { id, model } => {
record_source_identity(state, id, model);
if state.emitted_message_start {
Vec::new()
@@ -138,7 +139,7 @@ fn encode_anthropic_stream(
})]
}
}
- ConversationStreamEvent::TextDelta { text, .. } => {
+ LlmStreamEvent::TextDelta { text, .. } => {
state.output_tokens_seen += 1;
let mut out = ensure_anthropic_text_block(state);
out.push(json!({
@@ -148,7 +149,7 @@ fn encode_anthropic_stream(
}));
out
}
- ConversationStreamEvent::ReasoningDelta { text, .. } => {
+ LlmStreamEvent::ReasoningDelta { text, .. } => {
let mut out = ensure_anthropic_reasoning_block(state);
out.push(json!({
"type": "content_block_delta",
@@ -157,22 +158,22 @@ fn encode_anthropic_stream(
}));
out
}
- ConversationStreamEvent::ToolCallDelta {
+ LlmStreamEvent::ToolCallDelta {
index,
id,
name,
arguments_delta,
} => encode_anthropic_tool_delta(state, index, id, name, arguments_delta),
- ConversationStreamEvent::Usage(usage) => {
+ LlmStreamEvent::Usage(usage) => {
state.usage = usage;
state.saw_backend_usage = true;
Vec::new()
}
- ConversationStreamEvent::MessageStop { reason } => {
+ LlmStreamEvent::MessageStop { reason } => {
state.stop_reason = reason.or_else(|| state.stop_reason.clone());
Vec::new()
}
- ConversationStreamEvent::Error { message } => {
+ LlmStreamEvent::Error { message } => {
vec![json!({"type": "error", "error": {"message": message}})]
}
}
@@ -184,7 +185,7 @@ fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec<Value> {
if !state.emitted_message_start {
out.extend(encode_anthropic_stream(
state,
- ConversationStreamEvent::MessageStart {
+ LlmStreamEvent::MessageStart {
id: state.message_id.clone(),
model: state.model.clone(),
},
@@ -232,9 +233,7 @@ fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec<Value> {
}
// Converts Anthropic content-block starts into text or tool-call deltas.
-fn decode_anthropic_content_block_start(
- object: &Map<String, Value>,
-) -> Vec<ConversationStreamEvent> {
+fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmStreamEvent> {
let index = object.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
let block = object.get("content_block").and_then(Value::as_object);
match block
@@ -246,7 +245,7 @@ fn decode_anthropic_content_block_start(
.and_then(Value::as_str)
.filter(|text| !text.is_empty())
.map(|text| {
- vec![ConversationStreamEvent::TextDelta {
+ vec![LlmStreamEvent::TextDelta {
index,
text: text.to_string(),
}]
@@ -257,7 +256,7 @@ fn decode_anthropic_content_block_start(
.and_then(Value::as_str)
.filter(|text| !text.is_empty())
.map(|text| {
- vec![ConversationStreamEvent::ReasoningDelta {
+ vec![LlmStreamEvent::ReasoningDelta {
index,
text: text.to_string(),
}]
@@ -267,7 +266,7 @@ fn decode_anthropic_content_block_start(
let Some(block) = block else {
return Vec::new();
};
- vec![ConversationStreamEvent::ToolCallDelta {
+ vec![LlmStreamEvent::ToolCallDelta {
index,
id: block
.get("id")
@@ -285,9 +284,7 @@ fn decode_anthropic_content_block_start(
}
// Converts Anthropic content-block deltas into neutral text or argument deltas.
-fn decode_anthropic_content_block_delta(
- object: &Map<String, Value>,
-) -> Vec<ConversationStreamEvent> {
+fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmStreamEvent> {
let index = object.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
let Some(delta) = object.get("delta").and_then(Value::as_object) else {
return Vec::new();
@@ -297,7 +294,7 @@ fn decode_anthropic_content_block_delta(
.get("text")
.and_then(Value::as_str)
.map(|text| {
- vec![ConversationStreamEvent::TextDelta {
+ vec![LlmStreamEvent::TextDelta {
index,
text: text.to_string(),
}]
@@ -307,7 +304,7 @@ fn decode_anthropic_content_block_delta(
.get("thinking")
.and_then(Value::as_str)
.map(|text| {
- vec![ConversationStreamEvent::ReasoningDelta {
+ vec![LlmStreamEvent::ReasoningDelta {
index,
text: text.to_string(),
}]
@@ -318,7 +315,7 @@ fn decode_anthropic_content_block_delta(
.get("partial_json")
.and_then(Value::as_str)
.map(|partial_json| {
- vec![ConversationStreamEvent::ToolCallDelta {
+ vec![LlmStreamEvent::ToolCallDelta {
index,
id: None,
name: None,
diff --git a/crates/switchyard-translation/src/codecs/common.rs b/crates/switchyard-translation/src/codecs/common.rs
index ead93f13d..ffd8e34c6 100644
--- a/crates/switchyard-translation/src/codecs/common.rs
+++ b/crates/switchyard-translation/src/codecs/common.rs
@@ -5,7 +5,15 @@
use serde_json::{Map, Value};
-use crate::ir::ContentBlock;
+use crate::llm::ContentBlock;
+
+/// Returns whether a role name is recognized by a supported provider API.
+pub(crate) fn is_known_role_name(name: &str) -> bool {
+ matches!(
+ name,
+ "system" | "developer" | "user" | "assistant" | "tool" | "function"
+ )
+}
/// Extracts text-like blocks and joins them for text-only provider fields.
pub(crate) fn text_from_blocks(content: &[ContentBlock], separator: &str) -> String {
diff --git a/crates/switchyard-translation/src/codecs/mod.rs b/crates/switchyard-translation/src/codecs/mod.rs
index 64c07dbf5..fbcda4ab1 100644
--- a/crates/switchyard-translation/src/codecs/mod.rs
+++ b/crates/switchyard-translation/src/codecs/mod.rs
@@ -14,12 +14,12 @@ use serde_json::Value;
use crate::diagnostic::TranslationDiagnostic;
use crate::error::Result;
use crate::format::FormatId;
-use crate::ir::{ConversationRequest, ConversationResponse};
+use crate::llm::{LlmRequest, LlmResponse};
use crate::policy::TranslationPolicy;
/// Result of decoding a request into neutral IR.
pub struct DecodedRequest {
- pub request: ConversationRequest,
+ pub request: LlmRequest,
pub diagnostics: Vec<TranslationDiagnostic>,
}
@@ -31,7 +31,7 @@ pub struct EncodedRequest {
/// Result of decoding a response into neutral IR.
pub struct DecodedResponse {
- pub response: ConversationResponse,
+ pub response: LlmResponse,
pub diagnostics: Vec<TranslationDiagnostic>,
}
@@ -52,7 +52,7 @@ pub trait FormatCodec: Send + Sync {
/// Encodes a neutral request into provider JSON.
fn encode_request(
&self,
- request: &ConversationRequest,
+ request: &LlmRequest,
policy: &TranslationPolicy,
) -> Result<EncodedRequest>;
@@ -62,7 +62,7 @@ pub trait FormatCodec: Send + Sync {
/// Encodes a neutral response into provider JSON.
fn encode_response(
&self,
- response: &ConversationResponse,
+ response: &LlmResponse,
policy: &TranslationPolicy,
) -> Result<EncodedResponse>;
}
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
index 234949cb7..fbcd1ad42 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
@@ -5,18 +5,19 @@
use serde_json::{json, Map, Value};
-use crate::codecs::common::{provider_extensions, reasoning_text_from_blocks, text_from_blocks};
+use crate::codecs::common::{
+ is_known_role_name, provider_extensions, reasoning_text_from_blocks, text_from_blocks,
+};
use crate::codecs::{
DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec,
};
use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
-use crate::ir::{
- is_known_role_name, ContentBlock, ConversationRequest, ConversationResponse, FileSource,
- ImageSource, InstructionBlock, MediaSource, Message, OutputParams, ProviderExtensions,
- ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice,
- ToolDefinition, ToolResult, Usage,
+use crate::llm::{
+ ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, LlmResponse, MediaSource,
+ Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,
+ SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,
};
use crate::policy::{DeterministicIdPolicy, TranslationPolicy};
use crate::util::{
@@ -36,7 +37,7 @@ impl FormatCodec for OpenAiChatCodec {
fn decode_request(&self, body: &Value, policy: &TranslationPolicy) -> Result<DecodedRequest> {
let body = object(body, "$")?;
let mut diagnostics = Vec::new();
- let mut request = ConversationRequest {
+ let mut request = LlmRequest {
model: body
.get("model")
.and_then(Value::as_str)
@@ -67,7 +68,7 @@ impl FormatCodec for OpenAiChatCodec {
&Value::Object(body.clone()),
policy,
),
- ..ConversationRequest::default()
+ ..LlmRequest::default()
};
if let Some(messages) = body.get("messages").and_then(Value::as_array) {
@@ -172,7 +173,7 @@ impl FormatCodec for OpenAiChatCodec {
fn encode_request(
&self,
- request: &ConversationRequest,
+ request: &LlmRequest,
policy: &TranslationPolicy,
) -> Result<EncodedRequest> {
if let Some(body) =
@@ -245,7 +246,7 @@ impl FormatCodec for OpenAiChatCodec {
_policy: &TranslationPolicy,
) -> Result<DecodedResponse> {
let object = object(body, "$")?;
- let mut response = ConversationResponse {
+ let mut response = LlmResponse {
id: object
.get("id")
.and_then(Value::as_str)
@@ -312,7 +313,7 @@ impl FormatCodec for OpenAiChatCodec {
fn encode_response(
&self,
- response: &ConversationResponse,
+ response: &LlmResponse,
_policy: &TranslationPolicy,
) -> Result<EncodedResponse> {
if let Some(body) =
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
index 0ae488b28..501b40de7 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
@@ -6,11 +6,11 @@
use serde_json::{json, Map, Value};
use crate::codecs::stream::{
- record_source_identity, source_model_or_unknown, state_source_is, string_field,
- ConversationStreamEvent, StreamCodec, StreamTranslationState,
+ record_source_identity, source_model_or_unknown, state_source_is, string_field, StreamCodec,
+ StreamTranslationState,
};
use crate::format::{FormatId, WireFormat};
-use crate::ir::Usage;
+use crate::llm::{LlmStreamEvent, Usage};
/// Stream codec for OpenAI Chat Completions chunks.
pub struct OpenAiChatStreamCodec;
@@ -24,14 +24,14 @@ impl StreamCodec for OpenAiChatStreamCodec {
&self,
state: &mut StreamTranslationState,
event: &Value,
- ) -> Vec<ConversationStreamEvent> {
+ ) -> Vec<LlmStreamEvent> {
decode_openai_chat_stream(state, event)
}
fn encode_event(
&self,
state: &mut StreamTranslationState,
- event: ConversationStreamEvent,
+ event: LlmStreamEvent,
) -> Vec<Value> {
encode_openai_chat_stream(state, event)
}
@@ -45,9 +45,9 @@ impl StreamCodec for OpenAiChatStreamCodec {
fn decode_openai_chat_stream(
state: &mut StreamTranslationState,
event: &Value,
-) -> Vec<ConversationStreamEvent> {
+) -> Vec<LlmStreamEvent> {
let Some(object) = event.as_object() else {
- return vec![ConversationStreamEvent::Error {
+ return vec![LlmStreamEvent::Error {
message: "OpenAI stream event is not an object".to_string(),
}];
};
@@ -61,7 +61,7 @@ fn decode_openai_chat_stream(
if let Some(id) = string_field(object, "id") {
state.message_id = Some(id);
}
- out.push(ConversationStreamEvent::MessageStart {
+ out.push(LlmStreamEvent::MessageStart {
id: state.message_id.clone(),
model: state.model.clone(),
});
@@ -72,7 +72,7 @@ fn decode_openai_chat_stream(
capture_openai_usage_extras(state, object.get("usage"));
state.usage = usage.clone();
state.saw_backend_usage = true;
- out.push(ConversationStreamEvent::Usage(usage));
+ out.push(LlmStreamEvent::Usage(usage));
}
for choice in object
@@ -87,7 +87,7 @@ fn decode_openai_chat_stream(
if let Some(delta) = choice.get("delta").and_then(Value::as_object) {
if let Some(text) = delta.get("content").and_then(Value::as_str) {
if !text.is_empty() {
- out.push(ConversationStreamEvent::TextDelta {
+ out.push(LlmStreamEvent::TextDelta {
index: 0,
text: text.to_string(),
});
@@ -96,7 +96,7 @@ fn decode_openai_chat_stream(
for reasoning_key in ["reasoning_content", "reasoning"] {
if let Some(text) = delta.get(reasoning_key).and_then(Value::as_str) {
if !text.is_empty() {
- out.push(ConversationStreamEvent::ReasoningDelta {
+ out.push(LlmStreamEvent::ReasoningDelta {
index: 0,
text: text.to_string(),
});
@@ -107,7 +107,7 @@ fn decode_openai_chat_stream(
for tool_call in tool_calls {
if let Some(tool_call) = tool_call.as_object() {
let function = tool_call.get("function").and_then(Value::as_object);
- out.push(ConversationStreamEvent::ToolCallDelta {
+ out.push(LlmStreamEvent::ToolCallDelta {
index: tool_call.get("index").and_then(Value::as_u64).unwrap_or(0)
as usize,
id: tool_call
@@ -128,7 +128,7 @@ fn decode_openai_chat_stream(
}
}
if let Some(reason) = choice.get("finish_reason").and_then(Value::as_str) {
- out.push(ConversationStreamEvent::MessageStop {
+ out.push(LlmStreamEvent::MessageStop {
reason: Some(reason.to_string()),
});
}
@@ -139,10 +139,10 @@ fn decode_openai_chat_stream(
// Encodes neutral streaming events into OpenAI Chat chunks.
fn encode_openai_chat_stream(
state: &mut StreamTranslationState,
- event: ConversationStreamEvent,
+ event: LlmStreamEvent,
) -> Vec<Value> {
match event {
- ConversationStreamEvent::MessageStart { id, model } => {
+ LlmStreamEvent::MessageStart { id, model } => {
record_source_identity(state, id, model);
if state.emitted_message_start
|| (!state_source_is(state, WireFormat::AnthropicMessages)
@@ -159,7 +159,7 @@ fn encode_openai_chat_stream(
)]
}
}
- ConversationStreamEvent::TextDelta { text, .. } => {
+ LlmStreamEvent::TextDelta { text, .. } => {
vec![openai_stream_chunk(
state,
json!({"content": text}),
@@ -167,7 +167,7 @@ fn encode_openai_chat_stream(
None,
)]
}
- ConversationStreamEvent::ReasoningDelta { text, .. } => {
+ LlmStreamEvent::ReasoningDelta { text, .. } => {
vec![openai_stream_chunk(
state,
json!({"reasoning_content": text}),
@@ -175,7 +175,7 @@ fn encode_openai_chat_stream(
None,
)]
}
- ConversationStreamEvent::ToolCallDelta {
+ LlmStreamEvent::ToolCallDelta {
index,
id,
name,
@@ -187,12 +187,12 @@ fn encode_openai_chat_stream(
name,
arguments_delta,
)],
- ConversationStreamEvent::Usage(usage) => {
+ LlmStreamEvent::Usage(usage) => {
state.usage = usage;
state.saw_backend_usage = true;
Vec::new()
}
- ConversationStreamEvent::MessageStop { reason } => {
+ LlmStreamEvent::MessageStop { reason } => {
if state.finished {
return Vec::new();
}
@@ -204,7 +204,7 @@ fn encode_openai_chat_stream(
Some(openai_usage_value(state)),
)]
}
- ConversationStreamEvent::Error { message } => vec![json!({"error": {"message": message}})],
+ LlmStreamEvent::Error { message } => vec![json!({"error": {"message": message}})],
}
}
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index c708c8d92..71f41e09c 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -7,7 +7,9 @@ use std::collections::HashSet;
use serde_json::{json, Map, Value};
-use crate::codecs::common::{provider_extensions, reasoning_text_from_blocks, text_from_blocks};
+use crate::codecs::common::{
+ is_known_role_name, provider_extensions, reasoning_text_from_blocks, text_from_blocks,
+};
use crate::codecs::openai_chat::{decode_file_source, decode_image_source};
use crate::codecs::{
DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec,
@@ -15,10 +17,10 @@ use crate::codecs::{
use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
-use crate::ir::{
- is_known_role_name, ContentBlock, ConversationRequest, ConversationResponse, MediaSource,
- Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,
- SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,
+use crate::llm::{
+ ContentBlock, LlmRequest, LlmResponse, MediaSource, Message, OutputParams, ProviderExtensions,
+ ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice,
+ ToolDefinition, ToolResult, Usage,
};
use crate::policy::{DeterministicIdPolicy, TranslationPolicy};
use crate::util::{
@@ -38,7 +40,7 @@ impl FormatCodec for OpenAiResponsesCodec {
fn decode_request(&self, body: &Value, policy: &TranslationPolicy) -> Result<DecodedRequest> {
let body = crate::util::object(body, "$")?;
let mut diagnostics = Vec::new();
- let mut request = ConversationRequest {
+ let mut request = LlmRequest {
model: body
.get("model")
.and_then(Value::as_str)
@@ -68,11 +70,11 @@ impl FormatCodec for OpenAiResponsesCodec {
&Value::Object(body.clone()),
policy,
),
- ..ConversationRequest::default()
+ ..LlmRequest::default()
};
if let Some(instructions) = body.get("instructions").and_then(Value::as_str) {
if !instructions.is_empty() {
- request.instructions.push(crate::ir::InstructionBlock {
+ request.instructions.push(crate::llm::InstructionBlock {
role: Role::System,
content: vec![ContentBlock::Text {
text: instructions.to_string(),
@@ -113,7 +115,7 @@ impl FormatCodec for OpenAiResponsesCodec {
fn encode_request(
&self,
- request: &ConversationRequest,
+ request: &LlmRequest,
_policy: &TranslationPolicy,
) -> Result<EncodedRequest> {
if let Some(body) =
@@ -198,7 +200,7 @@ impl FormatCodec for OpenAiResponsesCodec {
stop_reason: Some(StopReason::EndTurn),
});
}
- let response = ConversationResponse {
+ let response = LlmResponse {
id: body
.get("id")
.and_then(Value::as_str)
@@ -229,7 +231,7 @@ impl FormatCodec for OpenAiResponsesCodec {
fn encode_response(
&self,
- response: &ConversationResponse,
+ response: &LlmResponse,
_policy: &TranslationPolicy,
) -> Result<EncodedResponse> {
if let Some(body) =
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs
index b11c219ca..48e6f1636 100644
--- a/crates/switchyard-translation/src/codecs/responses/stream.rs
+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs
@@ -7,10 +7,10 @@ use serde_json::{json, Value};
use crate::codecs::stream::{
record_source_identity, target_message_id_or_source_message_id, target_model_or_source_model,
- ConversationStreamEvent, StreamCodec, StreamTranslationState,
+ StreamCodec, StreamTranslationState,
};
use crate::format::{FormatId, WireFormat};
-use crate::ir::Usage;
+use crate::llm::{LlmStreamEvent, Usage};
/// Stream codec for OpenAI Responses API events.
pub struct OpenAiResponsesStreamCodec;
@@ -24,14 +24,14 @@ impl StreamCodec for OpenAiResponsesStreamCodec {
&self,
state: &mut StreamTranslationState,
event: &Value,
- ) -> Vec<ConversationStreamEvent> {
+ ) -> Vec<LlmStreamEvent> {
decode_responses_stream(state, event)
}
fn encode_event(
&self,
state: &mut StreamTranslationState,
- event: ConversationStreamEvent,
+ event: LlmStreamEvent,
) -> Vec<Value> {
encode_responses_stream(state, event)
}
@@ -45,7 +45,7 @@ impl StreamCodec for OpenAiResponsesStreamCodec {
fn decode_responses_stream(
state: &mut StreamTranslationState,
event: &Value,
-) -> Vec<ConversationStreamEvent> {
+) -> Vec<LlmStreamEvent> {
let event_type = event
.get("type")
.or_else(|| event.get("event"))
@@ -66,7 +66,7 @@ fn decode_responses_stream(
{
state.message_id = Some(id.to_string());
}
- vec![ConversationStreamEvent::MessageStart {
+ vec![LlmStreamEvent::MessageStart {
id: state.message_id.clone(),
model: state.model.clone(),
}]
@@ -75,7 +75,7 @@ fn decode_responses_stream(
.get("delta")
.and_then(Value::as_str)
.map(|text| {
- vec![ConversationStreamEvent::TextDelta {
+ vec![LlmStreamEvent::TextDelta {
index: event
.get("output_index")
.and_then(Value::as_u64)
@@ -90,7 +90,7 @@ fn decode_responses_stream(
.or_else(|| event.get("text"))
.and_then(Value::as_str)
.map(|text| {
- vec![ConversationStreamEvent::ReasoningDelta {
+ vec![LlmStreamEvent::ReasoningDelta {
index: event
.get("output_index")
.and_then(Value::as_u64)
@@ -110,7 +110,7 @@ fn decode_responses_stream(
.get("delta")
.and_then(Value::as_str)
.map(|delta| {
- vec![ConversationStreamEvent::ToolCallDelta {
+ vec![LlmStreamEvent::ToolCallDelta {
index: output_index as usize,
id: None,
name: None,
@@ -131,12 +131,12 @@ fn decode_responses_stream(
let usage = responses_usage(usage);
state.usage = usage.clone();
state.saw_backend_usage = true;
- out.push(ConversationStreamEvent::Usage(usage));
+ out.push(LlmStreamEvent::Usage(usage));
}
- out.push(ConversationStreamEvent::MessageStop { reason: None });
+ out.push(LlmStreamEvent::MessageStop { reason: None });
out
}
- Some("error") => vec![ConversationStreamEvent::Error {
+ Some("error") => vec![LlmStreamEvent::Error {
message: event
.get("message")
.and_then(Value::as_str)
@@ -150,33 +150,33 @@ fn decode_responses_stream(
// Encodes neutral streaming events into OpenAI Responses events.
fn encode_responses_stream(
state: &mut StreamTranslationState,
- event: ConversationStreamEvent,
+ event: LlmStreamEvent,
) -> Vec<Value> {
match event {
- ConversationStreamEvent::MessageStart { id, model } => {
+ LlmStreamEvent::MessageStart { id, model } => {
record_source_identity(state, id, model);
ensure_responses_created(state)
}
- ConversationStreamEvent::TextDelta { text, .. } => encode_responses_text_delta(state, text),
- ConversationStreamEvent::ReasoningDelta { text, .. } => {
+ LlmStreamEvent::TextDelta { text, .. } => encode_responses_text_delta(state, text),
+ LlmStreamEvent::ReasoningDelta { text, .. } => {
encode_responses_reasoning_delta(state, text)
}
- ConversationStreamEvent::ToolCallDelta {
+ LlmStreamEvent::ToolCallDelta {
index,
id,
name,
arguments_delta,
} => encode_responses_tool_delta(state, index, id, name, arguments_delta),
- ConversationStreamEvent::Usage(usage) => {
+ LlmStreamEvent::Usage(usage) => {
state.usage = usage;
state.saw_backend_usage = true;
Vec::new()
}
- ConversationStreamEvent::MessageStop { reason } => {
+ LlmStreamEvent::MessageStop { reason } => {
state.stop_reason = reason.or_else(|| state.stop_reason.clone());
Vec::new()
}
- ConversationStreamEvent::Error { message } => {
+ LlmStreamEvent::Error { message } => {
vec![json!({"type": "error", "message": message})]
}
}
@@ -295,7 +295,7 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec<Value> {
}
// Converts Responses function-call item creation into a neutral tool-call delta.
-fn decode_responses_output_item_added(event: &Value) -> Vec<ConversationStreamEvent> {
+fn decode_responses_output_item_added(event: &Value) -> Vec<LlmStreamEvent> {
let Some(item) = event.get("item").and_then(Value::as_object) else {
return Vec::new();
};
@@ -306,7 +306,7 @@ fn decode_responses_output_item_added(event: &Value) -> Vec<ConversationStreamEv
.get("output_index")
.and_then(Value::as_u64)
.unwrap_or(0) as usize;
- vec![ConversationStreamEvent::ToolCallDelta {
+ vec![LlmStreamEvent::ToolCallDelta {
index,
id: item
.get("call_id")
@@ -329,7 +329,7 @@ fn decode_responses_output_item_added(event: &Value) -> Vec<ConversationStreamEv
fn decode_responses_output_item_done(
event: &Value,
state: &StreamTranslationState,
-) -> Vec<ConversationStreamEvent> {
+) -> Vec<LlmStreamEvent> {
let Some(item) = event.get("item").and_then(Value::as_object) else {
return Vec::new();
};
@@ -348,7 +348,7 @@ fn decode_responses_output_item_done(
.map(|tool| tool.arguments.as_str())
.unwrap_or("");
if !arguments.is_empty() && arguments != existing {
- return vec![ConversationStreamEvent::ToolCallDelta {
+ return vec![LlmStreamEvent::ToolCallDelta {
index,
id: None,
name: None,
diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs
index a612f9d94..710d79dd3 100644
--- a/crates/switchyard-translation/src/codecs/stream.rs
+++ b/crates/switchyard-translation/src/codecs/stream.rs
@@ -15,7 +15,7 @@ use crate::codecs::responses::OpenAiResponsesStreamCodec;
use crate::engine::{FormatRegistry, TranslationEngine};
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
-use crate::ir::Usage;
+use crate::llm::{LlmStreamEvent, Usage};
/// Mutable state accumulated while translating one streaming response.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
@@ -83,36 +83,6 @@ impl StreamTranslationState {
}
}
-/// Provider-neutral event used between stream decoders and encoders.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub enum ConversationStreamEvent {
- MessageStart {
- id: Option<String>,
- model: Option<String>,
- },
- TextDelta {
- index: usize,
- text: String,
- },
- ReasoningDelta {
- index: usize,
- text: String,
- },
- ToolCallDelta {
- index: usize,
- id: Option<String>,
- name: Option<String>,
- arguments_delta: Option<String>,
- },
- Usage(Usage),
- MessageStop {
- reason: Option<String>,
- },
- Error {
- message: String,
- },
-}
-
/// Registry-backed streaming translator.
#[derive(Default)]
pub struct StreamTranslationEngine {
@@ -129,14 +99,11 @@ pub trait StreamCodec: Send + Sync {
&self,
state: &mut StreamTranslationState,
event: &Value,
- ) -> Vec<ConversationStreamEvent>;
+ ) -> Vec<LlmStreamEvent>;
/// Encodes one neutral event into zero or more provider events.
- fn encode_event(
- &self,
- state: &mut StreamTranslationState,
- event: ConversationStreamEvent,
- ) -> Vec<Value>;
+ fn encode_event(&self, state: &mut StreamTranslationState, event: LlmStreamEvent)
+ -> Vec<Value>;
/// Emits any terminal provider events needed after the source stream ends.
///
@@ -230,13 +197,13 @@ pub fn decode_stream_event(
state: &mut StreamTranslationState,
source: impl Into<FormatId>,
event: &Value,
-) -> Vec<ConversationStreamEvent> {
+) -> Vec<LlmStreamEvent> {
let source = source.into();
StreamCodecRegistry::with_builtins()
.codec(source)
.map(|codec| codec.decode_event(state, event))
.unwrap_or_else(|error| {
- vec![ConversationStreamEvent::Error {
+ vec![LlmStreamEvent::Error {
message: error.to_string(),
}]
})
@@ -246,7 +213,7 @@ pub fn decode_stream_event(
pub fn encode_stream_event(
state: &mut StreamTranslationState,
target: impl Into<FormatId>,
- event: ConversationStreamEvent,
+ event: LlmStreamEvent,
) -> Vec<Value> {
StreamCodecRegistry::with_builtins()
.codec(target)
diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs
index b7159649d..9dddc8db5 100644
--- a/crates/switchyard-translation/src/engine.rs
+++ b/crates/switchyard-translation/src/engine.rs
@@ -16,7 +16,7 @@ use crate::codecs::FormatCodec;
use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::FormatId;
-use crate::ir::{ConversationRequest, ConversationResponse};
+use crate::llm::{LlmRequest, LlmResponse};
use crate::policy::TranslationPolicy;
/// Encoded translation result with any diagnostics emitted along the way.
@@ -29,14 +29,14 @@ pub struct TranslationOutput {
/// Decoded request IR plus diagnostics.
#[derive(Debug)]
pub struct RequestIrOutput {
- pub request: ConversationRequest,
+ pub request: LlmRequest,
pub diagnostics: Vec<TranslationDiagnostic>,
}
/// Decoded response IR plus diagnostics.
#[derive(Debug)]
pub struct ResponseIrOutput {
- pub response: ConversationResponse,
+ pub response: LlmResponse,
pub diagnostics: Vec<TranslationDiagnostic>,
}
@@ -127,7 +127,7 @@ impl TranslationEngine {
pub fn encode_request(
&self,
target: impl Into<FormatId>,
- request: &ConversationRequest,
+ request: &LlmRequest,
policy: &TranslationPolicy,
) -> Result<TranslationOutput> {
let target = target.into();
@@ -184,7 +184,7 @@ impl TranslationEngine {
pub fn encode_response(
&self,
target: impl Into<FormatId>,
- response: &ConversationResponse,
+ response: &LlmResponse,
policy: &TranslationPolicy,
) -> Result<TranslationOutput> {
let target = target.into();
diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rs
index fdee0f2ce..da3187ca3 100644
--- a/crates/switchyard-translation/src/lib.rs
+++ b/crates/switchyard-translation/src/lib.rs
@@ -11,17 +11,17 @@ pub mod codecs;
pub mod diagnostic;
pub mod engine;
pub mod error;
-pub mod format;
-pub mod ir;
pub mod policy;
pub mod stream;
pub mod util;
+pub use libsy_protocol::{format, llm};
+
pub use diagnostic::*;
pub use engine::*;
pub use error::*;
pub use format::*;
-pub use ir::*;
+pub use llm::*;
pub use policy::*;
pub use stream::*;
pub use util::{
diff --git a/crates/switchyard-translation/src/util.rs b/crates/switchyard-translation/src/util.rs
index ac9e0b9bc..5f5f8d0f6 100644
--- a/crates/switchyard-translation/src/util.rs
+++ b/crates/switchyard-translation/src/util.rs
@@ -10,7 +10,7 @@ use serde_json::{json, Map, Value};
use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::FormatId;
-use crate::ir::{ContentBlock, ConversationRequest, Message, PreservationMetadata};
+use crate::llm::{ContentBlock, LlmRequest, Message, PreservationMetadata};
use crate::policy::{
LossyConversionPolicy, PreservationPolicy, TranslationPolicy, UnknownFieldPolicy,
};
@@ -122,7 +122,7 @@ pub fn compact_text_blocks<'a>(
/// Checks a request against declared target capabilities.
pub fn validate_request_capabilities(
- request: &ConversationRequest,
+ request: &LlmRequest,
diagnostics: &mut Vec<TranslationDiagnostic>,
policy: &TranslationPolicy,
) -> Result<()> {
diff --git a/crates/switchyard-translation/tests/extension_points.rs b/crates/switchyard-translation/tests/extension_points.rs
index 493920b5f..49a89334b 100644
--- a/crates/switchyard-translation/tests/extension_points.rs
+++ b/crates/switchyard-translation/tests/extension_points.rs
@@ -13,10 +13,10 @@ use switchyard_translation::util::{
exact_preserved_response,
};
use switchyard_translation::{
- ConversationRequest, ConversationResponse, ConversationStreamEvent, FormatId, FormatRegistry,
- LossyConversionPolicy, Message, PreservationPolicy, ResponseOutput, Role, StreamCodec,
- StreamCodecRegistry, StreamTranslationState, TargetCapabilities, TranslationEngine,
- TranslationPolicy, Usage, WireFormat,
+ FormatId, FormatRegistry, LlmRequest, LlmResponse, LlmStreamEvent, LossyConversionPolicy,
+ Message, PreservationPolicy, ResponseOutput, Role, StreamCodec, StreamCodecRegistry,
+ StreamTranslationState, TargetCapabilities, TranslationEngine, TranslationPolicy, Usage,
+ WireFormat,
};
type TestResult = std::result::Result<(), Box<dyn std::error::Error + Send + Sync>>;
@@ -133,7 +133,7 @@ impl FormatCodec for MinimalCustomCodec {
policy: &TranslationPolicy,
) -> switchyard_translation::Result<DecodedRequest> {
Ok(DecodedRequest {
- request: ConversationRequest {
+ request: LlmRequest {
model: body
.get("model")
.and_then(Value::as_str)
@@ -145,7 +145,7 @@ impl FormatCodec for MinimalCustomCodec {
.unwrap_or_default(),
)],
preservation: capture_request_preservation(self.format(), body, policy),
- ..ConversationRequest::default()
+ ..LlmRequest::default()
},
diagnostics: Vec::new(),
})
@@ -153,7 +153,7 @@ impl FormatCodec for MinimalCustomCodec {
fn encode_request(
&self,
- request: &ConversationRequest,
+ request: &LlmRequest,
policy: &TranslationPolicy,
) -> switchyard_translation::Result<EncodedRequest> {
if let Some(body) = exact_preserved_request(&request.preservation, self.format(), policy) {
@@ -181,7 +181,7 @@ impl FormatCodec for MinimalCustomCodec {
policy: &TranslationPolicy,
) -> switchyard_translation::Result<DecodedResponse> {
Ok(DecodedResponse {
- response: ConversationResponse {
+ response: LlmResponse {
id: body
.get("id")
.and_then(Value::as_str)
@@ -203,7 +203,7 @@ impl FormatCodec for MinimalCustomCodec {
}],
usage: Usage::default(),
preservation: capture_response_preservation(self.format(), body, policy),
- ..ConversationResponse::default()
+ ..LlmResponse::default()
},
diagnostics: Vec::new(),
})
@@ -211,7 +211,7 @@ impl FormatCodec for MinimalCustomCodec {
fn encode_response(
&self,
- response: &ConversationResponse,
+ response: &LlmResponse,
policy: &TranslationPolicy,
) -> switchyard_translation::Result<EncodedResponse> {
if let Some(body) = exact_preserved_response(&response.preservation, self.format(), policy)
@@ -254,9 +254,9 @@ impl StreamCodec for CustomStreamCodec {
&self,
_state: &mut StreamTranslationState,
event: &Value,
- ) -> Vec<ConversationStreamEvent> {
+ ) -> Vec<LlmStreamEvent> {
match event.get("kind").and_then(Value::as_str) {
- Some("start") => vec![ConversationStreamEvent::MessageStart {
+ Some("start") => vec![LlmStreamEvent::MessageStart {
id: event
.get("id")
.and_then(Value::as_str)
@@ -270,7 +270,7 @@ impl StreamCodec for CustomStreamCodec {
.get("text")
.and_then(Value::as_str)
.map(|text| {
- vec![ConversationStreamEvent::TextDelta {
+ vec![LlmStreamEvent::TextDelta {
index: 0,
text: text.to_string(),
}]
@@ -283,10 +283,10 @@ impl StreamCodec for CustomStreamCodec {
fn encode_event(
&self,
_state: &mut StreamTranslationState,
- event: ConversationStreamEvent,
+ event: LlmStreamEvent,
) -> Vec<Value> {
match event {
- ConversationStreamEvent::TextDelta { text, .. } => {
+ LlmStreamEvent::TextDelta { text, .. } => {
vec![json!({"kind": "delta", "text": text})]
}
_ => Vec::new(),