Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy): support streaming responses (#73)

Greg Clarkc7a6475f4566
diff --git a/Cargo.lock b/Cargo.lock
index 0dbf90623..aa3ce13f0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -869,6 +869,7 @@ dependencies = [
"futures",
"libsy",
"rand 0.8.6",
+ "switchyard-protocol",
"tokio",
"tokio-stream",
]
@@ -1632,6 +1633,7 @@ dependencies = [
name = "switchyard-protocol"
version = "0.1.0"
dependencies = [
+ "futures",
"serde",
"serde_json",
]
diff --git a/crates/libsy-examples/Cargo.toml b/crates/libsy-examples/Cargo.toml
index 082bec560..3e8de0653 100644
--- a/crates/libsy-examples/Cargo.toml
+++ b/crates/libsy-examples/Cargo.toml
@@ -14,6 +14,7 @@ rust-version.workspace = true
[dependencies]
libsy = { path = "../libsy" }
+switchyard-protocol = { path = "../libsy-protocol" }
async-trait = "0.1"
rand = "0.8"
futures = "0.3"
diff --git a/crates/libsy-examples/examples/research_agent.rs b/crates/libsy-examples/examples/research_agent.rs
index 0ea5804ca..a21f3ceae 100644
--- a/crates/libsy-examples/examples/research_agent.rs
+++ b/crates/libsy-examples/examples/research_agent.rs
@@ -9,17 +9,18 @@
//! multi-step routing (classify -> route) happens inside the classifier algorithm; the
//! agent never sees it. To drive the step stream yourself instead, use
//! `Algorithm::run_stream`. Run with:
-//! cargo run -p libsy --example research_agent
+//! cargo run -p libsy-examples --example research_agent
use std::error::Error;
use std::sync::Arc;
use async_trait::async_trait;
use libsy::{
- Algorithm, ContentBlock, Context, LlmClient, LlmRequest, LlmResponse, LlmTarget, LlmTargetSet,
- Message, Request, Response, ResponseOutput, Role, RoutedRequest,
+ Algorithm, Context, LlmClient, LlmResponse, LlmTarget, LlmTargetSet, Request, Response,
+ RoutedRequest,
};
use libsy_examples::llm_class::LlmClassifierOrchAlgo;
+use switchyard_protocol::{completion_text, text_request, text_response};
const CLASSIFIER: &str = "classifier/model";
const STRONG: &str = "strong/model";
@@ -41,14 +42,7 @@ impl LlmClient for StubClient {
format!("answer from {model}")
};
Ok(Response {
- llm_response: LlmResponse {
- outputs: vec![ResponseOutput {
- role: Role::Assistant,
- content: vec![ContentBlock::Text { text: completion }],
- stop_reason: None,
- }],
- ..LlmResponse::default()
- },
+ llm_response: LlmResponse::Agg(text_response(None, completion)),
metadata: None,
})
}
@@ -77,32 +71,13 @@ impl ResearchAgent {
let mut notes = Vec::new();
for step in self.plan(question) {
let request = Request {
- llm_request: LlmRequest {
- model: Some("auto".to_string()),
- messages: vec![Message::text(Role::User, step)],
- ..LlmRequest::default()
- },
+ llm_request: text_request(Some("auto".to_string()), step),
raw_request: None,
metadata: None,
};
let (_trace, response) = self.algo.clone().run(Context::default(), request).await?;
- notes.push(
- response
- .llm_response
- .outputs
- .iter()
- .flat_map(|output| output.content.iter())
- .filter_map(|block| match block {
- ContentBlock::Text { text }
- | ContentBlock::Refusal { text }
- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
- ContentBlock::Unknown { raw, .. } => raw.as_str(),
- _ => None,
- })
- .collect::<Vec<_>>()
- .join("\n"),
- );
+ notes.push(completion_text(&response.llm_response.into_agg().await?));
}
Ok(notes.join("\n"))
}
diff --git a/crates/libsy-examples/examples/research_agent_core.rs b/crates/libsy-examples/examples/research_agent_core.rs
index e6b3bfed6..16fa959af 100644
--- a/crates/libsy-examples/examples/research_agent_core.rs
+++ b/crates/libsy-examples/examples/research_agent_core.rs
@@ -7,16 +7,16 @@
//! surfaces as a `CallLlm` step. The agent makes the "real" model call itself and
//! fulfills the promise — this is the offload/streaming path ("ask, don't call").
//! The classifier's two steps show up as two `model call:` lines. Run with:
-//! cargo run -p libsy --example research_agent_core
+//! cargo run -p libsy-examples --example research_agent_core
use std::error::Error;
use std::sync::Arc;
use libsy::{
- Algorithm, ContentBlock, Context, Decision, LlmRequest, LlmResponse, LlmTarget, LlmTargetSet,
- Message, Request, Response, ResponseOutput, Role, Step,
+ Algorithm, Context, Decision, LlmResponse, LlmTarget, LlmTargetSet, Request, Response, Step,
};
use libsy_examples::llm_class::LlmClassifierOrchAlgo;
+use switchyard_protocol::{completion_text, text_request, text_response};
use tokio_stream::StreamExt;
const CLASSIFIER: &str = "classifier/model";
@@ -34,14 +34,7 @@ async fn call_model(model: &str) -> Response {
format!("answer from {model}")
};
Response {
- llm_response: LlmResponse {
- outputs: vec![ResponseOutput {
- role: Role::Assistant,
- content: vec![ContentBlock::Text { text: completion }],
- stop_reason: None,
- }],
- ..LlmResponse::default()
- },
+ llm_response: LlmResponse::Agg(text_response(None, completion)),
metadata: None,
}
}
@@ -69,11 +62,7 @@ impl ResearchAgent {
let mut notes = Vec::new();
for step in self.plan(question) {
let request = Request {
- llm_request: LlmRequest {
- model: Some("auto".to_string()),
- messages: vec![Message::text(Role::User, step)],
- ..LlmRequest::default()
- },
+ llm_request: text_request(Some("auto".to_string()), step),
raw_request: None,
metadata: None,
};
@@ -89,22 +78,7 @@ impl ResearchAgent {
// Decisions stream in as the algorithm makes them.
Step::Decision(decision) => print_decision(decision.as_ref()),
Step::ReturnToAgent(response) => {
- notes.push(
- response
- .llm_response
- .outputs
- .iter()
- .flat_map(|output| output.content.iter())
- .filter_map(|block| match block {
- ContentBlock::Text { text }
- | ContentBlock::Refusal { text }
- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
- ContentBlock::Unknown { raw, .. } => raw.as_str(),
- _ => None,
- })
- .collect::<Vec<_>>()
- .join("\n"),
- );
+ notes.push(completion_text(&response.llm_response.into_agg().await?));
}
}
}
diff --git a/crates/libsy-examples/examples/streaming_agent.rs b/crates/libsy-examples/examples/streaming_agent.rs
new file mode 100644
index 000000000..aa14f9e57
--- /dev/null
+++ b/crates/libsy-examples/examples/streaming_agent.rs
@@ -0,0 +1,104 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Agent that forwards a streamed response to the caller token-by-token.
+//!
+//! Drives [`Algorithm::run_stream`] with a **client-less** target, so each model call is
+//! offloaded as a [`Step::CallLlm`]. The agent serves it with a *streaming* response
+//! ([`LlmResponse::Stream`]) rather than a buffered one. That stream rides untouched
+//! through the algorithm and returns as [`Step::ReturnToAgent`], where the agent drives it
+//! and prints each [`LlmResponseChunk`] as it arrives — true token streaming end to end.
+//! Contrast [`Algorithm::run`], which would aggregate the same stream into one buffered
+//! answer. Run with:
+//! cargo run -p libsy-examples --example streaming_agent
+
+use std::error::Error;
+use std::io::Write;
+use std::sync::Arc;
+
+use futures::StreamExt;
+use libsy::{
+ Algorithm, Context, LlmResponse, LlmResponseChunk, LlmResponseStream, LlmTarget, LlmTargetSet,
+ Request, Response, Step,
+};
+use libsy_examples::rand::RandomOrchAlgo;
+use switchyard_protocol::{completion_text, text_request};
+
+/// The "real" model call the agent makes to fulfill an offloaded promise: a response
+/// whose body is a live token stream (`MessageStart`, one `TextDelta` per token,
+/// `MessageStop`). A real integrator would map an upstream provider SSE stream here.
+fn streaming_response(model: &str, tokens: &[&str]) -> Response {
+ let mut chunks = vec![LlmResponseChunk::MessageStart {
+ id: None,
+ model: Some(model.to_string()),
+ }];
+ for token in tokens {
+ chunks.push(LlmResponseChunk::TextDelta {
+ index: 0,
+ text: token.to_string(),
+ });
+ }
+ chunks.push(LlmResponseChunk::MessageStop {
+ reason: Some("stop".to_string()),
+ });
+
+ let stream: LlmResponseStream = futures::stream::iter(chunks.into_iter().map(Ok)).boxed();
+ Response {
+ llm_response: LlmResponse::Stream(stream),
+ metadata: None,
+ }
+}
+
+#[tokio::main(flavor = "current_thread")]
+async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
+ // One client-less target -> its call is offloaded for us to serve.
+ let targets = LlmTargetSet::new(vec![LlmTarget {
+ semantic_name: "stream/model".to_string(),
+ llm_client: None,
+ }]);
+ let algo: Arc<dyn Algorithm> = Arc::new(RandomOrchAlgo::new(targets));
+
+ let request = Request {
+ llm_request: text_request(Some("auto".to_string()), "tell me about switchyard"),
+ raw_request: None,
+ metadata: None,
+ };
+ let stream = algo.run_stream(Context::default(), request);
+ tokio::pin!(stream);
+
+ while let Some(step) = stream.next().await {
+ match step? {
+ Step::CallLlm(call) => {
+ // Serve the offloaded call with a streaming response.
+ let model = call.get_decision().selected_model().to_string();
+ let tokens = ["Switch", "yard ", "routes ", "LLM ", "traffic."];
+ call.respond(Ok(streaming_response(&model, &tokens)))?;
+ }
+ Step::Decision(decision) => {
+ println!(
+ "decision: {} ({})",
+ decision.selected_model(),
+ decision.reasoning().unwrap_or_default()
+ );
+ }
+ Step::ReturnToAgent(response) => match response.llm_response {
+ // The stream reached the agent untouched: print each token as it arrives.
+ LlmResponse::Stream(mut chunks) => {
+ print!("agent sees: ");
+ while let Some(chunk) = chunks.next().await {
+ if let LlmResponseChunk::TextDelta { text, .. } = chunk? {
+ print!("{text}");
+ std::io::stdout().flush().ok();
+ }
+ }
+ println!();
+ }
+ // A buffered backend would land here instead.
+ LlmResponse::Agg(agg) => {
+ println!("agent sees (buffered): {}", completion_text(&agg))
+ }
+ },
+ }
+ }
+ Ok(())
+}
diff --git a/crates/libsy-examples/src/ensemble.rs b/crates/libsy-examples/src/ensemble.rs
index e558ac098..d12f16e2e 100644
--- a/crates/libsy-examples/src/ensemble.rs
+++ b/crates/libsy-examples/src/ensemble.rs
@@ -22,10 +22,8 @@ use std::sync::{Arc, Mutex};
use async_trait::async_trait;
-use libsy::{
- Algorithm, ContentBlock, Context, Decision, Driver, LlmRequest, LlmTargetSet, Message, Request,
- Response, Role, Signals,
-};
+use libsy::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response, Signals};
+use switchyard_protocol::{completion_text, prompt_text, text_request};
/// Which step of the ensemble flow produced a decision.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -172,6 +170,7 @@ impl EnsembleOrchAlgo {
async fn route_committed(
&self,
driver: &Driver,
+ ctx: Context,
request: Request,
model: String,
) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {
@@ -184,13 +183,18 @@ impl EnsembleOrchAlgo {
selected_model: model.clone(),
phase: EnsemblePhase::Committed,
});
+ // The agent's inbound name rides through; the committed model is on the
+ // decision, not stamped onto the request.
let routed = Request {
- llm_request: request.llm_request,
+ llm_request: text_request(
+ request.llm_request.model.clone(),
+ prompt_text(&request.llm_request),
+ ),
raw_request: request.raw_request,
metadata: request.metadata,
};
let response = driver
- .call_llm_target(&target, routed, decision.clone())
+ .call_llm_target(ctx, &target, routed, decision.clone())
.await?;
Ok((vec![decision], response))
}
@@ -200,29 +204,13 @@ impl EnsembleOrchAlgo {
async fn ensemble_turn(
&self,
driver: &Driver,
+ ctx: Context,
request: Request,
) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {
- let user_prompt = request
- .llm_request
- .instructions
- .iter()
- .flat_map(|instruction| instruction.content.iter())
- .chain(
- request
- .llm_request
- .messages
- .iter()
- .flat_map(|message| message.content.iter()),
- )
- .filter_map(|block| match block {
- ContentBlock::Text { text }
- | ContentBlock::Refusal { text }
- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
- ContentBlock::Unknown { raw, .. } => raw.as_str(),
- _ => None,
- })
- .collect::<Vec<_>>()
- .join("\n");
+ let user_prompt = prompt_text(&request.llm_request);
+ // The agent's inbound name rides through every sub-call unchanged; the model
+ // each call hits is carried by its decision, not stamped onto the request.
+ let inbound = request.llm_request.model.clone();
// Fan out to all candidates concurrently with the same user prompt. Each
// call is annotated with its own candidate decision so the caller can see
@@ -238,16 +226,17 @@ impl EnsembleOrchAlgo {
});
candidate_decisions.push(decision.clone());
let call_request = Request {
- llm_request: request.llm_request.clone(),
+ llm_request: text_request(inbound.clone(), user_prompt.clone()),
raw_request: request.raw_request.clone(),
metadata: request.metadata.clone(),
};
let model = model.clone();
+ let ctx = ctx.clone();
calls.push(async move {
(
model,
driver
- .call_llm_target(&target, call_request, decision)
+ .call_llm_target(ctx, &target, call_request, decision)
.await,
)
});
@@ -282,33 +271,22 @@ impl EnsembleOrchAlgo {
phase: EnsemblePhase::Judge,
});
let judge_request = Request {
- llm_request: LlmRequest {
- model: request.llm_request.model.clone(),
- messages: vec![Message::text(Role::User, judge_prompt)],
- ..LlmRequest::default()
- },
- raw_request: None,
+ llm_request: text_request(inbound.clone(), judge_prompt),
+ raw_request: request.raw_request.clone(),
metadata: request.metadata.clone(),
};
let judge_response = driver
- .call_llm_target(&judge_target, judge_request, judge_decision.clone())
+ .call_llm_target(ctx, &judge_target, judge_request, judge_decision.clone())
.await?;
// Fail open: an unparseable pick falls back to the first response.
- let judge_text = judge_response
- .llm_response
- .outputs
- .iter()
- .flat_map(|output| output.content.iter())
- .filter_map(|block| match block {
- ContentBlock::Text { text }
- | ContentBlock::Refusal { text }
- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
- ContentBlock::Unknown { raw, .. } => raw.as_str(),
- _ => None,
- })
- .collect::<Vec<_>>()
- .join("\n");
- let choice = parse_choice(&judge_text, survivors.len());
+ let choice = parse_choice(
+ &judge_response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ survivors.len(),
+ );
let (model, response) = survivors
.into_iter()
.nth(choice)
@@ -353,21 +331,15 @@ fn build_judge_prompt(user_prompt: &str, survivors: &[(String, Response)]) -> St
prompt.push_str(user_prompt);
prompt.push_str("\n\n");
for (i, (_model, response)) in survivors.iter().enumerate() {
- let candidate_text = response
- .llm_response
- .outputs
- .iter()
- .flat_map(|output| output.content.iter())
- .filter_map(|block| match block {
- ContentBlock::Text { text }
- | ContentBlock::Refusal { text }
- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
- ContentBlock::Unknown { raw, .. } => raw.as_str(),
- _ => None,
- })
- .collect::<Vec<_>>()
- .join("\n");
- prompt.push_str(&format!("Response {}:\n{}\n\n", i + 1, candidate_text));
+ prompt.push_str(&format!(
+ "Response {}:\n{}\n\n",
+ i + 1,
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default()
+ ));
}
prompt.push_str(&format!(
"Reply with only the number (1-{}) of the best response.",
@@ -398,7 +370,7 @@ fn parse_choice(completion: &str, count: usize) -> usize {
impl Algorithm for EnsembleOrchAlgo {
async fn create_run_task(
self: Arc<Self>,
- _ctx: Context,
+ ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response, Box<dyn Error + Send + Sync>> {
@@ -406,14 +378,15 @@ impl Algorithm for EnsembleOrchAlgo {
// otherwise run a full ensemble turn. Both return a decision trace plus the
// final response.
let (trace, response) = if let Some(model) = self.resolve_committed()? {
- self.route_committed(&driver, request, model).await?
+ self.route_committed(&driver, ctx.clone(), request, model)
+ .await?
} else {
- self.ensemble_turn(&driver, request).await?
+ self.ensemble_turn(&driver, ctx.clone(), request).await?
};
// Publish the trace to the stream (candidate..., judge?, winner). The
// candidate decisions also rode along on their offloaded `CallLlm` steps.
for decision in trace {
- driver.info(decision).await?;
+ driver.info(ctx.clone(), decision).await?;
}
Ok(response)
}
@@ -430,8 +403,9 @@ impl Algorithm for EnsembleOrchAlgo {
#[cfg(test)]
mod tests {
use super::*;
- use libsy::{LlmClient, LlmResponse, LlmTarget, Response, ResponseOutput, RoutedRequest};
+ use libsy::{LlmClient, LlmResponse, LlmTarget, Response, RoutedRequest};
use std::sync::Mutex as StdMutex;
+ use switchyard_protocol::text_response;
/// Mock client that answers candidate calls with `answer from {model}` and,
/// for the judge model, returns the 1-based number of the response whose
@@ -455,22 +429,12 @@ mod tests {
.map_err(|_| "lock poisoned")?
.push(name.clone());
let completion = if name == self.judge_model {
- let prompt = routed.request.llm_request.messages[0]
- .text_content("\n")
- .unwrap_or_default();
- judge_pick(&prompt, &self.prefer)
+ judge_pick(&prompt_text(&routed.request.llm_request), &self.prefer)
} else {
format!("answer from {name}")
};
Ok(Response {
- llm_response: LlmResponse {
- outputs: vec![ResponseOutput {
- role: Role::Assistant,
- content: vec![ContentBlock::Text { text: completion }],
- stop_reason: None,
- }],
- ..LlmResponse::default()
- },
+ llm_response: LlmResponse::Agg(text_response(None, completion)),
metadata: None,
})
}
@@ -547,11 +511,7 @@ mod tests {
fn request(prompt: &str) -> Request {
Request {
- llm_request: LlmRequest {
- model: Some("auto".to_string()),
- messages: vec![Message::text(Role::User, prompt)],
- ..LlmRequest::default()
- },
+ llm_request: text_request(Some("auto".to_string()), prompt),
raw_request: None,
metadata: None,
}
@@ -580,10 +540,12 @@ mod tests {
.run(Context::default(), request("solve it"))
.await?;
assert_eq!(
- response.llm_response.outputs[0].content,
- vec![ContentBlock::Text {
- text: "answer from b/model".to_string()
- }]
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from b/model"
);
// Both candidates and the judge were called.
@@ -625,10 +587,12 @@ mod tests {
// fan-out to a/model and no judge call.
let (trace, response) = orch.clone().run(Context::default(), request("t3")).await?;
assert_eq!(
- response.llm_response.outputs[0].content,
- vec![ContentBlock::Text {
- text: "answer from b/model".to_string()
- }]
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from b/model"
);
assert_eq!(trace.len(), 1);
let decision = as_ensemble(&trace[0])?;
@@ -648,10 +612,12 @@ mod tests {
let (algo, calls) = algo(&["only/model"], "judge/haiku", "only/model", 100);
let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;
assert_eq!(
- response.llm_response.outputs[0].content,
- vec![ContentBlock::Text {
- text: "answer from only/model".to_string()
- }]
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from only/model"
);
// No judge call for a lone candidate.
assert!(!calls
@@ -768,24 +734,14 @@ mod tests {
let name = routed.decision.selected_model().to_string();
let completion = if name == self.judge_model {
// Judge runs after the barrier releases; it must not wait.
- let prompt = routed.request.llm_request.messages[0]
- .text_content("\n")
- .unwrap_or_default();
- judge_pick(&prompt, &self.prefer)
+ judge_pick(&prompt_text(&routed.request.llm_request), &self.prefer)
} else {
// Hold every candidate call until all sessions have fanned out.
self.barrier.wait().await;
format!("answer from {name}")
};
Ok(Response {
- llm_response: LlmResponse {
- outputs: vec![ResponseOutput {
- role: Role::Assistant,
- content: vec![ContentBlock::Text { text: completion }],
- stop_reason: None,
- }],
- ..LlmResponse::default()
- },
+ llm_response: LlmResponse::Agg(text_response(None, completion)),
metadata: None,
})
}
@@ -821,16 +777,11 @@ mod tests {
.run(Context::default(), request(prompt))
.await
.map(|(_, response)| {
- match response
+ response
.llm_response
- .outputs
- .into_iter()
- .next()
- .and_then(|output| output.content.into_iter().next())
- {
- Some(ContentBlock::Text { text }) => text,
- _ => String::new(),
- }
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default()
})
})
};
@@ -877,17 +828,14 @@ mod tests {
.and_then(|d| d.as_any().downcast_ref::<EnsembleDecision>())
.map(|d| d.phase)
.ok_or("missing final decision")?;
- let completion = match response
- .llm_response
- .outputs
- .into_iter()
- .next()
- .and_then(|output| output.content.into_iter().next())
- {
- Some(ContentBlock::Text { text }) => text,
- _ => String::new(),
- };
- Ok::<(String, EnsemblePhase), Box<dyn Error + Send + Sync>>((completion, phase))
+ Ok::<(String, EnsemblePhase), Box<dyn Error + Send + Sync>>((
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ phase,
+ ))
})
};
// The two sessions run in parallel; each committed independently.
diff --git a/crates/libsy-examples/src/llm_class.rs b/crates/libsy-examples/src/llm_class.rs
index 66e2008b5..a6a82dff8 100644
--- a/crates/libsy-examples/src/llm_class.rs
+++ b/crates/libsy-examples/src/llm_class.rs
@@ -15,10 +15,8 @@ use std::sync::Arc;
use async_trait::async_trait;
-use libsy::{
- Algorithm, ContentBlock, Context, Decision, Driver, LlmRequest, LlmTargetSet, Message, Request,
- Response, Role, Signals,
-};
+use libsy::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response, Signals};
+use switchyard_protocol::{completion_text, prompt_text, text_request};
/// Preamble prepended to the user prompt when asking the classifier target for a
/// strong-win-rate score.
@@ -102,44 +100,23 @@ impl LlmClassifierOrchAlgo {
impl Algorithm for LlmClassifierOrchAlgo {
async fn create_run_task(
self: Arc<Self>,
- _ctx: Context,
+ ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response, Box<dyn Error + Send + Sync>> {
- let user_prompt = request
- .llm_request
- .instructions
- .iter()
- .flat_map(|instruction| instruction.content.iter())
- .chain(
- request
- .llm_request
- .messages
- .iter()
- .flat_map(|message| message.content.iter()),
- )
- .filter_map(|block| match block {
- ContentBlock::Text { text }
- | ContentBlock::Refusal { text }
- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
- ContentBlock::Unknown { raw, .. } => raw.as_str(),
- _ => None,
- })
- .collect::<Vec<_>>()
- .join("\n");
+ let user_prompt = prompt_text(&request.llm_request);
+ // The agent's inbound name rides through unchanged on every sub-call; the
+ // model each sub-call actually hits is carried by its decision instead.
+ let inbound = request.llm_request.model.clone();
// 1. Classify: call the classifier target with the score-eliciting prompt.
let classifier_target = self.target_set.get_target(&self.classifier_model)?;
let classify_request = Request {
- llm_request: LlmRequest {
- model: request.llm_request.model.clone(),
- messages: vec![Message::text(
- Role::User,
- format!("{CLASSIFIER_PROMPT_PREAMBLE}{user_prompt}"),
- )],
- ..LlmRequest::default()
- },
- raw_request: None,
+ llm_request: text_request(
+ inbound.clone(),
+ format!("{CLASSIFIER_PROMPT_PREAMBLE}{user_prompt}"),
+ ),
+ raw_request: request.raw_request.clone(),
metadata: request.metadata.clone(),
};
let classify_decision: Arc<dyn Decision> = Arc::new(ClassifierDecision {
@@ -148,25 +125,23 @@ impl Algorithm for LlmClassifierOrchAlgo {
score: None,
tier: None,
});
- driver.info(classify_decision.clone()).await?;
+ driver.info(ctx.clone(), classify_decision.clone()).await?;
let classify_response = driver
- .call_llm_target(&classifier_target, classify_request, classify_decision)
+ .call_llm_target(
+ ctx.clone(),
+ &classifier_target,
+ classify_request,
+ classify_decision,
+ )
.await?;
- let score_text = classify_response
+ let score = classify_response
.llm_response
- .outputs
- .iter()
- .flat_map(|output| output.content.iter())
- .filter_map(|block| match block {
- ContentBlock::Text { text }
- | ContentBlock::Refusal { text }
- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),
- ContentBlock::Unknown { raw, .. } => raw.as_str(),
- _ => None,
- })
- .collect::<Vec<_>>()
- .join("\n");
- let score = score_text.trim().parse::<f64>().ok();
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default()
+ .trim()
+ .parse::<f64>()
+ .ok();
// 2. Route: pick strong/weak. Fail open — an unparseable score routes strong.
let (tier, model) = match score {
@@ -186,13 +161,13 @@ impl Algorithm for LlmClassifierOrchAlgo {
tier: Some(tier),
});
let routed_request = Request {
- llm_request: request.llm_request,
+ llm_request: text_request(inbound, user_prompt),
raw_request: request.raw_request,
metadata: request.metadata,
};
- driver.info(route_decision.clone()).await?;
+ driver.info(ctx.clone(), route_decision.clone()).await?;
driver
- .call_llm_target(&routed_target, routed_request, route_decision)
+ .call_llm_target(ctx, &routed_target, routed_request, route_decision)
.await
}
@@ -208,8 +183,9 @@ impl Algorithm for LlmClassifierOrchAlgo {
#[cfg(test)]
mod tests {
use super::*;
- use libsy::{LlmClient, LlmResponse, LlmTarget, Response, ResponseOutput, RoutedRequest};
+ use libsy::{LlmClient, LlmResponse, LlmTarget, Response, RoutedRequest};
use std::sync::Mutex;
+ use switchyard_protocol::text_response;
/// Returns `score` for the classifier target, an answer tagged with the model
/// otherwise; records the requests it saw so a test can inspect the classifier
@@ -227,27 +203,17 @@ mod tests {
routed: RoutedRequest,
) -> Result<Response, Box<dyn Error + Send + Sync>> {
let name = routed.decision.selected_model().to_string();
- let outputs = if name == self.classifier_model {
- vec![ResponseOutput {
- role: Role::Assistant,
- content: vec![ContentBlock::Text {
- text: self.score.clone(),
- }],
- stop_reason: None,
- }]
+ let completion = if name == self.classifier_model {
+ self.score.clone()
} else {
- Vec::new()
+ format!("answer from {name}")
};
self.seen
.lock()
.map_err(|_| "lock poisoned")?
.push(routed.request);
Ok(Response {
- llm_response: LlmResponse {
- model: Some(name),
- outputs,
- ..LlmResponse::default()
- },
+ llm_response: LlmResponse::Agg(text_response(None, completion)),
metadata: None,
})
}
@@ -282,11 +248,7 @@ mod tests {
fn request(prompt: &str) -> Request {
Request {
- llm_request: LlmRequest {
- model: Some("auto".to_string()),
- messages: vec![Message::text(Role::User, prompt)],
- ..LlmRequest::default()
- },
+ llm_request: text_request(Some("auto".to_string()), prompt),
raw_request: None,
metadata: None,
}
@@ -314,8 +276,12 @@ mod tests {
.run(Context::default(), request("solve this proof"))
.await?;
assert_eq!(
- response.llm_response.model.as_deref(),
- Some("frontier/model")
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from frontier/model"
);
// Trace: [classify, route].
assert_eq!(trace[0].selected_model(), "router/classifier");
@@ -332,7 +298,14 @@ mod tests {
let (trace, response) = orch(algo)
.run(Context::default(), request("say hello"))
.await?;
- assert_eq!(response.llm_response.model.as_deref(), Some("cheap/model"));
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from cheap/model"
+ );
let routed = as_classifier(&trace[1])?;
assert_eq!(routed.tier, Some(ClassifierTier::Weak));
assert_eq!(routed.score, Some(0.2));
@@ -347,8 +320,12 @@ mod tests {
.run(Context::default(), request("borderline"))
.await?;
assert_eq!(
- response.llm_response.model.as_deref(),
- Some("frontier/model")
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from frontier/model"
);
Ok(())
}
@@ -358,8 +335,12 @@ mod tests {
let (algo, _) = algo(0.5, "not-a-number");
let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;
assert_eq!(
- response.llm_response.model.as_deref(),
- Some("frontier/model")
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from frontier/model"
);
let routed = as_classifier(&trace[1])?;
assert_eq!(routed.tier, Some(ClassifierTier::Strong));
@@ -377,15 +358,9 @@ mod tests {
let seen = seen.lock().map_err(|_| "lock poisoned")?;
// Two calls: the classifier (preamble + user text), then the routed model.
assert_eq!(seen.len(), 2);
- let classifier_prompt = seen[0].llm_request.messages[0]
- .text_content("\n")
- .ok_or("classifier prompt is not text")?;
- assert!(classifier_prompt.contains("prove it"));
- assert!(classifier_prompt.contains("frontier model"));
- assert_eq!(
- seen[1].llm_request.messages[0].text_content("\n"),
- Some("prove it".to_string())
- );
+ assert!(prompt_text(&seen[0].llm_request).contains("prove it"));
+ assert!(prompt_text(&seen[0].llm_request).contains("frontier model"));
+ assert_eq!(prompt_text(&seen[1].llm_request), "prove it");
Ok(())
}
}
diff --git a/crates/libsy-examples/src/rand.rs b/crates/libsy-examples/src/rand.rs
index bebaa3803..ef28205d2 100644
--- a/crates/libsy-examples/src/rand.rs
+++ b/crates/libsy-examples/src/rand.rs
@@ -55,7 +55,7 @@ impl RandomOrchAlgo {
impl Algorithm for RandomOrchAlgo {
async fn create_run_task(
self: Arc<Self>,
- _ctx: Context,
+ ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response, Box<dyn Error + Send + Sync>> {
@@ -80,8 +80,10 @@ impl Algorithm for RandomOrchAlgo {
});
// Publish the decision to the stream, then offload the call.
- driver.info(decision.clone()).await?;
- driver.call_llm_target(&target, request, decision).await
+ driver.info(ctx.clone(), decision.clone()).await?;
+ driver
+ .call_llm_target(ctx, &target, request, decision)
+ .await
}
async fn process_signals(
@@ -96,10 +98,9 @@ impl Algorithm for RandomOrchAlgo {
#[cfg(test)]
mod tests {
use super::*;
- use libsy::{
- LlmClient, LlmRequest, LlmResponse, LlmTarget, Message, Response, Role, RoutedRequest,
- };
+ use libsy::{LlmClient, LlmResponse, LlmTarget, Response, RoutedRequest};
use std::collections::HashSet;
+ use switchyard_protocol::{completion_text, text_request, text_response};
/// Echoes back the target name it was called with, so a test can tell which
/// target the algo selected.
@@ -112,10 +113,10 @@ mod tests {
routed: RoutedRequest,
) -> Result<Response, Box<dyn Error + Send + Sync>> {
Ok(Response {
- llm_response: LlmResponse {
- model: Some(routed.decision.selected_model().to_string()),
- ..LlmResponse::default()
- },
+ llm_response: LlmResponse::Agg(text_response(
+ None,
+ routed.decision.selected_model(),
+ )),
metadata: None,
})
}
@@ -123,11 +124,7 @@ mod tests {
fn request() -> Request {
Request {
- llm_request: LlmRequest {
- model: Some("auto".to_string()),
- messages: vec![Message::text(Role::User, "hi")],
- ..LlmRequest::default()
- },
+ llm_request: text_request(Some("auto".to_string()), "hi"),
raw_request: None,
metadata: None,
}
@@ -154,7 +151,14 @@ mod tests {
) -> Result<(), Box<dyn Error + Send + Sync>> {
let orch = orch(&["only/model"]);
let (trace, response) = orch.clone().run(Context::default(), request()).await?;
- assert_eq!(response.llm_response.model.as_deref(), Some("only/model"));
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "only/model"
+ );
assert_eq!(trace.len(), 1);
assert_eq!(trace[0].selected_model(), "only/model");
Ok(())
@@ -167,7 +171,11 @@ mod tests {
let orch = orch(&names);
for _ in 0..50 {
let (trace, response) = orch.clone().run(Context::default(), request()).await?;
- let selected = response.llm_response.model.unwrap_or_default();
+ let selected = response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default();
assert!(
names.contains(&selected.as_str()),
"selected {selected} not in target set"
@@ -185,7 +193,13 @@ mod tests {
let mut seen = HashSet::new();
for _ in 0..100 {
let (_, response) = orch.clone().run(Context::default(), request()).await?;
- seen.insert(response.llm_response.model.unwrap_or_default());
+ seen.insert(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ );
}
// 100 uniform draws over two targets: both should appear (miss ~ 2^-99).
assert_eq!(
diff --git a/crates/libsy-protocol/Cargo.toml b/crates/libsy-protocol/Cargo.toml
index 51e255da3..6e4ec82c8 100644
--- a/crates/libsy-protocol/Cargo.toml
+++ b/crates/libsy-protocol/Cargo.toml
@@ -17,5 +17,6 @@ keywords = ["llm", "protocol", "openai", "anthropic"]
publish = ["crates-io"]
[dependencies]
+futures = "0.3"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
diff --git a/crates/libsy-protocol/src/envelope.rs b/crates/libsy-protocol/src/envelope.rs
new file mode 100644
index 000000000..33479cdeb
--- /dev/null
+++ b/crates/libsy-protocol/src/envelope.rs
@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! The request/response envelope: the normalized [`LlmRequest`]/[`LlmResponse`] paired
+//! with the original provider payload and correlation [`Metadata`].
+
+use crate::{LlmRequest, LlmResponse};
+
+/// Per-request state threaded to an algorithm alongside a request. A placeholder
+/// for cross-cutting state (correlation ids, budgets, deadlines) an algorithm will
+/// read; empty today.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct Context {}
+
+/// Correlation and routing metadata attached to a request or response.
+///
+/// All fields are optional; algorithms and observers use whichever are present
+/// (e.g. to key per-session state or emit correlated telemetry). `extra_metadata`
+/// is a free-form escape hatch for host-specific keys.
+#[derive(Clone)]
+pub struct Metadata {
+ /// Stable id for a multi-request session/conversation.
+ pub session_id: Option<String>,
+ /// Id of the agent making the request.
+ pub agent_id: Option<String>,
+ /// Id of the task the request belongs to.
+ pub task_id: Option<String>,
+ /// External trace/request id for joining with the host's telemetry.
+ pub correlation_id: Option<String>,
+ /// Arbitrary host-defined key/value metadata.
+ pub extra_metadata: Option<std::collections::BTreeMap<String, String>>,
+ /// HTTP headers to attach when forwarding the request/response, if any.
+ pub http_headers: Option<std::collections::BTreeMap<String, String>>,
+}
+
+/// A request an algorithm routes: the normalized [`LlmRequest`] plus the original
+/// provider payload and correlation [`Metadata`].
+#[derive(Clone)]
+pub struct Request {
+ /// The normalized request an algorithm routes.
+ pub llm_request: LlmRequest,
+ /// The original provider-shaped request body, if the host wants to forward it
+ /// verbatim (e.g. a proxy preserving messages/params). libsy does not read it.
+ pub raw_request: Option<serde_json::Value>,
+ /// Correlation metadata carried through the request.
+ pub metadata: Option<Metadata>,
+}
+
+/// A response an algorithm returns: the [`LlmResponse`] (streamed or aggregate) plus
+/// optional correlation [`Metadata`].
+///
+/// Not `Clone` — `llm_response` may own a live stream.
+pub struct Response {
+ /// The neutral model response — a chunk stream or the buffered aggregate.
+ pub llm_response: LlmResponse,
+ /// Correlation metadata carried through the response.
+ pub metadata: Option<Metadata>,
+}
diff --git a/crates/libsy-protocol/src/lib.rs b/crates/libsy-protocol/src/lib.rs
index c844a2d72..f2f3076b3 100644
--- a/crates/libsy-protocol/src/lib.rs
+++ b/crates/libsy-protocol/src/lib.rs
@@ -1,10 +1,110 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Provider-neutral LLM protocol types shared across Switchyard crates.
+//! Shared request/response protocol for libsy — the vocabulary an [`Algorithm`] reasons
+//! over, decoupled from libsy's orchestration.
+//!
+//! This crate owns Switchyard's neutral conversation IR: [`LlmRequest`] (model, messages,
+//! tools, sampling, …), the buffered [`AggLlmResponse`] (outputs, usage, …), and its
+//! streaming counterpart [`LlmResponseChunk`]; the [`Request`]/[`Response`] envelope that
+//! pairs them with correlation [`Metadata`]; plus the wire-[`format`] identifiers
+//! translation keys off. `switchyard-translation` re-exports the IR types under its own
+//! `ConversationRequest` / `ConversationResponse` / `ConversationStreamEvent` names. The IR
+//! carries no bare `prompt`/`completion`; the [`text_request`] / [`prompt_text`] /
+//! [`text_response`] / [`completion_text`] helpers bridge to and from plain text for the
+//! common single-turn case.
+//!
+//! The streamed-response type itself — a live stream of chunks *or* the terminal
+//! aggregate — is the [`LlmResponse`] enum; it owns a `futures::Stream`, so it is the one
+//! non-`Clone`, non-data type in this crate.
+//!
+//! [`Algorithm`]: https://docs.rs/libsy
+pub mod envelope;
pub mod format;
pub mod llm;
+pub mod stream;
+pub use envelope::*;
pub use format::*;
pub use llm::*;
+pub use stream::*;
+
+/// Build a single-turn request: one user message carrying `prompt`, for `model`.
+pub fn text_request(model: Option<String>, prompt: impl Into<String>) -> LlmRequest {
+ LlmRequest {
+ model,
+ messages: vec![Message::text(Role::User, prompt)],
+ ..LlmRequest::default()
+ }
+}
+
+/// The user's prompt text — the text of every user message, joined by newlines. Empty
+/// when the request has no user text.
+pub fn prompt_text(request: &LlmRequest) -> String {
+ request
+ .messages
+ .iter()
+ .filter(|message| message.role == Role::User)
+ .filter_map(|message| message.text_content("\n"))
+ .collect::<Vec<_>>()
+ .join("\n")
+}
+
+/// Build a single-turn response: one assistant message carrying `completion`, for `model`.
+pub fn text_response(model: Option<String>, completion: impl Into<String>) -> AggLlmResponse {
+ AggLlmResponse {
+ model,
+ outputs: vec![ResponseOutput {
+ role: Role::Assistant,
+ content: vec![ContentBlock::Text {
+ text: completion.into(),
+ }],
+ stop_reason: None,
+ }],
+ ..AggLlmResponse::default()
+ }
+}
+
+/// The assistant's completion text — the text blocks of the first output, concatenated.
+/// Empty when the response has no textual output.
+pub fn completion_text(response: &AggLlmResponse) -> String {
+ response
+ .outputs
+ .first()
+ .map(|output| {
+ output
+ .content
+ .iter()
+ .filter_map(|block| match block {
+ ContentBlock::Text { text } => Some(text.as_str()),
+ _ => None,
+ })
+ .collect::<String>()
+ })
+ .unwrap_or_default()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn request_round_trips_prompt_text() {
+ let req = text_request(Some("m".to_string()), "hello world");
+ assert_eq!(req.model.as_deref(), Some("m"));
+ assert_eq!(prompt_text(&req), "hello world");
+ }
+
+ #[test]
+ fn response_round_trips_completion_text() {
+ let resp = text_response(None, "the answer");
+ assert_eq!(completion_text(&resp), "the answer");
+ }
+
+ #[test]
+ fn empty_text_helpers_are_empty_strings() {
+ assert_eq!(prompt_text(&LlmRequest::default()), "");
+ assert_eq!(completion_text(&AggLlmResponse::default()), "");
+ }
+}
diff --git a/crates/libsy-protocol/src/llm.rs b/crates/libsy-protocol/src/llm.rs
index 9cb3ef874..6d873e794 100644
--- a/crates/libsy-protocol/src/llm.rs
+++ b/crates/libsy-protocol/src/llm.rs
@@ -229,36 +229,6 @@ pub struct Usage {
pub reasoning_tokens: Option<u64>,
}
-/// Provider-neutral event used between stream decoders and encoders.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub enum LlmStreamEvent {
- MessageStart {
- id: Option<String>,
- model: Option<String>,
- },
- TextDelta {
- index: usize,
- text: String,
- },
- ReasoningDelta {
- index: usize,
- text: String,
- },
- ToolCallDelta {
- index: usize,
- id: Option<String>,
- name: Option<String>,
- arguments_delta: Option<String>,
- },
- Usage(Usage),
- MessageStop {
- reason: Option<String>,
- },
- Error {
- message: String,
- },
-}
-
/// Normalized reason a model stopped producing output.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub enum StopReason {
@@ -278,9 +248,12 @@ pub struct ResponseOutput {
pub stop_reason: Option<StopReason>,
}
-/// Normalized response representation shared by Switchyard components.
+/// Normalized, fully-buffered response — the aggregate of a completed generation.
+/// libsy refers to this as an `AggLlmResponse` (the terminal form of a streamed
+/// [`LlmResponse`](crate::LlmResponse)); `switchyard-translation` re-exports it as
+/// `ConversationResponse`.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
-pub struct LlmResponse {
+pub struct AggLlmResponse {
pub id: Option<String>,
pub model: Option<String>,
pub outputs: Vec<ResponseOutput>,
@@ -289,7 +262,7 @@ pub struct LlmResponse {
pub preservation: PreservationMetadata,
}
-impl LlmResponse {
+impl AggLlmResponse {
/// Returns the first output item when a response has any output.
pub fn first_output(&self) -> Option<&ResponseOutput> {
self.outputs.first()
diff --git a/crates/libsy-protocol/src/stream.rs b/crates/libsy-protocol/src/stream.rs
new file mode 100644
index 000000000..175fd1d19
--- /dev/null
+++ b/crates/libsy-protocol/src/stream.rs
@@ -0,0 +1,333 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Streaming half of the neutral IR: incremental response chunks ([`LlmResponseChunk`])
+//! and the streamed response ([`LlmResponse`]) that carries either a live stream of them
+//! or the terminal [`AggLlmResponse`].
+
+use std::collections::BTreeMap;
+use std::error::Error;
+use std::pin::Pin;
+
+use futures::{Stream, StreamExt};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use crate::llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage};
+
+/// Boxed, thread-safe error carried by a stream item.
+type BoxError = Box<dyn Error + Send + Sync>;
+
+/// A boxed, `Send` stream of [`LlmResponseChunk`]s — the token-by-token output of a
+/// streaming backend. Each item may fail independently mid-stream.
+pub type LlmResponseStream = Pin<Box<dyn Stream<Item = Result<LlmResponseChunk, BoxError>> + Send>>;
+
+/// A model response: either a live [`Stream`](LlmResponse::Stream) of chunks or the
+/// terminal buffered [`Agg`](LlmResponse::Agg)regate.
+///
+/// Not `Clone` — the `Stream` variant owns a single-consumption stream. A buffered
+/// backend returns `Agg` directly; a streaming one returns `Stream` and the consumer
+/// drives it, folding to an [`AggLlmResponse`] when it needs the whole response.
+pub enum LlmResponse {
+ Stream(LlmResponseStream),
+ Agg(AggLlmResponse),
+}
+
+impl LlmResponse {
+ /// Borrow the aggregate; `None` while this is still a stream.
+ pub fn as_agg(&self) -> Option<&AggLlmResponse> {
+ match self {
+ LlmResponse::Agg(agg) => Some(agg),
+ LlmResponse::Stream(_) => None,
+ }
+ }
+
+ /// Reduce to the buffered aggregate: return an `Agg` unchanged, or drive a `Stream`
+ /// to completion, folding its chunks into an [`AggLlmResponse`] via
+ /// [`ResponseAccumulator`]. A stream item error, or an in-band
+ /// [`LlmResponseChunk::Error`], aborts with `Err`.
+ pub async fn into_agg(self) -> Result<AggLlmResponse, BoxError> {
+ match self {
+ LlmResponse::Agg(agg) => Ok(agg),
+ LlmResponse::Stream(mut stream) => {
+ let mut accumulator = ResponseAccumulator::new();
+ while let Some(item) = stream.next().await {
+ match item? {
+ LlmResponseChunk::Error { message } => return Err(message.into()),
+ chunk => accumulator.push(chunk),
+ }
+ }
+ Ok(accumulator.finish())
+ }
+ }
+ }
+}
+
+/// One provider-neutral streaming event — the normalized counterpart to
+/// [`AggLlmResponse`](crate::AggLlmResponse), sitting between stream decoders and
+/// encoders. `switchyard-translation` re-exports it as `ConversationStreamEvent`.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub enum LlmResponseChunk {
+ MessageStart {
+ id: Option<String>,
+ model: Option<String>,
+ },
+ TextDelta {
+ index: usize,
+ text: String,
+ },
+ ReasoningDelta {
+ index: usize,
+ text: String,
+ },
+ ToolCallDelta {
+ index: usize,
+ id: Option<String>,
+ name: Option<String>,
+ arguments_delta: Option<String>,
+ },
+ Usage(Usage),
+ MessageStop {
+ reason: Option<String>,
+ },
+ Error {
+ message: String,
+ },
+}
+
+/// Folds a sequence of [`LlmResponseChunk`]s into the terminal [`AggLlmResponse`].
+///
+/// Text and reasoning deltas concatenate; tool-call deltas assemble by index (name,
+/// id, and a growing arguments string parsed as JSON at the end); `MessageStart`,
+/// `Usage`, and `MessageStop` set the corresponding fields. `Error` chunks are
+/// ignored here — a driver consuming the stream is expected to surface them.
+///
+/// Drive it by `push`-ing each chunk in order, then call [`finish`](Self::finish).
+#[derive(Default)]
+pub struct ResponseAccumulator {
+ id: Option<String>,
+ model: Option<String>,
+ text: String,
+ reasoning: Option<String>,
+ tool_calls: BTreeMap<usize, PartialToolCall>,
+ usage: Usage,
+ stop_reason: Option<StopReason>,
+}
+
+/// A tool call being assembled from streamed [`LlmResponseChunk::ToolCallDelta`]s.
+#[derive(Default)]
+struct PartialToolCall {
+ id: Option<String>,
+ name: Option<String>,
+ arguments: String,
+}
+
+impl ResponseAccumulator {
+ /// A fresh accumulator with no chunks applied.
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Apply one chunk. Later `MessageStart`/`Usage`/`MessageStop` fields overwrite
+ /// earlier ones; text, reasoning, and tool-call arguments append.
+ pub fn push(&mut self, chunk: LlmResponseChunk) {
+ match chunk {
+ LlmResponseChunk::MessageStart { id, model } => {
+ if id.is_some() {
+ self.id = id;
+ }
+ if model.is_some() {
+ self.model = model;
+ }
+ }
+ LlmResponseChunk::TextDelta { text, .. } => self.text.push_str(&text),
+ LlmResponseChunk::ReasoningDelta { text, .. } => {
+ self.reasoning
+ .get_or_insert_with(String::new)
+ .push_str(&text);
+ }
+ LlmResponseChunk::ToolCallDelta {
+ index,
+ id,
+ name,
+ arguments_delta,
+ } => {
+ let call = self.tool_calls.entry(index).or_default();
+ if id.is_some() {
+ call.id = id;
+ }
+ if name.is_some() {
+ call.name = name;
+ }
+ if let Some(delta) = arguments_delta {
+ call.arguments.push_str(&delta);
+ }
+ }
+ LlmResponseChunk::Usage(usage) => self.usage = usage,
+ LlmResponseChunk::MessageStop { reason } => {
+ self.stop_reason = Some(stop_reason_from_str(reason.as_deref()));
+ }
+ LlmResponseChunk::Error { .. } => {}
+ }
+ }
+
+ /// Build the buffered response. Content is ordered reasoning, then text, then
+ /// tool calls (by ascending delta index) — a single assistant output.
+ pub fn finish(self) -> AggLlmResponse {
+ let mut content = Vec::new();
+ if let Some(reasoning) = self.reasoning {
+ content.push(ContentBlock::Reasoning {
+ text: reasoning,
+ signature: None,
+ });
+ }
+ if !self.text.is_empty() {
+ content.push(ContentBlock::Text { text: self.text });
+ }
+ for call in self.tool_calls.into_values() {
+ content.push(ContentBlock::ToolCall(ToolCall {
+ id: call.id.unwrap_or_default(),
+ name: call.name.unwrap_or_default(),
+ arguments: parse_tool_arguments(&call.arguments),
+ }));
+ }
+ AggLlmResponse {
+ id: self.id,
+ model: self.model,
+ outputs: vec![ResponseOutput {
+ role: Role::Assistant,
+ content,
+ stop_reason: self.stop_reason,
+ }],
+ usage: self.usage,
+ ..AggLlmResponse::default()
+ }
+ }
+}
+
+/// Parse an assembled tool-call arguments string as JSON, falling back to a JSON
+/// string when it is not valid JSON and to an empty object when it is empty.
+fn parse_tool_arguments(arguments: &str) -> Value {
+ if arguments.is_empty() {
+ return Value::Object(serde_json::Map::new());
+ }
+ serde_json::from_str(arguments).unwrap_or_else(|_| Value::String(arguments.to_string()))
+}
+
+/// Map a provider stop-reason string (as carried by [`LlmResponseChunk::MessageStop`])
+/// to a normalized [`StopReason`], covering the common OpenAI and Anthropic spellings.
+fn stop_reason_from_str(reason: Option<&str>) -> StopReason {
+ match reason {
+ Some("length" | "max_tokens") => StopReason::MaxTokens,
+ Some("tool_calls" | "function_call" | "tool_use") => StopReason::ToolUse,
+ Some("content_filter") => StopReason::ContentFilter,
+ Some("stop" | "end_turn" | "stop_sequence") | None => StopReason::EndTurn,
+ Some(_) => StopReason::Unknown,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ fn fold(chunks: Vec<LlmResponseChunk>) -> AggLlmResponse {
+ let mut accumulator = ResponseAccumulator::new();
+ for chunk in chunks {
+ accumulator.push(chunk);
+ }
+ accumulator.finish()
+ }
+
+ #[test]
+ fn folds_text_usage_and_stop_reason() {
+ let agg = fold(vec![
+ LlmResponseChunk::MessageStart {
+ id: Some("id1".to_string()),
+ model: Some("m".to_string()),
+ },
+ LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "Hel".to_string(),
+ },
+ LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "lo".to_string(),
+ },
+ LlmResponseChunk::Usage(Usage {
+ output_tokens: Some(2),
+ ..Usage::default()
+ }),
+ LlmResponseChunk::MessageStop {
+ reason: Some("length".to_string()),
+ },
+ ]);
+ assert_eq!(agg.id.as_deref(), Some("id1"));
+ assert_eq!(agg.model.as_deref(), Some("m"));
+ assert_eq!(agg.usage.output_tokens, Some(2));
+ assert_eq!(agg.outputs[0].stop_reason, Some(StopReason::MaxTokens));
+ assert_eq!(
+ agg.outputs[0].content,
+ vec![ContentBlock::Text {
+ text: "Hello".to_string()
+ }]
+ );
+ }
+
+ #[test]
+ fn assembles_tool_calls_by_index() {
+ // id/name arrive once, arguments stream across deltas and parse as JSON.
+ let agg = fold(vec![
+ LlmResponseChunk::ToolCallDelta {
+ index: 0,
+ id: Some("call_1".to_string()),
+ name: Some("lookup".to_string()),
+ arguments_delta: Some("{\"q\":".to_string()),
+ },
+ LlmResponseChunk::ToolCallDelta {
+ index: 0,
+ id: None,
+ name: None,
+ arguments_delta: Some("\"rust\"}".to_string()),
+ },
+ LlmResponseChunk::MessageStop {
+ reason: Some("tool_calls".to_string()),
+ },
+ ]);
+ assert_eq!(agg.outputs[0].stop_reason, Some(StopReason::ToolUse));
+ assert_eq!(
+ agg.outputs[0].content,
+ vec![ContentBlock::ToolCall(ToolCall {
+ id: "call_1".to_string(),
+ name: "lookup".to_string(),
+ arguments: json!({"q": "rust"}),
+ })]
+ );
+ }
+
+ #[test]
+ fn reasoning_precedes_text_in_content() {
+ let agg = fold(vec![
+ LlmResponseChunk::ReasoningDelta {
+ index: 0,
+ text: "think".to_string(),
+ },
+ LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "answer".to_string(),
+ },
+ ]);
+ assert_eq!(
+ agg.outputs[0].content,
+ vec![
+ ContentBlock::Reasoning {
+ text: "think".to_string(),
+ signature: None,
+ },
+ ContentBlock::Text {
+ text: "answer".to_string(),
+ },
+ ]
+ );
+ }
+}
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index d4eced886..99a8771ee 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -11,11 +11,9 @@ so it drops into a proxy, gateway, or agent runtime.
Build a target set, pick an algorithm, run a request:
```rust
-use libsy::{
- Algorithm, ContentBlock, Context, LlmRequest, LlmClient, LlmTarget, LlmTargetSet,
- Message, Request, Role,
-};
+use libsy::{Algorithm, Context, LlmClient, LlmTarget, LlmTargetSet, Request};
use libsy_examples::llm_class::LlmClassifierOrchAlgo;
+use switchyard_protocol::{completion_text, text_request};
use std::sync::Arc;
// Targets the algorithm routes among, each backed by your LlmClient (see below).
@@ -28,16 +26,16 @@ let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(
));
let req = Request {
- llm_request: LlmRequest {
- model: Some("auto".into()),
- messages: vec![Message::text(Role::User, "explain tail latency")],
- ..LlmRequest::default()
- },
+ // `text_request` is the single-turn shortcut; build an `LlmRequest` directly for
+ // multi-turn conversations, tools, or sampling parameters.
+ llm_request: text_request(Some("auto".into()), "explain tail latency"),
raw_request: None,
metadata: None,
};
let (trace, response) = algo.clone().run(Context::default(), req).await?; // calls in, trace + response out
println!("routed to {}", trace.last().unwrap().selected_model());
+// `response.llm_response` is buffered or streamed; fold it to the aggregate to read text.
+println!("answer: {}", completion_text(&response.llm_response.aggregate().await?));
```
Runnable: [`research_agent`](../libsy-examples/examples/research_agent.rs) (in the `libsy-examples` crate).
@@ -51,18 +49,29 @@ pub struct Request {
pub metadata: Option<Metadata>, // correlation: session / agent / task / correlation_id / extra
}
+// `Response` is a plain struct. The buffered-or-streamed choice lives one level down,
+// in its `llm_response` — which *is* an enum.
pub struct Response {
pub llm_response: LlmResponse,
pub metadata: Option<Metadata>,
}
+
+pub enum LlmResponse {
+ Stream(LlmResponseStream), // a live stream of `LlmResponseChunk`s (token-by-token)
+ Agg(AggLlmResponse), // the buffered aggregate — outputs, usage, stop reason
+}
```
-`switchyard-protocol` owns `LlmRequest`, `LlmResponse`, `Message`,
-`ContentBlock`, `ResponseOutput`, and `Role`. `libsy` re-exports those canonical types
-unchanged. Construct and inspect the conversation model directly so tools, sampling
-parameters, reasoning, and provider extensions remain visible instead of being hidden
-behind a second convenience API. `raw_request` remains available when a host needs exact
-source-body fidelity.
+`switchyard-protocol` owns the conversation IR: `LlmRequest`, the buffered
+`AggLlmResponse`, the streaming `LlmResponseChunk`, and the building blocks `Message`,
+`ContentBlock`, `ResponseOutput`, and `Role`. `libsy` re-exports the envelope
+(`Request`/`Response`/`Metadata`) plus `LlmRequest`, `LlmResponse`, `AggLlmResponse`,
+`LlmResponseChunk`, and `LlmResponseStream`; import the rest (`Message`, `Role`, …) and the
+`text_request` / `text_response` / `completion_text` helpers from `switchyard_protocol`.
+Construct and inspect the conversation model directly so tools, sampling parameters,
+reasoning, and provider extensions stay visible instead of being hidden behind a second
+convenience API. `raw_request` remains available when a host needs exact source-body
+fidelity.
## Targets and clients
@@ -82,20 +91,19 @@ impl LlmClient for MyClient {
// ... POST to your endpoint, read the completion ...
let completion = String::from("provider response text");
Ok(Response {
- llm_response: LlmResponse {
- outputs: vec![ResponseOutput {
- role: Role::Assistant,
- content: vec![ContentBlock::Text { text: completion }],
- stop_reason: None,
- }],
- ..LlmResponse::default()
- },
+ // Buffered: wrap the aggregate in `LlmResponse::Agg`. `text_response` builds a
+ // single-turn `AggLlmResponse`; construct one directly for tool calls, usage, etc.
+ llm_response: LlmResponse::Agg(text_response(None, completion)),
metadata: None,
})
}
}
```
+To stream instead, return `LlmResponse::Stream(..)` — a boxed
+`Stream<Item = Result<LlmResponseChunk, _>>` — and emit chunks as they arrive from your
+upstream. See [Streaming responses](#streaming-responses) below.
+
A `RoutedRequest` bundles the `request` with the routing `decision` and the target's
`default_client`; the model to call is `decision.selected_model()`, never a mutated
request field. `semantic_name` is the label an algorithm routes by; the client maps it to
@@ -118,10 +126,51 @@ Under the hood every model call is *offloaded* to the request's `Step` stream; `
the convenience that serves each one via the target's client. The step stream is bounded,
so pulling it paces the algorithm; each run is independent, so many run concurrently.
-## Streaming — you own the model calls (`run_stream`)
+## Streaming responses
+
+A `Response` carries an `LlmResponse` that is either buffered (`Agg`) or a live token
+stream (`Stream`). An `LlmClient` — or an algorithm — chooses: return
+`LlmResponse::Agg(..)` for a whole answer, or `LlmResponse::Stream(..)` to forward tokens
+as they arrive. libsy never buffers a stream on your behalf; it flows through the algorithm
+untouched, so `run` / `run_stream` hand back whatever was produced and **the caller
+decides**:
+
+```rust
+use futures::StreamExt;
+
+let (_trace, response) = algo.clone().run(Context::default(), req).await?;
+match response.llm_response {
+ // Forward tokens as they arrive — e.g. re-encode each chunk to your client's SSE.
+ LlmResponse::Stream(mut chunks) => {
+ while let Some(chunk) = chunks.next().await {
+ let chunk: LlmResponseChunk = chunk?; // TextDelta / ToolCallDelta / Usage / MessageStop / ..
+ /* emit chunk */
+ }
+ }
+ // Already buffered — read it directly.
+ LlmResponse::Agg(agg) => println!("{}", completion_text(&agg)),
+}
+```
+
+Need the whole answer regardless of how it arrived? `aggregate()` folds a `Stream` (or
+returns an `Agg` unchanged) into a single `AggLlmResponse`, surfacing any mid-stream error:
+
+```rust
+let agg = response.llm_response.aggregate().await?; // drives the stream to completion
+println!("{}", completion_text(&agg));
+```
+
+An `LlmResponseChunk` is the provider-neutral streaming event (`MessageStart`, `TextDelta`,
+`ReasoningDelta`, `ToolCallDelta`, `Usage`, `MessageStop`, `Error`); `ResponseAccumulator`
+is the fold behind `aggregate()` if you need to assemble one yourself. Runnable end-to-end
+demo: [`streaming_agent`](../libsy-examples/examples/streaming_agent.rs).
+
+## Driving the calls yourself (`run_stream`)
`run_stream` yields `CallLlm` promises you fulfill with your own transport (or the call's
`default_client`), streams each `Decision` as it happens, and ends with `ReturnToAgent`.
+This is the *step* stream (one `Step` at a time); it is orthogonal to whether any single
+response is itself streamed — see [Streaming responses](#streaming-responses).
Runnable: [`research_agent_core`](../libsy-examples/examples/research_agent_core.rs) (in the `libsy-examples` crate).
```rust
@@ -173,27 +222,27 @@ Example — the LLM classifier (classify, then route; full version in
```rust
#[async_trait]
impl Algorithm for LlmClassifierOrchAlgo {
- async fn create_run_task(self: Arc<Self>, _ctx: Context, driver: Driver, request: Request)
+ async fn create_run_task(self: Arc<Self>, ctx: Context, driver: Driver, request: Request)
-> Result<Response, Box<dyn Error + Send + Sync>> {
+ // Thread `ctx` into every offloaded call and decision — it carries the request's
+ // cross-cutting state (correlation ids, budgets) for observers downstream.
+
// 1. Classify: ask the classifier target for a score.
let classifier = self.target_set.get_target(&self.classifier_model)?;
- driver.info(classify_decision.clone()).await?;
+ driver.info(ctx.clone(), classify_decision.clone()).await?;
let classify_response =
- driver.call_llm_target(&classifier, classify_req, classify_decision).await?;
- // Abbreviated: this assumes a text score in the first output block. The full
- // implementation scans all outputs and text-like blocks.
- let score = classify_response.llm_response.first_output()
- .and_then(|output| output.content.first())
- .and_then(|block| match block {
- ContentBlock::Text { text } => text.trim().parse::<f64>().ok(),
- _ => None,
- });
+ driver.call_llm_target(ctx.clone(), &classifier, classify_req, classify_decision).await?;
+ // Abbreviated: fold the (buffered or streamed) response to its aggregate and read
+ // the completion text as a score. `.agg()` alone is `None` for an unfolded stream.
+ let score = classify_response.llm_response.aggregate().await
+ .ok()
+ .and_then(|agg| completion_text(&agg).trim().parse::<f64>().ok());
// 2. Route: strong if score >= threshold, else weak (fail open on None).
let model = if score.map_or(true, |s| s >= self.threshold) { &self.strong_model } else { &self.weak_model };
let routed = self.target_set.get_target(model)?;
- driver.info(route_decision.clone()).await?;
- driver.call_llm_target(&routed, routed_req, route_decision).await
+ driver.info(ctx.clone(), route_decision.clone()).await?;
+ driver.call_llm_target(ctx, &routed, routed_req, route_decision).await
}
async fn process_signals(self: Arc<Self>, _s: Signals) -> Result<(), Box<dyn Error + Send + Sync>> { Ok(()) }
@@ -220,6 +269,8 @@ tested — `cargo test -p libsy-examples`).
targets, `run` (libsy makes the calls).
- [`research_agent_core`](../libsy-examples/examples/research_agent_core.rs) — client-less
targets, `run_stream` (the agent makes the calls).
+- [`streaming_agent`](../libsy-examples/examples/streaming_agent.rs) — a target that streams
+ its response; the agent forwards each `LlmResponseChunk` to the caller token-by-token.
## Not yet built
diff --git a/crates/libsy/src/driver.rs b/crates/libsy/src/driver.rs
index 410f713d9..b0bb0c770 100644
--- a/crates/libsy/src/driver.rs
+++ b/crates/libsy/src/driver.rs
@@ -52,6 +52,8 @@ use std::{
sync::{Arc, Mutex},
};
+use crate::Context;
+
use futures::{Stream, StreamExt};
use tokio::sync::{mpsc, oneshot};
use tokio_stream::wrappers::ReceiverStream;
@@ -139,7 +141,7 @@ impl TypeErasedDriver {
/// Enqueue `req` as a [`DriverStep::Request`], await the consumer's response, and
/// downcast it to `RES`. Errors if the stream is closed, the promise is dropped
/// unfulfilled, the consumer responded with `Err`, or the response was not a `RES`.
- pub async fn fulfill_request<REQ, RES>(&self, req: REQ) -> Result<RES, BoxErr>
+ pub async fn fulfill_request<REQ, RES>(&self, _ctx: Context, req: REQ) -> Result<RES, BoxErr>
where
REQ: Any + Send + 'static,
RES: Any + Send + 'static,
@@ -170,7 +172,7 @@ impl TypeErasedDriver {
/// Push a fire-and-forget [`DriverStep::Info`] payload; there is no promise to
/// await for a response. Awaits channel capacity (the consumer pacing the stream)
/// and errors only if the stream is closed.
- pub async fn info<INFO>(&self, info: INFO) -> Result<(), BoxErr>
+ pub async fn info<INFO>(&self, _ctx: Context, info: INFO) -> Result<(), BoxErr>
where
INFO: Any + Send + 'static,
{
@@ -185,7 +187,7 @@ impl TypeErasedDriver {
/// stream (that happens when every `TypeErasedDriver` clone drops); the consumer treats it
/// as the last meaningful step. Awaits channel capacity and errors only if the
/// stream is closed.
- pub async fn done<T>(&self, payload: T) -> Result<(), BoxErr>
+ pub async fn done<T>(&self, _ctx: Context, payload: T) -> Result<(), BoxErr>
where
T: Any + Send + 'static,
{
@@ -200,7 +202,7 @@ impl TypeErasedDriver {
/// failure to the consumer (mirrors how the crate's `run_stream` yields an `Err`
/// step). Awaits channel capacity and errors only if the stream is
/// already closed.
- pub async fn fail(&self, err: BoxErr) -> Result<(), BoxErr> {
+ pub async fn fail(&self, _ctx: Context, err: BoxErr) -> Result<(), BoxErr> {
self.inner
.step_tx
.send(Err(err))
@@ -244,8 +246,11 @@ mod tests {
// Producer asks for a u32 -> String on its own task.
let producer = driver.clone();
- let handle =
- tokio::spawn(async move { producer.fulfill_request::<u32, String>(7u32).await });
+ let handle = tokio::spawn(async move {
+ producer
+ .fulfill_request::<u32, String>(Context::default(), 7u32)
+ .await
+ });
tokio::pin!(stream);
match stream.next().await.ok_or("no step")?? {
@@ -265,7 +270,7 @@ mod tests {
async fn info_pushes_a_typed_payload() -> Result<(), BoxErr> {
let driver = TypeErasedDriver::new();
let stream = driver.stream();
- driver.info(42u64).await?;
+ driver.info(Context::default(), 42u64).await?;
tokio::pin!(stream);
match stream.next().await.ok_or("no step")?? {
@@ -282,7 +287,9 @@ mod tests {
async fn done_emits_the_terminal_payload() -> Result<(), BoxErr> {
let driver = TypeErasedDriver::new();
let stream = driver.stream();
- driver.done("finished".to_string()).await?;
+ driver
+ .done(Context::default(), "finished".to_string())
+ .await?;
tokio::pin!(stream);
match stream.next().await.ok_or("no step")?? {
@@ -303,7 +310,11 @@ mod tests {
let stream = driver.stream();
let producer = driver.clone();
- let handle = tokio::spawn(async move { producer.fulfill_request::<u32, u32>(1u32).await });
+ let handle = tokio::spawn(async move {
+ producer
+ .fulfill_request::<u32, u32>(Context::default(), 1u32)
+ .await
+ });
tokio::pin!(stream);
match stream.next().await.ok_or("no step")?? {
@@ -329,8 +340,11 @@ mod tests {
// Producer expects a String back.
let producer = driver.clone();
- let handle =
- tokio::spawn(async move { producer.fulfill_request::<u32, String>(1u32).await });
+ let handle = tokio::spawn(async move {
+ producer
+ .fulfill_request::<u32, String>(Context::default(), 1u32)
+ .await
+ });
tokio::pin!(stream);
match stream.next().await.ok_or("no step")?? {
@@ -356,7 +370,11 @@ mod tests {
let stream = driver.stream();
let producer = driver.clone();
- let handle = tokio::spawn(async move { producer.fulfill_request::<u32, u32>(5u32).await });
+ let handle = tokio::spawn(async move {
+ producer
+ .fulfill_request::<u32, u32>(Context::default(), 5u32)
+ .await
+ });
tokio::pin!(stream);
match stream.next().await.ok_or("no step")?? {
@@ -378,9 +396,12 @@ mod tests {
// Drop the consumer stream (and its receiver) before producing anything.
drop(driver.stream());
- assert!(driver.fulfill_request::<u32, u32>(1u32).await.is_err());
- assert!(driver.info(1u32).await.is_err());
- assert!(driver.done(1u32).await.is_err());
+ assert!(driver
+ .fulfill_request::<u32, u32>(Context::default(), 1u32)
+ .await
+ .is_err());
+ assert!(driver.info(Context::default(), 1u32).await.is_err());
+ assert!(driver.done(Context::default(), 1u32).await.is_err());
Ok(())
}
@@ -390,7 +411,11 @@ mod tests {
let stream = driver.stream();
let producer = driver.clone();
- let handle = tokio::spawn(async move { producer.fulfill_request::<u32, u32>(1u32).await });
+ let handle = tokio::spawn(async move {
+ producer
+ .fulfill_request::<u32, u32>(Context::default(), 1u32)
+ .await
+ });
tokio::pin!(stream);
match stream.next().await.ok_or("no step")?? {
@@ -415,7 +440,11 @@ mod tests {
let producer = driver.clone();
handles.push((
i,
- tokio::spawn(async move { producer.fulfill_request::<usize, usize>(i).await }),
+ tokio::spawn(async move {
+ producer
+ .fulfill_request::<usize, usize>(Context::default(), i)
+ .await
+ }),
));
}
@@ -460,7 +489,7 @@ mod tests {
async fn fail_surfaces_an_error_item_on_the_stream() -> Result<(), BoxErr> {
let driver = TypeErasedDriver::new();
let stream = driver.stream();
- driver.fail("kaboom".into()).await?;
+ driver.fail(Context::default(), "kaboom".into()).await?;
tokio::pin!(stream);
match stream.next().await.ok_or("no item")? {
@@ -484,7 +513,7 @@ mod tests {
let producer = driver.clone();
let handle = tokio::spawn(async move {
let mut sent = 0usize;
- while producer.info(sent).await.is_ok() {
+ while producer.info(Context::default(), sent).await.is_ok() {
sent += 1;
}
sent
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index b35b8117c..d30766dd8 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -64,12 +64,17 @@ use std::{error::Error, pin::Pin, sync::Arc};
use async_trait::async_trait;
use futures::{Stream, StreamExt};
-use crate::driver::{DriverRequest, DriverStep, TypeErasedDriver};
-
+/// The request/response protocol types, re-exported from [`switchyard_protocol`].
+/// [`LlmRequest`] is the normalized request; [`AggLlmResponse`] is the buffered response;
+/// [`LlmResponseChunk`] is one streaming event; [`LlmResponse`] is the streamed response
+/// (a live [`LlmResponseStream`] or the terminal aggregate).
pub use switchyard_protocol::{
- ContentBlock, LlmRequest, LlmResponse, Message, ResponseOutput, Role,
+ AggLlmResponse, Context, LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStream,
+ Metadata, Request, Response,
};
+use crate::driver::{DriverRequest, DriverStep, TypeErasedDriver};
+
/// Shorthand for the crate's boxed, thread-safe error type.
type BoxErr = Box<dyn Error + Send + Sync>;
@@ -78,37 +83,6 @@ type BoxErr = Box<dyn Error + Send + Sync>;
/// `Arc<dyn Algorithm>` object-safe.
pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step, BoxErr>> + Send>>;
-/// Correlation and routing metadata attached to a request or response.
-///
-/// All fields are optional; algorithms and observers use whichever are present
-/// (e.g. to key per-session state or emit correlated telemetry). `extra_metadata`
-/// is a free-form escape hatch for host-specific keys.
-#[derive(Clone)]
-pub struct Metadata {
- /// Stable id for a multi-request session/conversation.
- pub session_id: Option<String>,
- /// Id of the agent making the request.
- pub agent_id: Option<String>,
- /// Id of the task the request belongs to.
- pub task_id: Option<String>,
- /// External trace/request id for joining with the host's telemetry.
- pub correlation_id: Option<String>,
- /// Arbitrary host-defined key/value metadata.
- pub extra_metadata: Option<std::collections::BTreeMap<String, String>>,
-}
-
-/// A request entering the orchestrator: the normalized [`LlmRequest`] plus the
-/// original provider payload and correlation [`Metadata`].
-#[derive(Clone)]
-pub struct Request {
- /// The normalized request an algorithm routes.
- pub llm_request: LlmRequest,
- /// Optional source provider body, carried only for hosts that need exact fidelity.
- pub raw_request: Option<serde_json::Value>,
- /// Correlation metadata carried through the request.
- pub metadata: Option<Metadata>,
-}
-
/// Agentic-stack events fed to an algorithm out of band via
/// [`Algorithm::process_signals`] (e.g. tool results, budget updates).
///
@@ -117,16 +91,6 @@ pub struct Request {
#[derive(Clone)]
pub struct Signals {}
-/// A response leaving the orchestrator: the neutral [`LlmResponse`] plus optional
-/// correlation [`Metadata`].
-#[derive(Clone)]
-pub struct Response {
- /// The neutral model response.
- pub llm_response: LlmResponse,
- /// Correlation metadata carried through the response.
- pub metadata: Option<Metadata>,
-}
-
/// A decision/trace object produced by an algorithm.
///
/// Carried as a trait object (not a generic parameter) so a stream consumer can
@@ -148,11 +112,12 @@ pub trait Decision: Send + Sync {
///
/// The two model identifiers live in separate, unambiguous places: the model to
/// call is [`decision.selected_model()`](Decision::selected_model), while
-/// `request.llm_request.model` is the inbound name the agent asked for. A client
-/// maps `selected_model()` to the provider model id it hits.
+/// `request.llm_request.model` is the *inbound* name the agent asked for (libsy
+/// never overwrites it). A client maps `selected_model()` to the provider model
+/// id it hits.
#[derive(Clone)]
pub struct RoutedRequest {
- /// The request to serve; its `model` is the agent's original inbound model.
+ /// The request to serve; its `model` is the agent's original name.
pub request: Request,
/// The routing decision behind this call; `selected_model()` is the model to hit.
pub decision: Arc<dyn Decision>,
@@ -171,22 +136,27 @@ pub struct RoutedRequest {
/// [`Driver::call_llm`] on the other side.
pub struct CallLlmRequest {
inner: DriverRequest,
+ routed: RoutedRequest,
}
impl CallLlmRequest {
- /// Wrap a driver request whose payload is a [`RoutedRequest`].
+ /// Wrap a driver request whose payload is a [`RoutedRequest`]. Caches an owned copy
+ /// so the accessors are plain field reads.
fn new(inner: DriverRequest) -> Self {
- Self { inner }
+ // The payload is always a `RoutedRequest` (set by `Driver::call_llm`); a
+ // mismatch would be a libsy bug, not a runtime condition.
+ let routed = match inner.request::<RoutedRequest>() {
+ Ok(routed) => routed.clone(),
+ Err(_) => unreachable!("CallLlmRequest payload is always a RoutedRequest"),
+ };
+ Self { inner, routed }
}
/// The routed request the host should serve. Its
/// [`default_client`](RoutedRequest::default_client) serves the call by default,
/// and its `decision.selected_model()` names the model to hit.
pub fn get_routed(&self) -> &RoutedRequest {
- match self.inner.request::<RoutedRequest>() {
- Ok(routed) => routed,
- Err(_) => unreachable!("CallLlmRequest payload is always a RoutedRequest"),
- }
+ &self.routed
}
/// The model request to perform (the [`Request`] inside the routed request).
@@ -229,9 +199,9 @@ impl Driver {
/// Offload a model call: publish `routed` as a [`Step::CallLlm`] and await the
/// consumer's [`Response`]. Errors if the stream is closed or the call failed.
- pub async fn call_llm(&self, routed: RoutedRequest) -> Result<Response, BoxErr> {
+ pub async fn call_llm(&self, ctx: Context, routed: RoutedRequest) -> Result<Response, BoxErr> {
self.driver
- .fulfill_request::<RoutedRequest, Response>(routed)
+ .fulfill_request::<RoutedRequest, Response>(ctx, routed)
.await
}
@@ -239,33 +209,41 @@ impl Driver {
/// default client into a [`RoutedRequest`], then publish it (see
/// [`call_llm`](Self::call_llm)). The convenience most algorithms use;
/// `decision.selected_model()` names the model to hit, and `request`'s
- /// inbound `model` is left untouched.
+ /// `model` is left untouched.
pub async fn call_llm_target(
&self,
+ ctx: Context,
target: &LlmTarget,
request: Request,
decision: Arc<dyn Decision>,
) -> Result<Response, BoxErr> {
- self.call_llm(RoutedRequest {
- request,
- decision,
- default_client: target.llm_client.clone(),
- })
+ self.call_llm(
+ ctx,
+ RoutedRequest {
+ request,
+ decision,
+ default_client: target.llm_client.clone(),
+ },
+ )
.await
}
/// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.
- pub async fn info(&self, decision: Arc<dyn Decision>) -> Result<(), BoxErr> {
- self.driver.info(decision).await
+ pub async fn info(&self, ctx: Context, decision: Arc<dyn Decision>) -> Result<(), BoxErr> {
+ self.driver.info(ctx, decision).await
}
/// Emit the terminal step: [`Step::ReturnToAgent`] on `Ok`, or an `Err` stream
/// item on failure. Internal: called once by [`run_stream`](Algorithm::run_stream)
/// when the algorithm finishes.
- pub(crate) async fn finish(&self, result: Result<Response, BoxErr>) -> Result<(), BoxErr> {
+ pub(crate) async fn finish(
+ &self,
+ ctx: Context,
+ result: Result<Response, BoxErr>,
+ ) -> Result<(), BoxErr> {
match result {
- Ok(response) => self.driver.done(response).await,
- Err(err) => self.driver.fail(err).await,
+ Ok(response) => self.driver.done(ctx, response).await,
+ Err(err) => self.driver.fail(ctx, err).await,
}
}
@@ -274,7 +252,7 @@ impl Driver {
/// payload that does not match the expected type for its step becomes an `Err` item.
pub(crate) fn stream(&self) -> impl Stream<Item = Result<Step, BoxErr>> {
self.driver.stream().map(|item| match item? {
- DriverStep::Request(req) => Ok(Step::CallLlm(CallLlmRequest::new(req))),
+ DriverStep::Request(req) => Ok(Step::CallLlm(Box::new(CallLlmRequest::new(req)))),
DriverStep::Info(payload) => payload
.downcast::<Arc<dyn Decision>>()
.map(|decision| Step::Decision(*decision))
@@ -293,25 +271,12 @@ impl Default for Driver {
}
}
-/// Per-request state threaded to an algorithm alongside its [`Driver`]. A placeholder
-/// for cross-cutting state (correlation ids, budgets, deadlines) an algorithm will
-/// read; empty today. It does not carry the offload driver, so it is safe to share.
-#[derive(Clone, Default)]
-pub struct Context {}
-
-impl Context {
- /// Build an empty context.
- pub fn new() -> Self {
- Self {}
- }
-}
-
/// One item in the stream returned by [`Driver::stream`] / [`Algorithm::run_stream`].
pub enum Step {
/// The algorithm needs this model call performed. The host serves it (optionally
/// via [`RoutedRequest::default_client`]) and fulfills it with
- /// [`CallLlmRequest::respond`].
- CallLlm(CallLlmRequest),
+ /// [`CallLlmRequest::respond`]. Boxed: it is by far the largest variant.
+ CallLlm(Box<CallLlmRequest>),
/// A routing decision the algorithm made, published via [`Driver::info`] as it
/// happens (rather than collected into a trace returned at the end).
Decision(Arc<dyn Decision>),
@@ -329,7 +294,7 @@ pub trait LlmClient: Send + Sync {
/// [`routed.decision.selected_model()`](Decision::selected_model) — the target
/// the algorithm routed to — mapping it to whatever provider model id this
/// client hits. `routed.request.llm_request.model` is the agent's original
- /// inbound name, carried through for reference, not a call target.
+ /// name, carried through for reference, not a call target.
async fn call(&self, request: RoutedRequest) -> Result<Response, Box<dyn Error + Send + Sync>>;
}
@@ -419,8 +384,10 @@ pub trait Algorithm: Send + Sync + 'static {
let driver = Driver::new();
let stream = driver.stream();
tokio::spawn(async move {
- let outcome = self.create_run_task(ctx, driver.clone(), request).await;
- let _ = driver.finish(outcome).await;
+ let outcome = self
+ .create_run_task(ctx.clone(), driver.clone(), request)
+ .await;
+ let _ = driver.finish(ctx, outcome).await;
});
Box::pin(stream)
}
@@ -430,8 +397,11 @@ pub trait Algorithm: Send + Sync + 'static {
/// [`Response`]. Provided: drives [`run_stream`](Self::run_stream) internally,
/// collecting each [`Step::Decision`]. Use it when the algorithm holds its own model
/// clients and the host wants the answer (and the decisions behind it); drive
- /// [`run_stream`](Self::run_stream) instead to serve the calls yourself. Errors
- /// if a routed target has no client to serve its call, or the algorithm fails.
+ /// [`run_stream`](Self::run_stream) instead to serve the calls yourself. The returned
+ /// [`Response`] is whatever the algorithm produced — a buffered [`LlmResponse::Agg`]
+ /// or a live [`LlmResponse::Stream`]; the caller decides whether to
+ /// [`aggregate`](LlmResponse::aggregate) it or forward its chunks. Errors if a routed
+ /// target has no client to serve its call, or the algorithm fails.
async fn run(
self: Arc<Self>,
ctx: Context,
@@ -472,7 +442,7 @@ pub trait Algorithm: Send + Sync + 'static {
match step {
None => stream_open = false,
Some(item) => match item? {
- Step::CallLlm(call) => in_flight.push(serve(call)),
+ Step::CallLlm(call) => in_flight.push(serve(*call)),
Step::Decision(decision) => trace.push(decision),
Step::ReturnToAgent(response) => {
final_response = Some(*response);
@@ -484,6 +454,8 @@ pub trait Algorithm: Send + Sync + 'static {
}
}
+ // Return whatever the algorithm produced — buffered or streamed. The caller
+ // decides whether to aggregate the response or forward its chunks.
final_response
.map(|response| (trace, response))
.ok_or_else(|| "run: stream ended without a final response".into())
@@ -494,6 +466,7 @@ pub trait Algorithm: Send + Sync + 'static {
mod tests {
use super::*;
use futures::StreamExt;
+ use switchyard_protocol::{completion_text, text_request, text_response};
/// Mock client that echoes back the target name it was called with.
struct EchoClient;
@@ -506,10 +479,10 @@ mod tests {
) -> Result<Response, Box<dyn Error + Send + Sync>> {
// Echo back the model the algorithm routed to (the decision's selection).
Ok(Response {
- llm_response: LlmResponse {
- model: Some(routed.decision.selected_model().to_string()),
- ..LlmResponse::default()
- },
+ llm_response: LlmResponse::Agg(text_response(
+ None,
+ routed.decision.selected_model().to_string(),
+ )),
metadata: None,
})
}
@@ -541,7 +514,7 @@ mod tests {
impl Algorithm for TestAlgo {
async fn create_run_task(
self: Arc<Self>,
- _ctx: Context,
+ ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response, Box<dyn Error + Send + Sync>> {
@@ -554,8 +527,10 @@ mod tests {
let decision: Arc<dyn Decision> = Arc::new(TestDecision {
model: target.semantic_name.clone(),
});
- driver.info(decision.clone()).await?;
- driver.call_llm_target(&target, request, decision).await
+ driver.info(ctx.clone(), decision.clone()).await?;
+ driver
+ .call_llm_target(ctx, &target, request, decision)
+ .await
}
async fn process_signals(
@@ -573,11 +548,7 @@ mod tests {
fn request() -> Request {
Request {
- llm_request: LlmRequest {
- model: Some("auto".to_string()),
- messages: vec![Message::text(Role::User, "hi")],
- ..LlmRequest::default()
- },
+ llm_request: text_request(Some("auto".to_string()), "hi".to_string()),
raw_request: None,
metadata: None,
}
@@ -595,6 +566,90 @@ mod tests {
LlmTargetSet::new(targets)
}
+ /// Client that serves a call as a token stream — its `call` returns
+ /// [`LlmResponse::Stream`] replaying `chunks` in order (as `Ok` items).
+ struct StreamingClient {
+ chunks: Vec<LlmResponseChunk>,
+ }
+
+ #[async_trait]
+ impl LlmClient for StreamingClient {
+ async fn call(
+ &self,
+ _routed: RoutedRequest,
+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {
+ let stream = futures::stream::iter(self.chunks.clone().into_iter().map(Ok)).boxed();
+ Ok(Response {
+ llm_response: LlmResponse::Stream(stream),
+ metadata: None,
+ })
+ }
+ }
+
+ /// Build a single-target algo whose one target streams `chunks`.
+ fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> Arc<dyn Algorithm> {
+ let target = LlmTarget {
+ semantic_name: "stream/model".to_string(),
+ llm_client: Some(Arc::new(StreamingClient { chunks }) as Arc<dyn LlmClient>),
+ };
+ orch(LlmTargetSet::new(vec![target]))
+ }
+
+ #[tokio::test]
+ async fn run_returns_a_streamed_response_the_caller_aggregates(
+ ) -> Result<(), Box<dyn Error + Send + Sync>> {
+ // A streaming client -> its chunks flow through the promise and `ReturnToAgent`,
+ // and `run` returns the live stream untouched for the caller to fold.
+ let orch = streaming_orch(vec![
+ LlmResponseChunk::MessageStart {
+ id: Some("m1".to_string()),
+ model: Some("stream/model".to_string()),
+ },
+ LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "hel".to_string(),
+ },
+ LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "lo".to_string(),
+ },
+ LlmResponseChunk::MessageStop {
+ reason: Some("stop".to_string()),
+ },
+ ]);
+ let (trace, response) = orch.run(Context::default(), request()).await?;
+ // `run` handed back the live stream; the caller folds it to a buffered aggregate.
+ let agg = response.llm_response.into_agg().await?;
+ assert_eq!(completion_text(&agg), "hello");
+ assert_eq!(agg.model.as_deref(), Some("stream/model"));
+ assert_eq!(trace.len(), 1);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn aggregating_a_streamed_response_propagates_a_mid_stream_error(
+ ) -> Result<(), Box<dyn Error + Send + Sync>> {
+ // `run` succeeds and returns the stream; the in-band `Error` chunk surfaces only
+ // when the caller aggregates it.
+ let orch = streaming_orch(vec![
+ LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "partial".to_string(),
+ },
+ LlmResponseChunk::Error {
+ message: "upstream exploded".to_string(),
+ },
+ ]);
+ let (_, response) = orch.run(Context::default(), request()).await?;
+ match response.llm_response.into_agg().await {
+ Ok(_) => Err("expected a mid-stream error, got an aggregate".into()),
+ Err(err) => {
+ assert!(err.to_string().contains("upstream exploded"));
+ Ok(())
+ }
+ }
+ }
+
#[tokio::test]
async fn run_offloads_via_promise_then_returns_to_agent(
) -> Result<(), Box<dyn Error + Send + Sync>> {
@@ -614,10 +669,10 @@ mod tests {
assert_eq!(call.get_decision().selected_model(), "offload/model");
// Fulfilling the promise is the "real" model call the caller makes.
call.respond(Ok(Response {
- llm_response: LlmResponse {
- model: Some("fulfilled".to_string()),
- ..LlmResponse::default()
- },
+ llm_response: LlmResponse::Agg(text_response(
+ None,
+ "fulfilled".to_string(),
+ )),
metadata: None,
}))?;
}
@@ -625,7 +680,13 @@ mod tests {
assert_eq!(decision.selected_model(), "offload/model");
}
Step::ReturnToAgent(response) => {
- final_completion = response.llm_response.model.clone();
+ final_completion = Some(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ );
}
}
}
@@ -661,7 +722,13 @@ mod tests {
}
Step::Decision(_) => {}
Step::ReturnToAgent(response) => {
- final_completion = response.llm_response.model.clone();
+ final_completion = Some(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ );
}
}
}
@@ -680,7 +747,14 @@ mod tests {
.run(Context::default(), request())
.await?;
// TestAlgo calls the first target; EchoClient echoes its name.
- assert_eq!(response.llm_response.model.as_deref(), Some("direct/model"));
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "direct/model"
+ );
assert_eq!(trace[0].selected_model(), "direct/model");
Ok(())
}
@@ -720,10 +794,10 @@ mod tests {
) -> Result<Response, Box<dyn Error + Send + Sync>> {
self.barrier.wait().await;
Ok(Response {
- llm_response: LlmResponse {
- model: Some(routed.decision.selected_model().to_string()),
- ..LlmResponse::default()
- },
+ llm_response: LlmResponse::Agg(text_response(
+ None,
+ routed.decision.selected_model().to_string(),
+ )),
metadata: None,
})
}
@@ -745,7 +819,13 @@ mod tests {
handles.push(tokio::spawn(async move {
algo.run(Context::default(), request())
.await
- .map(|(_, response)| response.llm_response.model.unwrap_or_default())
+ .map(|(_, response)| {
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default()
+ })
}));
}
@@ -823,16 +903,10 @@ mod tests {
drop(permit);
self.current.fetch_sub(1, Ordering::SeqCst);
Ok(Response {
- llm_response: LlmResponse {
- outputs: vec![ResponseOutput {
- role: Role::Assistant,
- content: vec![ContentBlock::Text {
- text: routed.decision.selected_model().to_string(),
- }],
- stop_reason: None,
- }],
- ..LlmResponse::default()
- },
+ llm_response: LlmResponse::Agg(text_response(
+ None,
+ routed.decision.selected_model().to_string(),
+ )),
metadata: None,
})
}
@@ -848,32 +922,26 @@ mod tests {
impl Algorithm for FanOut {
async fn create_run_task(
self: Arc<Self>,
- _ctx: Context,
+ ctx: Context,
driver: Driver,
_request: Request,
) -> Result<Response, Box<dyn Error + Send + Sync>> {
let calls = (0..self.n).map(|i| {
let driver = driver.clone();
+ let ctx = ctx.clone();
let target = self.target.clone();
async move {
let decision: Arc<dyn Decision> = Arc::new(TestDecision {
model: format!("m{i}"),
});
- driver.call_llm_target(&target, request(), decision).await
+ driver
+ .call_llm_target(ctx, &target, request(), decision)
+ .await
}
});
futures::future::join_all(calls).await;
Ok(Response {
- llm_response: LlmResponse {
- outputs: vec![ResponseOutput {
- role: Role::Assistant,
- content: vec![ContentBlock::Text {
- text: "done".to_string(),
- }],
- stop_reason: None,
- }],
- ..LlmResponse::default()
- },
+ llm_response: LlmResponse::Agg(text_response(None, "done".to_string())),
metadata: None,
})
}
@@ -934,11 +1002,10 @@ mod tests {
assert_eq!(
response
.llm_response
- .first_output()
- .and_then(|output| output.content.first()),
- Some(&ContentBlock::Text {
- text: "done".to_string()
- })
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "done"
);
assert_eq!(
max.load(Ordering::SeqCst),
diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
index 453292ba5..0542082f6 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
@@ -14,8 +14,8 @@ use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
use crate::llm::{
- ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, LlmResponse, MediaSource,
- Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,
+ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest,
+ MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,
SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,
};
use crate::policy::{DeterministicIdPolicy, TranslationPolicy};
@@ -252,7 +252,7 @@ impl FormatCodec for AnthropicMessagesCodec {
text: String::new(),
});
}
- let response = LlmResponse {
+ let response = AggLlmResponse {
id: body
.get("id")
.and_then(Value::as_str)
@@ -297,7 +297,7 @@ impl FormatCodec for AnthropicMessagesCodec {
fn encode_response(
&self,
- response: &LlmResponse,
+ response: &AggLlmResponse,
_policy: &TranslationPolicy,
) -> Result<EncodedResponse> {
if let Some(body) = exact_preserved_response(
diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
index 215173b4b..077152752 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
@@ -10,8 +10,8 @@ use crate::codecs::stream::{
StreamCodec, StreamTranslationState,
};
use crate::format::{FormatId, WireFormat};
-use crate::llm::LlmStreamEvent;
use crate::util::sanitize_anthropic_tool_use_id;
+use crate::LlmResponseChunk;
/// Stream codec for Anthropic Messages events.
pub struct AnthropicMessagesStreamCodec;
@@ -25,14 +25,14 @@ impl StreamCodec for AnthropicMessagesStreamCodec {
&self,
state: &mut StreamTranslationState,
event: &Value,
- ) -> Vec<LlmStreamEvent> {
+ ) -> Vec<LlmResponseChunk> {
decode_anthropic_stream(state, event)
}
fn encode_event(
&self,
state: &mut StreamTranslationState,
- event: LlmStreamEvent,
+ event: LlmResponseChunk,
) -> Vec<Value> {
encode_anthropic_stream(state, event)
}
@@ -46,9 +46,9 @@ impl StreamCodec for AnthropicMessagesStreamCodec {
fn decode_anthropic_stream(
state: &mut StreamTranslationState,
event: &Value,
-) -> Vec<LlmStreamEvent> {
+) -> Vec<LlmResponseChunk> {
let Some(object) = event.as_object() else {
- return vec![LlmStreamEvent::Error {
+ return vec![LlmResponseChunk::Error {
message: "Anthropic stream event is not an object".to_string(),
}];
};
@@ -73,7 +73,7 @@ fn decode_anthropic_stream(
capture_anthropic_usage(state, usage);
}
}
- vec![LlmStreamEvent::MessageStart {
+ vec![LlmResponseChunk::MessageStart {
id: state.message_id.clone(),
model: state.model.clone(),
}]
@@ -84,7 +84,7 @@ fn decode_anthropic_stream(
let mut out = Vec::new();
if let Some(usage) = object.get("usage") {
capture_anthropic_usage(state, usage);
- out.push(LlmStreamEvent::Usage(state.usage.clone()));
+ out.push(LlmResponseChunk::Usage(state.usage.clone()));
}
if let Some(stop_reason) = object
.get("delta")
@@ -92,14 +92,22 @@ fn decode_anthropic_stream(
.and_then(|delta| delta.get("stop_reason"))
.and_then(Value::as_str)
{
- out.push(LlmStreamEvent::MessageStop {
+ // Remember the provider stop reason: Anthropic delivers it here, on
+ // `message_delta`, while the terminal `message_stop` carries none of its own.
+ state.stop_reason = Some(stop_reason.to_string());
+ out.push(LlmResponseChunk::MessageStop {
reason: Some(stop_reason.to_string()),
});
}
out
}
- Some("message_stop") => vec![LlmStreamEvent::MessageStop { reason: None }],
- Some("error") => vec![LlmStreamEvent::Error {
+ // Anthropic's terminal `message_stop` carries no reason of its own; replay the one
+ // remembered from `message_delta` so a chunk accumulator keeps the real stop reason
+ // (e.g. `max_tokens`) instead of overwriting it with a reasonless `EndTurn`.
+ Some("message_stop") => vec![LlmResponseChunk::MessageStop {
+ reason: state.stop_reason.clone(),
+ }],
+ Some("error") => vec![LlmResponseChunk::Error {
message: object
.get("error")
.and_then(Value::as_object)
@@ -115,10 +123,10 @@ fn decode_anthropic_stream(
// Encodes neutral streaming events into Anthropic Messages events.
fn encode_anthropic_stream(
state: &mut StreamTranslationState,
- event: LlmStreamEvent,
+ event: LlmResponseChunk,
) -> Vec<Value> {
match event {
- LlmStreamEvent::MessageStart { id, model } => {
+ LlmResponseChunk::MessageStart { id, model } => {
record_source_identity(state, id, model);
if state.emitted_message_start {
Vec::new()
@@ -139,7 +147,7 @@ fn encode_anthropic_stream(
})]
}
}
- LlmStreamEvent::TextDelta { text, .. } => {
+ LlmResponseChunk::TextDelta { text, .. } => {
state.output_tokens_seen += 1;
let mut out = ensure_anthropic_text_block(state);
out.push(json!({
@@ -149,7 +157,7 @@ fn encode_anthropic_stream(
}));
out
}
- LlmStreamEvent::ReasoningDelta { text, .. } => {
+ LlmResponseChunk::ReasoningDelta { text, .. } => {
let mut out = ensure_anthropic_reasoning_block(state);
out.push(json!({
"type": "content_block_delta",
@@ -158,22 +166,22 @@ fn encode_anthropic_stream(
}));
out
}
- LlmStreamEvent::ToolCallDelta {
+ LlmResponseChunk::ToolCallDelta {
index,
id,
name,
arguments_delta,
} => encode_anthropic_tool_delta(state, index, id, name, arguments_delta),
- LlmStreamEvent::Usage(usage) => {
+ LlmResponseChunk::Usage(usage) => {
state.usage = usage;
state.saw_backend_usage = true;
Vec::new()
}
- LlmStreamEvent::MessageStop { reason } => {
+ LlmResponseChunk::MessageStop { reason } => {
state.stop_reason = reason.or_else(|| state.stop_reason.clone());
Vec::new()
}
- LlmStreamEvent::Error { message } => {
+ LlmResponseChunk::Error { message } => {
vec![json!({"type": "error", "error": {"message": message}})]
}
}
@@ -185,7 +193,7 @@ fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec<Value> {
if !state.emitted_message_start {
out.extend(encode_anthropic_stream(
state,
- LlmStreamEvent::MessageStart {
+ LlmResponseChunk::MessageStart {
id: state.message_id.clone(),
model: state.model.clone(),
},
@@ -233,7 +241,7 @@ fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec<Value> {
}
// Converts Anthropic content-block starts into text or tool-call deltas.
-fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmStreamEvent> {
+fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmResponseChunk> {
let index = object.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
let block = object.get("content_block").and_then(Value::as_object);
match block
@@ -245,7 +253,7 @@ fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmS
.and_then(Value::as_str)
.filter(|text| !text.is_empty())
.map(|text| {
- vec![LlmStreamEvent::TextDelta {
+ vec![LlmResponseChunk::TextDelta {
index,
text: text.to_string(),
}]
@@ -256,7 +264,7 @@ fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmS
.and_then(Value::as_str)
.filter(|text| !text.is_empty())
.map(|text| {
- vec![LlmStreamEvent::ReasoningDelta {
+ vec![LlmResponseChunk::ReasoningDelta {
index,
text: text.to_string(),
}]
@@ -266,7 +274,7 @@ fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmS
let Some(block) = block else {
return Vec::new();
};
- vec![LlmStreamEvent::ToolCallDelta {
+ vec![LlmResponseChunk::ToolCallDelta {
index,
id: block
.get("id")
@@ -284,7 +292,7 @@ fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmS
}
// Converts Anthropic content-block deltas into neutral text or argument deltas.
-fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmStreamEvent> {
+fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmResponseChunk> {
let index = object.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
let Some(delta) = object.get("delta").and_then(Value::as_object) else {
return Vec::new();
@@ -294,7 +302,7 @@ fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmS
.get("text")
.and_then(Value::as_str)
.map(|text| {
- vec![LlmStreamEvent::TextDelta {
+ vec![LlmResponseChunk::TextDelta {
index,
text: text.to_string(),
}]
@@ -304,7 +312,7 @@ fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmS
.get("thinking")
.and_then(Value::as_str)
.map(|text| {
- vec![LlmStreamEvent::ReasoningDelta {
+ vec![LlmResponseChunk::ReasoningDelta {
index,
text: text.to_string(),
}]
@@ -315,7 +323,7 @@ fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmS
.get("partial_json")
.and_then(Value::as_str)
.map(|partial_json| {
- vec![LlmStreamEvent::ToolCallDelta {
+ vec![LlmResponseChunk::ToolCallDelta {
index,
id: None,
name: None,
diff --git a/crates/switchyard-translation/src/codecs/mod.rs b/crates/switchyard-translation/src/codecs/mod.rs
index fbcda4ab1..78d3a007c 100644
--- a/crates/switchyard-translation/src/codecs/mod.rs
+++ b/crates/switchyard-translation/src/codecs/mod.rs
@@ -14,7 +14,7 @@ use serde_json::Value;
use crate::diagnostic::TranslationDiagnostic;
use crate::error::Result;
use crate::format::FormatId;
-use crate::llm::{LlmRequest, LlmResponse};
+use crate::llm::{AggLlmResponse, LlmRequest};
use crate::policy::TranslationPolicy;
/// Result of decoding a request into neutral IR.
@@ -31,7 +31,7 @@ pub struct EncodedRequest {
/// Result of decoding a response into neutral IR.
pub struct DecodedResponse {
- pub response: LlmResponse,
+ pub response: AggLlmResponse,
pub diagnostics: Vec<TranslationDiagnostic>,
}
@@ -62,7 +62,7 @@ pub trait FormatCodec: Send + Sync {
/// Encodes a neutral response into provider JSON.
fn encode_response(
&self,
- response: &LlmResponse,
+ response: &AggLlmResponse,
policy: &TranslationPolicy,
) -> Result<EncodedResponse>;
}
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
index fbcd1ad42..c6d823102 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
@@ -15,8 +15,8 @@ use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
use crate::llm::{
- ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, LlmResponse, MediaSource,
- Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,
+ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest,
+ MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,
SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,
};
use crate::policy::{DeterministicIdPolicy, TranslationPolicy};
@@ -246,7 +246,7 @@ impl FormatCodec for OpenAiChatCodec {
_policy: &TranslationPolicy,
) -> Result<DecodedResponse> {
let object = object(body, "$")?;
- let mut response = LlmResponse {
+ let mut response = AggLlmResponse {
id: object
.get("id")
.and_then(Value::as_str)
@@ -313,7 +313,7 @@ impl FormatCodec for OpenAiChatCodec {
fn encode_response(
&self,
- response: &LlmResponse,
+ response: &AggLlmResponse,
_policy: &TranslationPolicy,
) -> Result<EncodedResponse> {
if let Some(body) =
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
index 501b40de7..d2e5f6e74 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
@@ -10,7 +10,8 @@ use crate::codecs::stream::{
StreamTranslationState,
};
use crate::format::{FormatId, WireFormat};
-use crate::llm::{LlmStreamEvent, Usage};
+use crate::llm::Usage;
+use crate::LlmResponseChunk;
/// Stream codec for OpenAI Chat Completions chunks.
pub struct OpenAiChatStreamCodec;
@@ -24,14 +25,14 @@ impl StreamCodec for OpenAiChatStreamCodec {
&self,
state: &mut StreamTranslationState,
event: &Value,
- ) -> Vec<LlmStreamEvent> {
+ ) -> Vec<LlmResponseChunk> {
decode_openai_chat_stream(state, event)
}
fn encode_event(
&self,
state: &mut StreamTranslationState,
- event: LlmStreamEvent,
+ event: LlmResponseChunk,
) -> Vec<Value> {
encode_openai_chat_stream(state, event)
}
@@ -45,9 +46,9 @@ impl StreamCodec for OpenAiChatStreamCodec {
fn decode_openai_chat_stream(
state: &mut StreamTranslationState,
event: &Value,
-) -> Vec<LlmStreamEvent> {
+) -> Vec<LlmResponseChunk> {
let Some(object) = event.as_object() else {
- return vec![LlmStreamEvent::Error {
+ return vec![LlmResponseChunk::Error {
message: "OpenAI stream event is not an object".to_string(),
}];
};
@@ -61,7 +62,7 @@ fn decode_openai_chat_stream(
if let Some(id) = string_field(object, "id") {
state.message_id = Some(id);
}
- out.push(LlmStreamEvent::MessageStart {
+ out.push(LlmResponseChunk::MessageStart {
id: state.message_id.clone(),
model: state.model.clone(),
});
@@ -72,7 +73,7 @@ fn decode_openai_chat_stream(
capture_openai_usage_extras(state, object.get("usage"));
state.usage = usage.clone();
state.saw_backend_usage = true;
- out.push(LlmStreamEvent::Usage(usage));
+ out.push(LlmResponseChunk::Usage(usage));
}
for choice in object
@@ -87,7 +88,7 @@ fn decode_openai_chat_stream(
if let Some(delta) = choice.get("delta").and_then(Value::as_object) {
if let Some(text) = delta.get("content").and_then(Value::as_str) {
if !text.is_empty() {
- out.push(LlmStreamEvent::TextDelta {
+ out.push(LlmResponseChunk::TextDelta {
index: 0,
text: text.to_string(),
});
@@ -96,7 +97,7 @@ fn decode_openai_chat_stream(
for reasoning_key in ["reasoning_content", "reasoning"] {
if let Some(text) = delta.get(reasoning_key).and_then(Value::as_str) {
if !text.is_empty() {
- out.push(LlmStreamEvent::ReasoningDelta {
+ out.push(LlmResponseChunk::ReasoningDelta {
index: 0,
text: text.to_string(),
});
@@ -107,7 +108,7 @@ fn decode_openai_chat_stream(
for tool_call in tool_calls {
if let Some(tool_call) = tool_call.as_object() {
let function = tool_call.get("function").and_then(Value::as_object);
- out.push(LlmStreamEvent::ToolCallDelta {
+ out.push(LlmResponseChunk::ToolCallDelta {
index: tool_call.get("index").and_then(Value::as_u64).unwrap_or(0)
as usize,
id: tool_call
@@ -128,7 +129,7 @@ fn decode_openai_chat_stream(
}
}
if let Some(reason) = choice.get("finish_reason").and_then(Value::as_str) {
- out.push(LlmStreamEvent::MessageStop {
+ out.push(LlmResponseChunk::MessageStop {
reason: Some(reason.to_string()),
});
}
@@ -139,10 +140,10 @@ fn decode_openai_chat_stream(
// Encodes neutral streaming events into OpenAI Chat chunks.
fn encode_openai_chat_stream(
state: &mut StreamTranslationState,
- event: LlmStreamEvent,
+ event: LlmResponseChunk,
) -> Vec<Value> {
match event {
- LlmStreamEvent::MessageStart { id, model } => {
+ LlmResponseChunk::MessageStart { id, model } => {
record_source_identity(state, id, model);
if state.emitted_message_start
|| (!state_source_is(state, WireFormat::AnthropicMessages)
@@ -159,7 +160,7 @@ fn encode_openai_chat_stream(
)]
}
}
- LlmStreamEvent::TextDelta { text, .. } => {
+ LlmResponseChunk::TextDelta { text, .. } => {
vec![openai_stream_chunk(
state,
json!({"content": text}),
@@ -167,7 +168,7 @@ fn encode_openai_chat_stream(
None,
)]
}
- LlmStreamEvent::ReasoningDelta { text, .. } => {
+ LlmResponseChunk::ReasoningDelta { text, .. } => {
vec![openai_stream_chunk(
state,
json!({"reasoning_content": text}),
@@ -175,7 +176,7 @@ fn encode_openai_chat_stream(
None,
)]
}
- LlmStreamEvent::ToolCallDelta {
+ LlmResponseChunk::ToolCallDelta {
index,
id,
name,
@@ -187,12 +188,12 @@ fn encode_openai_chat_stream(
name,
arguments_delta,
)],
- LlmStreamEvent::Usage(usage) => {
+ LlmResponseChunk::Usage(usage) => {
state.usage = usage;
state.saw_backend_usage = true;
Vec::new()
}
- LlmStreamEvent::MessageStop { reason } => {
+ LlmResponseChunk::MessageStop { reason } => {
if state.finished {
return Vec::new();
}
@@ -204,7 +205,7 @@ fn encode_openai_chat_stream(
Some(openai_usage_value(state)),
)]
}
- LlmStreamEvent::Error { message } => vec![json!({"error": {"message": message}})],
+ LlmResponseChunk::Error { message } => vec![json!({"error": {"message": message}})],
}
}
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index 71f41e09c..e92ec45b9 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -18,9 +18,9 @@ use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
use crate::llm::{
- ContentBlock, LlmRequest, LlmResponse, MediaSource, Message, OutputParams, ProviderExtensions,
- ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice,
- ToolDefinition, ToolResult, Usage,
+ AggLlmResponse, ContentBlock, LlmRequest, MediaSource, Message, OutputParams,
+ ProviderExtensions, ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason,
+ ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,
};
use crate::policy::{DeterministicIdPolicy, TranslationPolicy};
use crate::util::{
@@ -200,7 +200,7 @@ impl FormatCodec for OpenAiResponsesCodec {
stop_reason: Some(StopReason::EndTurn),
});
}
- let response = LlmResponse {
+ let response = AggLlmResponse {
id: body
.get("id")
.and_then(Value::as_str)
@@ -231,7 +231,7 @@ impl FormatCodec for OpenAiResponsesCodec {
fn encode_response(
&self,
- response: &LlmResponse,
+ response: &AggLlmResponse,
_policy: &TranslationPolicy,
) -> Result<EncodedResponse> {
if let Some(body) =
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs
index 48e6f1636..42234b9d2 100644
--- a/crates/switchyard-translation/src/codecs/responses/stream.rs
+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs
@@ -10,7 +10,8 @@ use crate::codecs::stream::{
StreamCodec, StreamTranslationState,
};
use crate::format::{FormatId, WireFormat};
-use crate::llm::{LlmStreamEvent, Usage};
+use crate::llm::Usage;
+use crate::LlmResponseChunk;
/// Stream codec for OpenAI Responses API events.
pub struct OpenAiResponsesStreamCodec;
@@ -24,14 +25,14 @@ impl StreamCodec for OpenAiResponsesStreamCodec {
&self,
state: &mut StreamTranslationState,
event: &Value,
- ) -> Vec<LlmStreamEvent> {
+ ) -> Vec<LlmResponseChunk> {
decode_responses_stream(state, event)
}
fn encode_event(
&self,
state: &mut StreamTranslationState,
- event: LlmStreamEvent,
+ event: LlmResponseChunk,
) -> Vec<Value> {
encode_responses_stream(state, event)
}
@@ -45,7 +46,7 @@ impl StreamCodec for OpenAiResponsesStreamCodec {
fn decode_responses_stream(
state: &mut StreamTranslationState,
event: &Value,
-) -> Vec<LlmStreamEvent> {
+) -> Vec<LlmResponseChunk> {
let event_type = event
.get("type")
.or_else(|| event.get("event"))
@@ -66,7 +67,7 @@ fn decode_responses_stream(
{
state.message_id = Some(id.to_string());
}
- vec![LlmStreamEvent::MessageStart {
+ vec![LlmResponseChunk::MessageStart {
id: state.message_id.clone(),
model: state.model.clone(),
}]
@@ -75,7 +76,7 @@ fn decode_responses_stream(
.get("delta")
.and_then(Value::as_str)
.map(|text| {
- vec![LlmStreamEvent::TextDelta {
+ vec![LlmResponseChunk::TextDelta {
index: event
.get("output_index")
.and_then(Value::as_u64)
@@ -90,7 +91,7 @@ fn decode_responses_stream(
.or_else(|| event.get("text"))
.and_then(Value::as_str)
.map(|text| {
- vec![LlmStreamEvent::ReasoningDelta {
+ vec![LlmResponseChunk::ReasoningDelta {
index: event
.get("output_index")
.and_then(Value::as_u64)
@@ -110,7 +111,7 @@ fn decode_responses_stream(
.get("delta")
.and_then(Value::as_str)
.map(|delta| {
- vec![LlmStreamEvent::ToolCallDelta {
+ vec![LlmResponseChunk::ToolCallDelta {
index: output_index as usize,
id: None,
name: None,
@@ -131,12 +132,12 @@ fn decode_responses_stream(
let usage = responses_usage(usage);
state.usage = usage.clone();
state.saw_backend_usage = true;
- out.push(LlmStreamEvent::Usage(usage));
+ out.push(LlmResponseChunk::Usage(usage));
}
- out.push(LlmStreamEvent::MessageStop { reason: None });
+ out.push(LlmResponseChunk::MessageStop { reason: None });
out
}
- Some("error") => vec![LlmStreamEvent::Error {
+ Some("error") => vec![LlmResponseChunk::Error {
message: event
.get("message")
.and_then(Value::as_str)
@@ -150,33 +151,33 @@ fn decode_responses_stream(
// Encodes neutral streaming events into OpenAI Responses events.
fn encode_responses_stream(
state: &mut StreamTranslationState,
- event: LlmStreamEvent,
+ event: LlmResponseChunk,
) -> Vec<Value> {
match event {
- LlmStreamEvent::MessageStart { id, model } => {
+ LlmResponseChunk::MessageStart { id, model } => {
record_source_identity(state, id, model);
ensure_responses_created(state)
}
- LlmStreamEvent::TextDelta { text, .. } => encode_responses_text_delta(state, text),
- LlmStreamEvent::ReasoningDelta { text, .. } => {
+ LlmResponseChunk::TextDelta { text, .. } => encode_responses_text_delta(state, text),
+ LlmResponseChunk::ReasoningDelta { text, .. } => {
encode_responses_reasoning_delta(state, text)
}
- LlmStreamEvent::ToolCallDelta {
+ LlmResponseChunk::ToolCallDelta {
index,
id,
name,
arguments_delta,
} => encode_responses_tool_delta(state, index, id, name, arguments_delta),
- LlmStreamEvent::Usage(usage) => {
+ LlmResponseChunk::Usage(usage) => {
state.usage = usage;
state.saw_backend_usage = true;
Vec::new()
}
- LlmStreamEvent::MessageStop { reason } => {
+ LlmResponseChunk::MessageStop { reason } => {
state.stop_reason = reason.or_else(|| state.stop_reason.clone());
Vec::new()
}
- LlmStreamEvent::Error { message } => {
+ LlmResponseChunk::Error { message } => {
vec![json!({"type": "error", "message": message})]
}
}
@@ -295,7 +296,7 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec<Value> {
}
// Converts Responses function-call item creation into a neutral tool-call delta.
-fn decode_responses_output_item_added(event: &Value) -> Vec<LlmStreamEvent> {
+fn decode_responses_output_item_added(event: &Value) -> Vec<LlmResponseChunk> {
let Some(item) = event.get("item").and_then(Value::as_object) else {
return Vec::new();
};
@@ -306,7 +307,7 @@ fn decode_responses_output_item_added(event: &Value) -> Vec<LlmStreamEvent> {
.get("output_index")
.and_then(Value::as_u64)
.unwrap_or(0) as usize;
- vec![LlmStreamEvent::ToolCallDelta {
+ vec![LlmResponseChunk::ToolCallDelta {
index,
id: item
.get("call_id")
@@ -329,7 +330,7 @@ fn decode_responses_output_item_added(event: &Value) -> Vec<LlmStreamEvent> {
fn decode_responses_output_item_done(
event: &Value,
state: &StreamTranslationState,
-) -> Vec<LlmStreamEvent> {
+) -> Vec<LlmResponseChunk> {
let Some(item) = event.get("item").and_then(Value::as_object) else {
return Vec::new();
};
@@ -348,7 +349,7 @@ fn decode_responses_output_item_done(
.map(|tool| tool.arguments.as_str())
.unwrap_or("");
if !arguments.is_empty() && arguments != existing {
- return vec![LlmStreamEvent::ToolCallDelta {
+ return vec![LlmResponseChunk::ToolCallDelta {
index,
id: None,
name: None,
diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs
index 710d79dd3..730e224b7 100644
--- a/crates/switchyard-translation/src/codecs/stream.rs
+++ b/crates/switchyard-translation/src/codecs/stream.rs
@@ -15,7 +15,8 @@ use crate::codecs::responses::OpenAiResponsesStreamCodec;
use crate::engine::{FormatRegistry, TranslationEngine};
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
-use crate::llm::{LlmStreamEvent, Usage};
+use crate::llm::Usage;
+use crate::LlmResponseChunk;
/// Mutable state accumulated while translating one streaming response.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
@@ -99,11 +100,14 @@ pub trait StreamCodec: Send + Sync {
&self,
state: &mut StreamTranslationState,
event: &Value,
- ) -> Vec<LlmStreamEvent>;
+ ) -> Vec<LlmResponseChunk>;
/// Encodes one neutral event into zero or more provider events.
- fn encode_event(&self, state: &mut StreamTranslationState, event: LlmStreamEvent)
- -> Vec<Value>;
+ fn encode_event(
+ &self,
+ state: &mut StreamTranslationState,
+ event: LlmResponseChunk,
+ ) -> Vec<Value>;
/// Emits any terminal provider events needed after the source stream ends.
///
@@ -197,13 +201,13 @@ pub fn decode_stream_event(
state: &mut StreamTranslationState,
source: impl Into<FormatId>,
event: &Value,
-) -> Vec<LlmStreamEvent> {
+) -> Vec<LlmResponseChunk> {
let source = source.into();
StreamCodecRegistry::with_builtins()
.codec(source)
.map(|codec| codec.decode_event(state, event))
.unwrap_or_else(|error| {
- vec![LlmStreamEvent::Error {
+ vec![LlmResponseChunk::Error {
message: error.to_string(),
}]
})
@@ -213,7 +217,7 @@ pub fn decode_stream_event(
pub fn encode_stream_event(
state: &mut StreamTranslationState,
target: impl Into<FormatId>,
- event: LlmStreamEvent,
+ event: LlmResponseChunk,
) -> Vec<Value> {
StreamCodecRegistry::with_builtins()
.codec(target)
diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs
index 9dddc8db5..5303c3ed6 100644
--- a/crates/switchyard-translation/src/engine.rs
+++ b/crates/switchyard-translation/src/engine.rs
@@ -16,7 +16,7 @@ use crate::codecs::FormatCodec;
use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::FormatId;
-use crate::llm::{LlmRequest, LlmResponse};
+use crate::llm::{AggLlmResponse, LlmRequest};
use crate::policy::TranslationPolicy;
/// Encoded translation result with any diagnostics emitted along the way.
@@ -36,7 +36,7 @@ pub struct RequestIrOutput {
/// Decoded response IR plus diagnostics.
#[derive(Debug)]
pub struct ResponseIrOutput {
- pub response: LlmResponse,
+ pub response: AggLlmResponse,
pub diagnostics: Vec<TranslationDiagnostic>,
}
@@ -184,7 +184,7 @@ impl TranslationEngine {
pub fn encode_response(
&self,
target: impl Into<FormatId>,
- response: &LlmResponse,
+ response: &AggLlmResponse,
policy: &TranslationPolicy,
) -> Result<TranslationOutput> {
let target = target.into();
diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rs
index 96ed486fd..3eaaf21a9 100644
--- a/crates/switchyard-translation/src/lib.rs
+++ b/crates/switchyard-translation/src/lib.rs
@@ -15,6 +15,7 @@ pub mod policy;
pub mod stream;
pub mod util;
+pub use switchyard_protocol::stream::LlmResponseChunk;
pub use switchyard_protocol::{format, llm};
pub use diagnostic::*;
diff --git a/crates/switchyard-translation/tests/extension_points.rs b/crates/switchyard-translation/tests/extension_points.rs
index 49a89334b..2b70283a1 100644
--- a/crates/switchyard-translation/tests/extension_points.rs
+++ b/crates/switchyard-translation/tests/extension_points.rs
@@ -13,7 +13,7 @@ use switchyard_translation::util::{
exact_preserved_response,
};
use switchyard_translation::{
- FormatId, FormatRegistry, LlmRequest, LlmResponse, LlmStreamEvent, LossyConversionPolicy,
+ AggLlmResponse, FormatId, FormatRegistry, LlmRequest, LlmResponseChunk, LossyConversionPolicy,
Message, PreservationPolicy, ResponseOutput, Role, StreamCodec, StreamCodecRegistry,
StreamTranslationState, TargetCapabilities, TranslationEngine, TranslationPolicy, Usage,
WireFormat,
@@ -181,7 +181,7 @@ impl FormatCodec for MinimalCustomCodec {
policy: &TranslationPolicy,
) -> switchyard_translation::Result<DecodedResponse> {
Ok(DecodedResponse {
- response: LlmResponse {
+ response: AggLlmResponse {
id: body
.get("id")
.and_then(Value::as_str)
@@ -203,7 +203,7 @@ impl FormatCodec for MinimalCustomCodec {
}],
usage: Usage::default(),
preservation: capture_response_preservation(self.format(), body, policy),
- ..LlmResponse::default()
+ ..AggLlmResponse::default()
},
diagnostics: Vec::new(),
})
@@ -211,7 +211,7 @@ impl FormatCodec for MinimalCustomCodec {
fn encode_response(
&self,
- response: &LlmResponse,
+ response: &AggLlmResponse,
policy: &TranslationPolicy,
) -> switchyard_translation::Result<EncodedResponse> {
if let Some(body) = exact_preserved_response(&response.preservation, self.format(), policy)
@@ -254,9 +254,9 @@ impl StreamCodec for CustomStreamCodec {
&self,
_state: &mut StreamTranslationState,
event: &Value,
- ) -> Vec<LlmStreamEvent> {
+ ) -> Vec<LlmResponseChunk> {
match event.get("kind").and_then(Value::as_str) {
- Some("start") => vec![LlmStreamEvent::MessageStart {
+ Some("start") => vec![LlmResponseChunk::MessageStart {
id: event
.get("id")
.and_then(Value::as_str)
@@ -270,7 +270,7 @@ impl StreamCodec for CustomStreamCodec {
.get("text")
.and_then(Value::as_str)
.map(|text| {
- vec![LlmStreamEvent::TextDelta {
+ vec![LlmResponseChunk::TextDelta {
index: 0,
text: text.to_string(),
}]
@@ -283,10 +283,10 @@ impl StreamCodec for CustomStreamCodec {
fn encode_event(
&self,
_state: &mut StreamTranslationState,
- event: LlmStreamEvent,
+ event: LlmResponseChunk,
) -> Vec<Value> {
match event {
- LlmStreamEvent::TextDelta { text, .. } => {
+ LlmResponseChunk::TextDelta { text, .. } => {
vec![json!({"kind": "delta", "text": text})]
}
_ => Vec::new(),
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index b128a8629..3c4f5e658 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -5,7 +5,10 @@
use pretty_assertions::assert_eq;
use serde_json::json;
-use switchyard_translation::{StreamTranslationState, TranslationEngine, WireFormat};
+use switchyard_protocol::{ResponseAccumulator, StopReason};
+use switchyard_translation::{
+ decode_stream_event, StreamTranslationState, TranslationEngine, WireFormat,
+};
type TestResult = std::result::Result<(), Box<dyn std::error::Error + Send + Sync>>;
@@ -416,3 +419,35 @@ fn anthropic_thinking_stream_deltas_do_not_become_openai_chat_content() -> TestR
.any(|event| { event["choices"][0]["delta"]["content"] == "private chain of thought" }));
Ok(())
}
+
+// A real Anthropic stream carries its stop reason on `message_delta` and then always
+// sends a reasonless `message_stop`. Decoding both and folding the chunks must preserve
+// the provider reason (here `max_tokens`) — the terminal `message_stop` must not emit a
+// second, reasonless `MessageStop` that the accumulator would fold to `EndTurn`.
+#[test]
+fn anthropic_message_stop_does_not_overwrite_max_tokens_stop_reason() -> TestResult {
+ let mut state =
+ StreamTranslationState::new(WireFormat::AnthropicMessages, WireFormat::AnthropicMessages);
+ let events = vec![
+ json!({"type": "message_start", "message": {"id": "msg_1", "model": "claude", "usage": {"input_tokens": 3}}}),
+ json!({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),
+ json!({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hi"}}),
+ json!({"type": "message_delta", "delta": {"stop_reason": "max_tokens"}, "usage": {"output_tokens": 5}}),
+ json!({"type": "message_stop"}),
+ ];
+
+ let mut accumulator = ResponseAccumulator::new();
+ for event in &events {
+ for chunk in decode_stream_event(&mut state, WireFormat::AnthropicMessages, event) {
+ accumulator.push(chunk);
+ }
+ }
+ let aggregate = accumulator.finish();
+
+ assert_eq!(
+ aggregate.outputs[0].stop_reason,
+ Some(StopReason::MaxTokens),
+ "the reasonless message_stop must not overwrite the max_tokens stop reason"
+ );
+ Ok(())
+}