Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(libsy): support streaming responses (#73)
Greg Clarkc7a6475f4566
diff --git a/Cargo.lock b/Cargo.lockindex 0dbf90623..aa3ce13f0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -869,6 +869,7 @@ dependencies = ["futures","libsy","rand 0.8.6",+ "switchyard-protocol","tokio","tokio-stream",]@@ -1632,6 +1633,7 @@ dependencies = [name = "switchyard-protocol"version = "0.1.0"dependencies = [+ "futures","serde","serde_json",]diff --git a/crates/libsy-examples/Cargo.toml b/crates/libsy-examples/Cargo.tomlindex 082bec560..3e8de0653 100644--- a/crates/libsy-examples/Cargo.toml+++ b/crates/libsy-examples/Cargo.toml@@ -14,6 +14,7 @@ rust-version.workspace = true[dependencies]libsy = { path = "../libsy" }+switchyard-protocol = { path = "../libsy-protocol" }async-trait = "0.1"rand = "0.8"futures = "0.3"diff --git a/crates/libsy-examples/examples/research_agent.rs b/crates/libsy-examples/examples/research_agent.rsindex 0ea5804ca..a21f3ceae 100644--- a/crates/libsy-examples/examples/research_agent.rs+++ b/crates/libsy-examples/examples/research_agent.rs@@ -9,17 +9,18 @@//! multi-step routing (classify -> route) happens inside the classifier algorithm; the//! agent never sees it. To drive the step stream yourself instead, use//! `Algorithm::run_stream`. Run with:-//! cargo run -p libsy --example research_agent+//! cargo run -p libsy-examples --example research_agentuse std::error::Error;use std::sync::Arc;use async_trait::async_trait;use libsy::{- Algorithm, ContentBlock, Context, LlmClient, LlmRequest, LlmResponse, LlmTarget, LlmTargetSet,- Message, Request, Response, ResponseOutput, Role, RoutedRequest,+ Algorithm, Context, LlmClient, LlmResponse, LlmTarget, LlmTargetSet, Request, Response,+ RoutedRequest,};use libsy_examples::llm_class::LlmClassifierOrchAlgo;+use switchyard_protocol::{completion_text, text_request, text_response};const CLASSIFIER: &str = "classifier/model";const STRONG: &str = "strong/model";@@ -41,14 +42,7 @@ impl LlmClient for StubClient {format!("answer from {model}")};Ok(Response {- llm_response: LlmResponse {- outputs: vec![ResponseOutput {- role: Role::Assistant,- content: vec![ContentBlock::Text { text: completion }],- stop_reason: None,- }],- ..LlmResponse::default()- },+ llm_response: LlmResponse::Agg(text_response(None, completion)),metadata: None,})}@@ -77,32 +71,13 @@ impl ResearchAgent {let mut notes = Vec::new();for step in self.plan(question) {let request = Request {- llm_request: LlmRequest {- model: Some("auto".to_string()),- messages: vec![Message::text(Role::User, step)],- ..LlmRequest::default()- },+ llm_request: text_request(Some("auto".to_string()), step),raw_request: None,metadata: None,};let (_trace, response) = self.algo.clone().run(Context::default(), request).await?;- notes.push(- response- .llm_response- .outputs- .iter()- .flat_map(|output| output.content.iter())- .filter_map(|block| match block {- ContentBlock::Text { text }- | ContentBlock::Refusal { text }- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),- ContentBlock::Unknown { raw, .. } => raw.as_str(),- _ => None,- })- .collect::<Vec<_>>()- .join("\n"),- );+ notes.push(completion_text(&response.llm_response.into_agg().await?));}Ok(notes.join("\n"))}diff --git a/crates/libsy-examples/examples/research_agent_core.rs b/crates/libsy-examples/examples/research_agent_core.rsindex e6b3bfed6..16fa959af 100644--- a/crates/libsy-examples/examples/research_agent_core.rs+++ b/crates/libsy-examples/examples/research_agent_core.rs@@ -7,16 +7,16 @@//! surfaces as a `CallLlm` step. The agent makes the "real" model call itself and//! fulfills the promise — this is the offload/streaming path ("ask, don't call").//! The classifier's two steps show up as two `model call:` lines. Run with:-//! cargo run -p libsy --example research_agent_core+//! cargo run -p libsy-examples --example research_agent_coreuse std::error::Error;use std::sync::Arc;use libsy::{- Algorithm, ContentBlock, Context, Decision, LlmRequest, LlmResponse, LlmTarget, LlmTargetSet,- Message, Request, Response, ResponseOutput, Role, Step,+ Algorithm, Context, Decision, LlmResponse, LlmTarget, LlmTargetSet, Request, Response, Step,};use libsy_examples::llm_class::LlmClassifierOrchAlgo;+use switchyard_protocol::{completion_text, text_request, text_response};use tokio_stream::StreamExt;const CLASSIFIER: &str = "classifier/model";@@ -34,14 +34,7 @@ async fn call_model(model: &str) -> Response {format!("answer from {model}")};Response {- llm_response: LlmResponse {- outputs: vec![ResponseOutput {- role: Role::Assistant,- content: vec![ContentBlock::Text { text: completion }],- stop_reason: None,- }],- ..LlmResponse::default()- },+ llm_response: LlmResponse::Agg(text_response(None, completion)),metadata: None,}}@@ -69,11 +62,7 @@ impl ResearchAgent {let mut notes = Vec::new();for step in self.plan(question) {let request = Request {- llm_request: LlmRequest {- model: Some("auto".to_string()),- messages: vec![Message::text(Role::User, step)],- ..LlmRequest::default()- },+ llm_request: text_request(Some("auto".to_string()), step),raw_request: None,metadata: None,};@@ -89,22 +78,7 @@ impl ResearchAgent {// Decisions stream in as the algorithm makes them.Step::Decision(decision) => print_decision(decision.as_ref()),Step::ReturnToAgent(response) => {- notes.push(- response- .llm_response- .outputs- .iter()- .flat_map(|output| output.content.iter())- .filter_map(|block| match block {- ContentBlock::Text { text }- | ContentBlock::Refusal { text }- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),- ContentBlock::Unknown { raw, .. } => raw.as_str(),- _ => None,- })- .collect::<Vec<_>>()- .join("\n"),- );+ notes.push(completion_text(&response.llm_response.into_agg().await?));}}}diff --git a/crates/libsy-examples/examples/streaming_agent.rs b/crates/libsy-examples/examples/streaming_agent.rsnew file mode 100644index 000000000..aa14f9e57--- /dev/null+++ b/crates/libsy-examples/examples/streaming_agent.rs@@ -0,0 +1,104 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Agent that forwards a streamed response to the caller token-by-token.+//!+//! Drives [`Algorithm::run_stream`] with a **client-less** target, so each model call is+//! offloaded as a [`Step::CallLlm`]. The agent serves it with a *streaming* response+//! ([`LlmResponse::Stream`]) rather than a buffered one. That stream rides untouched+//! through the algorithm and returns as [`Step::ReturnToAgent`], where the agent drives it+//! and prints each [`LlmResponseChunk`] as it arrives — true token streaming end to end.+//! Contrast [`Algorithm::run`], which would aggregate the same stream into one buffered+//! answer. Run with:+//! cargo run -p libsy-examples --example streaming_agent++use std::error::Error;+use std::io::Write;+use std::sync::Arc;++use futures::StreamExt;+use libsy::{+ Algorithm, Context, LlmResponse, LlmResponseChunk, LlmResponseStream, LlmTarget, LlmTargetSet,+ Request, Response, Step,+};+use libsy_examples::rand::RandomOrchAlgo;+use switchyard_protocol::{completion_text, text_request};++/// The "real" model call the agent makes to fulfill an offloaded promise: a response+/// whose body is a live token stream (`MessageStart`, one `TextDelta` per token,+/// `MessageStop`). A real integrator would map an upstream provider SSE stream here.+fn streaming_response(model: &str, tokens: &[&str]) -> Response {+ let mut chunks = vec![LlmResponseChunk::MessageStart {+ id: None,+ model: Some(model.to_string()),+ }];+ for token in tokens {+ chunks.push(LlmResponseChunk::TextDelta {+ index: 0,+ text: token.to_string(),+ });+ }+ chunks.push(LlmResponseChunk::MessageStop {+ reason: Some("stop".to_string()),+ });++ let stream: LlmResponseStream = futures::stream::iter(chunks.into_iter().map(Ok)).boxed();+ Response {+ llm_response: LlmResponse::Stream(stream),+ metadata: None,+ }+}++#[tokio::main(flavor = "current_thread")]+async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {+ // One client-less target -> its call is offloaded for us to serve.+ let targets = LlmTargetSet::new(vec![LlmTarget {+ semantic_name: "stream/model".to_string(),+ llm_client: None,+ }]);+ let algo: Arc<dyn Algorithm> = Arc::new(RandomOrchAlgo::new(targets));++ let request = Request {+ llm_request: text_request(Some("auto".to_string()), "tell me about switchyard"),+ raw_request: None,+ metadata: None,+ };+ let stream = algo.run_stream(Context::default(), request);+ tokio::pin!(stream);++ while let Some(step) = stream.next().await {+ match step? {+ Step::CallLlm(call) => {+ // Serve the offloaded call with a streaming response.+ let model = call.get_decision().selected_model().to_string();+ let tokens = ["Switch", "yard ", "routes ", "LLM ", "traffic."];+ call.respond(Ok(streaming_response(&model, &tokens)))?;+ }+ Step::Decision(decision) => {+ println!(+ "decision: {} ({})",+ decision.selected_model(),+ decision.reasoning().unwrap_or_default()+ );+ }+ Step::ReturnToAgent(response) => match response.llm_response {+ // The stream reached the agent untouched: print each token as it arrives.+ LlmResponse::Stream(mut chunks) => {+ print!("agent sees: ");+ while let Some(chunk) = chunks.next().await {+ if let LlmResponseChunk::TextDelta { text, .. } = chunk? {+ print!("{text}");+ std::io::stdout().flush().ok();+ }+ }+ println!();+ }+ // A buffered backend would land here instead.+ LlmResponse::Agg(agg) => {+ println!("agent sees (buffered): {}", completion_text(&agg))+ }+ },+ }+ }+ Ok(())+}diff --git a/crates/libsy-examples/src/ensemble.rs b/crates/libsy-examples/src/ensemble.rsindex e558ac098..d12f16e2e 100644--- a/crates/libsy-examples/src/ensemble.rs+++ b/crates/libsy-examples/src/ensemble.rs@@ -22,10 +22,8 @@ use std::sync::{Arc, Mutex};use async_trait::async_trait;-use libsy::{- Algorithm, ContentBlock, Context, Decision, Driver, LlmRequest, LlmTargetSet, Message, Request,- Response, Role, Signals,-};+use libsy::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response, Signals};+use switchyard_protocol::{completion_text, prompt_text, text_request};/// Which step of the ensemble flow produced a decision.#[derive(Clone, Copy, Debug, PartialEq, Eq)]@@ -172,6 +170,7 @@ impl EnsembleOrchAlgo {async fn route_committed(&self,driver: &Driver,+ ctx: Context,request: Request,model: String,) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {@@ -184,13 +183,18 @@ impl EnsembleOrchAlgo {selected_model: model.clone(),phase: EnsemblePhase::Committed,});+ // The agent's inbound name rides through; the committed model is on the+ // decision, not stamped onto the request.let routed = Request {- llm_request: request.llm_request,+ llm_request: text_request(+ request.llm_request.model.clone(),+ prompt_text(&request.llm_request),+ ),raw_request: request.raw_request,metadata: request.metadata,};let response = driver- .call_llm_target(&target, routed, decision.clone())+ .call_llm_target(ctx, &target, routed, decision.clone()).await?;Ok((vec![decision], response))}@@ -200,29 +204,13 @@ impl EnsembleOrchAlgo {async fn ensemble_turn(&self,driver: &Driver,+ ctx: Context,request: Request,) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {- let user_prompt = request- .llm_request- .instructions- .iter()- .flat_map(|instruction| instruction.content.iter())- .chain(- request- .llm_request- .messages- .iter()- .flat_map(|message| message.content.iter()),- )- .filter_map(|block| match block {- ContentBlock::Text { text }- | ContentBlock::Refusal { text }- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),- ContentBlock::Unknown { raw, .. } => raw.as_str(),- _ => None,- })- .collect::<Vec<_>>()- .join("\n");+ let user_prompt = prompt_text(&request.llm_request);+ // The agent's inbound name rides through every sub-call unchanged; the model+ // each call hits is carried by its decision, not stamped onto the request.+ let inbound = request.llm_request.model.clone();// Fan out to all candidates concurrently with the same user prompt. Each// call is annotated with its own candidate decision so the caller can see@@ -238,16 +226,17 @@ impl EnsembleOrchAlgo {});candidate_decisions.push(decision.clone());let call_request = Request {- llm_request: request.llm_request.clone(),+ llm_request: text_request(inbound.clone(), user_prompt.clone()),raw_request: request.raw_request.clone(),metadata: request.metadata.clone(),};let model = model.clone();+ let ctx = ctx.clone();calls.push(async move {(model,driver- .call_llm_target(&target, call_request, decision)+ .call_llm_target(ctx, &target, call_request, decision).await,)});@@ -282,33 +271,22 @@ impl EnsembleOrchAlgo {phase: EnsemblePhase::Judge,});let judge_request = Request {- llm_request: LlmRequest {- model: request.llm_request.model.clone(),- messages: vec![Message::text(Role::User, judge_prompt)],- ..LlmRequest::default()- },- raw_request: None,+ llm_request: text_request(inbound.clone(), judge_prompt),+ raw_request: request.raw_request.clone(),metadata: request.metadata.clone(),};let judge_response = driver- .call_llm_target(&judge_target, judge_request, judge_decision.clone())+ .call_llm_target(ctx, &judge_target, judge_request, judge_decision.clone()).await?;// Fail open: an unparseable pick falls back to the first response.- let judge_text = judge_response- .llm_response- .outputs- .iter()- .flat_map(|output| output.content.iter())- .filter_map(|block| match block {- ContentBlock::Text { text }- | ContentBlock::Refusal { text }- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),- ContentBlock::Unknown { raw, .. } => raw.as_str(),- _ => None,- })- .collect::<Vec<_>>()- .join("\n");- let choice = parse_choice(&judge_text, survivors.len());+ let choice = parse_choice(+ &judge_response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ survivors.len(),+ );let (model, response) = survivors.into_iter().nth(choice)@@ -353,21 +331,15 @@ fn build_judge_prompt(user_prompt: &str, survivors: &[(String, Response)]) -> Stprompt.push_str(user_prompt);prompt.push_str("\n\n");for (i, (_model, response)) in survivors.iter().enumerate() {- let candidate_text = response- .llm_response- .outputs- .iter()- .flat_map(|output| output.content.iter())- .filter_map(|block| match block {- ContentBlock::Text { text }- | ContentBlock::Refusal { text }- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),- ContentBlock::Unknown { raw, .. } => raw.as_str(),- _ => None,- })- .collect::<Vec<_>>()- .join("\n");- prompt.push_str(&format!("Response {}:\n{}\n\n", i + 1, candidate_text));+ prompt.push_str(&format!(+ "Response {}:\n{}\n\n",+ i + 1,+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default()+ ));}prompt.push_str(&format!("Reply with only the number (1-{}) of the best response.",@@ -398,7 +370,7 @@ fn parse_choice(completion: &str, count: usize) -> usize {impl Algorithm for EnsembleOrchAlgo {async fn create_run_task(self: Arc<Self>,- _ctx: Context,+ ctx: Context,driver: Driver,request: Request,) -> Result<Response, Box<dyn Error + Send + Sync>> {@@ -406,14 +378,15 @@ impl Algorithm for EnsembleOrchAlgo {// otherwise run a full ensemble turn. Both return a decision trace plus the// final response.let (trace, response) = if let Some(model) = self.resolve_committed()? {- self.route_committed(&driver, request, model).await?+ self.route_committed(&driver, ctx.clone(), request, model)+ .await?} else {- self.ensemble_turn(&driver, request).await?+ self.ensemble_turn(&driver, ctx.clone(), request).await?};// Publish the trace to the stream (candidate..., judge?, winner). The// candidate decisions also rode along on their offloaded `CallLlm` steps.for decision in trace {- driver.info(decision).await?;+ driver.info(ctx.clone(), decision).await?;}Ok(response)}@@ -430,8 +403,9 @@ impl Algorithm for EnsembleOrchAlgo {#[cfg(test)]mod tests {use super::*;- use libsy::{LlmClient, LlmResponse, LlmTarget, Response, ResponseOutput, RoutedRequest};+ use libsy::{LlmClient, LlmResponse, LlmTarget, Response, RoutedRequest};use std::sync::Mutex as StdMutex;+ use switchyard_protocol::text_response;/// Mock client that answers candidate calls with `answer from {model}` and,/// for the judge model, returns the 1-based number of the response whose@@ -455,22 +429,12 @@ mod tests {.map_err(|_| "lock poisoned")?.push(name.clone());let completion = if name == self.judge_model {- let prompt = routed.request.llm_request.messages[0]- .text_content("\n")- .unwrap_or_default();- judge_pick(&prompt, &self.prefer)+ judge_pick(&prompt_text(&routed.request.llm_request), &self.prefer)} else {format!("answer from {name}")};Ok(Response {- llm_response: LlmResponse {- outputs: vec![ResponseOutput {- role: Role::Assistant,- content: vec![ContentBlock::Text { text: completion }],- stop_reason: None,- }],- ..LlmResponse::default()- },+ llm_response: LlmResponse::Agg(text_response(None, completion)),metadata: None,})}@@ -547,11 +511,7 @@ mod tests {fn request(prompt: &str) -> Request {Request {- llm_request: LlmRequest {- model: Some("auto".to_string()),- messages: vec![Message::text(Role::User, prompt)],- ..LlmRequest::default()- },+ llm_request: text_request(Some("auto".to_string()), prompt),raw_request: None,metadata: None,}@@ -580,10 +540,12 @@ mod tests {.run(Context::default(), request("solve it")).await?;assert_eq!(- response.llm_response.outputs[0].content,- vec![ContentBlock::Text {- text: "answer from b/model".to_string()- }]+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from b/model");// Both candidates and the judge were called.@@ -625,10 +587,12 @@ mod tests {// fan-out to a/model and no judge call.let (trace, response) = orch.clone().run(Context::default(), request("t3")).await?;assert_eq!(- response.llm_response.outputs[0].content,- vec![ContentBlock::Text {- text: "answer from b/model".to_string()- }]+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from b/model");assert_eq!(trace.len(), 1);let decision = as_ensemble(&trace[0])?;@@ -648,10 +612,12 @@ mod tests {let (algo, calls) = algo(&["only/model"], "judge/haiku", "only/model", 100);let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;assert_eq!(- response.llm_response.outputs[0].content,- vec![ContentBlock::Text {- text: "answer from only/model".to_string()- }]+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from only/model");// No judge call for a lone candidate.assert!(!calls@@ -768,24 +734,14 @@ mod tests {let name = routed.decision.selected_model().to_string();let completion = if name == self.judge_model {// Judge runs after the barrier releases; it must not wait.- let prompt = routed.request.llm_request.messages[0]- .text_content("\n")- .unwrap_or_default();- judge_pick(&prompt, &self.prefer)+ judge_pick(&prompt_text(&routed.request.llm_request), &self.prefer)} else {// Hold every candidate call until all sessions have fanned out.self.barrier.wait().await;format!("answer from {name}")};Ok(Response {- llm_response: LlmResponse {- outputs: vec![ResponseOutput {- role: Role::Assistant,- content: vec![ContentBlock::Text { text: completion }],- stop_reason: None,- }],- ..LlmResponse::default()- },+ llm_response: LlmResponse::Agg(text_response(None, completion)),metadata: None,})}@@ -821,16 +777,11 @@ mod tests {.run(Context::default(), request(prompt)).await.map(|(_, response)| {- match response+ response.llm_response- .outputs- .into_iter()- .next()- .and_then(|output| output.content.into_iter().next())- {- Some(ContentBlock::Text { text }) => text,- _ => String::new(),- }+ .as_agg()+ .map(completion_text)+ .unwrap_or_default()})})};@@ -877,17 +828,14 @@ mod tests {.and_then(|d| d.as_any().downcast_ref::<EnsembleDecision>()).map(|d| d.phase).ok_or("missing final decision")?;- let completion = match response- .llm_response- .outputs- .into_iter()- .next()- .and_then(|output| output.content.into_iter().next())- {- Some(ContentBlock::Text { text }) => text,- _ => String::new(),- };- Ok::<(String, EnsemblePhase), Box<dyn Error + Send + Sync>>((completion, phase))+ Ok::<(String, EnsemblePhase), Box<dyn Error + Send + Sync>>((+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ phase,+ ))})};// The two sessions run in parallel; each committed independently.diff --git a/crates/libsy-examples/src/llm_class.rs b/crates/libsy-examples/src/llm_class.rsindex 66e2008b5..a6a82dff8 100644--- a/crates/libsy-examples/src/llm_class.rs+++ b/crates/libsy-examples/src/llm_class.rs@@ -15,10 +15,8 @@ use std::sync::Arc;use async_trait::async_trait;-use libsy::{- Algorithm, ContentBlock, Context, Decision, Driver, LlmRequest, LlmTargetSet, Message, Request,- Response, Role, Signals,-};+use libsy::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response, Signals};+use switchyard_protocol::{completion_text, prompt_text, text_request};/// Preamble prepended to the user prompt when asking the classifier target for a/// strong-win-rate score.@@ -102,44 +100,23 @@ impl LlmClassifierOrchAlgo {impl Algorithm for LlmClassifierOrchAlgo {async fn create_run_task(self: Arc<Self>,- _ctx: Context,+ ctx: Context,driver: Driver,request: Request,) -> Result<Response, Box<dyn Error + Send + Sync>> {- let user_prompt = request- .llm_request- .instructions- .iter()- .flat_map(|instruction| instruction.content.iter())- .chain(- request- .llm_request- .messages- .iter()- .flat_map(|message| message.content.iter()),- )- .filter_map(|block| match block {- ContentBlock::Text { text }- | ContentBlock::Refusal { text }- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),- ContentBlock::Unknown { raw, .. } => raw.as_str(),- _ => None,- })- .collect::<Vec<_>>()- .join("\n");+ let user_prompt = prompt_text(&request.llm_request);+ // The agent's inbound name rides through unchanged on every sub-call; the+ // model each sub-call actually hits is carried by its decision instead.+ let inbound = request.llm_request.model.clone();// 1. Classify: call the classifier target with the score-eliciting prompt.let classifier_target = self.target_set.get_target(&self.classifier_model)?;let classify_request = Request {- llm_request: LlmRequest {- model: request.llm_request.model.clone(),- messages: vec![Message::text(- Role::User,- format!("{CLASSIFIER_PROMPT_PREAMBLE}{user_prompt}"),- )],- ..LlmRequest::default()- },- raw_request: None,+ llm_request: text_request(+ inbound.clone(),+ format!("{CLASSIFIER_PROMPT_PREAMBLE}{user_prompt}"),+ ),+ raw_request: request.raw_request.clone(),metadata: request.metadata.clone(),};let classify_decision: Arc<dyn Decision> = Arc::new(ClassifierDecision {@@ -148,25 +125,23 @@ impl Algorithm for LlmClassifierOrchAlgo {score: None,tier: None,});- driver.info(classify_decision.clone()).await?;+ driver.info(ctx.clone(), classify_decision.clone()).await?;let classify_response = driver- .call_llm_target(&classifier_target, classify_request, classify_decision)+ .call_llm_target(+ ctx.clone(),+ &classifier_target,+ classify_request,+ classify_decision,+ ).await?;- let score_text = classify_response+ let score = classify_response.llm_response- .outputs- .iter()- .flat_map(|output| output.content.iter())- .filter_map(|block| match block {- ContentBlock::Text { text }- | ContentBlock::Refusal { text }- | ContentBlock::Reasoning { text, .. } => Some(text.as_str()),- ContentBlock::Unknown { raw, .. } => raw.as_str(),- _ => None,- })- .collect::<Vec<_>>()- .join("\n");- let score = score_text.trim().parse::<f64>().ok();+ .as_agg()+ .map(completion_text)+ .unwrap_or_default()+ .trim()+ .parse::<f64>()+ .ok();// 2. Route: pick strong/weak. Fail open — an unparseable score routes strong.let (tier, model) = match score {@@ -186,13 +161,13 @@ impl Algorithm for LlmClassifierOrchAlgo {tier: Some(tier),});let routed_request = Request {- llm_request: request.llm_request,+ llm_request: text_request(inbound, user_prompt),raw_request: request.raw_request,metadata: request.metadata,};- driver.info(route_decision.clone()).await?;+ driver.info(ctx.clone(), route_decision.clone()).await?;driver- .call_llm_target(&routed_target, routed_request, route_decision)+ .call_llm_target(ctx, &routed_target, routed_request, route_decision).await}@@ -208,8 +183,9 @@ impl Algorithm for LlmClassifierOrchAlgo {#[cfg(test)]mod tests {use super::*;- use libsy::{LlmClient, LlmResponse, LlmTarget, Response, ResponseOutput, RoutedRequest};+ use libsy::{LlmClient, LlmResponse, LlmTarget, Response, RoutedRequest};use std::sync::Mutex;+ use switchyard_protocol::text_response;/// Returns `score` for the classifier target, an answer tagged with the model/// otherwise; records the requests it saw so a test can inspect the classifier@@ -227,27 +203,17 @@ mod tests {routed: RoutedRequest,) -> Result<Response, Box<dyn Error + Send + Sync>> {let name = routed.decision.selected_model().to_string();- let outputs = if name == self.classifier_model {- vec![ResponseOutput {- role: Role::Assistant,- content: vec![ContentBlock::Text {- text: self.score.clone(),- }],- stop_reason: None,- }]+ let completion = if name == self.classifier_model {+ self.score.clone()} else {- Vec::new()+ format!("answer from {name}")};self.seen.lock().map_err(|_| "lock poisoned")?.push(routed.request);Ok(Response {- llm_response: LlmResponse {- model: Some(name),- outputs,- ..LlmResponse::default()- },+ llm_response: LlmResponse::Agg(text_response(None, completion)),metadata: None,})}@@ -282,11 +248,7 @@ mod tests {fn request(prompt: &str) -> Request {Request {- llm_request: LlmRequest {- model: Some("auto".to_string()),- messages: vec![Message::text(Role::User, prompt)],- ..LlmRequest::default()- },+ llm_request: text_request(Some("auto".to_string()), prompt),raw_request: None,metadata: None,}@@ -314,8 +276,12 @@ mod tests {.run(Context::default(), request("solve this proof")).await?;assert_eq!(- response.llm_response.model.as_deref(),- Some("frontier/model")+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from frontier/model");// Trace: [classify, route].assert_eq!(trace[0].selected_model(), "router/classifier");@@ -332,7 +298,14 @@ mod tests {let (trace, response) = orch(algo).run(Context::default(), request("say hello")).await?;- assert_eq!(response.llm_response.model.as_deref(), Some("cheap/model"));+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from cheap/model"+ );let routed = as_classifier(&trace[1])?;assert_eq!(routed.tier, Some(ClassifierTier::Weak));assert_eq!(routed.score, Some(0.2));@@ -347,8 +320,12 @@ mod tests {.run(Context::default(), request("borderline")).await?;assert_eq!(- response.llm_response.model.as_deref(),- Some("frontier/model")+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from frontier/model");Ok(())}@@ -358,8 +335,12 @@ mod tests {let (algo, _) = algo(0.5, "not-a-number");let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;assert_eq!(- response.llm_response.model.as_deref(),- Some("frontier/model")+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from frontier/model");let routed = as_classifier(&trace[1])?;assert_eq!(routed.tier, Some(ClassifierTier::Strong));@@ -377,15 +358,9 @@ mod tests {let seen = seen.lock().map_err(|_| "lock poisoned")?;// Two calls: the classifier (preamble + user text), then the routed model.assert_eq!(seen.len(), 2);- let classifier_prompt = seen[0].llm_request.messages[0]- .text_content("\n")- .ok_or("classifier prompt is not text")?;- assert!(classifier_prompt.contains("prove it"));- assert!(classifier_prompt.contains("frontier model"));- assert_eq!(- seen[1].llm_request.messages[0].text_content("\n"),- Some("prove it".to_string())- );+ assert!(prompt_text(&seen[0].llm_request).contains("prove it"));+ assert!(prompt_text(&seen[0].llm_request).contains("frontier model"));+ assert_eq!(prompt_text(&seen[1].llm_request), "prove it");Ok(())}}diff --git a/crates/libsy-examples/src/rand.rs b/crates/libsy-examples/src/rand.rsindex bebaa3803..ef28205d2 100644--- a/crates/libsy-examples/src/rand.rs+++ b/crates/libsy-examples/src/rand.rs@@ -55,7 +55,7 @@ impl RandomOrchAlgo {impl Algorithm for RandomOrchAlgo {async fn create_run_task(self: Arc<Self>,- _ctx: Context,+ ctx: Context,driver: Driver,request: Request,) -> Result<Response, Box<dyn Error + Send + Sync>> {@@ -80,8 +80,10 @@ impl Algorithm for RandomOrchAlgo {});// Publish the decision to the stream, then offload the call.- driver.info(decision.clone()).await?;- driver.call_llm_target(&target, request, decision).await+ driver.info(ctx.clone(), decision.clone()).await?;+ driver+ .call_llm_target(ctx, &target, request, decision)+ .await}async fn process_signals(@@ -96,10 +98,9 @@ impl Algorithm for RandomOrchAlgo {#[cfg(test)]mod tests {use super::*;- use libsy::{- LlmClient, LlmRequest, LlmResponse, LlmTarget, Message, Response, Role, RoutedRequest,- };+ use libsy::{LlmClient, LlmResponse, LlmTarget, Response, RoutedRequest};use std::collections::HashSet;+ use switchyard_protocol::{completion_text, text_request, text_response};/// Echoes back the target name it was called with, so a test can tell which/// target the algo selected.@@ -112,10 +113,10 @@ mod tests {routed: RoutedRequest,) -> Result<Response, Box<dyn Error + Send + Sync>> {Ok(Response {- llm_response: LlmResponse {- model: Some(routed.decision.selected_model().to_string()),- ..LlmResponse::default()- },+ llm_response: LlmResponse::Agg(text_response(+ None,+ routed.decision.selected_model(),+ )),metadata: None,})}@@ -123,11 +124,7 @@ mod tests {fn request() -> Request {Request {- llm_request: LlmRequest {- model: Some("auto".to_string()),- messages: vec![Message::text(Role::User, "hi")],- ..LlmRequest::default()- },+ llm_request: text_request(Some("auto".to_string()), "hi"),raw_request: None,metadata: None,}@@ -154,7 +151,14 @@ mod tests {) -> Result<(), Box<dyn Error + Send + Sync>> {let orch = orch(&["only/model"]);let (trace, response) = orch.clone().run(Context::default(), request()).await?;- assert_eq!(response.llm_response.model.as_deref(), Some("only/model"));+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "only/model"+ );assert_eq!(trace.len(), 1);assert_eq!(trace[0].selected_model(), "only/model");Ok(())@@ -167,7 +171,11 @@ mod tests {let orch = orch(&names);for _ in 0..50 {let (trace, response) = orch.clone().run(Context::default(), request()).await?;- let selected = response.llm_response.model.unwrap_or_default();+ let selected = response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default();assert!(names.contains(&selected.as_str()),"selected {selected} not in target set"@@ -185,7 +193,13 @@ mod tests {let mut seen = HashSet::new();for _ in 0..100 {let (_, response) = orch.clone().run(Context::default(), request()).await?;- seen.insert(response.llm_response.model.unwrap_or_default());+ seen.insert(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ );}// 100 uniform draws over two targets: both should appear (miss ~ 2^-99).assert_eq!(diff --git a/crates/libsy-protocol/Cargo.toml b/crates/libsy-protocol/Cargo.tomlindex 51e255da3..6e4ec82c8 100644--- a/crates/libsy-protocol/Cargo.toml+++ b/crates/libsy-protocol/Cargo.toml@@ -17,5 +17,6 @@ keywords = ["llm", "protocol", "openai", "anthropic"]publish = ["crates-io"][dependencies]+futures = "0.3"serde = { version = "1", features = ["derive"] }serde_json = "1"diff --git a/crates/libsy-protocol/src/envelope.rs b/crates/libsy-protocol/src/envelope.rsnew file mode 100644index 000000000..33479cdeb--- /dev/null+++ b/crates/libsy-protocol/src/envelope.rs@@ -0,0 +1,58 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! The request/response envelope: the normalized [`LlmRequest`]/[`LlmResponse`] paired+//! with the original provider payload and correlation [`Metadata`].++use crate::{LlmRequest, LlmResponse};++/// Per-request state threaded to an algorithm alongside a request. A placeholder+/// for cross-cutting state (correlation ids, budgets, deadlines) an algorithm will+/// read; empty today.+#[derive(Clone, Debug, Default, PartialEq, Eq)]+pub struct Context {}++/// Correlation and routing metadata attached to a request or response.+///+/// All fields are optional; algorithms and observers use whichever are present+/// (e.g. to key per-session state or emit correlated telemetry). `extra_metadata`+/// is a free-form escape hatch for host-specific keys.+#[derive(Clone)]+pub struct Metadata {+ /// Stable id for a multi-request session/conversation.+ pub session_id: Option<String>,+ /// Id of the agent making the request.+ pub agent_id: Option<String>,+ /// Id of the task the request belongs to.+ pub task_id: Option<String>,+ /// External trace/request id for joining with the host's telemetry.+ pub correlation_id: Option<String>,+ /// Arbitrary host-defined key/value metadata.+ pub extra_metadata: Option<std::collections::BTreeMap<String, String>>,+ /// HTTP headers to attach when forwarding the request/response, if any.+ pub http_headers: Option<std::collections::BTreeMap<String, String>>,+}++/// A request an algorithm routes: the normalized [`LlmRequest`] plus the original+/// provider payload and correlation [`Metadata`].+#[derive(Clone)]+pub struct Request {+ /// The normalized request an algorithm routes.+ pub llm_request: LlmRequest,+ /// The original provider-shaped request body, if the host wants to forward it+ /// verbatim (e.g. a proxy preserving messages/params). libsy does not read it.+ pub raw_request: Option<serde_json::Value>,+ /// Correlation metadata carried through the request.+ pub metadata: Option<Metadata>,+}++/// A response an algorithm returns: the [`LlmResponse`] (streamed or aggregate) plus+/// optional correlation [`Metadata`].+///+/// Not `Clone` — `llm_response` may own a live stream.+pub struct Response {+ /// The neutral model response — a chunk stream or the buffered aggregate.+ pub llm_response: LlmResponse,+ /// Correlation metadata carried through the response.+ pub metadata: Option<Metadata>,+}diff --git a/crates/libsy-protocol/src/lib.rs b/crates/libsy-protocol/src/lib.rsindex c844a2d72..f2f3076b3 100644--- a/crates/libsy-protocol/src/lib.rs+++ b/crates/libsy-protocol/src/lib.rs@@ -1,10 +1,110 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Provider-neutral LLM protocol types shared across Switchyard crates.+//! Shared request/response protocol for libsy — the vocabulary an [`Algorithm`] reasons+//! over, decoupled from libsy's orchestration.+//!+//! This crate owns Switchyard's neutral conversation IR: [`LlmRequest`] (model, messages,+//! tools, sampling, …), the buffered [`AggLlmResponse`] (outputs, usage, …), and its+//! streaming counterpart [`LlmResponseChunk`]; the [`Request`]/[`Response`] envelope that+//! pairs them with correlation [`Metadata`]; plus the wire-[`format`] identifiers+//! translation keys off. `switchyard-translation` re-exports the IR types under its own+//! `ConversationRequest` / `ConversationResponse` / `ConversationStreamEvent` names. The IR+//! carries no bare `prompt`/`completion`; the [`text_request`] / [`prompt_text`] /+//! [`text_response`] / [`completion_text`] helpers bridge to and from plain text for the+//! common single-turn case.+//!+//! The streamed-response type itself — a live stream of chunks *or* the terminal+//! aggregate — is the [`LlmResponse`] enum; it owns a `futures::Stream`, so it is the one+//! non-`Clone`, non-data type in this crate.+//!+//! [`Algorithm`]: https://docs.rs/libsy+pub mod envelope;pub mod format;pub mod llm;+pub mod stream;+pub use envelope::*;pub use format::*;pub use llm::*;+pub use stream::*;++/// Build a single-turn request: one user message carrying `prompt`, for `model`.+pub fn text_request(model: Option<String>, prompt: impl Into<String>) -> LlmRequest {+ LlmRequest {+ model,+ messages: vec![Message::text(Role::User, prompt)],+ ..LlmRequest::default()+ }+}++/// The user's prompt text — the text of every user message, joined by newlines. Empty+/// when the request has no user text.+pub fn prompt_text(request: &LlmRequest) -> String {+ request+ .messages+ .iter()+ .filter(|message| message.role == Role::User)+ .filter_map(|message| message.text_content("\n"))+ .collect::<Vec<_>>()+ .join("\n")+}++/// Build a single-turn response: one assistant message carrying `completion`, for `model`.+pub fn text_response(model: Option<String>, completion: impl Into<String>) -> AggLlmResponse {+ AggLlmResponse {+ model,+ outputs: vec![ResponseOutput {+ role: Role::Assistant,+ content: vec![ContentBlock::Text {+ text: completion.into(),+ }],+ stop_reason: None,+ }],+ ..AggLlmResponse::default()+ }+}++/// The assistant's completion text — the text blocks of the first output, concatenated.+/// Empty when the response has no textual output.+pub fn completion_text(response: &AggLlmResponse) -> String {+ response+ .outputs+ .first()+ .map(|output| {+ output+ .content+ .iter()+ .filter_map(|block| match block {+ ContentBlock::Text { text } => Some(text.as_str()),+ _ => None,+ })+ .collect::<String>()+ })+ .unwrap_or_default()+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn request_round_trips_prompt_text() {+ let req = text_request(Some("m".to_string()), "hello world");+ assert_eq!(req.model.as_deref(), Some("m"));+ assert_eq!(prompt_text(&req), "hello world");+ }++ #[test]+ fn response_round_trips_completion_text() {+ let resp = text_response(None, "the answer");+ assert_eq!(completion_text(&resp), "the answer");+ }++ #[test]+ fn empty_text_helpers_are_empty_strings() {+ assert_eq!(prompt_text(&LlmRequest::default()), "");+ assert_eq!(completion_text(&AggLlmResponse::default()), "");+ }+}diff --git a/crates/libsy-protocol/src/llm.rs b/crates/libsy-protocol/src/llm.rsindex 9cb3ef874..6d873e794 100644--- a/crates/libsy-protocol/src/llm.rs+++ b/crates/libsy-protocol/src/llm.rs@@ -229,36 +229,6 @@ pub struct Usage {pub reasoning_tokens: Option<u64>,}-/// Provider-neutral event used between stream decoders and encoders.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub enum LlmStreamEvent {- MessageStart {- id: Option<String>,- model: Option<String>,- },- TextDelta {- index: usize,- text: String,- },- ReasoningDelta {- index: usize,- text: String,- },- ToolCallDelta {- index: usize,- id: Option<String>,- name: Option<String>,- arguments_delta: Option<String>,- },- Usage(Usage),- MessageStop {- reason: Option<String>,- },- Error {- message: String,- },-}-/// Normalized reason a model stopped producing output.#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]pub enum StopReason {@@ -278,9 +248,12 @@ pub struct ResponseOutput {pub stop_reason: Option<StopReason>,}-/// Normalized response representation shared by Switchyard components.+/// Normalized, fully-buffered response — the aggregate of a completed generation.+/// libsy refers to this as an `AggLlmResponse` (the terminal form of a streamed+/// [`LlmResponse`](crate::LlmResponse)); `switchyard-translation` re-exports it as+/// `ConversationResponse`.#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]-pub struct LlmResponse {+pub struct AggLlmResponse {pub id: Option<String>,pub model: Option<String>,pub outputs: Vec<ResponseOutput>,@@ -289,7 +262,7 @@ pub struct LlmResponse {pub preservation: PreservationMetadata,}-impl LlmResponse {+impl AggLlmResponse {/// Returns the first output item when a response has any output.pub fn first_output(&self) -> Option<&ResponseOutput> {self.outputs.first()diff --git a/crates/libsy-protocol/src/stream.rs b/crates/libsy-protocol/src/stream.rsnew file mode 100644index 000000000..175fd1d19--- /dev/null+++ b/crates/libsy-protocol/src/stream.rs@@ -0,0 +1,333 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Streaming half of the neutral IR: incremental response chunks ([`LlmResponseChunk`])+//! and the streamed response ([`LlmResponse`]) that carries either a live stream of them+//! or the terminal [`AggLlmResponse`].++use std::collections::BTreeMap;+use std::error::Error;+use std::pin::Pin;++use futures::{Stream, StreamExt};+use serde::{Deserialize, Serialize};+use serde_json::Value;++use crate::llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage};++/// Boxed, thread-safe error carried by a stream item.+type BoxError = Box<dyn Error + Send + Sync>;++/// A boxed, `Send` stream of [`LlmResponseChunk`]s — the token-by-token output of a+/// streaming backend. Each item may fail independently mid-stream.+pub type LlmResponseStream = Pin<Box<dyn Stream<Item = Result<LlmResponseChunk, BoxError>> + Send>>;++/// A model response: either a live [`Stream`](LlmResponse::Stream) of chunks or the+/// terminal buffered [`Agg`](LlmResponse::Agg)regate.+///+/// Not `Clone` — the `Stream` variant owns a single-consumption stream. A buffered+/// backend returns `Agg` directly; a streaming one returns `Stream` and the consumer+/// drives it, folding to an [`AggLlmResponse`] when it needs the whole response.+pub enum LlmResponse {+ Stream(LlmResponseStream),+ Agg(AggLlmResponse),+}++impl LlmResponse {+ /// Borrow the aggregate; `None` while this is still a stream.+ pub fn as_agg(&self) -> Option<&AggLlmResponse> {+ match self {+ LlmResponse::Agg(agg) => Some(agg),+ LlmResponse::Stream(_) => None,+ }+ }++ /// Reduce to the buffered aggregate: return an `Agg` unchanged, or drive a `Stream`+ /// to completion, folding its chunks into an [`AggLlmResponse`] via+ /// [`ResponseAccumulator`]. A stream item error, or an in-band+ /// [`LlmResponseChunk::Error`], aborts with `Err`.+ pub async fn into_agg(self) -> Result<AggLlmResponse, BoxError> {+ match self {+ LlmResponse::Agg(agg) => Ok(agg),+ LlmResponse::Stream(mut stream) => {+ let mut accumulator = ResponseAccumulator::new();+ while let Some(item) = stream.next().await {+ match item? {+ LlmResponseChunk::Error { message } => return Err(message.into()),+ chunk => accumulator.push(chunk),+ }+ }+ Ok(accumulator.finish())+ }+ }+ }+}++/// One provider-neutral streaming event — the normalized counterpart to+/// [`AggLlmResponse`](crate::AggLlmResponse), sitting between stream decoders and+/// encoders. `switchyard-translation` re-exports it as `ConversationStreamEvent`.+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]+pub enum LlmResponseChunk {+ MessageStart {+ id: Option<String>,+ model: Option<String>,+ },+ TextDelta {+ index: usize,+ text: String,+ },+ ReasoningDelta {+ index: usize,+ text: String,+ },+ ToolCallDelta {+ index: usize,+ id: Option<String>,+ name: Option<String>,+ arguments_delta: Option<String>,+ },+ Usage(Usage),+ MessageStop {+ reason: Option<String>,+ },+ Error {+ message: String,+ },+}++/// Folds a sequence of [`LlmResponseChunk`]s into the terminal [`AggLlmResponse`].+///+/// Text and reasoning deltas concatenate; tool-call deltas assemble by index (name,+/// id, and a growing arguments string parsed as JSON at the end); `MessageStart`,+/// `Usage`, and `MessageStop` set the corresponding fields. `Error` chunks are+/// ignored here — a driver consuming the stream is expected to surface them.+///+/// Drive it by `push`-ing each chunk in order, then call [`finish`](Self::finish).+#[derive(Default)]+pub struct ResponseAccumulator {+ id: Option<String>,+ model: Option<String>,+ text: String,+ reasoning: Option<String>,+ tool_calls: BTreeMap<usize, PartialToolCall>,+ usage: Usage,+ stop_reason: Option<StopReason>,+}++/// A tool call being assembled from streamed [`LlmResponseChunk::ToolCallDelta`]s.+#[derive(Default)]+struct PartialToolCall {+ id: Option<String>,+ name: Option<String>,+ arguments: String,+}++impl ResponseAccumulator {+ /// A fresh accumulator with no chunks applied.+ pub fn new() -> Self {+ Self::default()+ }++ /// Apply one chunk. Later `MessageStart`/`Usage`/`MessageStop` fields overwrite+ /// earlier ones; text, reasoning, and tool-call arguments append.+ pub fn push(&mut self, chunk: LlmResponseChunk) {+ match chunk {+ LlmResponseChunk::MessageStart { id, model } => {+ if id.is_some() {+ self.id = id;+ }+ if model.is_some() {+ self.model = model;+ }+ }+ LlmResponseChunk::TextDelta { text, .. } => self.text.push_str(&text),+ LlmResponseChunk::ReasoningDelta { text, .. } => {+ self.reasoning+ .get_or_insert_with(String::new)+ .push_str(&text);+ }+ LlmResponseChunk::ToolCallDelta {+ index,+ id,+ name,+ arguments_delta,+ } => {+ let call = self.tool_calls.entry(index).or_default();+ if id.is_some() {+ call.id = id;+ }+ if name.is_some() {+ call.name = name;+ }+ if let Some(delta) = arguments_delta {+ call.arguments.push_str(&delta);+ }+ }+ LlmResponseChunk::Usage(usage) => self.usage = usage,+ LlmResponseChunk::MessageStop { reason } => {+ self.stop_reason = Some(stop_reason_from_str(reason.as_deref()));+ }+ LlmResponseChunk::Error { .. } => {}+ }+ }++ /// Build the buffered response. Content is ordered reasoning, then text, then+ /// tool calls (by ascending delta index) — a single assistant output.+ pub fn finish(self) -> AggLlmResponse {+ let mut content = Vec::new();+ if let Some(reasoning) = self.reasoning {+ content.push(ContentBlock::Reasoning {+ text: reasoning,+ signature: None,+ });+ }+ if !self.text.is_empty() {+ content.push(ContentBlock::Text { text: self.text });+ }+ for call in self.tool_calls.into_values() {+ content.push(ContentBlock::ToolCall(ToolCall {+ id: call.id.unwrap_or_default(),+ name: call.name.unwrap_or_default(),+ arguments: parse_tool_arguments(&call.arguments),+ }));+ }+ AggLlmResponse {+ id: self.id,+ model: self.model,+ outputs: vec![ResponseOutput {+ role: Role::Assistant,+ content,+ stop_reason: self.stop_reason,+ }],+ usage: self.usage,+ ..AggLlmResponse::default()+ }+ }+}++/// Parse an assembled tool-call arguments string as JSON, falling back to a JSON+/// string when it is not valid JSON and to an empty object when it is empty.+fn parse_tool_arguments(arguments: &str) -> Value {+ if arguments.is_empty() {+ return Value::Object(serde_json::Map::new());+ }+ serde_json::from_str(arguments).unwrap_or_else(|_| Value::String(arguments.to_string()))+}++/// Map a provider stop-reason string (as carried by [`LlmResponseChunk::MessageStop`])+/// to a normalized [`StopReason`], covering the common OpenAI and Anthropic spellings.+fn stop_reason_from_str(reason: Option<&str>) -> StopReason {+ match reason {+ Some("length" | "max_tokens") => StopReason::MaxTokens,+ Some("tool_calls" | "function_call" | "tool_use") => StopReason::ToolUse,+ Some("content_filter") => StopReason::ContentFilter,+ Some("stop" | "end_turn" | "stop_sequence") | None => StopReason::EndTurn,+ Some(_) => StopReason::Unknown,+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ fn fold(chunks: Vec<LlmResponseChunk>) -> AggLlmResponse {+ let mut accumulator = ResponseAccumulator::new();+ for chunk in chunks {+ accumulator.push(chunk);+ }+ accumulator.finish()+ }++ #[test]+ fn folds_text_usage_and_stop_reason() {+ let agg = fold(vec![+ LlmResponseChunk::MessageStart {+ id: Some("id1".to_string()),+ model: Some("m".to_string()),+ },+ LlmResponseChunk::TextDelta {+ index: 0,+ text: "Hel".to_string(),+ },+ LlmResponseChunk::TextDelta {+ index: 0,+ text: "lo".to_string(),+ },+ LlmResponseChunk::Usage(Usage {+ output_tokens: Some(2),+ ..Usage::default()+ }),+ LlmResponseChunk::MessageStop {+ reason: Some("length".to_string()),+ },+ ]);+ assert_eq!(agg.id.as_deref(), Some("id1"));+ assert_eq!(agg.model.as_deref(), Some("m"));+ assert_eq!(agg.usage.output_tokens, Some(2));+ assert_eq!(agg.outputs[0].stop_reason, Some(StopReason::MaxTokens));+ assert_eq!(+ agg.outputs[0].content,+ vec![ContentBlock::Text {+ text: "Hello".to_string()+ }]+ );+ }++ #[test]+ fn assembles_tool_calls_by_index() {+ // id/name arrive once, arguments stream across deltas and parse as JSON.+ let agg = fold(vec![+ LlmResponseChunk::ToolCallDelta {+ index: 0,+ id: Some("call_1".to_string()),+ name: Some("lookup".to_string()),+ arguments_delta: Some("{\"q\":".to_string()),+ },+ LlmResponseChunk::ToolCallDelta {+ index: 0,+ id: None,+ name: None,+ arguments_delta: Some("\"rust\"}".to_string()),+ },+ LlmResponseChunk::MessageStop {+ reason: Some("tool_calls".to_string()),+ },+ ]);+ assert_eq!(agg.outputs[0].stop_reason, Some(StopReason::ToolUse));+ assert_eq!(+ agg.outputs[0].content,+ vec![ContentBlock::ToolCall(ToolCall {+ id: "call_1".to_string(),+ name: "lookup".to_string(),+ arguments: json!({"q": "rust"}),+ })]+ );+ }++ #[test]+ fn reasoning_precedes_text_in_content() {+ let agg = fold(vec![+ LlmResponseChunk::ReasoningDelta {+ index: 0,+ text: "think".to_string(),+ },+ LlmResponseChunk::TextDelta {+ index: 0,+ text: "answer".to_string(),+ },+ ]);+ assert_eq!(+ agg.outputs[0].content,+ vec![+ ContentBlock::Reasoning {+ text: "think".to_string(),+ signature: None,+ },+ ContentBlock::Text {+ text: "answer".to_string(),+ },+ ]+ );+ }+}diff --git a/crates/libsy/README.md b/crates/libsy/README.mdindex d4eced886..99a8771ee 100644--- a/crates/libsy/README.md+++ b/crates/libsy/README.md@@ -11,11 +11,9 @@ so it drops into a proxy, gateway, or agent runtime.Build a target set, pick an algorithm, run a request:```rust-use libsy::{- Algorithm, ContentBlock, Context, LlmRequest, LlmClient, LlmTarget, LlmTargetSet,- Message, Request, Role,-};+use libsy::{Algorithm, Context, LlmClient, LlmTarget, LlmTargetSet, Request};use libsy_examples::llm_class::LlmClassifierOrchAlgo;+use switchyard_protocol::{completion_text, text_request};use std::sync::Arc;// Targets the algorithm routes among, each backed by your LlmClient (see below).@@ -28,16 +26,16 @@ let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new());let req = Request {- llm_request: LlmRequest {- model: Some("auto".into()),- messages: vec![Message::text(Role::User, "explain tail latency")],- ..LlmRequest::default()- },+ // `text_request` is the single-turn shortcut; build an `LlmRequest` directly for+ // multi-turn conversations, tools, or sampling parameters.+ llm_request: text_request(Some("auto".into()), "explain tail latency"),raw_request: None,metadata: None,};let (trace, response) = algo.clone().run(Context::default(), req).await?; // calls in, trace + response outprintln!("routed to {}", trace.last().unwrap().selected_model());+// `response.llm_response` is buffered or streamed; fold it to the aggregate to read text.+println!("answer: {}", completion_text(&response.llm_response.aggregate().await?));```Runnable: [`research_agent`](../libsy-examples/examples/research_agent.rs) (in the `libsy-examples` crate).@@ -51,18 +49,29 @@ pub struct Request {pub metadata: Option<Metadata>, // correlation: session / agent / task / correlation_id / extra}+// `Response` is a plain struct. The buffered-or-streamed choice lives one level down,+// in its `llm_response` — which *is* an enum.pub struct Response {pub llm_response: LlmResponse,pub metadata: Option<Metadata>,}++pub enum LlmResponse {+ Stream(LlmResponseStream), // a live stream of `LlmResponseChunk`s (token-by-token)+ Agg(AggLlmResponse), // the buffered aggregate — outputs, usage, stop reason+}```-`switchyard-protocol` owns `LlmRequest`, `LlmResponse`, `Message`,-`ContentBlock`, `ResponseOutput`, and `Role`. `libsy` re-exports those canonical types-unchanged. Construct and inspect the conversation model directly so tools, sampling-parameters, reasoning, and provider extensions remain visible instead of being hidden-behind a second convenience API. `raw_request` remains available when a host needs exact-source-body fidelity.+`switchyard-protocol` owns the conversation IR: `LlmRequest`, the buffered+`AggLlmResponse`, the streaming `LlmResponseChunk`, and the building blocks `Message`,+`ContentBlock`, `ResponseOutput`, and `Role`. `libsy` re-exports the envelope+(`Request`/`Response`/`Metadata`) plus `LlmRequest`, `LlmResponse`, `AggLlmResponse`,+`LlmResponseChunk`, and `LlmResponseStream`; import the rest (`Message`, `Role`, …) and the+`text_request` / `text_response` / `completion_text` helpers from `switchyard_protocol`.+Construct and inspect the conversation model directly so tools, sampling parameters,+reasoning, and provider extensions stay visible instead of being hidden behind a second+convenience API. `raw_request` remains available when a host needs exact source-body+fidelity.## Targets and clients@@ -82,20 +91,19 @@ impl LlmClient for MyClient {// ... POST to your endpoint, read the completion ...let completion = String::from("provider response text");Ok(Response {- llm_response: LlmResponse {- outputs: vec![ResponseOutput {- role: Role::Assistant,- content: vec![ContentBlock::Text { text: completion }],- stop_reason: None,- }],- ..LlmResponse::default()- },+ // Buffered: wrap the aggregate in `LlmResponse::Agg`. `text_response` builds a+ // single-turn `AggLlmResponse`; construct one directly for tool calls, usage, etc.+ llm_response: LlmResponse::Agg(text_response(None, completion)),metadata: None,})}}```+To stream instead, return `LlmResponse::Stream(..)` — a boxed+`Stream<Item = Result<LlmResponseChunk, _>>` — and emit chunks as they arrive from your+upstream. See [Streaming responses](#streaming-responses) below.+A `RoutedRequest` bundles the `request` with the routing `decision` and the target's`default_client`; the model to call is `decision.selected_model()`, never a mutatedrequest field. `semantic_name` is the label an algorithm routes by; the client maps it to@@ -118,10 +126,51 @@ Under the hood every model call is *offloaded* to the request's `Step` stream; `the convenience that serves each one via the target's client. The step stream is bounded,so pulling it paces the algorithm; each run is independent, so many run concurrently.-## Streaming — you own the model calls (`run_stream`)+## Streaming responses++A `Response` carries an `LlmResponse` that is either buffered (`Agg`) or a live token+stream (`Stream`). An `LlmClient` — or an algorithm — chooses: return+`LlmResponse::Agg(..)` for a whole answer, or `LlmResponse::Stream(..)` to forward tokens+as they arrive. libsy never buffers a stream on your behalf; it flows through the algorithm+untouched, so `run` / `run_stream` hand back whatever was produced and **the caller+decides**:++```rust+use futures::StreamExt;++let (_trace, response) = algo.clone().run(Context::default(), req).await?;+match response.llm_response {+ // Forward tokens as they arrive — e.g. re-encode each chunk to your client's SSE.+ LlmResponse::Stream(mut chunks) => {+ while let Some(chunk) = chunks.next().await {+ let chunk: LlmResponseChunk = chunk?; // TextDelta / ToolCallDelta / Usage / MessageStop / ..+ /* emit chunk */+ }+ }+ // Already buffered — read it directly.+ LlmResponse::Agg(agg) => println!("{}", completion_text(&agg)),+}+```++Need the whole answer regardless of how it arrived? `aggregate()` folds a `Stream` (or+returns an `Agg` unchanged) into a single `AggLlmResponse`, surfacing any mid-stream error:++```rust+let agg = response.llm_response.aggregate().await?; // drives the stream to completion+println!("{}", completion_text(&agg));+```++An `LlmResponseChunk` is the provider-neutral streaming event (`MessageStart`, `TextDelta`,+`ReasoningDelta`, `ToolCallDelta`, `Usage`, `MessageStop`, `Error`); `ResponseAccumulator`+is the fold behind `aggregate()` if you need to assemble one yourself. Runnable end-to-end+demo: [`streaming_agent`](../libsy-examples/examples/streaming_agent.rs).++## Driving the calls yourself (`run_stream`)`run_stream` yields `CallLlm` promises you fulfill with your own transport (or the call's`default_client`), streams each `Decision` as it happens, and ends with `ReturnToAgent`.+This is the *step* stream (one `Step` at a time); it is orthogonal to whether any single+response is itself streamed — see [Streaming responses](#streaming-responses).Runnable: [`research_agent_core`](../libsy-examples/examples/research_agent_core.rs) (in the `libsy-examples` crate).```rust@@ -173,27 +222,27 @@ Example — the LLM classifier (classify, then route; full version in```rust#[async_trait]impl Algorithm for LlmClassifierOrchAlgo {- async fn create_run_task(self: Arc<Self>, _ctx: Context, driver: Driver, request: Request)+ async fn create_run_task(self: Arc<Self>, ctx: Context, driver: Driver, request: Request)-> Result<Response, Box<dyn Error + Send + Sync>> {+ // Thread `ctx` into every offloaded call and decision — it carries the request's+ // cross-cutting state (correlation ids, budgets) for observers downstream.+// 1. Classify: ask the classifier target for a score.let classifier = self.target_set.get_target(&self.classifier_model)?;- driver.info(classify_decision.clone()).await?;+ driver.info(ctx.clone(), classify_decision.clone()).await?;let classify_response =- driver.call_llm_target(&classifier, classify_req, classify_decision).await?;- // Abbreviated: this assumes a text score in the first output block. The full- // implementation scans all outputs and text-like blocks.- let score = classify_response.llm_response.first_output()- .and_then(|output| output.content.first())- .and_then(|block| match block {- ContentBlock::Text { text } => text.trim().parse::<f64>().ok(),- _ => None,- });+ driver.call_llm_target(ctx.clone(), &classifier, classify_req, classify_decision).await?;+ // Abbreviated: fold the (buffered or streamed) response to its aggregate and read+ // the completion text as a score. `.agg()` alone is `None` for an unfolded stream.+ let score = classify_response.llm_response.aggregate().await+ .ok()+ .and_then(|agg| completion_text(&agg).trim().parse::<f64>().ok());// 2. Route: strong if score >= threshold, else weak (fail open on None).let model = if score.map_or(true, |s| s >= self.threshold) { &self.strong_model } else { &self.weak_model };let routed = self.target_set.get_target(model)?;- driver.info(route_decision.clone()).await?;- driver.call_llm_target(&routed, routed_req, route_decision).await+ driver.info(ctx.clone(), route_decision.clone()).await?;+ driver.call_llm_target(ctx, &routed, routed_req, route_decision).await}async fn process_signals(self: Arc<Self>, _s: Signals) -> Result<(), Box<dyn Error + Send + Sync>> { Ok(()) }@@ -220,6 +269,8 @@ tested — `cargo test -p libsy-examples`).targets, `run` (libsy makes the calls).- [`research_agent_core`](../libsy-examples/examples/research_agent_core.rs) — client-lesstargets, `run_stream` (the agent makes the calls).+- [`streaming_agent`](../libsy-examples/examples/streaming_agent.rs) — a target that streams+ its response; the agent forwards each `LlmResponseChunk` to the caller token-by-token.## Not yet builtdiff --git a/crates/libsy/src/driver.rs b/crates/libsy/src/driver.rsindex 410f713d9..b0bb0c770 100644--- a/crates/libsy/src/driver.rs+++ b/crates/libsy/src/driver.rs@@ -52,6 +52,8 @@ use std::{sync::{Arc, Mutex},};+use crate::Context;+use futures::{Stream, StreamExt};use tokio::sync::{mpsc, oneshot};use tokio_stream::wrappers::ReceiverStream;@@ -139,7 +141,7 @@ impl TypeErasedDriver {/// Enqueue `req` as a [`DriverStep::Request`], await the consumer's response, and/// downcast it to `RES`. Errors if the stream is closed, the promise is dropped/// unfulfilled, the consumer responded with `Err`, or the response was not a `RES`.- pub async fn fulfill_request<REQ, RES>(&self, req: REQ) -> Result<RES, BoxErr>+ pub async fn fulfill_request<REQ, RES>(&self, _ctx: Context, req: REQ) -> Result<RES, BoxErr>whereREQ: Any + Send + 'static,RES: Any + Send + 'static,@@ -170,7 +172,7 @@ impl TypeErasedDriver {/// Push a fire-and-forget [`DriverStep::Info`] payload; there is no promise to/// await for a response. Awaits channel capacity (the consumer pacing the stream)/// and errors only if the stream is closed.- pub async fn info<INFO>(&self, info: INFO) -> Result<(), BoxErr>+ pub async fn info<INFO>(&self, _ctx: Context, info: INFO) -> Result<(), BoxErr>whereINFO: Any + Send + 'static,{@@ -185,7 +187,7 @@ impl TypeErasedDriver {/// stream (that happens when every `TypeErasedDriver` clone drops); the consumer treats it/// as the last meaningful step. Awaits channel capacity and errors only if the/// stream is closed.- pub async fn done<T>(&self, payload: T) -> Result<(), BoxErr>+ pub async fn done<T>(&self, _ctx: Context, payload: T) -> Result<(), BoxErr>whereT: Any + Send + 'static,{@@ -200,7 +202,7 @@ impl TypeErasedDriver {/// failure to the consumer (mirrors how the crate's `run_stream` yields an `Err`/// step). Awaits channel capacity and errors only if the stream is/// already closed.- pub async fn fail(&self, err: BoxErr) -> Result<(), BoxErr> {+ pub async fn fail(&self, _ctx: Context, err: BoxErr) -> Result<(), BoxErr> {self.inner.step_tx.send(Err(err))@@ -244,8 +246,11 @@ mod tests {// Producer asks for a u32 -> String on its own task.let producer = driver.clone();- let handle =- tokio::spawn(async move { producer.fulfill_request::<u32, String>(7u32).await });+ let handle = tokio::spawn(async move {+ producer+ .fulfill_request::<u32, String>(Context::default(), 7u32)+ .await+ });tokio::pin!(stream);match stream.next().await.ok_or("no step")?? {@@ -265,7 +270,7 @@ mod tests {async fn info_pushes_a_typed_payload() -> Result<(), BoxErr> {let driver = TypeErasedDriver::new();let stream = driver.stream();- driver.info(42u64).await?;+ driver.info(Context::default(), 42u64).await?;tokio::pin!(stream);match stream.next().await.ok_or("no step")?? {@@ -282,7 +287,9 @@ mod tests {async fn done_emits_the_terminal_payload() -> Result<(), BoxErr> {let driver = TypeErasedDriver::new();let stream = driver.stream();- driver.done("finished".to_string()).await?;+ driver+ .done(Context::default(), "finished".to_string())+ .await?;tokio::pin!(stream);match stream.next().await.ok_or("no step")?? {@@ -303,7 +310,11 @@ mod tests {let stream = driver.stream();let producer = driver.clone();- let handle = tokio::spawn(async move { producer.fulfill_request::<u32, u32>(1u32).await });+ let handle = tokio::spawn(async move {+ producer+ .fulfill_request::<u32, u32>(Context::default(), 1u32)+ .await+ });tokio::pin!(stream);match stream.next().await.ok_or("no step")?? {@@ -329,8 +340,11 @@ mod tests {// Producer expects a String back.let producer = driver.clone();- let handle =- tokio::spawn(async move { producer.fulfill_request::<u32, String>(1u32).await });+ let handle = tokio::spawn(async move {+ producer+ .fulfill_request::<u32, String>(Context::default(), 1u32)+ .await+ });tokio::pin!(stream);match stream.next().await.ok_or("no step")?? {@@ -356,7 +370,11 @@ mod tests {let stream = driver.stream();let producer = driver.clone();- let handle = tokio::spawn(async move { producer.fulfill_request::<u32, u32>(5u32).await });+ let handle = tokio::spawn(async move {+ producer+ .fulfill_request::<u32, u32>(Context::default(), 5u32)+ .await+ });tokio::pin!(stream);match stream.next().await.ok_or("no step")?? {@@ -378,9 +396,12 @@ mod tests {// Drop the consumer stream (and its receiver) before producing anything.drop(driver.stream());- assert!(driver.fulfill_request::<u32, u32>(1u32).await.is_err());- assert!(driver.info(1u32).await.is_err());- assert!(driver.done(1u32).await.is_err());+ assert!(driver+ .fulfill_request::<u32, u32>(Context::default(), 1u32)+ .await+ .is_err());+ assert!(driver.info(Context::default(), 1u32).await.is_err());+ assert!(driver.done(Context::default(), 1u32).await.is_err());Ok(())}@@ -390,7 +411,11 @@ mod tests {let stream = driver.stream();let producer = driver.clone();- let handle = tokio::spawn(async move { producer.fulfill_request::<u32, u32>(1u32).await });+ let handle = tokio::spawn(async move {+ producer+ .fulfill_request::<u32, u32>(Context::default(), 1u32)+ .await+ });tokio::pin!(stream);match stream.next().await.ok_or("no step")?? {@@ -415,7 +440,11 @@ mod tests {let producer = driver.clone();handles.push((i,- tokio::spawn(async move { producer.fulfill_request::<usize, usize>(i).await }),+ tokio::spawn(async move {+ producer+ .fulfill_request::<usize, usize>(Context::default(), i)+ .await+ }),));}@@ -460,7 +489,7 @@ mod tests {async fn fail_surfaces_an_error_item_on_the_stream() -> Result<(), BoxErr> {let driver = TypeErasedDriver::new();let stream = driver.stream();- driver.fail("kaboom".into()).await?;+ driver.fail(Context::default(), "kaboom".into()).await?;tokio::pin!(stream);match stream.next().await.ok_or("no item")? {@@ -484,7 +513,7 @@ mod tests {let producer = driver.clone();let handle = tokio::spawn(async move {let mut sent = 0usize;- while producer.info(sent).await.is_ok() {+ while producer.info(Context::default(), sent).await.is_ok() {sent += 1;}sentdiff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsindex b35b8117c..d30766dd8 100644--- a/crates/libsy/src/lib.rs+++ b/crates/libsy/src/lib.rs@@ -64,12 +64,17 @@ use std::{error::Error, pin::Pin, sync::Arc};use async_trait::async_trait;use futures::{Stream, StreamExt};-use crate::driver::{DriverRequest, DriverStep, TypeErasedDriver};-+/// The request/response protocol types, re-exported from [`switchyard_protocol`].+/// [`LlmRequest`] is the normalized request; [`AggLlmResponse`] is the buffered response;+/// [`LlmResponseChunk`] is one streaming event; [`LlmResponse`] is the streamed response+/// (a live [`LlmResponseStream`] or the terminal aggregate).pub use switchyard_protocol::{- ContentBlock, LlmRequest, LlmResponse, Message, ResponseOutput, Role,+ AggLlmResponse, Context, LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStream,+ Metadata, Request, Response,};+use crate::driver::{DriverRequest, DriverStep, TypeErasedDriver};+/// Shorthand for the crate's boxed, thread-safe error type.type BoxErr = Box<dyn Error + Send + Sync>;@@ -78,37 +83,6 @@ type BoxErr = Box<dyn Error + Send + Sync>;/// `Arc<dyn Algorithm>` object-safe.pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step, BoxErr>> + Send>>;-/// Correlation and routing metadata attached to a request or response.-///-/// All fields are optional; algorithms and observers use whichever are present-/// (e.g. to key per-session state or emit correlated telemetry). `extra_metadata`-/// is a free-form escape hatch for host-specific keys.-#[derive(Clone)]-pub struct Metadata {- /// Stable id for a multi-request session/conversation.- pub session_id: Option<String>,- /// Id of the agent making the request.- pub agent_id: Option<String>,- /// Id of the task the request belongs to.- pub task_id: Option<String>,- /// External trace/request id for joining with the host's telemetry.- pub correlation_id: Option<String>,- /// Arbitrary host-defined key/value metadata.- pub extra_metadata: Option<std::collections::BTreeMap<String, String>>,-}--/// A request entering the orchestrator: the normalized [`LlmRequest`] plus the-/// original provider payload and correlation [`Metadata`].-#[derive(Clone)]-pub struct Request {- /// The normalized request an algorithm routes.- pub llm_request: LlmRequest,- /// Optional source provider body, carried only for hosts that need exact fidelity.- pub raw_request: Option<serde_json::Value>,- /// Correlation metadata carried through the request.- pub metadata: Option<Metadata>,-}-/// Agentic-stack events fed to an algorithm out of band via/// [`Algorithm::process_signals`] (e.g. tool results, budget updates).///@@ -117,16 +91,6 @@ pub struct Request {#[derive(Clone)]pub struct Signals {}-/// A response leaving the orchestrator: the neutral [`LlmResponse`] plus optional-/// correlation [`Metadata`].-#[derive(Clone)]-pub struct Response {- /// The neutral model response.- pub llm_response: LlmResponse,- /// Correlation metadata carried through the response.- pub metadata: Option<Metadata>,-}-/// A decision/trace object produced by an algorithm.////// Carried as a trait object (not a generic parameter) so a stream consumer can@@ -148,11 +112,12 @@ pub trait Decision: Send + Sync {////// The two model identifiers live in separate, unambiguous places: the model to/// call is [`decision.selected_model()`](Decision::selected_model), while-/// `request.llm_request.model` is the inbound name the agent asked for. A client-/// maps `selected_model()` to the provider model id it hits.+/// `request.llm_request.model` is the *inbound* name the agent asked for (libsy+/// never overwrites it). A client maps `selected_model()` to the provider model+/// id it hits.#[derive(Clone)]pub struct RoutedRequest {- /// The request to serve; its `model` is the agent's original inbound model.+ /// The request to serve; its `model` is the agent's original name.pub request: Request,/// The routing decision behind this call; `selected_model()` is the model to hit.pub decision: Arc<dyn Decision>,@@ -171,22 +136,27 @@ pub struct RoutedRequest {/// [`Driver::call_llm`] on the other side.pub struct CallLlmRequest {inner: DriverRequest,+ routed: RoutedRequest,}impl CallLlmRequest {- /// Wrap a driver request whose payload is a [`RoutedRequest`].+ /// Wrap a driver request whose payload is a [`RoutedRequest`]. Caches an owned copy+ /// so the accessors are plain field reads.fn new(inner: DriverRequest) -> Self {- Self { inner }+ // The payload is always a `RoutedRequest` (set by `Driver::call_llm`); a+ // mismatch would be a libsy bug, not a runtime condition.+ let routed = match inner.request::<RoutedRequest>() {+ Ok(routed) => routed.clone(),+ Err(_) => unreachable!("CallLlmRequest payload is always a RoutedRequest"),+ };+ Self { inner, routed }}/// The routed request the host should serve. Its/// [`default_client`](RoutedRequest::default_client) serves the call by default,/// and its `decision.selected_model()` names the model to hit.pub fn get_routed(&self) -> &RoutedRequest {- match self.inner.request::<RoutedRequest>() {- Ok(routed) => routed,- Err(_) => unreachable!("CallLlmRequest payload is always a RoutedRequest"),- }+ &self.routed}/// The model request to perform (the [`Request`] inside the routed request).@@ -229,9 +199,9 @@ impl Driver {/// Offload a model call: publish `routed` as a [`Step::CallLlm`] and await the/// consumer's [`Response`]. Errors if the stream is closed or the call failed.- pub async fn call_llm(&self, routed: RoutedRequest) -> Result<Response, BoxErr> {+ pub async fn call_llm(&self, ctx: Context, routed: RoutedRequest) -> Result<Response, BoxErr> {self.driver- .fulfill_request::<RoutedRequest, Response>(routed)+ .fulfill_request::<RoutedRequest, Response>(ctx, routed).await}@@ -239,33 +209,41 @@ impl Driver {/// default client into a [`RoutedRequest`], then publish it (see/// [`call_llm`](Self::call_llm)). The convenience most algorithms use;/// `decision.selected_model()` names the model to hit, and `request`'s- /// inbound `model` is left untouched.+ /// `model` is left untouched.pub async fn call_llm_target(&self,+ ctx: Context,target: &LlmTarget,request: Request,decision: Arc<dyn Decision>,) -> Result<Response, BoxErr> {- self.call_llm(RoutedRequest {- request,- decision,- default_client: target.llm_client.clone(),- })+ self.call_llm(+ ctx,+ RoutedRequest {+ request,+ decision,+ default_client: target.llm_client.clone(),+ },+ ).await}/// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.- pub async fn info(&self, decision: Arc<dyn Decision>) -> Result<(), BoxErr> {- self.driver.info(decision).await+ pub async fn info(&self, ctx: Context, decision: Arc<dyn Decision>) -> Result<(), BoxErr> {+ self.driver.info(ctx, decision).await}/// Emit the terminal step: [`Step::ReturnToAgent`] on `Ok`, or an `Err` stream/// item on failure. Internal: called once by [`run_stream`](Algorithm::run_stream)/// when the algorithm finishes.- pub(crate) async fn finish(&self, result: Result<Response, BoxErr>) -> Result<(), BoxErr> {+ pub(crate) async fn finish(+ &self,+ ctx: Context,+ result: Result<Response, BoxErr>,+ ) -> Result<(), BoxErr> {match result {- Ok(response) => self.driver.done(response).await,- Err(err) => self.driver.fail(err).await,+ Ok(response) => self.driver.done(ctx, response).await,+ Err(err) => self.driver.fail(ctx, err).await,}}@@ -274,7 +252,7 @@ impl Driver {/// payload that does not match the expected type for its step becomes an `Err` item.pub(crate) fn stream(&self) -> impl Stream<Item = Result<Step, BoxErr>> {self.driver.stream().map(|item| match item? {- DriverStep::Request(req) => Ok(Step::CallLlm(CallLlmRequest::new(req))),+ DriverStep::Request(req) => Ok(Step::CallLlm(Box::new(CallLlmRequest::new(req)))),DriverStep::Info(payload) => payload.downcast::<Arc<dyn Decision>>().map(|decision| Step::Decision(*decision))@@ -293,25 +271,12 @@ impl Default for Driver {}}-/// Per-request state threaded to an algorithm alongside its [`Driver`]. A placeholder-/// for cross-cutting state (correlation ids, budgets, deadlines) an algorithm will-/// read; empty today. It does not carry the offload driver, so it is safe to share.-#[derive(Clone, Default)]-pub struct Context {}--impl Context {- /// Build an empty context.- pub fn new() -> Self {- Self {}- }-}-/// One item in the stream returned by [`Driver::stream`] / [`Algorithm::run_stream`].pub enum Step {/// The algorithm needs this model call performed. The host serves it (optionally/// via [`RoutedRequest::default_client`]) and fulfills it with- /// [`CallLlmRequest::respond`].- CallLlm(CallLlmRequest),+ /// [`CallLlmRequest::respond`]. Boxed: it is by far the largest variant.+ CallLlm(Box<CallLlmRequest>),/// A routing decision the algorithm made, published via [`Driver::info`] as it/// happens (rather than collected into a trace returned at the end).Decision(Arc<dyn Decision>),@@ -329,7 +294,7 @@ pub trait LlmClient: Send + Sync {/// [`routed.decision.selected_model()`](Decision::selected_model) — the target/// the algorithm routed to — mapping it to whatever provider model id this/// client hits. `routed.request.llm_request.model` is the agent's original- /// inbound name, carried through for reference, not a call target.+ /// name, carried through for reference, not a call target.async fn call(&self, request: RoutedRequest) -> Result<Response, Box<dyn Error + Send + Sync>>;}@@ -419,8 +384,10 @@ pub trait Algorithm: Send + Sync + 'static {let driver = Driver::new();let stream = driver.stream();tokio::spawn(async move {- let outcome = self.create_run_task(ctx, driver.clone(), request).await;- let _ = driver.finish(outcome).await;+ let outcome = self+ .create_run_task(ctx.clone(), driver.clone(), request)+ .await;+ let _ = driver.finish(ctx, outcome).await;});Box::pin(stream)}@@ -430,8 +397,11 @@ pub trait Algorithm: Send + Sync + 'static {/// [`Response`]. Provided: drives [`run_stream`](Self::run_stream) internally,/// collecting each [`Step::Decision`]. Use it when the algorithm holds its own model/// clients and the host wants the answer (and the decisions behind it); drive- /// [`run_stream`](Self::run_stream) instead to serve the calls yourself. Errors- /// if a routed target has no client to serve its call, or the algorithm fails.+ /// [`run_stream`](Self::run_stream) instead to serve the calls yourself. The returned+ /// [`Response`] is whatever the algorithm produced — a buffered [`LlmResponse::Agg`]+ /// or a live [`LlmResponse::Stream`]; the caller decides whether to+ /// [`aggregate`](LlmResponse::aggregate) it or forward its chunks. Errors if a routed+ /// target has no client to serve its call, or the algorithm fails.async fn run(self: Arc<Self>,ctx: Context,@@ -472,7 +442,7 @@ pub trait Algorithm: Send + Sync + 'static {match step {None => stream_open = false,Some(item) => match item? {- Step::CallLlm(call) => in_flight.push(serve(call)),+ Step::CallLlm(call) => in_flight.push(serve(*call)),Step::Decision(decision) => trace.push(decision),Step::ReturnToAgent(response) => {final_response = Some(*response);@@ -484,6 +454,8 @@ pub trait Algorithm: Send + Sync + 'static {}}+ // Return whatever the algorithm produced — buffered or streamed. The caller+ // decides whether to aggregate the response or forward its chunks.final_response.map(|response| (trace, response)).ok_or_else(|| "run: stream ended without a final response".into())@@ -494,6 +466,7 @@ pub trait Algorithm: Send + Sync + 'static {mod tests {use super::*;use futures::StreamExt;+ use switchyard_protocol::{completion_text, text_request, text_response};/// Mock client that echoes back the target name it was called with.struct EchoClient;@@ -506,10 +479,10 @@ mod tests {) -> Result<Response, Box<dyn Error + Send + Sync>> {// Echo back the model the algorithm routed to (the decision's selection).Ok(Response {- llm_response: LlmResponse {- model: Some(routed.decision.selected_model().to_string()),- ..LlmResponse::default()- },+ llm_response: LlmResponse::Agg(text_response(+ None,+ routed.decision.selected_model().to_string(),+ )),metadata: None,})}@@ -541,7 +514,7 @@ mod tests {impl Algorithm for TestAlgo {async fn create_run_task(self: Arc<Self>,- _ctx: Context,+ ctx: Context,driver: Driver,request: Request,) -> Result<Response, Box<dyn Error + Send + Sync>> {@@ -554,8 +527,10 @@ mod tests {let decision: Arc<dyn Decision> = Arc::new(TestDecision {model: target.semantic_name.clone(),});- driver.info(decision.clone()).await?;- driver.call_llm_target(&target, request, decision).await+ driver.info(ctx.clone(), decision.clone()).await?;+ driver+ .call_llm_target(ctx, &target, request, decision)+ .await}async fn process_signals(@@ -573,11 +548,7 @@ mod tests {fn request() -> Request {Request {- llm_request: LlmRequest {- model: Some("auto".to_string()),- messages: vec![Message::text(Role::User, "hi")],- ..LlmRequest::default()- },+ llm_request: text_request(Some("auto".to_string()), "hi".to_string()),raw_request: None,metadata: None,}@@ -595,6 +566,90 @@ mod tests {LlmTargetSet::new(targets)}+ /// Client that serves a call as a token stream — its `call` returns+ /// [`LlmResponse::Stream`] replaying `chunks` in order (as `Ok` items).+ struct StreamingClient {+ chunks: Vec<LlmResponseChunk>,+ }++ #[async_trait]+ impl LlmClient for StreamingClient {+ async fn call(+ &self,+ _routed: RoutedRequest,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let stream = futures::stream::iter(self.chunks.clone().into_iter().map(Ok)).boxed();+ Ok(Response {+ llm_response: LlmResponse::Stream(stream),+ metadata: None,+ })+ }+ }++ /// Build a single-target algo whose one target streams `chunks`.+ fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> Arc<dyn Algorithm> {+ let target = LlmTarget {+ semantic_name: "stream/model".to_string(),+ llm_client: Some(Arc::new(StreamingClient { chunks }) as Arc<dyn LlmClient>),+ };+ orch(LlmTargetSet::new(vec![target]))+ }++ #[tokio::test]+ async fn run_returns_a_streamed_response_the_caller_aggregates(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // A streaming client -> its chunks flow through the promise and `ReturnToAgent`,+ // and `run` returns the live stream untouched for the caller to fold.+ let orch = streaming_orch(vec![+ LlmResponseChunk::MessageStart {+ id: Some("m1".to_string()),+ model: Some("stream/model".to_string()),+ },+ LlmResponseChunk::TextDelta {+ index: 0,+ text: "hel".to_string(),+ },+ LlmResponseChunk::TextDelta {+ index: 0,+ text: "lo".to_string(),+ },+ LlmResponseChunk::MessageStop {+ reason: Some("stop".to_string()),+ },+ ]);+ let (trace, response) = orch.run(Context::default(), request()).await?;+ // `run` handed back the live stream; the caller folds it to a buffered aggregate.+ let agg = response.llm_response.into_agg().await?;+ assert_eq!(completion_text(&agg), "hello");+ assert_eq!(agg.model.as_deref(), Some("stream/model"));+ assert_eq!(trace.len(), 1);+ Ok(())+ }++ #[tokio::test]+ async fn aggregating_a_streamed_response_propagates_a_mid_stream_error(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // `run` succeeds and returns the stream; the in-band `Error` chunk surfaces only+ // when the caller aggregates it.+ let orch = streaming_orch(vec![+ LlmResponseChunk::TextDelta {+ index: 0,+ text: "partial".to_string(),+ },+ LlmResponseChunk::Error {+ message: "upstream exploded".to_string(),+ },+ ]);+ let (_, response) = orch.run(Context::default(), request()).await?;+ match response.llm_response.into_agg().await {+ Ok(_) => Err("expected a mid-stream error, got an aggregate".into()),+ Err(err) => {+ assert!(err.to_string().contains("upstream exploded"));+ Ok(())+ }+ }+ }+#[tokio::test]async fn run_offloads_via_promise_then_returns_to_agent() -> Result<(), Box<dyn Error + Send + Sync>> {@@ -614,10 +669,10 @@ mod tests {assert_eq!(call.get_decision().selected_model(), "offload/model");// Fulfilling the promise is the "real" model call the caller makes.call.respond(Ok(Response {- llm_response: LlmResponse {- model: Some("fulfilled".to_string()),- ..LlmResponse::default()- },+ llm_response: LlmResponse::Agg(text_response(+ None,+ "fulfilled".to_string(),+ )),metadata: None,}))?;}@@ -625,7 +680,13 @@ mod tests {assert_eq!(decision.selected_model(), "offload/model");}Step::ReturnToAgent(response) => {- final_completion = response.llm_response.model.clone();+ final_completion = Some(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ );}}}@@ -661,7 +722,13 @@ mod tests {}Step::Decision(_) => {}Step::ReturnToAgent(response) => {- final_completion = response.llm_response.model.clone();+ final_completion = Some(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ );}}}@@ -680,7 +747,14 @@ mod tests {.run(Context::default(), request()).await?;// TestAlgo calls the first target; EchoClient echoes its name.- assert_eq!(response.llm_response.model.as_deref(), Some("direct/model"));+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "direct/model"+ );assert_eq!(trace[0].selected_model(), "direct/model");Ok(())}@@ -720,10 +794,10 @@ mod tests {) -> Result<Response, Box<dyn Error + Send + Sync>> {self.barrier.wait().await;Ok(Response {- llm_response: LlmResponse {- model: Some(routed.decision.selected_model().to_string()),- ..LlmResponse::default()- },+ llm_response: LlmResponse::Agg(text_response(+ None,+ routed.decision.selected_model().to_string(),+ )),metadata: None,})}@@ -745,7 +819,13 @@ mod tests {handles.push(tokio::spawn(async move {algo.run(Context::default(), request()).await- .map(|(_, response)| response.llm_response.model.unwrap_or_default())+ .map(|(_, response)| {+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default()+ })}));}@@ -823,16 +903,10 @@ mod tests {drop(permit);self.current.fetch_sub(1, Ordering::SeqCst);Ok(Response {- llm_response: LlmResponse {- outputs: vec![ResponseOutput {- role: Role::Assistant,- content: vec![ContentBlock::Text {- text: routed.decision.selected_model().to_string(),- }],- stop_reason: None,- }],- ..LlmResponse::default()- },+ llm_response: LlmResponse::Agg(text_response(+ None,+ routed.decision.selected_model().to_string(),+ )),metadata: None,})}@@ -848,32 +922,26 @@ mod tests {impl Algorithm for FanOut {async fn create_run_task(self: Arc<Self>,- _ctx: Context,+ ctx: Context,driver: Driver,_request: Request,) -> Result<Response, Box<dyn Error + Send + Sync>> {let calls = (0..self.n).map(|i| {let driver = driver.clone();+ let ctx = ctx.clone();let target = self.target.clone();async move {let decision: Arc<dyn Decision> = Arc::new(TestDecision {model: format!("m{i}"),});- driver.call_llm_target(&target, request(), decision).await+ driver+ .call_llm_target(ctx, &target, request(), decision)+ .await}});futures::future::join_all(calls).await;Ok(Response {- llm_response: LlmResponse {- outputs: vec![ResponseOutput {- role: Role::Assistant,- content: vec![ContentBlock::Text {- text: "done".to_string(),- }],- stop_reason: None,- }],- ..LlmResponse::default()- },+ llm_response: LlmResponse::Agg(text_response(None, "done".to_string())),metadata: None,})}@@ -934,11 +1002,10 @@ mod tests {assert_eq!(response.llm_response- .first_output()- .and_then(|output| output.content.first()),- Some(&ContentBlock::Text {- text: "done".to_string()- })+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "done");assert_eq!(max.load(Ordering::SeqCst),diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rsindex 453292ba5..0542082f6 100644--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs@@ -14,8 +14,8 @@ use crate::diagnostic::TranslationDiagnostic;use crate::error::{Result, TranslationError};use crate::format::{FormatId, WireFormat};use crate::llm::{- ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, LlmResponse, MediaSource,- Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,+ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest,+ MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,};use crate::policy::{DeterministicIdPolicy, TranslationPolicy};@@ -252,7 +252,7 @@ impl FormatCodec for AnthropicMessagesCodec {text: String::new(),});}- let response = LlmResponse {+ let response = AggLlmResponse {id: body.get("id").and_then(Value::as_str)@@ -297,7 +297,7 @@ impl FormatCodec for AnthropicMessagesCodec {fn encode_response(&self,- response: &LlmResponse,+ response: &AggLlmResponse,_policy: &TranslationPolicy,) -> Result<EncodedResponse> {if let Some(body) = exact_preserved_response(diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rsindex 215173b4b..077152752 100644--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs@@ -10,8 +10,8 @@ use crate::codecs::stream::{StreamCodec, StreamTranslationState,};use crate::format::{FormatId, WireFormat};-use crate::llm::LlmStreamEvent;use crate::util::sanitize_anthropic_tool_use_id;+use crate::LlmResponseChunk;/// Stream codec for Anthropic Messages events.pub struct AnthropicMessagesStreamCodec;@@ -25,14 +25,14 @@ impl StreamCodec for AnthropicMessagesStreamCodec {&self,state: &mut StreamTranslationState,event: &Value,- ) -> Vec<LlmStreamEvent> {+ ) -> Vec<LlmResponseChunk> {decode_anthropic_stream(state, event)}fn encode_event(&self,state: &mut StreamTranslationState,- event: LlmStreamEvent,+ event: LlmResponseChunk,) -> Vec<Value> {encode_anthropic_stream(state, event)}@@ -46,9 +46,9 @@ impl StreamCodec for AnthropicMessagesStreamCodec {fn decode_anthropic_stream(state: &mut StreamTranslationState,event: &Value,-) -> Vec<LlmStreamEvent> {+) -> Vec<LlmResponseChunk> {let Some(object) = event.as_object() else {- return vec![LlmStreamEvent::Error {+ return vec![LlmResponseChunk::Error {message: "Anthropic stream event is not an object".to_string(),}];};@@ -73,7 +73,7 @@ fn decode_anthropic_stream(capture_anthropic_usage(state, usage);}}- vec![LlmStreamEvent::MessageStart {+ vec![LlmResponseChunk::MessageStart {id: state.message_id.clone(),model: state.model.clone(),}]@@ -84,7 +84,7 @@ fn decode_anthropic_stream(let mut out = Vec::new();if let Some(usage) = object.get("usage") {capture_anthropic_usage(state, usage);- out.push(LlmStreamEvent::Usage(state.usage.clone()));+ out.push(LlmResponseChunk::Usage(state.usage.clone()));}if let Some(stop_reason) = object.get("delta")@@ -92,14 +92,22 @@ fn decode_anthropic_stream(.and_then(|delta| delta.get("stop_reason")).and_then(Value::as_str){- out.push(LlmStreamEvent::MessageStop {+ // Remember the provider stop reason: Anthropic delivers it here, on+ // `message_delta`, while the terminal `message_stop` carries none of its own.+ state.stop_reason = Some(stop_reason.to_string());+ out.push(LlmResponseChunk::MessageStop {reason: Some(stop_reason.to_string()),});}out}- Some("message_stop") => vec![LlmStreamEvent::MessageStop { reason: None }],- Some("error") => vec![LlmStreamEvent::Error {+ // Anthropic's terminal `message_stop` carries no reason of its own; replay the one+ // remembered from `message_delta` so a chunk accumulator keeps the real stop reason+ // (e.g. `max_tokens`) instead of overwriting it with a reasonless `EndTurn`.+ Some("message_stop") => vec![LlmResponseChunk::MessageStop {+ reason: state.stop_reason.clone(),+ }],+ Some("error") => vec![LlmResponseChunk::Error {message: object.get("error").and_then(Value::as_object)@@ -115,10 +123,10 @@ fn decode_anthropic_stream(// Encodes neutral streaming events into Anthropic Messages events.fn encode_anthropic_stream(state: &mut StreamTranslationState,- event: LlmStreamEvent,+ event: LlmResponseChunk,) -> Vec<Value> {match event {- LlmStreamEvent::MessageStart { id, model } => {+ LlmResponseChunk::MessageStart { id, model } => {record_source_identity(state, id, model);if state.emitted_message_start {Vec::new()@@ -139,7 +147,7 @@ fn encode_anthropic_stream(})]}}- LlmStreamEvent::TextDelta { text, .. } => {+ LlmResponseChunk::TextDelta { text, .. } => {state.output_tokens_seen += 1;let mut out = ensure_anthropic_text_block(state);out.push(json!({@@ -149,7 +157,7 @@ fn encode_anthropic_stream(}));out}- LlmStreamEvent::ReasoningDelta { text, .. } => {+ LlmResponseChunk::ReasoningDelta { text, .. } => {let mut out = ensure_anthropic_reasoning_block(state);out.push(json!({"type": "content_block_delta",@@ -158,22 +166,22 @@ fn encode_anthropic_stream(}));out}- LlmStreamEvent::ToolCallDelta {+ LlmResponseChunk::ToolCallDelta {index,id,name,arguments_delta,} => encode_anthropic_tool_delta(state, index, id, name, arguments_delta),- LlmStreamEvent::Usage(usage) => {+ LlmResponseChunk::Usage(usage) => {state.usage = usage;state.saw_backend_usage = true;Vec::new()}- LlmStreamEvent::MessageStop { reason } => {+ LlmResponseChunk::MessageStop { reason } => {state.stop_reason = reason.or_else(|| state.stop_reason.clone());Vec::new()}- LlmStreamEvent::Error { message } => {+ LlmResponseChunk::Error { message } => {vec![json!({"type": "error", "error": {"message": message}})]}}@@ -185,7 +193,7 @@ fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec<Value> {if !state.emitted_message_start {out.extend(encode_anthropic_stream(state,- LlmStreamEvent::MessageStart {+ LlmResponseChunk::MessageStart {id: state.message_id.clone(),model: state.model.clone(),},@@ -233,7 +241,7 @@ fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec<Value> {}// Converts Anthropic content-block starts into text or tool-call deltas.-fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmStreamEvent> {+fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmResponseChunk> {let index = object.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;let block = object.get("content_block").and_then(Value::as_object);match block@@ -245,7 +253,7 @@ fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmS.and_then(Value::as_str).filter(|text| !text.is_empty()).map(|text| {- vec![LlmStreamEvent::TextDelta {+ vec![LlmResponseChunk::TextDelta {index,text: text.to_string(),}]@@ -256,7 +264,7 @@ fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmS.and_then(Value::as_str).filter(|text| !text.is_empty()).map(|text| {- vec![LlmStreamEvent::ReasoningDelta {+ vec![LlmResponseChunk::ReasoningDelta {index,text: text.to_string(),}]@@ -266,7 +274,7 @@ fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmSlet Some(block) = block else {return Vec::new();};- vec![LlmStreamEvent::ToolCallDelta {+ vec![LlmResponseChunk::ToolCallDelta {index,id: block.get("id")@@ -284,7 +292,7 @@ fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmS}// Converts Anthropic content-block deltas into neutral text or argument deltas.-fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmStreamEvent> {+fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmResponseChunk> {let index = object.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;let Some(delta) = object.get("delta").and_then(Value::as_object) else {return Vec::new();@@ -294,7 +302,7 @@ fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmS.get("text").and_then(Value::as_str).map(|text| {- vec![LlmStreamEvent::TextDelta {+ vec![LlmResponseChunk::TextDelta {index,text: text.to_string(),}]@@ -304,7 +312,7 @@ fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmS.get("thinking").and_then(Value::as_str).map(|text| {- vec![LlmStreamEvent::ReasoningDelta {+ vec![LlmResponseChunk::ReasoningDelta {index,text: text.to_string(),}]@@ -315,7 +323,7 @@ fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmS.get("partial_json").and_then(Value::as_str).map(|partial_json| {- vec![LlmStreamEvent::ToolCallDelta {+ vec![LlmResponseChunk::ToolCallDelta {index,id: None,name: None,diff --git a/crates/switchyard-translation/src/codecs/mod.rs b/crates/switchyard-translation/src/codecs/mod.rsindex fbcda4ab1..78d3a007c 100644--- a/crates/switchyard-translation/src/codecs/mod.rs+++ b/crates/switchyard-translation/src/codecs/mod.rs@@ -14,7 +14,7 @@ use serde_json::Value;use crate::diagnostic::TranslationDiagnostic;use crate::error::Result;use crate::format::FormatId;-use crate::llm::{LlmRequest, LlmResponse};+use crate::llm::{AggLlmResponse, LlmRequest};use crate::policy::TranslationPolicy;/// Result of decoding a request into neutral IR.@@ -31,7 +31,7 @@ pub struct EncodedRequest {/// Result of decoding a response into neutral IR.pub struct DecodedResponse {- pub response: LlmResponse,+ pub response: AggLlmResponse,pub diagnostics: Vec<TranslationDiagnostic>,}@@ -62,7 +62,7 @@ pub trait FormatCodec: Send + Sync {/// Encodes a neutral response into provider JSON.fn encode_response(&self,- response: &LlmResponse,+ response: &AggLlmResponse,policy: &TranslationPolicy,) -> Result<EncodedResponse>;}diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rsindex fbcd1ad42..c6d823102 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs@@ -15,8 +15,8 @@ use crate::diagnostic::TranslationDiagnostic;use crate::error::{Result, TranslationError};use crate::format::{FormatId, WireFormat};use crate::llm::{- ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, LlmResponse, MediaSource,- Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,+ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest,+ MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,};use crate::policy::{DeterministicIdPolicy, TranslationPolicy};@@ -246,7 +246,7 @@ impl FormatCodec for OpenAiChatCodec {_policy: &TranslationPolicy,) -> Result<DecodedResponse> {let object = object(body, "$")?;- let mut response = LlmResponse {+ let mut response = AggLlmResponse {id: object.get("id").and_then(Value::as_str)@@ -313,7 +313,7 @@ impl FormatCodec for OpenAiChatCodec {fn encode_response(&self,- response: &LlmResponse,+ response: &AggLlmResponse,_policy: &TranslationPolicy,) -> Result<EncodedResponse> {if let Some(body) =diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rsindex 501b40de7..d2e5f6e74 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs@@ -10,7 +10,8 @@ use crate::codecs::stream::{StreamTranslationState,};use crate::format::{FormatId, WireFormat};-use crate::llm::{LlmStreamEvent, Usage};+use crate::llm::Usage;+use crate::LlmResponseChunk;/// Stream codec for OpenAI Chat Completions chunks.pub struct OpenAiChatStreamCodec;@@ -24,14 +25,14 @@ impl StreamCodec for OpenAiChatStreamCodec {&self,state: &mut StreamTranslationState,event: &Value,- ) -> Vec<LlmStreamEvent> {+ ) -> Vec<LlmResponseChunk> {decode_openai_chat_stream(state, event)}fn encode_event(&self,state: &mut StreamTranslationState,- event: LlmStreamEvent,+ event: LlmResponseChunk,) -> Vec<Value> {encode_openai_chat_stream(state, event)}@@ -45,9 +46,9 @@ impl StreamCodec for OpenAiChatStreamCodec {fn decode_openai_chat_stream(state: &mut StreamTranslationState,event: &Value,-) -> Vec<LlmStreamEvent> {+) -> Vec<LlmResponseChunk> {let Some(object) = event.as_object() else {- return vec![LlmStreamEvent::Error {+ return vec![LlmResponseChunk::Error {message: "OpenAI stream event is not an object".to_string(),}];};@@ -61,7 +62,7 @@ fn decode_openai_chat_stream(if let Some(id) = string_field(object, "id") {state.message_id = Some(id);}- out.push(LlmStreamEvent::MessageStart {+ out.push(LlmResponseChunk::MessageStart {id: state.message_id.clone(),model: state.model.clone(),});@@ -72,7 +73,7 @@ fn decode_openai_chat_stream(capture_openai_usage_extras(state, object.get("usage"));state.usage = usage.clone();state.saw_backend_usage = true;- out.push(LlmStreamEvent::Usage(usage));+ out.push(LlmResponseChunk::Usage(usage));}for choice in object@@ -87,7 +88,7 @@ fn decode_openai_chat_stream(if let Some(delta) = choice.get("delta").and_then(Value::as_object) {if let Some(text) = delta.get("content").and_then(Value::as_str) {if !text.is_empty() {- out.push(LlmStreamEvent::TextDelta {+ out.push(LlmResponseChunk::TextDelta {index: 0,text: text.to_string(),});@@ -96,7 +97,7 @@ fn decode_openai_chat_stream(for reasoning_key in ["reasoning_content", "reasoning"] {if let Some(text) = delta.get(reasoning_key).and_then(Value::as_str) {if !text.is_empty() {- out.push(LlmStreamEvent::ReasoningDelta {+ out.push(LlmResponseChunk::ReasoningDelta {index: 0,text: text.to_string(),});@@ -107,7 +108,7 @@ fn decode_openai_chat_stream(for tool_call in tool_calls {if let Some(tool_call) = tool_call.as_object() {let function = tool_call.get("function").and_then(Value::as_object);- out.push(LlmStreamEvent::ToolCallDelta {+ out.push(LlmResponseChunk::ToolCallDelta {index: tool_call.get("index").and_then(Value::as_u64).unwrap_or(0)as usize,id: tool_call@@ -128,7 +129,7 @@ fn decode_openai_chat_stream(}}if let Some(reason) = choice.get("finish_reason").and_then(Value::as_str) {- out.push(LlmStreamEvent::MessageStop {+ out.push(LlmResponseChunk::MessageStop {reason: Some(reason.to_string()),});}@@ -139,10 +140,10 @@ fn decode_openai_chat_stream(// Encodes neutral streaming events into OpenAI Chat chunks.fn encode_openai_chat_stream(state: &mut StreamTranslationState,- event: LlmStreamEvent,+ event: LlmResponseChunk,) -> Vec<Value> {match event {- LlmStreamEvent::MessageStart { id, model } => {+ LlmResponseChunk::MessageStart { id, model } => {record_source_identity(state, id, model);if state.emitted_message_start|| (!state_source_is(state, WireFormat::AnthropicMessages)@@ -159,7 +160,7 @@ fn encode_openai_chat_stream()]}}- LlmStreamEvent::TextDelta { text, .. } => {+ LlmResponseChunk::TextDelta { text, .. } => {vec![openai_stream_chunk(state,json!({"content": text}),@@ -167,7 +168,7 @@ fn encode_openai_chat_stream(None,)]}- LlmStreamEvent::ReasoningDelta { text, .. } => {+ LlmResponseChunk::ReasoningDelta { text, .. } => {vec![openai_stream_chunk(state,json!({"reasoning_content": text}),@@ -175,7 +176,7 @@ fn encode_openai_chat_stream(None,)]}- LlmStreamEvent::ToolCallDelta {+ LlmResponseChunk::ToolCallDelta {index,id,name,@@ -187,12 +188,12 @@ fn encode_openai_chat_stream(name,arguments_delta,)],- LlmStreamEvent::Usage(usage) => {+ LlmResponseChunk::Usage(usage) => {state.usage = usage;state.saw_backend_usage = true;Vec::new()}- LlmStreamEvent::MessageStop { reason } => {+ LlmResponseChunk::MessageStop { reason } => {if state.finished {return Vec::new();}@@ -204,7 +205,7 @@ fn encode_openai_chat_stream(Some(openai_usage_value(state)),)]}- LlmStreamEvent::Error { message } => vec![json!({"error": {"message": message}})],+ LlmResponseChunk::Error { message } => vec![json!({"error": {"message": message}})],}}diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rsindex 71f41e09c..e92ec45b9 100644--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs@@ -18,9 +18,9 @@ use crate::diagnostic::TranslationDiagnostic;use crate::error::{Result, TranslationError};use crate::format::{FormatId, WireFormat};use crate::llm::{- ContentBlock, LlmRequest, LlmResponse, MediaSource, Message, OutputParams, ProviderExtensions,- ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, ToolCall, ToolChoice,- ToolDefinition, ToolResult, Usage,+ AggLlmResponse, ContentBlock, LlmRequest, MediaSource, Message, OutputParams,+ ProviderExtensions, ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason,+ ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,};use crate::policy::{DeterministicIdPolicy, TranslationPolicy};use crate::util::{@@ -200,7 +200,7 @@ impl FormatCodec for OpenAiResponsesCodec {stop_reason: Some(StopReason::EndTurn),});}- let response = LlmResponse {+ let response = AggLlmResponse {id: body.get("id").and_then(Value::as_str)@@ -231,7 +231,7 @@ impl FormatCodec for OpenAiResponsesCodec {fn encode_response(&self,- response: &LlmResponse,+ response: &AggLlmResponse,_policy: &TranslationPolicy,) -> Result<EncodedResponse> {if let Some(body) =diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rsindex 48e6f1636..42234b9d2 100644--- a/crates/switchyard-translation/src/codecs/responses/stream.rs+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs@@ -10,7 +10,8 @@ use crate::codecs::stream::{StreamCodec, StreamTranslationState,};use crate::format::{FormatId, WireFormat};-use crate::llm::{LlmStreamEvent, Usage};+use crate::llm::Usage;+use crate::LlmResponseChunk;/// Stream codec for OpenAI Responses API events.pub struct OpenAiResponsesStreamCodec;@@ -24,14 +25,14 @@ impl StreamCodec for OpenAiResponsesStreamCodec {&self,state: &mut StreamTranslationState,event: &Value,- ) -> Vec<LlmStreamEvent> {+ ) -> Vec<LlmResponseChunk> {decode_responses_stream(state, event)}fn encode_event(&self,state: &mut StreamTranslationState,- event: LlmStreamEvent,+ event: LlmResponseChunk,) -> Vec<Value> {encode_responses_stream(state, event)}@@ -45,7 +46,7 @@ impl StreamCodec for OpenAiResponsesStreamCodec {fn decode_responses_stream(state: &mut StreamTranslationState,event: &Value,-) -> Vec<LlmStreamEvent> {+) -> Vec<LlmResponseChunk> {let event_type = event.get("type").or_else(|| event.get("event"))@@ -66,7 +67,7 @@ fn decode_responses_stream({state.message_id = Some(id.to_string());}- vec![LlmStreamEvent::MessageStart {+ vec![LlmResponseChunk::MessageStart {id: state.message_id.clone(),model: state.model.clone(),}]@@ -75,7 +76,7 @@ fn decode_responses_stream(.get("delta").and_then(Value::as_str).map(|text| {- vec![LlmStreamEvent::TextDelta {+ vec![LlmResponseChunk::TextDelta {index: event.get("output_index").and_then(Value::as_u64)@@ -90,7 +91,7 @@ fn decode_responses_stream(.or_else(|| event.get("text")).and_then(Value::as_str).map(|text| {- vec![LlmStreamEvent::ReasoningDelta {+ vec![LlmResponseChunk::ReasoningDelta {index: event.get("output_index").and_then(Value::as_u64)@@ -110,7 +111,7 @@ fn decode_responses_stream(.get("delta").and_then(Value::as_str).map(|delta| {- vec![LlmStreamEvent::ToolCallDelta {+ vec![LlmResponseChunk::ToolCallDelta {index: output_index as usize,id: None,name: None,@@ -131,12 +132,12 @@ fn decode_responses_stream(let usage = responses_usage(usage);state.usage = usage.clone();state.saw_backend_usage = true;- out.push(LlmStreamEvent::Usage(usage));+ out.push(LlmResponseChunk::Usage(usage));}- out.push(LlmStreamEvent::MessageStop { reason: None });+ out.push(LlmResponseChunk::MessageStop { reason: None });out}- Some("error") => vec![LlmStreamEvent::Error {+ Some("error") => vec![LlmResponseChunk::Error {message: event.get("message").and_then(Value::as_str)@@ -150,33 +151,33 @@ fn decode_responses_stream(// Encodes neutral streaming events into OpenAI Responses events.fn encode_responses_stream(state: &mut StreamTranslationState,- event: LlmStreamEvent,+ event: LlmResponseChunk,) -> Vec<Value> {match event {- LlmStreamEvent::MessageStart { id, model } => {+ LlmResponseChunk::MessageStart { id, model } => {record_source_identity(state, id, model);ensure_responses_created(state)}- LlmStreamEvent::TextDelta { text, .. } => encode_responses_text_delta(state, text),- LlmStreamEvent::ReasoningDelta { text, .. } => {+ LlmResponseChunk::TextDelta { text, .. } => encode_responses_text_delta(state, text),+ LlmResponseChunk::ReasoningDelta { text, .. } => {encode_responses_reasoning_delta(state, text)}- LlmStreamEvent::ToolCallDelta {+ LlmResponseChunk::ToolCallDelta {index,id,name,arguments_delta,} => encode_responses_tool_delta(state, index, id, name, arguments_delta),- LlmStreamEvent::Usage(usage) => {+ LlmResponseChunk::Usage(usage) => {state.usage = usage;state.saw_backend_usage = true;Vec::new()}- LlmStreamEvent::MessageStop { reason } => {+ LlmResponseChunk::MessageStop { reason } => {state.stop_reason = reason.or_else(|| state.stop_reason.clone());Vec::new()}- LlmStreamEvent::Error { message } => {+ LlmResponseChunk::Error { message } => {vec![json!({"type": "error", "message": message})]}}@@ -295,7 +296,7 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec<Value> {}// Converts Responses function-call item creation into a neutral tool-call delta.-fn decode_responses_output_item_added(event: &Value) -> Vec<LlmStreamEvent> {+fn decode_responses_output_item_added(event: &Value) -> Vec<LlmResponseChunk> {let Some(item) = event.get("item").and_then(Value::as_object) else {return Vec::new();};@@ -306,7 +307,7 @@ fn decode_responses_output_item_added(event: &Value) -> Vec<LlmStreamEvent> {.get("output_index").and_then(Value::as_u64).unwrap_or(0) as usize;- vec![LlmStreamEvent::ToolCallDelta {+ vec![LlmResponseChunk::ToolCallDelta {index,id: item.get("call_id")@@ -329,7 +330,7 @@ fn decode_responses_output_item_added(event: &Value) -> Vec<LlmStreamEvent> {fn decode_responses_output_item_done(event: &Value,state: &StreamTranslationState,-) -> Vec<LlmStreamEvent> {+) -> Vec<LlmResponseChunk> {let Some(item) = event.get("item").and_then(Value::as_object) else {return Vec::new();};@@ -348,7 +349,7 @@ fn decode_responses_output_item_done(.map(|tool| tool.arguments.as_str()).unwrap_or("");if !arguments.is_empty() && arguments != existing {- return vec![LlmStreamEvent::ToolCallDelta {+ return vec![LlmResponseChunk::ToolCallDelta {index,id: None,name: None,diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rsindex 710d79dd3..730e224b7 100644--- a/crates/switchyard-translation/src/codecs/stream.rs+++ b/crates/switchyard-translation/src/codecs/stream.rs@@ -15,7 +15,8 @@ use crate::codecs::responses::OpenAiResponsesStreamCodec;use crate::engine::{FormatRegistry, TranslationEngine};use crate::error::{Result, TranslationError};use crate::format::{FormatId, WireFormat};-use crate::llm::{LlmStreamEvent, Usage};+use crate::llm::Usage;+use crate::LlmResponseChunk;/// Mutable state accumulated while translating one streaming response.#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]@@ -99,11 +100,14 @@ pub trait StreamCodec: Send + Sync {&self,state: &mut StreamTranslationState,event: &Value,- ) -> Vec<LlmStreamEvent>;+ ) -> Vec<LlmResponseChunk>;/// Encodes one neutral event into zero or more provider events.- fn encode_event(&self, state: &mut StreamTranslationState, event: LlmStreamEvent)- -> Vec<Value>;+ fn encode_event(+ &self,+ state: &mut StreamTranslationState,+ event: LlmResponseChunk,+ ) -> Vec<Value>;/// Emits any terminal provider events needed after the source stream ends.///@@ -197,13 +201,13 @@ pub fn decode_stream_event(state: &mut StreamTranslationState,source: impl Into<FormatId>,event: &Value,-) -> Vec<LlmStreamEvent> {+) -> Vec<LlmResponseChunk> {let source = source.into();StreamCodecRegistry::with_builtins().codec(source).map(|codec| codec.decode_event(state, event)).unwrap_or_else(|error| {- vec![LlmStreamEvent::Error {+ vec![LlmResponseChunk::Error {message: error.to_string(),}]})@@ -213,7 +217,7 @@ pub fn decode_stream_event(pub fn encode_stream_event(state: &mut StreamTranslationState,target: impl Into<FormatId>,- event: LlmStreamEvent,+ event: LlmResponseChunk,) -> Vec<Value> {StreamCodecRegistry::with_builtins().codec(target)diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rsindex 9dddc8db5..5303c3ed6 100644--- a/crates/switchyard-translation/src/engine.rs+++ b/crates/switchyard-translation/src/engine.rs@@ -16,7 +16,7 @@ use crate::codecs::FormatCodec;use crate::diagnostic::TranslationDiagnostic;use crate::error::{Result, TranslationError};use crate::format::FormatId;-use crate::llm::{LlmRequest, LlmResponse};+use crate::llm::{AggLlmResponse, LlmRequest};use crate::policy::TranslationPolicy;/// Encoded translation result with any diagnostics emitted along the way.@@ -36,7 +36,7 @@ pub struct RequestIrOutput {/// Decoded response IR plus diagnostics.#[derive(Debug)]pub struct ResponseIrOutput {- pub response: LlmResponse,+ pub response: AggLlmResponse,pub diagnostics: Vec<TranslationDiagnostic>,}@@ -184,7 +184,7 @@ impl TranslationEngine {pub fn encode_response(&self,target: impl Into<FormatId>,- response: &LlmResponse,+ response: &AggLlmResponse,policy: &TranslationPolicy,) -> Result<TranslationOutput> {let target = target.into();diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rsindex 96ed486fd..3eaaf21a9 100644--- a/crates/switchyard-translation/src/lib.rs+++ b/crates/switchyard-translation/src/lib.rs@@ -15,6 +15,7 @@ pub mod policy;pub mod stream;pub mod util;+pub use switchyard_protocol::stream::LlmResponseChunk;pub use switchyard_protocol::{format, llm};pub use diagnostic::*;diff --git a/crates/switchyard-translation/tests/extension_points.rs b/crates/switchyard-translation/tests/extension_points.rsindex 49a89334b..2b70283a1 100644--- a/crates/switchyard-translation/tests/extension_points.rs+++ b/crates/switchyard-translation/tests/extension_points.rs@@ -13,7 +13,7 @@ use switchyard_translation::util::{exact_preserved_response,};use switchyard_translation::{- FormatId, FormatRegistry, LlmRequest, LlmResponse, LlmStreamEvent, LossyConversionPolicy,+ AggLlmResponse, FormatId, FormatRegistry, LlmRequest, LlmResponseChunk, LossyConversionPolicy,Message, PreservationPolicy, ResponseOutput, Role, StreamCodec, StreamCodecRegistry,StreamTranslationState, TargetCapabilities, TranslationEngine, TranslationPolicy, Usage,WireFormat,@@ -181,7 +181,7 @@ impl FormatCodec for MinimalCustomCodec {policy: &TranslationPolicy,) -> switchyard_translation::Result<DecodedResponse> {Ok(DecodedResponse {- response: LlmResponse {+ response: AggLlmResponse {id: body.get("id").and_then(Value::as_str)@@ -203,7 +203,7 @@ impl FormatCodec for MinimalCustomCodec {}],usage: Usage::default(),preservation: capture_response_preservation(self.format(), body, policy),- ..LlmResponse::default()+ ..AggLlmResponse::default()},diagnostics: Vec::new(),})@@ -211,7 +211,7 @@ impl FormatCodec for MinimalCustomCodec {fn encode_response(&self,- response: &LlmResponse,+ response: &AggLlmResponse,policy: &TranslationPolicy,) -> switchyard_translation::Result<EncodedResponse> {if let Some(body) = exact_preserved_response(&response.preservation, self.format(), policy)@@ -254,9 +254,9 @@ impl StreamCodec for CustomStreamCodec {&self,_state: &mut StreamTranslationState,event: &Value,- ) -> Vec<LlmStreamEvent> {+ ) -> Vec<LlmResponseChunk> {match event.get("kind").and_then(Value::as_str) {- Some("start") => vec![LlmStreamEvent::MessageStart {+ Some("start") => vec![LlmResponseChunk::MessageStart {id: event.get("id").and_then(Value::as_str)@@ -270,7 +270,7 @@ impl StreamCodec for CustomStreamCodec {.get("text").and_then(Value::as_str).map(|text| {- vec![LlmStreamEvent::TextDelta {+ vec![LlmResponseChunk::TextDelta {index: 0,text: text.to_string(),}]@@ -283,10 +283,10 @@ impl StreamCodec for CustomStreamCodec {fn encode_event(&self,_state: &mut StreamTranslationState,- event: LlmStreamEvent,+ event: LlmResponseChunk,) -> Vec<Value> {match event {- LlmStreamEvent::TextDelta { text, .. } => {+ LlmResponseChunk::TextDelta { text, .. } => {vec![json!({"kind": "delta", "text": text})]}_ => Vec::new(),diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rsindex b128a8629..3c4f5e658 100644--- a/crates/switchyard-translation/tests/stream_translation.rs+++ b/crates/switchyard-translation/tests/stream_translation.rs@@ -5,7 +5,10 @@use pretty_assertions::assert_eq;use serde_json::json;-use switchyard_translation::{StreamTranslationState, TranslationEngine, WireFormat};+use switchyard_protocol::{ResponseAccumulator, StopReason};+use switchyard_translation::{+ decode_stream_event, StreamTranslationState, TranslationEngine, WireFormat,+};type TestResult = std::result::Result<(), Box<dyn std::error::Error + Send + Sync>>;@@ -416,3 +419,35 @@ fn anthropic_thinking_stream_deltas_do_not_become_openai_chat_content() -> TestR.any(|event| { event["choices"][0]["delta"]["content"] == "private chain of thought" }));Ok(())}++// A real Anthropic stream carries its stop reason on `message_delta` and then always+// sends a reasonless `message_stop`. Decoding both and folding the chunks must preserve+// the provider reason (here `max_tokens`) — the terminal `message_stop` must not emit a+// second, reasonless `MessageStop` that the accumulator would fold to `EndTurn`.+#[test]+fn anthropic_message_stop_does_not_overwrite_max_tokens_stop_reason() -> TestResult {+ let mut state =+ StreamTranslationState::new(WireFormat::AnthropicMessages, WireFormat::AnthropicMessages);+ let events = vec![+ json!({"type": "message_start", "message": {"id": "msg_1", "model": "claude", "usage": {"input_tokens": 3}}}),+ json!({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),+ json!({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hi"}}),+ json!({"type": "message_delta", "delta": {"stop_reason": "max_tokens"}, "usage": {"output_tokens": 5}}),+ json!({"type": "message_stop"}),+ ];++ let mut accumulator = ResponseAccumulator::new();+ for event in &events {+ for chunk in decode_stream_event(&mut state, WireFormat::AnthropicMessages, event) {+ accumulator.push(chunk);+ }+ }+ let aggregate = accumulator.finish();++ assert_eq!(+ aggregate.outputs[0].stop_reason,+ Some(StopReason::MaxTokens),+ "the reasonless message_stop must not overwrite the max_tokens stop reason"+ );+ Ok(())+}