Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(routing): preserve cache reuse across handoff notes (#1213)
Elyas Mehtabuddinda3dcf04d62f
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex 6158f42bf..4fa4db687 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -393,7 +393,18 @@ impl TranslatingLlmClient {if nvidia_inference_api_backend(backend) {strip_anthropic_tool_strict(&mut body);}- enable_anthropic_prompt_caching(&mut body);+ enable_anthropic_prompt_caching(&mut body, llm_request.temporary_note_count);+ }+ if matches!(+ backend,+ Backend::OpenAiChat(_) | Backend::OpenAiResponses(_)+ ) {+ enable_openai_guidance_caching(+ &mut body,+ wire_format,+ upstream_model,+ llm_request.temporary_note_count,+ );}if matches!(backend, Backend::OpenAiChat(_)) {ensure_openai_stream_usage(&mut body);@@ -1452,44 +1463,135 @@ fn count_cache_control_blocks(value: &Value) -> usize {}}-// Marks the final message content block as the Anthropic prompt-cache breakpoint.-fn enable_anthropic_prompt_caching(body: &mut Value) {- // Top-level cache control manages the final breakpoint and its TTL.- if body.get("cache_control").is_some() {+// This helper marks the final caller content block before any temporary notes, if policy permits.+fn enable_anthropic_prompt_caching(body: &mut Value, temporary_note_count: usize) {+ // Anthropic rejects a fifth cache marker with HTTP 400.+ // This check preserves the caller's markers when the request has reached that limit.+ if count_cache_control_blocks(body) >= MAX_CACHE_CONTROL_BLOCKS {return;}- // Abstain once the caller has spent the budget itself. Adding a fifth marker- // turns a request that was valid on arrival into an upstream HTTP 400, and a- // caller that placed four breakpoints deliberately needs them more than we- // need a fifth. When the final block is already marked this returns early- // and changes nothing, which is what the insert below would have done.- if count_cache_control_blocks(body) >= MAX_CACHE_CONTROL_BLOCKS {+ // Requests with only a top-level `1h` policy can return HTTP 400.+ // New block markers match the caller's TTL on turns with or without notes.+ let marker = body+ .get("cache_control")+ .cloned()+ .unwrap_or_else(|| serde_json::json!({"type":"ephemeral"}));+ let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else {return;+ };+ let mut remaining = temporary_note_count;+ // Temporary guidance is absent next turn, so the breakpoint must precede it.+ for message in messages.iter_mut().rev() {+ match message.get_mut("content") {+ Some(content @ Value::String(_)) if remaining == 0 => {+ let text = std::mem::take(content);+ *content = serde_json::json!([{+ "type": "text", "text": text, "cache_control": marker+ }]);+ return;+ }+ Some(Value::String(_)) => remaining = remaining.saturating_sub(1),+ Some(Value::Array(blocks)) => {+ if remaining >= blocks.len() {+ remaining -= blocks.len();+ continue;+ }+ let index = blocks.len() - remaining - 1;+ if let Some(block) = blocks[index].as_object_mut() {+ block.entry("cache_control".to_string()).or_insert(marker);+ }+ return;+ }+ _ => {}+ }}- let Some(content) = body- .get_mut("messages")- .and_then(Value::as_array_mut)- .and_then(|messages| messages.last_mut())- .and_then(|message| message.get_mut("content"))- else {+}++// GPT-5.6 and GPT-6 cache writes end at selected message or content boundaries.+// A temporary suffix needs a separate cache entry before it so the next request can reuse the prefix.+fn enable_openai_guidance_caching(+ body: &mut Value,+ format: WireFormat,+ model: &ModelId,+ suffix: usize,+) {+ let name = model.as_str().rsplit('/').next().unwrap_or_default();+ let supported = name == "gpt-5.6"+ || name.starts_with("gpt-5.6-")+ || name == "gpt-6"+ || name.starts_with("gpt-6-")+ || name.starts_with("gpt-6.");+ // Explicit-only callers choose their write locations.+ // This helper adds markers only for names recognized by its model-name check.+ if suffix == 0+ || !supported+ || body+ .pointer("/prompt_cache_options/mode")+ .and_then(Value::as_str)+ == Some("explicit")+ {+ return;+ }+ let key = if format == WireFormat::OpenAiResponses {+ "input"+ } else {+ "messages"+ };+ let Some(items) = body.get_mut(key).and_then(Value::as_array_mut) else {return;};- match content {- Value::String(text) => {- *content = serde_json::json!([{- "type": "text",- "text": std::mem::take(text),- "cache_control": {"type": "ephemeral"}- }]);+ let marker_count = items+ .iter()+ .map(|item| {+ ["content", "output"]+ .iter()+ .filter_map(|key| item.get(key).and_then(Value::as_array))+ .flatten()+ .filter(|block| block.get("prompt_cache_breakpoint").is_some())+ .count()+ })+ .sum::<usize>();+ // Implicit mode reserves one of the four write slots for the provider's final breakpoint.+ if marker_count >= 3 {+ return;+ }+ let Some(prefix_len) = items.len().checked_sub(suffix) else {+ return;+ };+ for item in items[..prefix_len].iter_mut().rev() {+ let field = if item.get("type").and_then(Value::as_str) == Some("function_call_output") {+ "output"+ } else if matches!(+ item.get("role").and_then(Value::as_str),+ Some("user" | "tool" | "developer" | "system")+ ) {+ "content"+ } else {+ continue;+ };+ let Some(content) = item.get_mut(field) else {+ continue;+ };+ if let Value::String(text) = content {+ let text = std::mem::take(text);+ let kind = if format == WireFormat::OpenAiResponses {+ "input_text"+ } else {+ "text"+ };+ *content = serde_json::json!([{"type":kind,"text":text}]);}- Value::Array(blocks) => {- if let Some(block) = blocks.last_mut().and_then(Value::as_object_mut) {- block- .entry("cache_control".to_string())- .or_insert_with(|| serde_json::json!({"type": "ephemeral"}));- }+ // Empty content arrays have no block to mark, so the search continues through earlier items.+ if let Some(block) = content+ .as_array_mut()+ .and_then(|blocks| blocks.last_mut())+ .and_then(Value::as_object_mut)+ {+ block+ .entry("prompt_cache_breakpoint".to_string())+ .or_insert_with(|| serde_json::json!({"mode":"explicit"}));+ return;}- _ => {}}}@@ -1884,14 +1986,21 @@ mod tests {"messages": [{"role": "user", "content": "hello"}]});let mut body = caller_managed.clone();- enable_anthropic_prompt_caching(&mut body);- assert_eq!(body, caller_managed);+ enable_anthropic_prompt_caching(&mut body, 0);+ assert_eq!(body["cache_control"], caller_managed["cache_control"]);+ assert_eq!(+ body["messages"][0]["content"],+ json!([{+ "type": "text", "text": "hello",+ "cache_control": {"type": "ephemeral", "ttl": "1h"}+ }])+ );let mut body = json!({"messages": [{"role": "user", "content": "hello"}]});- enable_anthropic_prompt_caching(&mut body);+ enable_anthropic_prompt_caching(&mut body, 0);assert_eq!(body["messages"][0]["content"][0]["cache_control"],@@ -1931,7 +2040,7 @@ mod tests {let mut body = body_at_the_cache_control_limit();assert_eq!(cache_control_blocks(&body), 4);- enable_anthropic_prompt_caching(&mut body);+ enable_anthropic_prompt_caching(&mut body, 0);// Anthropic and Bedrock both reject a fifth marker with HTTP 400, which// would fail a request that was valid before it reached us.@@ -1950,7 +2059,7 @@ mod tests {body["system"].as_array_mut().unwrap().pop();assert_eq!(cache_control_blocks(&body), 3);- enable_anthropic_prompt_caching(&mut body);+ enable_anthropic_prompt_caching(&mut body, 0);assert_eq!(cache_control_blocks(&body), 4);assert_eq!(diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.tomlindex 2e0a08169..6fdc9b408 100644--- a/crates/libsy/Cargo.toml+++ b/crates/libsy/Cargo.toml@@ -33,6 +33,7 @@ rand.workspace = trueregex.workspace = trueshlex = "2"switchyard-protocol.workspace = true+switchyard-translation.workspace = truethiserror.workspace = truetokio-stream = "0.1"tracing.workspace = true@@ -46,7 +47,6 @@ tokio.workspace = truetokio = { version = "1", features = ["rt", "sync", "macros", "time"] }[dev-dependencies]-switchyard-translation.workspace = true# SDK + in-memory exporter to assert what the observability layer records.opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }tokio.workspace = truediff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex 6b0a01906..bd0a1b757 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -176,12 +176,18 @@ impl FallThrough {let mut request = outcome.request.clone();request.llm_request.model = Some(target.to_string());if let Some(prompt) = prompts.get(target) {- super::util::prompts::prepend_system_prompt(&mut request, prompt);+ switchyard_translation::prepend_system_prompt(+ &mut request.llm_request,+ prompt,+ );}outcome.candidate_requests.insert(target.clone(), request);}if let Some(prompt) = prompts.get(&outcome.selected_model_ids[0]) {- super::util::prompts::prepend_system_prompt(&mut outcome.request, prompt);+ switchyard_translation::prepend_system_prompt(+ &mut outcome.request.llm_request,+ prompt,+ );}}Ok(outcome)@@ -551,7 +557,10 @@ mod tests {.filter_map(|message| message.text_content("|")).collect();- assert_eq!(messages, vec![format!("fix the build|{NOTE}")]);+ assert_eq!(+ messages,+ vec!["fix the build".to_string(), NOTE.to_string()]+ );assert!(request.llm_request.instructions.is_empty(),"a note is not an instruction"diff --git a/crates/libsy/src/algorithms/util/prompts.rs b/crates/libsy/src/algorithms/util/prompts.rsindex 1e66c4286..969833966 100644--- a/crates/libsy/src/algorithms/util/prompts.rs+++ b/crates/libsy/src/algorithms/util/prompts.rs@@ -1,170 +1,195 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Adding text to a request on its way to the model it was routed to.-//!-//! These helpers mutate the normalized request. A codec asked to encode for the format the-//! request arrived in replays the body captured at decode instead of reading that-//! request — so an addition that leaves exact replay in place never reaches the-//! model. This is not enforced: a future processor that mutates the request and-//! forgets the call reintroduces SWITCH-1224, silently and without a failing-//! test.+//! Router guidance uses the shared request edits so preserved provider content stays intact.-use switchyard_protocol::{ContentBlock, InstructionBlock, Message, Request, Role};+use switchyard_protocol::Request;-/// Appends `note` to the request as conversation text.-///-/// It joins the trailing user message when there is one, rather than opening a-/// turn of its own: Anthropic rejects two consecutive user messages, and a-/// `tool_result` must stay first within its message. A text block after it-/// satisfies both and keeps the addition a cache-safe suffix. Any other trailing-/// role — an empty conversation, or one ending on an assistant turn — takes a-/// fresh user message.+/// This helper appends temporary guidance and updates preserved provider bodies.pub fn append_note(request: &mut Request, note: &str) {- match request.llm_request.messages.last_mut() {- Some(last) if last.role == Role::User => last.content.push(ContentBlock::Text {- text: note.to_string(),- }),- _ => request- .llm_request- .messages- .push(Message::text(Role::User, note)),- }- drop_exact_replay(request);+ switchyard_translation::append_note(&mut request.llm_request, note);}-/// Gives up exact same-format replay for this turn.-///-/// A codec replays the preserved inbound body verbatim when the target format-/// matches the source, which is what keeps a same-format hop lossless. That body-/// predates anything added here, so leaving it in place would encode the request-/// as it arrived and silently drop the addition. Dropping it sends the codec down-/// its normal path, which encodes from the request itself.-///-/// Every stored body goes, not just the one for the inbound format: preservation-/// also carries bodies embedded by earlier hops, and the addition is missing from-/// all of them equally.-///-/// Call this from any new code that mutates the request. Nothing checks that you-/// have.+/// This helper clears preserved bodies and the note count after other conversation edits.+/// Subsequent encoding uses the edited request instead of replaying stale provider bodies.pub(crate) fn drop_exact_replay(request: &mut Request) {request.llm_request.preservation.requests.clear();-}--/// Prepends a system prompt and disables exact replay so the edit reaches the provider.-pub(crate) fn prepend_system_prompt(request: &mut Request, prompt: &str) {- request.llm_request.instructions.insert(- 0,- InstructionBlock {- role: Role::System,- content: vec![ContentBlock::Text {- text: prompt.to_string(),- }],- },- );- drop_exact_replay(request);+ request.llm_request.temporary_note_count = 0;}#[cfg(test)]mod tests {use super::*;- use switchyard_protocol::{LlmRequest, ToolResult, text_request};-- const NOTE: &str = "recovering from an error";-- /// Every test request carries the exact inbound body a codec keeps for- /// same-format replay, so each assertion below also says what happens to it.- fn request_with(messages: Vec<Message>) -> Request {- Request {- llm_request: LlmRequest {- messages,- preservation: preserved_body(),- ..LlmRequest::default()- },- raw_request: None,- metadata: None,- }- }-- fn preserved_body() -> switchyard_protocol::PreservationMetadata {- let mut preservation = switchyard_protocol::PreservationMetadata::default();- preservation.requests.insert(- "openai_chat".into(),- serde_json::json!({"model": "weak", "messages": [{"role": "user", "content": "hi"}]}),- );- preservation- }-- fn replays_exactly(request: &Request) -> bool {- !request.llm_request.preservation.requests.is_empty()- }+ use serde_json::json;+ use switchyard_protocol::WireFormat;+ use switchyard_translation::{TranslationEngine, TranslationPolicy};+ // Native bodies retain provider fields and tool argument text that decoding may not preserve.#[test]- fn a_note_joins_a_trailing_user_turn_after_its_tool_result() {- // The shape a coding-agent turn actually arrives in: the tool result- // leads the trailing user message, so the note has to follow it.- let tool_result = ContentBlock::ToolResult(ToolResult {- tool_call_id: "call_1".to_string(),- content: vec![ContentBlock::Text {- text: "exit 1".to_string(),- }],- is_error: Some(true),- });- let mut request = request_with(vec![Message {- role: Role::User,- content: vec![tool_result.clone()],- }]);-- append_note(&mut request, NOTE);-- let messages = &request.llm_request.messages;- assert_eq!(messages.len(), 1, "no second consecutive user turn");- assert_eq!(- messages[0].content,- vec![- tool_result,- ContentBlock::Text {- text: NOTE.to_string()- }- ]- );+ fn edits_preserve_native_prefixes_and_provider_fields() -> Result<(), Box<dyn std::error::Error>>+ {+ let engine = TranslationEngine::default();+ let policy = TranslationPolicy::default();+ let fixtures = [+ (+ WireFormat::OpenAiChat,+ json!({+ "model":"auto", "messages":[+ {"role":"developer","content":[{"type":"text","text":"Keep this instruction.","vendor":"keep"}]},+ {"role":"assistant","tool_calls":[{"id":"call_1","type":"function","function":{"name":"run","arguments":"{ \"cmd\": \"test\" }"}}]},+ {"role":"tool","tool_call_id":"call_1","content":"exit 1","vendor":"keep"}+ ], "metadata":{"caller":"keep"}, "temperature":0.2+ }),+ ),+ (+ WireFormat::OpenAiResponses,+ json!({+ "model":"auto", "instructions":"Keep this instruction.", "input":[+ {"type":"message","role":"developer","content":[{"type":"input_text","text":"Keep this too."}]},+ {"type":"function_call","call_id":"call_1","name":"run","arguments":"{ \"cmd\": \"test\" }"},+ {"type":"function_call_output","call_id":"call_1","output":"exit 1"}+ ], "metadata":{"caller":"keep"}, "store":false+ }),+ ),+ (+ WireFormat::AnthropicMessages,+ json!({+ "model":"auto", "max_tokens":32,+ "system":[{"type":"text","text":"Keep this instruction.","cache_control":{"type":"ephemeral","ttl":"1h"}}],+ "messages":[+ {"role":"assistant","content":[{"type":"tool_use","id":"call_1","name":"run","input":{"cmd":"test"}}]},+ {"role":"user","content":[{"type":"tool_result","tool_use_id":"call_1","content":"exit 1","is_error":true,"cache_control":{"type":"ephemeral"}}]}+ ], "metadata":{"user_id":"keep"}+ }),+ ),+ ];+ for (format, original) in fixtures {+ let mut request = Request {+ llm_request: engine.decode_request(format, &original, &policy)?.request,+ ..Default::default()+ };+ append_note(&mut request, "Reassess the failure.");+ let encoded = engine+ .encode_request(format, &request.llm_request, &policy)?+ .body;+ let mut expected = original.clone();+ match format {+ WireFormat::OpenAiChat => expected["messages"].as_array_mut().ok_or("messages")?.push(json!({"role":"user","content":[{"type":"text","text":"Reassess the failure."}]})),+ WireFormat::OpenAiResponses => expected["input"].as_array_mut().ok_or("input")?.push(json!({"role":"user","content":[{"type":"input_text","text":"Reassess the failure."}]})),+ WireFormat::AnthropicMessages => expected["messages"][1]["content"].as_array_mut().ok_or("content")?.push(json!({"type":"text","text":"Reassess the failure."})),+ }+ assert_eq!(encoded, expected, "{format}: only the note may change");+ switchyard_translation::prepend_system_prompt(+ &mut request.llm_request,+ "Use the capable tier.",+ );+ let edited = engine+ .encode_request(format, &request.llm_request, &policy)?+ .body;+ // The system prompt edit must retain the earlier note and all conversation and tool items.+ let key = if format == WireFormat::OpenAiResponses {+ "input"+ } else {+ "messages"+ };+ if format == WireFormat::OpenAiChat {+ assert_eq!(+ edited[key].as_array().ok_or("messages")?[1..],+ encoded[key].as_array().ok_or("messages")?[..]+ );+ } else {+ assert_eq!(edited[key], encoded[key]);+ }+ assert_eq!(edited["metadata"], original["metadata"]);+ }+ Ok(())}#[test]- fn a_note_opens_a_user_turn_after_an_assistant_turn() {- let mut request = request_with(vec![Message::text(Role::Assistant, "done")]);-- append_note(&mut request, NOTE);-- let messages = &request.llm_request.messages;- assert_eq!(messages.len(), 2);- assert_eq!(messages[1].role, Role::User);- assert_eq!(messages[1].text_content(""), Some(NOTE.to_string()));+ fn string_inputs_and_assistant_endings_keep_the_original_message()+ -> Result<(), Box<dyn std::error::Error>> {+ let engine = TranslationEngine::default();+ let policy = TranslationPolicy::default();+ for (format, body, key) in [+ (+ WireFormat::OpenAiChat,+ json!({"model":"m","messages":[{"role":"assistant","content":"done"}]}),+ "messages",+ ),+ (+ WireFormat::AnthropicMessages,+ json!({"model":"m","max_tokens":8,"messages":[{"role":"user","content":"question"},{"role":"assistant","content":"done"}]}),+ "messages",+ ),+ (+ WireFormat::OpenAiResponses,+ json!({"model":"m","input":"question"}),+ "input",+ ),+ ] {+ let mut request = Request {+ llm_request: engine.decode_request(format, &body, &policy)?.request,+ ..Default::default()+ };+ append_note(&mut request, "note one");+ append_note(&mut request, "note two");+ let encoded = engine+ .encode_request(format, &request.llm_request, &policy)?+ .body;+ let items = encoded[key].as_array().ok_or("items")?;+ if body[key].is_array() {+ let original = body[key].as_array().ok_or("original")?;+ assert_eq!(&items[..original.len()], original);+ } else {+ assert_eq!(items[0], json!({"role":"user","content":"question"}));+ }+ }+ Ok(())}-+ // Unsupported preserved bodies must not cause encoding to omit the note.#[test]- fn a_note_leaves_the_rest_of_the_conversation_untouched() {+ fn invalid_preservation_falls_back_without_losing_the_note()+ -> Result<(), Box<dyn std::error::Error>> {+ let engine = TranslationEngine::default();+ let policy = TranslationPolicy::default();+ let source = json!({"model":"m","messages":[{"role":"user","content":"original task"}]});let mut request = Request {- llm_request: LlmRequest {- preservation: preserved_body(),- ..text_request(Some("auto".to_string()), "fix the build")- },- raw_request: None,- metadata: None,+ llm_request: engine+ .decode_request(WireFormat::OpenAiChat, &source, &policy)?+ .request,+ ..Default::default()};-- append_note(&mut request, NOTE);-- let trail: Vec<String> = request+ request.llm_request- .messages- .iter()- .filter_map(|message| message.text_content("|"))- .collect();- assert_eq!(trail, vec![format!("fix the build|{NOTE}")]);- assert!(- !replays_exactly(&request),- "a same-format hop would replay the body captured before the note"+ .preservation+ .requests+ .insert("custom".into(), json!({"messages":"unsupported"}));+ request.llm_request.preservation.requests.insert(+ WireFormat::OpenAiChat.into(),+ json!({"model":"m","messages":"unsupported"}),);+ append_note(&mut request, "reassess the failure");+ assert!(request.llm_request.preservation.requests.is_empty());+ for target in [+ WireFormat::OpenAiChat,+ WireFormat::OpenAiResponses,+ WireFormat::AnthropicMessages,+ ] {+ let body = engine+ .encode_request(target, &request.llm_request, &policy)?+ .body;+ let decoded = engine.decode_request(target, &body, &policy)?.request;+ let text: Vec<_> = decoded+ .messages+ .iter()+ .filter_map(|message| message.text_content(""))+ .collect();+ assert_eq!(+ text,+ vec!["original task", "reassess the failure"],+ "{target}"+ );+ }+ Ok(())}}diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rsindex 2d1fcbf7c..33f4019e9 100644--- a/crates/libsy/src/algorithms/util/stage.rs+++ b/crates/libsy/src/algorithms/util/stage.rs@@ -1497,7 +1497,7 @@ mod tests {.score(&mut state, &mut request, &driver).await?;- assert_eq!(trailing_text(&request), Some(format!("hi|{ESCALATION}")));+ assert_eq!(trailing_text(&request), Some(ESCALATION.to_string()));Ok(())}@@ -1509,7 +1509,7 @@ mod tests {let mut first = request();classifier.score(&mut state, &mut first, &driver).await?;- assert_eq!(trailing_text(&first), Some(format!("hi|{ESCALATION}")));+ assert_eq!(trailing_text(&first), Some(ESCALATION.to_string()));for _ in 0..2 {let mut held = request();@@ -1535,7 +1535,7 @@ mod tests {.score(&mut state, &mut request, &driver).await?;- assert_eq!(trailing_text(&request), Some(format!("hi|{DEESCALATION}")));+ assert_eq!(trailing_text(&request), Some(DEESCALATION.to_string()));Ok(())}diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rsindex 5e62d8c15..fe9599559 100644--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs@@ -788,20 +788,9 @@ fn handoff_preserves_existing_messages_and_non_text_content() {classifier.hand_off(&mut task, &mut state, Some("unchecked result"));// A notice may be appended, but all original content stays unchanged.let messages = &task.llm_request.messages;- assert_eq!(messages.len(), original.len());- assert_eq!(- &messages[..original.len() - 1],- &original[..original.len() - 1]- );- let last = messages.last().expect("last user message");- assert_eq!(- &last.content[..original- .last()- .expect("original last message")- .content- .len()],- &original.last().expect("original last message").content- );+ assert_eq!(messages.len(), original.len() + 1);+ assert_eq!(&messages[..original.len()], original.as_slice());+ let last = messages.last().expect("handoff note");assert!(last.text_content("").is_some_and(|text| text.contains("unchecked result"))diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rsindex 122412615..edcf87689 100644--- a/crates/protocol/src/llm.rs+++ b/crates/protocol/src/llm.rs@@ -536,6 +536,14 @@ pub struct LlmRequest {pub instructions: Vec<InstructionBlock>,/// Ordered conversation messages.pub messages: Vec<Message>,+ /// This request-local count tracks temporary notes appended after the caller's conversation.+ /// The HTTP client uses it to choose a cache breakpoint before the notes, subject to+ /// provider support, caller policy, marker limits, and an eligible prefix.+ /// Rust moves and clones retain the count.+ /// Serialization omits the count, and deserialization resets it to zero.+ /// Note text and preserved provider bodies remain serialized.+ #[serde(skip)]+ pub temporary_note_count: usize,/// Whether this request performs compaction or resumes from a summary.#[serde(skip_serializing_if = "std::ops::Not::not")]pub is_compaction: bool,diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rsindex 8d4a5c3c7..7a450b535 100644--- a/crates/switchyard-translation/src/codecs/responses/stream.rs+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs@@ -304,17 +304,7 @@ fn decode_responses_stream(}}}- if let Some(usage) = event- .get("response")- .and_then(Value::as_object)- .and_then(|response| response.get("usage"))- .and_then(Value::as_object)- {- let usage = responses_usage(usage);- state.usage = usage.clone();- state.saw_backend_usage = true;- out.push(LlmResponseChunk::Usage(usage));- }+ decode_terminal_usage(state, event, &mut out);// A completed response that produced a tool call ended the turn to run that// tool, not because the assistant was done. The buffered decoder reports tool// use for such output; the stream must too, or stop-reason-driven tool loops@@ -325,9 +315,14 @@ fn decode_responses_stream(out}// Carries the Anthropic spelling because every encoder already maps it.- Some("response.incomplete") => vec![LlmResponseChunk::MessageStop {- reason: Some("max_tokens".to_string()),- }],+ Some("response.incomplete") => {+ let mut out = Vec::new();+ decode_terminal_usage(state, event, &mut out);+ out.push(LlmResponseChunk::MessageStop {+ reason: Some("max_tokens".to_string()),+ });+ out+ }Some("response.failed") => vec![LlmResponseChunk::StreamError {message: event.get("response")@@ -1167,6 +1162,25 @@ fn encode_responses_tool_delta(out}+// Both completed and token-limited Responses events can carry final usage and provider cost.+fn decode_terminal_usage(+ state: &mut StreamTranslationState,+ event: &Value,+ out: &mut Vec<LlmResponseChunk>,+) {+ if let Some(usage) = event+ .get("response")+ .and_then(Value::as_object)+ .and_then(|response| response.get("usage"))+ .and_then(Value::as_object)+ {+ let usage = responses_usage(usage);+ state.usage = usage.clone();+ state.saw_backend_usage = true;+ out.push(LlmResponseChunk::Usage(usage));+ }+}+// Normalizes OpenAI Responses token usage fields.fn responses_usage(usage: &serde_json::Map<String, Value>) -> Usage {let aggregate_input_tokens = usage.get("input_tokens").and_then(Value::as_u64);diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rsindex 3b23417cc..b221749cd 100644--- a/crates/switchyard-translation/src/lib.rs+++ b/crates/switchyard-translation/src/lib.rs@@ -12,6 +12,7 @@ pub mod engine;pub mod error;mod helpers;pub mod policy;+mod request_edit;mod sse;pub mod stream;pub mod util;@@ -31,6 +32,7 @@ pub use format::*;pub use helpers::*;pub use llm::*;pub use policy::*;+pub use request_edit::{append_note, prepend_system_prompt};pub use stream::*;pub use util::{PRESERVATION_METADATA_KEY, normalize_anthropic_tool_use_ids, prepare_request_for_target,diff --git a/crates/switchyard-translation/src/request_edit.rs b/crates/switchyard-translation/src/request_edit.rsnew file mode 100644index 000000000..9bc65ce25--- /dev/null+++ b/crates/switchyard-translation/src/request_edit.rs@@ -0,0 +1,124 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Request edits update normalized content and preserved provider bodies together.++use serde_json::{Map, Value, json};+use switchyard_protocol::{ContentBlock, InstructionBlock, LlmRequest, Message, Role, WireFormat};++/// This function appends temporary guidance and updates preserved provider bodies.+/// It removes bodies it cannot edit so encoding rebuilds them from the updated request.+pub fn append_note(request: &mut LlmRequest, note: &str) {+ // A separate message preserves the caller's final message boundary for OpenAI caching.+ request.messages.push(Message::text(Role::User, note));+ request.temporary_note_count += 1;+ request.preservation.requests.retain(|format, body| {+ let Some(body) = body.as_object_mut() else {+ return false;+ };+ let (key, content_type) = match format.as_str() {+ "openai_chat" | "anthropic_messages" => ("messages", "text"),+ "openai_responses" => ("input", "input_text"),+ _ => return false,+ };+ // Responses also accepts a string as a single user input.+ if key == "input"+ && let Some(Value::String(text)) = body.get_mut(key)+ {+ let original = std::mem::take(text);+ body.insert(key.into(), json!([{"role":"user", "content":original}]));+ }+ if key == "input" && body.get(key).is_none_or(Value::is_null) {+ body.insert(key.into(), Value::Array(Vec::new()));+ }+ let Some(messages) = body.get_mut(key).and_then(Value::as_array_mut) else {+ return false;+ };+ if format.as_str() == WireFormat::AnthropicMessages.as_str()+ && let Some(last) = messages.last_mut()+ && last.get("role").and_then(Value::as_str) == Some("user")+ {+ // Anthropic guidance follows existing content, including any tool result.+ match last.get_mut("content") {+ Some(Value::String(text)) => {+ let original = std::mem::take(text);+ last["content"] = json!([+ {"type":"text", "text":original},+ {"type":"text", "text":note}+ ]);+ }+ Some(Value::Array(blocks)) => blocks.push(json!({"type":"text", "text":note})),+ _ => return false,+ }+ } else {+ messages.push(json!({"role":"user", "content":[{"type":content_type,"text":note}]}));+ }+ true+ });+}++/// This function prepends a system prompt and updates supported preserved provider bodies.+/// It removes bodies it cannot edit so encoding rebuilds them from the updated request.+pub fn prepend_system_prompt(request: &mut LlmRequest, prompt: &str) {+ request.instructions.insert(+ 0,+ InstructionBlock {+ role: Role::System,+ content: vec![ContentBlock::Text {+ text: prompt.to_string(),+ }],+ },+ );+ request.preservation.requests.retain(|format, body| {+ let Some(body) = body.as_object_mut() else {+ return false;+ };+ match format.as_str() {+ "openai_chat" => prepend_openai_chat_system(body, prompt),+ "openai_responses" => prepend_text_field(body, "instructions", prompt),+ "anthropic_messages" => {+ body.remove("switchyard_anthropic_request");+ prepend_anthropic_system(body, prompt)+ }+ _ => false,+ }+ });+}++// This helper inserts a system message while keeping the existing Chat messages intact.+fn prepend_openai_chat_system(body: &mut Map<String, Value>, prompt: &str) -> bool {+ let message = json!({"role": "system", "content": prompt});+ match body.get_mut("messages") {+ None | Some(Value::Null) => {+ body.insert("messages".to_string(), Value::Array(vec![message]));+ }+ Some(Value::Array(messages)) => messages.insert(0, message),+ Some(_) => return false,+ }+ true+}++// Preserved Responses instructions and Anthropic system prompts can be strings, missing, or null.+fn prepend_text_field(body: &mut Map<String, Value>, key: &str, prompt: &str) -> bool {+ match body.get_mut(key) {+ None | Some(Value::Null) => {+ body.insert(key.to_string(), Value::String(prompt.to_string()));+ }+ Some(Value::String(text)) if text.is_empty() => {+ *text = prompt.to_string();+ }+ Some(Value::String(text)) => text.insert_str(0, &format!("{prompt}\n\n")),+ Some(_) => return false,+ }+ true+}++// This helper retains existing Anthropic system blocks when it prepends a prompt.+fn prepend_anthropic_system(body: &mut Map<String, Value>, prompt: &str) -> bool {+ if let Some(Value::Array(blocks)) = body.get_mut("system") {+ blocks.insert(0, json!({"type":"text","text":prompt}));+ true+ } else {+ prepend_text_field(body, "system", prompt)+ }+}diff --git a/crates/switchyard-translation/src/util.rs b/crates/switchyard-translation/src/util.rsindex b9006045d..79dadec9f 100644--- a/crates/switchyard-translation/src/util.rs+++ b/crates/switchyard-translation/src/util.rs@@ -12,7 +12,7 @@ use switchyard_protocol::ModelId;use crate::diagnostic::TranslationDiagnostic;use crate::error::{Result, TranslationError};use crate::format::{FormatId, WireFormat};-use crate::llm::{ContentBlock, InstructionBlock, LlmRequest, Message, PreservationMetadata, Role};+use crate::llm::{ContentBlock, LlmRequest, Message, PreservationMetadata};use crate::policy::{LossyConversionPolicy, PreservationPolicy, TranslationPolicy, UnknownFieldPolicy,};@@ -338,107 +338,26 @@ pub fn prepare_request_for_target(let target = target.to_string();request.model = Some(target.clone());if let Some(prompt) = prompt {- request.instructions.insert(- 0,- InstructionBlock {- role: Role::System,- content: vec![ContentBlock::Text {- text: prompt.to_string(),- }],- },- );+ crate::prepend_system_prompt(request, prompt);}- prepare_preserved_requests(&mut request.preservation, &target, prompt);-}--// Retains exact replay only where the built-in wire body can receive the target changes safely.-fn prepare_preserved_requests(- preservation: &mut PreservationMetadata,- target: &str,- prompt: Option<&str>,-) {- preservation.requests.retain(|format, body| {- let is_builtin = format.as_str() == WireFormat::OpenAiChat.as_str()- || format.as_str() == WireFormat::OpenAiResponses.as_str()- || format.as_str() == WireFormat::AnthropicMessages.as_str();- let Some(body) = is_builtin.then_some(body).and_then(Value::as_object_mut) else {+ request.preservation.requests.retain(|format, body| {+ if ![+ WireFormat::OpenAiChat.as_str(),+ WireFormat::OpenAiResponses.as_str(),+ WireFormat::AnthropicMessages.as_str(),+ ]+ .contains(&format.as_str())+ {return false;- };- body.insert("model".to_string(), Value::String(target.to_string()));- match prompt {- None => true,- Some(prompt) if format.as_str() == WireFormat::OpenAiChat.as_str() => {- prepend_openai_chat_system(body, prompt)- }- Some(prompt) if format.as_str() == WireFormat::OpenAiResponses.as_str() => {- prepend_openai_responses_instructions(body, prompt)- }- Some(prompt) if format.as_str() == WireFormat::AnthropicMessages.as_str() => {- body.remove(crate::codecs::common::ANTHROPIC_REQUEST_KEY);- prepend_anthropic_system(body, prompt)- }- Some(_) => false,}+ let Some(body) = body.as_object_mut() else {+ return false;+ };+ body.insert("model".to_string(), Value::String(target.clone()));+ true});}-// Prepends a target prompt without rebuilding or otherwise changing a Chat request.-fn prepend_openai_chat_system(body: &mut Map<String, Value>, prompt: &str) -> bool {- let message = json!({"role": "system", "content": prompt});- match body.get_mut("messages") {- None | Some(Value::Null) => {- body.insert("messages".to_string(), Value::Array(vec![message]));- }- Some(Value::Array(messages)) => messages.insert(0, message),- Some(_) => return false,- }- true-}--// Prepends a target prompt without rebuilding or otherwise changing a Responses request.-fn prepend_openai_responses_instructions(body: &mut Map<String, Value>, prompt: &str) -> bool {- match body.get_mut("instructions") {- None | Some(Value::Null) => {- body.insert(- "instructions".to_string(),- Value::String(prompt.to_string()),- );- }- Some(Value::String(instructions)) if instructions.is_empty() => {- *instructions = prompt.to_string();- }- Some(Value::String(instructions)) => {- instructions.insert_str(0, &format!("{prompt}\n\n"));- }- Some(_) => return false,- }- true-}--// Prepends a target prompt without rebuilding or otherwise changing an Anthropic request.-fn prepend_anthropic_system(body: &mut Map<String, Value>, prompt: &str) -> bool {- match body.get_mut("system") {- None | Some(Value::Null) => {- body.insert("system".to_string(), Value::String(prompt.to_string()));- }- Some(Value::String(system)) if system.is_empty() => {- *system = prompt.to_string();- }- Some(Value::String(system)) => {- system.insert_str(0, &format!("{prompt}\n\n"));- }- Some(Value::Array(blocks)) => blocks.insert(- 0,- json!({- "type": "text",- "text": prompt,- }),- ),- Some(_) => return false,- }- true-}-/// Embeds preservation metadata into a translated wire body when requested.pub fn embed_preservation(mut body: Value,diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rsindex 22899b029..9ac6058ae 100644--- a/crates/switchyard-translation/tests/stream_translation.rs+++ b/crates/switchyard-translation/tests/stream_translation.rs@@ -2525,12 +2525,14 @@ fn anthropic_max_tokens_stop_translates_to_responses_incomplete_event() -> TestR// Verifies a streamed response.incomplete from a Responses upstream reaches a Chat client.#[test]fn responses_incomplete_event_translates_to_chat_length_finish() -> TestResult {+ // A token-limited response must retain final accounting as well as the length stop reason.let engine = TranslationEngine::default();let mut state =StreamTranslationState::new(WireFormat::OpenAiResponses, WireFormat::OpenAiChat);let incomplete = json!({"type": "response.incomplete",- "response": {"status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}}+ "response": {"status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"},+ "usage":{"input_tokens":12000,"input_tokens_details":{"cached_tokens":4000,"cache_write_tokens":7000},"output_tokens":16,"total_tokens":12016,"cost":0.01515}}});let mut events = engine.translate_event(@@ -2541,10 +2543,21 @@ fn responses_incomplete_event_translates_to_chat_length_finish() -> TestResult {)?;events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiChat)?);- let Some(terminal) = events.last() else {+ let Some(terminal) = events+ .iter()+ .find(|event| event["choices"][0]["finish_reason"] == "length")+ else {return Err("finish should emit a terminal Chat chunk".into());};assert_eq!(terminal["choices"][0]["finish_reason"], "length");+ let usage = events+ .iter()+ .find_map(|event| event.get("usage"))+ .ok_or("missing terminal usage")?;+ assert_eq!(usage["prompt_tokens"], 12000);+ assert_eq!(usage["prompt_tokens_details"]["cached_tokens"], 4000);+ assert_eq!(usage["prompt_tokens_details"]["cache_write_tokens"], 7000);+ assert_eq!(usage["cost"], 0.01515);Ok(())}