Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(routing): preserve cache reuse across handoff notes (#1213)

Elyas Mehtabuddinda3dcf04d62f
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 6158f42bf..4fa4db687 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -393,7 +393,18 @@ impl TranslatingLlmClient {
if nvidia_inference_api_backend(backend) {
strip_anthropic_tool_strict(&mut body);
}
- enable_anthropic_prompt_caching(&mut body);
+ enable_anthropic_prompt_caching(&mut body, llm_request.temporary_note_count);
+ }
+ if matches!(
+ backend,
+ Backend::OpenAiChat(_) | Backend::OpenAiResponses(_)
+ ) {
+ enable_openai_guidance_caching(
+ &mut body,
+ wire_format,
+ upstream_model,
+ llm_request.temporary_note_count,
+ );
}
if matches!(backend, Backend::OpenAiChat(_)) {
ensure_openai_stream_usage(&mut body);
@@ -1452,44 +1463,135 @@ fn count_cache_control_blocks(value: &Value) -> usize {
}
}
-// Marks the final message content block as the Anthropic prompt-cache breakpoint.
-fn enable_anthropic_prompt_caching(body: &mut Value) {
- // Top-level cache control manages the final breakpoint and its TTL.
- if body.get("cache_control").is_some() {
+// This helper marks the final caller content block before any temporary notes, if policy permits.
+fn enable_anthropic_prompt_caching(body: &mut Value, temporary_note_count: usize) {
+ // Anthropic rejects a fifth cache marker with HTTP 400.
+ // This check preserves the caller's markers when the request has reached that limit.
+ if count_cache_control_blocks(body) >= MAX_CACHE_CONTROL_BLOCKS {
return;
}
- // Abstain once the caller has spent the budget itself. Adding a fifth marker
- // turns a request that was valid on arrival into an upstream HTTP 400, and a
- // caller that placed four breakpoints deliberately needs them more than we
- // need a fifth. When the final block is already marked this returns early
- // and changes nothing, which is what the insert below would have done.
- if count_cache_control_blocks(body) >= MAX_CACHE_CONTROL_BLOCKS {
+ // Requests with only a top-level `1h` policy can return HTTP 400.
+ // New block markers match the caller's TTL on turns with or without notes.
+ let marker = body
+ .get("cache_control")
+ .cloned()
+ .unwrap_or_else(|| serde_json::json!({"type":"ephemeral"}));
+ let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else {
return;
+ };
+ let mut remaining = temporary_note_count;
+ // Temporary guidance is absent next turn, so the breakpoint must precede it.
+ for message in messages.iter_mut().rev() {
+ match message.get_mut("content") {
+ Some(content @ Value::String(_)) if remaining == 0 => {
+ let text = std::mem::take(content);
+ *content = serde_json::json!([{
+ "type": "text", "text": text, "cache_control": marker
+ }]);
+ return;
+ }
+ Some(Value::String(_)) => remaining = remaining.saturating_sub(1),
+ Some(Value::Array(blocks)) => {
+ if remaining >= blocks.len() {
+ remaining -= blocks.len();
+ continue;
+ }
+ let index = blocks.len() - remaining - 1;
+ if let Some(block) = blocks[index].as_object_mut() {
+ block.entry("cache_control".to_string()).or_insert(marker);
+ }
+ return;
+ }
+ _ => {}
+ }
}
- let Some(content) = body
- .get_mut("messages")
- .and_then(Value::as_array_mut)
- .and_then(|messages| messages.last_mut())
- .and_then(|message| message.get_mut("content"))
- else {
+}
+
+// GPT-5.6 and GPT-6 cache writes end at selected message or content boundaries.
+// A temporary suffix needs a separate cache entry before it so the next request can reuse the prefix.
+fn enable_openai_guidance_caching(
+ body: &mut Value,
+ format: WireFormat,
+ model: &ModelId,
+ suffix: usize,
+) {
+ let name = model.as_str().rsplit('/').next().unwrap_or_default();
+ let supported = name == "gpt-5.6"
+ || name.starts_with("gpt-5.6-")
+ || name == "gpt-6"
+ || name.starts_with("gpt-6-")
+ || name.starts_with("gpt-6.");
+ // Explicit-only callers choose their write locations.
+ // This helper adds markers only for names recognized by its model-name check.
+ if suffix == 0
+ || !supported
+ || body
+ .pointer("/prompt_cache_options/mode")
+ .and_then(Value::as_str)
+ == Some("explicit")
+ {
+ return;
+ }
+ let key = if format == WireFormat::OpenAiResponses {
+ "input"
+ } else {
+ "messages"
+ };
+ let Some(items) = body.get_mut(key).and_then(Value::as_array_mut) else {
return;
};
- match content {
- Value::String(text) => {
- *content = serde_json::json!([{
- "type": "text",
- "text": std::mem::take(text),
- "cache_control": {"type": "ephemeral"}
- }]);
+ let marker_count = items
+ .iter()
+ .map(|item| {
+ ["content", "output"]
+ .iter()
+ .filter_map(|key| item.get(key).and_then(Value::as_array))
+ .flatten()
+ .filter(|block| block.get("prompt_cache_breakpoint").is_some())
+ .count()
+ })
+ .sum::<usize>();
+ // Implicit mode reserves one of the four write slots for the provider's final breakpoint.
+ if marker_count >= 3 {
+ return;
+ }
+ let Some(prefix_len) = items.len().checked_sub(suffix) else {
+ return;
+ };
+ for item in items[..prefix_len].iter_mut().rev() {
+ let field = if item.get("type").and_then(Value::as_str) == Some("function_call_output") {
+ "output"
+ } else if matches!(
+ item.get("role").and_then(Value::as_str),
+ Some("user" | "tool" | "developer" | "system")
+ ) {
+ "content"
+ } else {
+ continue;
+ };
+ let Some(content) = item.get_mut(field) else {
+ continue;
+ };
+ if let Value::String(text) = content {
+ let text = std::mem::take(text);
+ let kind = if format == WireFormat::OpenAiResponses {
+ "input_text"
+ } else {
+ "text"
+ };
+ *content = serde_json::json!([{"type":kind,"text":text}]);
}
- Value::Array(blocks) => {
- if let Some(block) = blocks.last_mut().and_then(Value::as_object_mut) {
- block
- .entry("cache_control".to_string())
- .or_insert_with(|| serde_json::json!({"type": "ephemeral"}));
- }
+ // Empty content arrays have no block to mark, so the search continues through earlier items.
+ if let Some(block) = content
+ .as_array_mut()
+ .and_then(|blocks| blocks.last_mut())
+ .and_then(Value::as_object_mut)
+ {
+ block
+ .entry("prompt_cache_breakpoint".to_string())
+ .or_insert_with(|| serde_json::json!({"mode":"explicit"}));
+ return;
}
- _ => {}
}
}
@@ -1884,14 +1986,21 @@ mod tests {
"messages": [{"role": "user", "content": "hello"}]
});
let mut body = caller_managed.clone();
- enable_anthropic_prompt_caching(&mut body);
- assert_eq!(body, caller_managed);
+ enable_anthropic_prompt_caching(&mut body, 0);
+ assert_eq!(body["cache_control"], caller_managed["cache_control"]);
+ assert_eq!(
+ body["messages"][0]["content"],
+ json!([{
+ "type": "text", "text": "hello",
+ "cache_control": {"type": "ephemeral", "ttl": "1h"}
+ }])
+ );
let mut body = json!({
"messages": [{"role": "user", "content": "hello"}]
});
- enable_anthropic_prompt_caching(&mut body);
+ enable_anthropic_prompt_caching(&mut body, 0);
assert_eq!(
body["messages"][0]["content"][0]["cache_control"],
@@ -1931,7 +2040,7 @@ mod tests {
let mut body = body_at_the_cache_control_limit();
assert_eq!(cache_control_blocks(&body), 4);
- enable_anthropic_prompt_caching(&mut body);
+ enable_anthropic_prompt_caching(&mut body, 0);
// Anthropic and Bedrock both reject a fifth marker with HTTP 400, which
// would fail a request that was valid before it reached us.
@@ -1950,7 +2059,7 @@ mod tests {
body["system"].as_array_mut().unwrap().pop();
assert_eq!(cache_control_blocks(&body), 3);
- enable_anthropic_prompt_caching(&mut body);
+ enable_anthropic_prompt_caching(&mut body, 0);
assert_eq!(cache_control_blocks(&body), 4);
assert_eq!(
diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml
index 2e0a08169..6fdc9b408 100644
--- a/crates/libsy/Cargo.toml
+++ b/crates/libsy/Cargo.toml
@@ -33,6 +33,7 @@ rand.workspace = true
regex.workspace = true
shlex = "2"
switchyard-protocol.workspace = true
+switchyard-translation.workspace = true
thiserror.workspace = true
tokio-stream = "0.1"
tracing.workspace = true
@@ -46,7 +47,6 @@ tokio.workspace = true
tokio = { version = "1", features = ["rt", "sync", "macros", "time"] }
[dev-dependencies]
-switchyard-translation.workspace = true
# SDK + in-memory exporter to assert what the observability layer records.
opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }
tokio.workspace = true
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 6b0a01906..bd0a1b757 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -176,12 +176,18 @@ impl FallThrough {
let mut request = outcome.request.clone();
request.llm_request.model = Some(target.to_string());
if let Some(prompt) = prompts.get(target) {
- super::util::prompts::prepend_system_prompt(&mut request, prompt);
+ switchyard_translation::prepend_system_prompt(
+ &mut request.llm_request,
+ prompt,
+ );
}
outcome.candidate_requests.insert(target.clone(), request);
}
if let Some(prompt) = prompts.get(&outcome.selected_model_ids[0]) {
- super::util::prompts::prepend_system_prompt(&mut outcome.request, prompt);
+ switchyard_translation::prepend_system_prompt(
+ &mut outcome.request.llm_request,
+ prompt,
+ );
}
}
Ok(outcome)
@@ -551,7 +557,10 @@ mod tests {
.filter_map(|message| message.text_content("|"))
.collect();
- assert_eq!(messages, vec![format!("fix the build|{NOTE}")]);
+ assert_eq!(
+ messages,
+ vec!["fix the build".to_string(), NOTE.to_string()]
+ );
assert!(
request.llm_request.instructions.is_empty(),
"a note is not an instruction"
diff --git a/crates/libsy/src/algorithms/util/prompts.rs b/crates/libsy/src/algorithms/util/prompts.rs
index 1e66c4286..969833966 100644
--- a/crates/libsy/src/algorithms/util/prompts.rs
+++ b/crates/libsy/src/algorithms/util/prompts.rs
@@ -1,170 +1,195 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Adding text to a request on its way to the model it was routed to.
-//!
-//! These helpers mutate the normalized request. A codec asked to encode for the format the
-//! request arrived in replays the body captured at decode instead of reading that
-//! request — so an addition that leaves exact replay in place never reaches the
-//! model. This is not enforced: a future processor that mutates the request and
-//! forgets the call reintroduces SWITCH-1224, silently and without a failing
-//! test.
+//! Router guidance uses the shared request edits so preserved provider content stays intact.
-use switchyard_protocol::{ContentBlock, InstructionBlock, Message, Request, Role};
+use switchyard_protocol::Request;
-/// Appends `note` to the request as conversation text.
-///
-/// It joins the trailing user message when there is one, rather than opening a
-/// turn of its own: Anthropic rejects two consecutive user messages, and a
-/// `tool_result` must stay first within its message. A text block after it
-/// satisfies both and keeps the addition a cache-safe suffix. Any other trailing
-/// role — an empty conversation, or one ending on an assistant turn — takes a
-/// fresh user message.
+/// This helper appends temporary guidance and updates preserved provider bodies.
pub fn append_note(request: &mut Request, note: &str) {
- match request.llm_request.messages.last_mut() {
- Some(last) if last.role == Role::User => last.content.push(ContentBlock::Text {
- text: note.to_string(),
- }),
- _ => request
- .llm_request
- .messages
- .push(Message::text(Role::User, note)),
- }
- drop_exact_replay(request);
+ switchyard_translation::append_note(&mut request.llm_request, note);
}
-/// Gives up exact same-format replay for this turn.
-///
-/// A codec replays the preserved inbound body verbatim when the target format
-/// matches the source, which is what keeps a same-format hop lossless. That body
-/// predates anything added here, so leaving it in place would encode the request
-/// as it arrived and silently drop the addition. Dropping it sends the codec down
-/// its normal path, which encodes from the request itself.
-///
-/// Every stored body goes, not just the one for the inbound format: preservation
-/// also carries bodies embedded by earlier hops, and the addition is missing from
-/// all of them equally.
-///
-/// Call this from any new code that mutates the request. Nothing checks that you
-/// have.
+/// This helper clears preserved bodies and the note count after other conversation edits.
+/// Subsequent encoding uses the edited request instead of replaying stale provider bodies.
pub(crate) fn drop_exact_replay(request: &mut Request) {
request.llm_request.preservation.requests.clear();
-}
-
-/// Prepends a system prompt and disables exact replay so the edit reaches the provider.
-pub(crate) fn prepend_system_prompt(request: &mut Request, prompt: &str) {
- request.llm_request.instructions.insert(
- 0,
- InstructionBlock {
- role: Role::System,
- content: vec![ContentBlock::Text {
- text: prompt.to_string(),
- }],
- },
- );
- drop_exact_replay(request);
+ request.llm_request.temporary_note_count = 0;
}
#[cfg(test)]
mod tests {
use super::*;
- use switchyard_protocol::{LlmRequest, ToolResult, text_request};
-
- const NOTE: &str = "recovering from an error";
-
- /// Every test request carries the exact inbound body a codec keeps for
- /// same-format replay, so each assertion below also says what happens to it.
- fn request_with(messages: Vec<Message>) -> Request {
- Request {
- llm_request: LlmRequest {
- messages,
- preservation: preserved_body(),
- ..LlmRequest::default()
- },
- raw_request: None,
- metadata: None,
- }
- }
-
- fn preserved_body() -> switchyard_protocol::PreservationMetadata {
- let mut preservation = switchyard_protocol::PreservationMetadata::default();
- preservation.requests.insert(
- "openai_chat".into(),
- serde_json::json!({"model": "weak", "messages": [{"role": "user", "content": "hi"}]}),
- );
- preservation
- }
-
- fn replays_exactly(request: &Request) -> bool {
- !request.llm_request.preservation.requests.is_empty()
- }
+ use serde_json::json;
+ use switchyard_protocol::WireFormat;
+ use switchyard_translation::{TranslationEngine, TranslationPolicy};
+ // Native bodies retain provider fields and tool argument text that decoding may not preserve.
#[test]
- fn a_note_joins_a_trailing_user_turn_after_its_tool_result() {
- // The shape a coding-agent turn actually arrives in: the tool result
- // leads the trailing user message, so the note has to follow it.
- let tool_result = ContentBlock::ToolResult(ToolResult {
- tool_call_id: "call_1".to_string(),
- content: vec![ContentBlock::Text {
- text: "exit 1".to_string(),
- }],
- is_error: Some(true),
- });
- let mut request = request_with(vec![Message {
- role: Role::User,
- content: vec![tool_result.clone()],
- }]);
-
- append_note(&mut request, NOTE);
-
- let messages = &request.llm_request.messages;
- assert_eq!(messages.len(), 1, "no second consecutive user turn");
- assert_eq!(
- messages[0].content,
- vec![
- tool_result,
- ContentBlock::Text {
- text: NOTE.to_string()
- }
- ]
- );
+ fn edits_preserve_native_prefixes_and_provider_fields() -> Result<(), Box<dyn std::error::Error>>
+ {
+ let engine = TranslationEngine::default();
+ let policy = TranslationPolicy::default();
+ let fixtures = [
+ (
+ WireFormat::OpenAiChat,
+ json!({
+ "model":"auto", "messages":[
+ {"role":"developer","content":[{"type":"text","text":"Keep this instruction.","vendor":"keep"}]},
+ {"role":"assistant","tool_calls":[{"id":"call_1","type":"function","function":{"name":"run","arguments":"{ \"cmd\": \"test\" }"}}]},
+ {"role":"tool","tool_call_id":"call_1","content":"exit 1","vendor":"keep"}
+ ], "metadata":{"caller":"keep"}, "temperature":0.2
+ }),
+ ),
+ (
+ WireFormat::OpenAiResponses,
+ json!({
+ "model":"auto", "instructions":"Keep this instruction.", "input":[
+ {"type":"message","role":"developer","content":[{"type":"input_text","text":"Keep this too."}]},
+ {"type":"function_call","call_id":"call_1","name":"run","arguments":"{ \"cmd\": \"test\" }"},
+ {"type":"function_call_output","call_id":"call_1","output":"exit 1"}
+ ], "metadata":{"caller":"keep"}, "store":false
+ }),
+ ),
+ (
+ WireFormat::AnthropicMessages,
+ json!({
+ "model":"auto", "max_tokens":32,
+ "system":[{"type":"text","text":"Keep this instruction.","cache_control":{"type":"ephemeral","ttl":"1h"}}],
+ "messages":[
+ {"role":"assistant","content":[{"type":"tool_use","id":"call_1","name":"run","input":{"cmd":"test"}}]},
+ {"role":"user","content":[{"type":"tool_result","tool_use_id":"call_1","content":"exit 1","is_error":true,"cache_control":{"type":"ephemeral"}}]}
+ ], "metadata":{"user_id":"keep"}
+ }),
+ ),
+ ];
+ for (format, original) in fixtures {
+ let mut request = Request {
+ llm_request: engine.decode_request(format, &original, &policy)?.request,
+ ..Default::default()
+ };
+ append_note(&mut request, "Reassess the failure.");
+ let encoded = engine
+ .encode_request(format, &request.llm_request, &policy)?
+ .body;
+ let mut expected = original.clone();
+ match format {
+ WireFormat::OpenAiChat => expected["messages"].as_array_mut().ok_or("messages")?.push(json!({"role":"user","content":[{"type":"text","text":"Reassess the failure."}]})),
+ WireFormat::OpenAiResponses => expected["input"].as_array_mut().ok_or("input")?.push(json!({"role":"user","content":[{"type":"input_text","text":"Reassess the failure."}]})),
+ WireFormat::AnthropicMessages => expected["messages"][1]["content"].as_array_mut().ok_or("content")?.push(json!({"type":"text","text":"Reassess the failure."})),
+ }
+ assert_eq!(encoded, expected, "{format}: only the note may change");
+ switchyard_translation::prepend_system_prompt(
+ &mut request.llm_request,
+ "Use the capable tier.",
+ );
+ let edited = engine
+ .encode_request(format, &request.llm_request, &policy)?
+ .body;
+ // The system prompt edit must retain the earlier note and all conversation and tool items.
+ let key = if format == WireFormat::OpenAiResponses {
+ "input"
+ } else {
+ "messages"
+ };
+ if format == WireFormat::OpenAiChat {
+ assert_eq!(
+ edited[key].as_array().ok_or("messages")?[1..],
+ encoded[key].as_array().ok_or("messages")?[..]
+ );
+ } else {
+ assert_eq!(edited[key], encoded[key]);
+ }
+ assert_eq!(edited["metadata"], original["metadata"]);
+ }
+ Ok(())
}
#[test]
- fn a_note_opens_a_user_turn_after_an_assistant_turn() {
- let mut request = request_with(vec![Message::text(Role::Assistant, "done")]);
-
- append_note(&mut request, NOTE);
-
- let messages = &request.llm_request.messages;
- assert_eq!(messages.len(), 2);
- assert_eq!(messages[1].role, Role::User);
- assert_eq!(messages[1].text_content(""), Some(NOTE.to_string()));
+ fn string_inputs_and_assistant_endings_keep_the_original_message()
+ -> Result<(), Box<dyn std::error::Error>> {
+ let engine = TranslationEngine::default();
+ let policy = TranslationPolicy::default();
+ for (format, body, key) in [
+ (
+ WireFormat::OpenAiChat,
+ json!({"model":"m","messages":[{"role":"assistant","content":"done"}]}),
+ "messages",
+ ),
+ (
+ WireFormat::AnthropicMessages,
+ json!({"model":"m","max_tokens":8,"messages":[{"role":"user","content":"question"},{"role":"assistant","content":"done"}]}),
+ "messages",
+ ),
+ (
+ WireFormat::OpenAiResponses,
+ json!({"model":"m","input":"question"}),
+ "input",
+ ),
+ ] {
+ let mut request = Request {
+ llm_request: engine.decode_request(format, &body, &policy)?.request,
+ ..Default::default()
+ };
+ append_note(&mut request, "note one");
+ append_note(&mut request, "note two");
+ let encoded = engine
+ .encode_request(format, &request.llm_request, &policy)?
+ .body;
+ let items = encoded[key].as_array().ok_or("items")?;
+ if body[key].is_array() {
+ let original = body[key].as_array().ok_or("original")?;
+ assert_eq!(&items[..original.len()], original);
+ } else {
+ assert_eq!(items[0], json!({"role":"user","content":"question"}));
+ }
+ }
+ Ok(())
}
-
+ // Unsupported preserved bodies must not cause encoding to omit the note.
#[test]
- fn a_note_leaves_the_rest_of_the_conversation_untouched() {
+ fn invalid_preservation_falls_back_without_losing_the_note()
+ -> Result<(), Box<dyn std::error::Error>> {
+ let engine = TranslationEngine::default();
+ let policy = TranslationPolicy::default();
+ let source = json!({"model":"m","messages":[{"role":"user","content":"original task"}]});
let mut request = Request {
- llm_request: LlmRequest {
- preservation: preserved_body(),
- ..text_request(Some("auto".to_string()), "fix the build")
- },
- raw_request: None,
- metadata: None,
+ llm_request: engine
+ .decode_request(WireFormat::OpenAiChat, &source, &policy)?
+ .request,
+ ..Default::default()
};
-
- append_note(&mut request, NOTE);
-
- let trail: Vec<String> = request
+ request
.llm_request
- .messages
- .iter()
- .filter_map(|message| message.text_content("|"))
- .collect();
- assert_eq!(trail, vec![format!("fix the build|{NOTE}")]);
- assert!(
- !replays_exactly(&request),
- "a same-format hop would replay the body captured before the note"
+ .preservation
+ .requests
+ .insert("custom".into(), json!({"messages":"unsupported"}));
+ request.llm_request.preservation.requests.insert(
+ WireFormat::OpenAiChat.into(),
+ json!({"model":"m","messages":"unsupported"}),
);
+ append_note(&mut request, "reassess the failure");
+ assert!(request.llm_request.preservation.requests.is_empty());
+ for target in [
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiResponses,
+ WireFormat::AnthropicMessages,
+ ] {
+ let body = engine
+ .encode_request(target, &request.llm_request, &policy)?
+ .body;
+ let decoded = engine.decode_request(target, &body, &policy)?.request;
+ let text: Vec<_> = decoded
+ .messages
+ .iter()
+ .filter_map(|message| message.text_content(""))
+ .collect();
+ assert_eq!(
+ text,
+ vec!["original task", "reassess the failure"],
+ "{target}"
+ );
+ }
+ Ok(())
}
}
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index 2d1fcbf7c..33f4019e9 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -1497,7 +1497,7 @@ mod tests {
.score(&mut state, &mut request, &driver)
.await?;
- assert_eq!(trailing_text(&request), Some(format!("hi|{ESCALATION}")));
+ assert_eq!(trailing_text(&request), Some(ESCALATION.to_string()));
Ok(())
}
@@ -1509,7 +1509,7 @@ mod tests {
let mut first = request();
classifier.score(&mut state, &mut first, &driver).await?;
- assert_eq!(trailing_text(&first), Some(format!("hi|{ESCALATION}")));
+ assert_eq!(trailing_text(&first), Some(ESCALATION.to_string()));
for _ in 0..2 {
let mut held = request();
@@ -1535,7 +1535,7 @@ mod tests {
.score(&mut state, &mut request, &driver)
.await?;
- assert_eq!(trailing_text(&request), Some(format!("hi|{DEESCALATION}")));
+ assert_eq!(trailing_text(&request), Some(DEESCALATION.to_string()));
Ok(())
}
diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
index 5e62d8c15..fe9599559 100644
--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
@@ -788,20 +788,9 @@ fn handoff_preserves_existing_messages_and_non_text_content() {
classifier.hand_off(&mut task, &mut state, Some("unchecked result"));
// A notice may be appended, but all original content stays unchanged.
let messages = &task.llm_request.messages;
- assert_eq!(messages.len(), original.len());
- assert_eq!(
- &messages[..original.len() - 1],
- &original[..original.len() - 1]
- );
- let last = messages.last().expect("last user message");
- assert_eq!(
- &last.content[..original
- .last()
- .expect("original last message")
- .content
- .len()],
- &original.last().expect("original last message").content
- );
+ assert_eq!(messages.len(), original.len() + 1);
+ assert_eq!(&messages[..original.len()], original.as_slice());
+ let last = messages.last().expect("handoff note");
assert!(
last.text_content("")
.is_some_and(|text| text.contains("unchecked result"))
diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs
index 122412615..edcf87689 100644
--- a/crates/protocol/src/llm.rs
+++ b/crates/protocol/src/llm.rs
@@ -536,6 +536,14 @@ pub struct LlmRequest {
pub instructions: Vec<InstructionBlock>,
/// Ordered conversation messages.
pub messages: Vec<Message>,
+ /// This request-local count tracks temporary notes appended after the caller's conversation.
+ /// The HTTP client uses it to choose a cache breakpoint before the notes, subject to
+ /// provider support, caller policy, marker limits, and an eligible prefix.
+ /// Rust moves and clones retain the count.
+ /// Serialization omits the count, and deserialization resets it to zero.
+ /// Note text and preserved provider bodies remain serialized.
+ #[serde(skip)]
+ pub temporary_note_count: usize,
/// Whether this request performs compaction or resumes from a summary.
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub is_compaction: bool,
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs
index 8d4a5c3c7..7a450b535 100644
--- a/crates/switchyard-translation/src/codecs/responses/stream.rs
+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs
@@ -304,17 +304,7 @@ fn decode_responses_stream(
}
}
}
- if let Some(usage) = event
- .get("response")
- .and_then(Value::as_object)
- .and_then(|response| response.get("usage"))
- .and_then(Value::as_object)
- {
- let usage = responses_usage(usage);
- state.usage = usage.clone();
- state.saw_backend_usage = true;
- out.push(LlmResponseChunk::Usage(usage));
- }
+ decode_terminal_usage(state, event, &mut out);
// A completed response that produced a tool call ended the turn to run that
// tool, not because the assistant was done. The buffered decoder reports tool
// use for such output; the stream must too, or stop-reason-driven tool loops
@@ -325,9 +315,14 @@ fn decode_responses_stream(
out
}
// Carries the Anthropic spelling because every encoder already maps it.
- Some("response.incomplete") => vec![LlmResponseChunk::MessageStop {
- reason: Some("max_tokens".to_string()),
- }],
+ Some("response.incomplete") => {
+ let mut out = Vec::new();
+ decode_terminal_usage(state, event, &mut out);
+ out.push(LlmResponseChunk::MessageStop {
+ reason: Some("max_tokens".to_string()),
+ });
+ out
+ }
Some("response.failed") => vec![LlmResponseChunk::StreamError {
message: event
.get("response")
@@ -1167,6 +1162,25 @@ fn encode_responses_tool_delta(
out
}
+// Both completed and token-limited Responses events can carry final usage and provider cost.
+fn decode_terminal_usage(
+ state: &mut StreamTranslationState,
+ event: &Value,
+ out: &mut Vec<LlmResponseChunk>,
+) {
+ if let Some(usage) = event
+ .get("response")
+ .and_then(Value::as_object)
+ .and_then(|response| response.get("usage"))
+ .and_then(Value::as_object)
+ {
+ let usage = responses_usage(usage);
+ state.usage = usage.clone();
+ state.saw_backend_usage = true;
+ out.push(LlmResponseChunk::Usage(usage));
+ }
+}
+
// Normalizes OpenAI Responses token usage fields.
fn responses_usage(usage: &serde_json::Map<String, Value>) -> Usage {
let aggregate_input_tokens = usage.get("input_tokens").and_then(Value::as_u64);
diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rs
index 3b23417cc..b221749cd 100644
--- a/crates/switchyard-translation/src/lib.rs
+++ b/crates/switchyard-translation/src/lib.rs
@@ -12,6 +12,7 @@ pub mod engine;
pub mod error;
mod helpers;
pub mod policy;
+mod request_edit;
mod sse;
pub mod stream;
pub mod util;
@@ -31,6 +32,7 @@ pub use format::*;
pub use helpers::*;
pub use llm::*;
pub use policy::*;
+pub use request_edit::{append_note, prepend_system_prompt};
pub use stream::*;
pub use util::{
PRESERVATION_METADATA_KEY, normalize_anthropic_tool_use_ids, prepare_request_for_target,
diff --git a/crates/switchyard-translation/src/request_edit.rs b/crates/switchyard-translation/src/request_edit.rs
new file mode 100644
index 000000000..9bc65ce25
--- /dev/null
+++ b/crates/switchyard-translation/src/request_edit.rs
@@ -0,0 +1,124 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Request edits update normalized content and preserved provider bodies together.
+
+use serde_json::{Map, Value, json};
+use switchyard_protocol::{ContentBlock, InstructionBlock, LlmRequest, Message, Role, WireFormat};
+
+/// This function appends temporary guidance and updates preserved provider bodies.
+/// It removes bodies it cannot edit so encoding rebuilds them from the updated request.
+pub fn append_note(request: &mut LlmRequest, note: &str) {
+ // A separate message preserves the caller's final message boundary for OpenAI caching.
+ request.messages.push(Message::text(Role::User, note));
+ request.temporary_note_count += 1;
+ request.preservation.requests.retain(|format, body| {
+ let Some(body) = body.as_object_mut() else {
+ return false;
+ };
+ let (key, content_type) = match format.as_str() {
+ "openai_chat" | "anthropic_messages" => ("messages", "text"),
+ "openai_responses" => ("input", "input_text"),
+ _ => return false,
+ };
+ // Responses also accepts a string as a single user input.
+ if key == "input"
+ && let Some(Value::String(text)) = body.get_mut(key)
+ {
+ let original = std::mem::take(text);
+ body.insert(key.into(), json!([{"role":"user", "content":original}]));
+ }
+ if key == "input" && body.get(key).is_none_or(Value::is_null) {
+ body.insert(key.into(), Value::Array(Vec::new()));
+ }
+ let Some(messages) = body.get_mut(key).and_then(Value::as_array_mut) else {
+ return false;
+ };
+ if format.as_str() == WireFormat::AnthropicMessages.as_str()
+ && let Some(last) = messages.last_mut()
+ && last.get("role").and_then(Value::as_str) == Some("user")
+ {
+ // Anthropic guidance follows existing content, including any tool result.
+ match last.get_mut("content") {
+ Some(Value::String(text)) => {
+ let original = std::mem::take(text);
+ last["content"] = json!([
+ {"type":"text", "text":original},
+ {"type":"text", "text":note}
+ ]);
+ }
+ Some(Value::Array(blocks)) => blocks.push(json!({"type":"text", "text":note})),
+ _ => return false,
+ }
+ } else {
+ messages.push(json!({"role":"user", "content":[{"type":content_type,"text":note}]}));
+ }
+ true
+ });
+}
+
+/// This function prepends a system prompt and updates supported preserved provider bodies.
+/// It removes bodies it cannot edit so encoding rebuilds them from the updated request.
+pub fn prepend_system_prompt(request: &mut LlmRequest, prompt: &str) {
+ request.instructions.insert(
+ 0,
+ InstructionBlock {
+ role: Role::System,
+ content: vec![ContentBlock::Text {
+ text: prompt.to_string(),
+ }],
+ },
+ );
+ request.preservation.requests.retain(|format, body| {
+ let Some(body) = body.as_object_mut() else {
+ return false;
+ };
+ match format.as_str() {
+ "openai_chat" => prepend_openai_chat_system(body, prompt),
+ "openai_responses" => prepend_text_field(body, "instructions", prompt),
+ "anthropic_messages" => {
+ body.remove("switchyard_anthropic_request");
+ prepend_anthropic_system(body, prompt)
+ }
+ _ => false,
+ }
+ });
+}
+
+// This helper inserts a system message while keeping the existing Chat messages intact.
+fn prepend_openai_chat_system(body: &mut Map<String, Value>, prompt: &str) -> bool {
+ let message = json!({"role": "system", "content": prompt});
+ match body.get_mut("messages") {
+ None | Some(Value::Null) => {
+ body.insert("messages".to_string(), Value::Array(vec![message]));
+ }
+ Some(Value::Array(messages)) => messages.insert(0, message),
+ Some(_) => return false,
+ }
+ true
+}
+
+// Preserved Responses instructions and Anthropic system prompts can be strings, missing, or null.
+fn prepend_text_field(body: &mut Map<String, Value>, key: &str, prompt: &str) -> bool {
+ match body.get_mut(key) {
+ None | Some(Value::Null) => {
+ body.insert(key.to_string(), Value::String(prompt.to_string()));
+ }
+ Some(Value::String(text)) if text.is_empty() => {
+ *text = prompt.to_string();
+ }
+ Some(Value::String(text)) => text.insert_str(0, &format!("{prompt}\n\n")),
+ Some(_) => return false,
+ }
+ true
+}
+
+// This helper retains existing Anthropic system blocks when it prepends a prompt.
+fn prepend_anthropic_system(body: &mut Map<String, Value>, prompt: &str) -> bool {
+ if let Some(Value::Array(blocks)) = body.get_mut("system") {
+ blocks.insert(0, json!({"type":"text","text":prompt}));
+ true
+ } else {
+ prepend_text_field(body, "system", prompt)
+ }
+}
diff --git a/crates/switchyard-translation/src/util.rs b/crates/switchyard-translation/src/util.rs
index b9006045d..79dadec9f 100644
--- a/crates/switchyard-translation/src/util.rs
+++ b/crates/switchyard-translation/src/util.rs
@@ -12,7 +12,7 @@ use switchyard_protocol::ModelId;
use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
-use crate::llm::{ContentBlock, InstructionBlock, LlmRequest, Message, PreservationMetadata, Role};
+use crate::llm::{ContentBlock, LlmRequest, Message, PreservationMetadata};
use crate::policy::{
LossyConversionPolicy, PreservationPolicy, TranslationPolicy, UnknownFieldPolicy,
};
@@ -338,107 +338,26 @@ pub fn prepare_request_for_target(
let target = target.to_string();
request.model = Some(target.clone());
if let Some(prompt) = prompt {
- request.instructions.insert(
- 0,
- InstructionBlock {
- role: Role::System,
- content: vec![ContentBlock::Text {
- text: prompt.to_string(),
- }],
- },
- );
+ crate::prepend_system_prompt(request, prompt);
}
- prepare_preserved_requests(&mut request.preservation, &target, prompt);
-}
-
-// Retains exact replay only where the built-in wire body can receive the target changes safely.
-fn prepare_preserved_requests(
- preservation: &mut PreservationMetadata,
- target: &str,
- prompt: Option<&str>,
-) {
- preservation.requests.retain(|format, body| {
- let is_builtin = format.as_str() == WireFormat::OpenAiChat.as_str()
- || format.as_str() == WireFormat::OpenAiResponses.as_str()
- || format.as_str() == WireFormat::AnthropicMessages.as_str();
- let Some(body) = is_builtin.then_some(body).and_then(Value::as_object_mut) else {
+ request.preservation.requests.retain(|format, body| {
+ if ![
+ WireFormat::OpenAiChat.as_str(),
+ WireFormat::OpenAiResponses.as_str(),
+ WireFormat::AnthropicMessages.as_str(),
+ ]
+ .contains(&format.as_str())
+ {
return false;
- };
- body.insert("model".to_string(), Value::String(target.to_string()));
- match prompt {
- None => true,
- Some(prompt) if format.as_str() == WireFormat::OpenAiChat.as_str() => {
- prepend_openai_chat_system(body, prompt)
- }
- Some(prompt) if format.as_str() == WireFormat::OpenAiResponses.as_str() => {
- prepend_openai_responses_instructions(body, prompt)
- }
- Some(prompt) if format.as_str() == WireFormat::AnthropicMessages.as_str() => {
- body.remove(crate::codecs::common::ANTHROPIC_REQUEST_KEY);
- prepend_anthropic_system(body, prompt)
- }
- Some(_) => false,
}
+ let Some(body) = body.as_object_mut() else {
+ return false;
+ };
+ body.insert("model".to_string(), Value::String(target.clone()));
+ true
});
}
-// Prepends a target prompt without rebuilding or otherwise changing a Chat request.
-fn prepend_openai_chat_system(body: &mut Map<String, Value>, prompt: &str) -> bool {
- let message = json!({"role": "system", "content": prompt});
- match body.get_mut("messages") {
- None | Some(Value::Null) => {
- body.insert("messages".to_string(), Value::Array(vec![message]));
- }
- Some(Value::Array(messages)) => messages.insert(0, message),
- Some(_) => return false,
- }
- true
-}
-
-// Prepends a target prompt without rebuilding or otherwise changing a Responses request.
-fn prepend_openai_responses_instructions(body: &mut Map<String, Value>, prompt: &str) -> bool {
- match body.get_mut("instructions") {
- None | Some(Value::Null) => {
- body.insert(
- "instructions".to_string(),
- Value::String(prompt.to_string()),
- );
- }
- Some(Value::String(instructions)) if instructions.is_empty() => {
- *instructions = prompt.to_string();
- }
- Some(Value::String(instructions)) => {
- instructions.insert_str(0, &format!("{prompt}\n\n"));
- }
- Some(_) => return false,
- }
- true
-}
-
-// Prepends a target prompt without rebuilding or otherwise changing an Anthropic request.
-fn prepend_anthropic_system(body: &mut Map<String, Value>, prompt: &str) -> bool {
- match body.get_mut("system") {
- None | Some(Value::Null) => {
- body.insert("system".to_string(), Value::String(prompt.to_string()));
- }
- Some(Value::String(system)) if system.is_empty() => {
- *system = prompt.to_string();
- }
- Some(Value::String(system)) => {
- system.insert_str(0, &format!("{prompt}\n\n"));
- }
- Some(Value::Array(blocks)) => blocks.insert(
- 0,
- json!({
- "type": "text",
- "text": prompt,
- }),
- ),
- Some(_) => return false,
- }
- true
-}
-
/// Embeds preservation metadata into a translated wire body when requested.
pub fn embed_preservation(
mut body: Value,
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index 22899b029..9ac6058ae 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -2525,12 +2525,14 @@ fn anthropic_max_tokens_stop_translates_to_responses_incomplete_event() -> TestR
// Verifies a streamed response.incomplete from a Responses upstream reaches a Chat client.
#[test]
fn responses_incomplete_event_translates_to_chat_length_finish() -> TestResult {
+ // A token-limited response must retain final accounting as well as the length stop reason.
let engine = TranslationEngine::default();
let mut state =
StreamTranslationState::new(WireFormat::OpenAiResponses, WireFormat::OpenAiChat);
let incomplete = json!({
"type": "response.incomplete",
- "response": {"status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}}
+ "response": {"status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"},
+ "usage":{"input_tokens":12000,"input_tokens_details":{"cached_tokens":4000,"cache_write_tokens":7000},"output_tokens":16,"total_tokens":12016,"cost":0.01515}}
});
let mut events = engine.translate_event(
@@ -2541,10 +2543,21 @@ fn responses_incomplete_event_translates_to_chat_length_finish() -> TestResult {
)?;
events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiChat)?);
- let Some(terminal) = events.last() else {
+ let Some(terminal) = events
+ .iter()
+ .find(|event| event["choices"][0]["finish_reason"] == "length")
+ else {
return Err("finish should emit a terminal Chat chunk".into());
};
assert_eq!(terminal["choices"][0]["finish_reason"], "length");
+ let usage = events
+ .iter()
+ .find_map(|event| event.get("usage"))
+ .ok_or("missing terminal usage")?;
+ assert_eq!(usage["prompt_tokens"], 12000);
+ assert_eq!(usage["prompt_tokens_details"]["cached_tokens"], 4000);
+ assert_eq!(usage["prompt_tokens_details"]["cache_write_tokens"], 7000);
+ assert_eq!(usage["cost"], 0.01515);
Ok(())
}