Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): preserve Anthropic target prompt requests (#732)

nachiketb-nvidia8156af1d2f1c
diff --git a/crates/switchyard-translation/src/util.rs b/crates/switchyard-translation/src/util.rs
index eaa744363..2469157c6 100644
--- a/crates/switchyard-translation/src/util.rs
+++ b/crates/switchyard-translation/src/util.rs
@@ -277,8 +277,8 @@ pub fn exact_preserved_response(
/// Applies a selected target model and optionally prepends its system prompt.
///
-/// Model-only preparation restamps built-in preserved bodies so exact replay uses the target.
-/// Adding a prompt invalidates preserved bodies because they predate that content mutation.
+/// Preserved built-in bodies are updated in their native format when possible so exact replay
+/// keeps caller fields that are not represented in the normalized request.
/// Call this once per candidate using a request that has not already received a target prompt.
pub fn prepare_request_for_target(
request: &mut LlmRequest,
@@ -297,14 +297,16 @@ pub fn prepare_request_for_target(
}],
},
);
- request.preservation.requests.clear();
- } else {
- stamp_preserved_request_models(&mut request.preservation, &target);
}
+ prepare_preserved_requests(&mut request.preservation, &target, prompt);
}
-// Retains exact replay only where the built-in wire model field can be updated safely.
-fn stamp_preserved_request_models(preservation: &mut PreservationMetadata, target: &str) {
+// Retains exact replay only where the built-in wire body can receive the target changes safely.
+fn prepare_preserved_requests(
+ preservation: &mut PreservationMetadata,
+ target: &str,
+ prompt: Option<&str>,
+) {
preservation.requests.retain(|format, body| {
let is_builtin = format.as_str() == WireFormat::OpenAiChat.as_str()
|| format.as_str() == WireFormat::OpenAiResponses.as_str()
@@ -313,10 +315,41 @@ fn stamp_preserved_request_models(preservation: &mut PreservationMetadata, targe
return false;
};
body.insert("model".to_string(), Value::String(target.to_string()));
- true
+ match prompt {
+ None => true,
+ Some(prompt) if format.as_str() == WireFormat::AnthropicMessages.as_str() => {
+ body.remove(crate::codecs::common::ANTHROPIC_REQUEST_KEY);
+ prepend_anthropic_system(body, prompt)
+ }
+ Some(_) => false,
+ }
});
}
+// Prepends a target prompt without rebuilding or otherwise changing an Anthropic request.
+fn prepend_anthropic_system(body: &mut Map<String, Value>, prompt: &str) -> bool {
+ match body.get_mut("system") {
+ None | Some(Value::Null) => {
+ body.insert("system".to_string(), Value::String(prompt.to_string()));
+ }
+ Some(Value::String(system)) if system.is_empty() => {
+ *system = prompt.to_string();
+ }
+ Some(Value::String(system)) => {
+ system.insert_str(0, &format!("{prompt}\n\n"));
+ }
+ Some(Value::Array(blocks)) => blocks.insert(
+ 0,
+ json!({
+ "type": "text",
+ "text": prompt,
+ }),
+ ),
+ Some(_) => return false,
+ }
+ true
+}
+
/// Embeds preservation metadata into a translated wire body when requested.
pub fn embed_preservation(
mut body: Value,
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index b09e3655c..a6bcccf1b 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -53,6 +53,74 @@ fn preparing_a_target_prompt_invalidates_exact_replay() -> TestResult {
fn anthropic_target_prompt_preserves_native_request_fields() -> TestResult {
let engine = TranslationEngine::default();
let policy = TranslationPolicy::default();
+ for thinking in [
+ Some(json!({"type": "enabled", "budget_tokens": 2048})),
+ Some(json!({"type": "adaptive"})),
+ Some(json!({"type": "disabled"})),
+ None,
+ ] {
+ let mut body = json!({
+ "model": "route",
+ "max_tokens": 32,
+ "system": [{
+ "type": "text",
+ "text": "caller prompt",
+ "cache_control": {"type": "ephemeral"}
+ }],
+ "messages": [{"role": "user", "content": "hi"}],
+ "mcp_servers": [{
+ "type": "url",
+ "name": "inventory",
+ "url": "https://fixture.invalid/mcp",
+ "authorization_token": "fixture-token-not-a-secret",
+ "tool_configuration": {
+ "enabled": true,
+ "allowed_tools": ["lookup_sku"]
+ }
+ }],
+ "container": {
+ "id": "container_fixture",
+ "skills": [{
+ "type": "custom",
+ "skill_id": "skill_fixture_qa",
+ "version": "latest"
+ }]
+ },
+ "output_config": {
+ "effort": "low",
+ "format": {
+ "type": "json_schema",
+ "schema": {
+ "type": "object",
+ "properties": {"marker": {"type": "string"}},
+ "required": ["marker"],
+ "additionalProperties": false
+ }
+ }
+ },
+ "stop_sequences": ["<END>"]
+ });
+ if let Some(thinking) = thinking {
+ body["thinking"] = thinking;
+ }
+ let mut expected = body.clone();
+ expected["model"] = json!("target/model");
+ expected["system"]
+ .as_array_mut()
+ .ok_or("expected system blocks")?
+ .insert(0, json!({"type": "text", "text": "target prompt"}));
+
+ let mut request = engine
+ .decode_request(WireFormat::AnthropicMessages, &body, &policy)?
+ .request;
+ prepare_request_for_target(&mut request, &"target/model".into(), Some("target prompt"));
+ let output = engine
+ .encode_request(WireFormat::AnthropicMessages, &request, &policy)?
+ .body;
+
+ assert_eq!(output, expected);
+ }
+
let fields = json!({
"inference_geo": "us",
"service_tier": "standard_only",
@@ -63,22 +131,15 @@ fn anthropic_target_prompt_preserves_native_request_fields() -> TestResult {
"speed": "fast",
"diagnostics": {"previous_message_id": "msg_previous"}
});
- for source in [
- WireFormat::AnthropicMessages,
- WireFormat::OpenAiChat,
- WireFormat::OpenAiResponses,
- ] {
+ for source in [WireFormat::OpenAiChat, WireFormat::OpenAiResponses] {
for has_fields in [true, false] {
let mut body = match source {
- WireFormat::AnthropicMessages => json!({
- "model": "route", "max_tokens": 32, "system": "caller prompt",
- "messages": [{"role": "user", "content": "hi"}]
- }),
WireFormat::OpenAiChat => json!({
"model": "route",
"messages": [{"role": "user", "content": "hi"}]
}),
- _ => json!({"model": "route", "input": "hi"}),
+ WireFormat::OpenAiResponses => json!({"model": "route", "input": "hi"}),
+ WireFormat::AnthropicMessages => unreachable!(),
};
if has_fields {
for (key, value) in fields.as_object().ok_or("expected fields object")? {
@@ -95,18 +156,9 @@ fn anthropic_target_prompt_preserves_native_request_fields() -> TestResult {
.body;
assert_eq!(output["model"], "target/model");
- assert_eq!(
- output["system"],
- if source == WireFormat::AnthropicMessages {
- "target prompt\n\ncaller prompt"
- } else {
- "target prompt"
- }
- );
- for (key, value) in fields.as_object().ok_or("expected fields object")? {
- if source == WireFormat::AnthropicMessages && has_fields {
- assert_eq!(&output[key], value, "{source:?}: {key}");
- } else if key == "stop_sequences" {
+ assert_eq!(output["system"], "target prompt");
+ for (key, _value) in fields.as_object().ok_or("expected fields object")? {
+ if key == "stop_sequences" {
assert_eq!(output[key], json!(["FALLBACK"]));
} else {
assert!(output.get(key).is_none(), "{source:?}: {key}");