Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): preserve native Anthropic request controls (#727)

Graham King8f9c67da95ff
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index d36f732ee..f9580a537 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -1067,6 +1067,10 @@ fn count_cache_control_blocks(value: &Value) -> usize {
// Marks the final message content block as the Anthropic prompt-cache breakpoint.
fn enable_anthropic_prompt_caching(body: &mut Value) {
+ // Top-level cache control manages the final breakpoint and its TTL.
+ if body.get("cache_control").is_some() {
+ return;
+ }
// Abstain once the caller has spent the budget itself. Adding a fifth marker
// turns a request that was valid on arrival into an upstream HTTP 400, and a
// caller that placed four breakpoints deliberately needs them more than we
@@ -1358,6 +1362,14 @@ mod tests {
#[test]
fn anthropic_prompt_caching_marks_final_message() {
+ let caller_managed = json!({
+ "cache_control": {"type": "ephemeral", "ttl": "1h"},
+ "messages": [{"role": "user", "content": "hello"}]
+ });
+ let mut body = caller_managed.clone();
+ enable_anthropic_prompt_caching(&mut body);
+ assert_eq!(body, caller_managed);
+
let mut body = json!({
"messages": [{"role": "user", "content": "hello"}]
});
diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
index d479c2dfb..5fe85769d 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
@@ -5,7 +5,9 @@
use serde_json::{Map, Value, json};
-use crate::codecs::common::{is_known_role_name, provider_extensions, text_from_blocks};
+use crate::codecs::common::{
+ ANTHROPIC_REQUEST_KEY, is_known_role_name, provider_extensions, text_from_blocks,
+};
use crate::codecs::openai_chat::{decode_file_source, decode_image_source};
use crate::codecs::{
DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec,
@@ -157,6 +159,10 @@ impl FormatCodec for AnthropicMessagesCodec {
"stream",
],
);
+ request
+ .extensions
+ .fields
+ .insert(ANTHROPIC_REQUEST_KEY.to_string(), Value::Bool(true));
Ok(DecodedRequest {
request,
@@ -215,10 +221,26 @@ impl FormatCodec for AnthropicMessagesCodec {
encode_anthropic_tool_choice(choice),
);
}
+ if request.extensions.fields.get(ANTHROPIC_REQUEST_KEY) == Some(&Value::Bool(true)) {
+ for field in [
+ "inference_geo",
+ "service_tier",
+ "stop_sequences",
+ "metadata",
+ "cache_control",
+ "container",
+ "speed",
+ "diagnostics",
+ ] {
+ if let Some(value) = request.extensions.fields.get(field) {
+ body.insert(field.to_string(), value.clone());
+ }
+ }
+ }
if let Some(stop_sequences) =
anthropic_stop_sequences_from_extensions(&request.extensions.fields)
{
- body.insert("stop_sequences".to_string(), stop_sequences);
+ body.entry("stop_sequences").or_insert(stop_sequences);
}
if let Some(max_tokens) = request.output.max_output_tokens {
body.insert("max_tokens".to_string(), json!(max_tokens));
diff --git a/crates/switchyard-translation/src/codecs/common.rs b/crates/switchyard-translation/src/codecs/common.rs
index 1f3a31c8c..bfcce5f41 100644
--- a/crates/switchyard-translation/src/codecs/common.rs
+++ b/crates/switchyard-translation/src/codecs/common.rs
@@ -7,6 +7,9 @@ use serde_json::{Map, Value};
use crate::llm::ContentBlock;
+// Internal provenance survives mutations that invalidate exact request replay.
+pub(crate) const ANTHROPIC_REQUEST_KEY: &str = "switchyard_anthropic_request";
+
/// Returns whether a role name is recognized by a supported provider API.
pub(crate) fn is_known_role_name(name: &str) -> bool {
matches!(
@@ -144,7 +147,7 @@ pub(crate) fn provider_extensions(
) -> Map<String, Value> {
let mut extensions = Map::new();
for (key, value) in object {
- if !known.contains(&key.as_str()) {
+ if key != ANTHROPIC_REQUEST_KEY && !known.contains(&key.as_str()) {
extensions.insert(key.clone(), value.clone());
}
}
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index f03a1622b..21f9e4ffb 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -49,6 +49,75 @@ fn preparing_a_target_prompt_invalidates_exact_replay() -> TestResult {
Ok(())
}
+#[test]
+fn anthropic_target_prompt_preserves_native_request_fields() -> TestResult {
+ let engine = TranslationEngine::default();
+ let policy = TranslationPolicy::default();
+ let fields = json!({
+ "inference_geo": "us",
+ "service_tier": "standard_only",
+ "stop_sequences": ["END"],
+ "metadata": {"user_id": "opaque-user"},
+ "cache_control": {"type": "ephemeral", "ttl": "1h"},
+ "container": {"id": "container_1"},
+ "speed": "fast",
+ "diagnostics": {"previous_message_id": "msg_previous"}
+ });
+ for source in [
+ WireFormat::AnthropicMessages,
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiResponses,
+ ] {
+ for has_fields in [true, false] {
+ let mut body = match source {
+ WireFormat::AnthropicMessages => json!({
+ "model": "route", "max_tokens": 32, "system": "caller prompt",
+ "messages": [{"role": "user", "content": "hi"}]
+ }),
+ WireFormat::OpenAiChat => json!({
+ "model": "route",
+ "messages": [{"role": "user", "content": "hi"}]
+ }),
+ _ => json!({"model": "route", "input": "hi"}),
+ };
+ if has_fields {
+ for (key, value) in fields.as_object().ok_or("expected fields object")? {
+ body[key] = value.clone();
+ }
+ }
+ // Caller-supplied extensions must not spoof Anthropic provenance.
+ body["switchyard_anthropic_request"] = json!(true);
+ body["stop"] = json!(["FALLBACK"]);
+ let mut request = engine.decode_request(source, &body, &policy)?.request;
+ prepare_request_for_target(&mut request, &"target/model".into(), Some("target prompt"));
+ let output = engine
+ .encode_request(WireFormat::AnthropicMessages, &request, &policy)?
+ .body;
+
+ assert_eq!(output["model"], "target/model");
+ assert_eq!(
+ output["system"],
+ if source == WireFormat::AnthropicMessages {
+ "target prompt\n\ncaller prompt"
+ } else {
+ "target prompt"
+ }
+ );
+ for (key, value) in fields.as_object().ok_or("expected fields object")? {
+ if source == WireFormat::AnthropicMessages && has_fields {
+ assert_eq!(&output[key], value, "{source:?}: {key}");
+ } else if key == "stop_sequences" {
+ assert_eq!(output[key], json!(["FALLBACK"]));
+ } else {
+ assert!(output.get(key).is_none(), "{source:?}: {key}");
+ }
+ }
+ assert!(output.get("switchyard_anthropic_request").is_none());
+ }
+ }
+ Ok(())
+}
+
// Rebuilding for target instructions must preserve the Responses conversation link.
#[test]
fn responses_previous_response_id_survives_target_prompt() -> TestResult {