Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): preserve empty Anthropic tool inputs in streams (#784)

nachiketb-nvidia1ff053aceb95
diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
index b3a0b5f7e..9117cb534 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
@@ -100,8 +100,21 @@ fn decode_anthropic_stream(
model: state.model.clone(),
}]
}
- Some("content_block_start") => decode_anthropic_content_block_start(object),
- Some("content_block_delta") => decode_anthropic_content_block_delta(object),
+ Some("content_block_start") => decode_anthropic_content_block_start(state, object),
+ Some("content_block_delta") => decode_anthropic_content_block_delta(state, object),
+ Some("content_block_stop") => {
+ let index = object.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
+ if state.empty_anthropic_tool_inputs.remove(&index) {
+ vec![LlmResponseChunk::ToolCallDelta {
+ index,
+ id: None,
+ name: None,
+ arguments_delta: Some("{}".to_string()),
+ }]
+ } else {
+ Vec::new()
+ }
+ }
Some("message_delta") => {
let mut out = Vec::new();
if let Some(usage) = object.get("usage") {
@@ -319,7 +332,10 @@ fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec<Value> {
}
// Converts Anthropic content-block starts into text or tool-call deltas.
-fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmResponseChunk> {
+fn decode_anthropic_content_block_start(
+ state: &mut StreamTranslationState,
+ object: &Map<String, Value>,
+) -> Vec<LlmResponseChunk> {
let index = object.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
let block = object.get("content_block").and_then(Value::as_object);
match block
@@ -362,6 +378,13 @@ fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmR
let Some(block) = block else {
return Vec::new();
};
+ if block
+ .get("input")
+ .and_then(Value::as_object)
+ .is_some_and(|input| input.is_empty())
+ {
+ state.empty_anthropic_tool_inputs.insert(index);
+ }
vec![LlmResponseChunk::ToolCallDelta {
index,
id: block
@@ -380,7 +403,10 @@ fn decode_anthropic_content_block_start(object: &Map<String, Value>) -> Vec<LlmR
}
// Converts Anthropic content-block deltas into neutral text or argument deltas.
-fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmResponseChunk> {
+fn decode_anthropic_content_block_delta(
+ state: &mut StreamTranslationState,
+ object: &Map<String, Value>,
+) -> Vec<LlmResponseChunk> {
let index = object.get("index").and_then(Value::as_u64).unwrap_or(0) as usize;
let Some(delta) = object.get("delta").and_then(Value::as_object) else {
return Vec::new();
@@ -415,6 +441,9 @@ fn decode_anthropic_content_block_delta(object: &Map<String, Value>) -> Vec<LlmR
.get("partial_json")
.and_then(Value::as_str)
.map(|partial_json| {
+ if !partial_json.is_empty() {
+ state.empty_anthropic_tool_inputs.remove(&index);
+ }
vec![LlmResponseChunk::ToolCallDelta {
index,
id: None,
diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs
index 3732ff2b6..a2333f610 100644
--- a/crates/switchyard-translation/src/codecs/stream.rs
+++ b/crates/switchyard-translation/src/codecs/stream.rs
@@ -54,6 +54,9 @@ pub struct StreamTranslationState {
pub(crate) tool_states: BTreeMap<usize, StreamToolState>,
#[serde(default)]
pub(crate) deferred_anthropic_tools: Vec<usize>,
+ /// Empty initial inputs remain placeholders until their blocks close without arguments.
+ #[serde(default)]
+ pub(crate) empty_anthropic_tool_inputs: std::collections::BTreeSet<usize>,
/// Reasoning text observed while DECODING, per output index, so a completed item
/// that repeats already-streamed text is not decoded twice.
pub(crate) decoded_reasoning: BTreeMap<usize, String>,
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index 7893ff22b..ccb3fca51 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -1617,6 +1617,167 @@ fn chat_tool_call_index_counts_tool_calls_not_content_blocks() -> TestResult {
Ok(())
}
+#[test]
+fn anthropic_empty_tool_input_survives_stream_translation() -> TestResult {
+ let engine = TranslationEngine::default();
+ let source = WireFormat::AnthropicMessages;
+ for empty_fragments in [vec![""], vec![], vec!["{}"]] {
+ let mut upstream = vec![
+ json!({"type": "message_start", "message": {"id": "msg_tools", "model": "claude"}}),
+ json!({"type": "content_block_start", "index": 1, "content_block": {
+ "type": "tool_use", "id": "call_empty", "name": "clock", "input": {}}}),
+ json!({"type": "content_block_start", "index": 2, "content_block": {
+ "type": "tool_use", "id": "call_lookup", "name": "lookup", "input": {}}}),
+ ];
+ for fragment in empty_fragments {
+ upstream.push(json!({"type": "content_block_delta", "index": 1,
+ "delta": {"type": "input_json_delta", "partial_json": fragment}}));
+ }
+ upstream.extend([
+ json!({"type": "content_block_delta", "index": 2,
+ "delta": {"type": "input_json_delta", "partial_json": ""}}),
+ json!({"type": "content_block_delta", "index": 2,
+ "delta": {"type": "input_json_delta", "partial_json": "{\"q\":"}}),
+ json!({"type": "content_block_stop", "index": 1}),
+ json!({"type": "content_block_delta", "index": 2,
+ "delta": {"type": "input_json_delta", "partial_json": "\"rust\"}"}}),
+ json!({"type": "content_block_stop", "index": 2}),
+ json!({"type": "message_delta", "delta": {"stop_reason": "tool_use"}}),
+ json!({"type": "message_stop"}),
+ ]);
+ let mut observed = Vec::new();
+ for separate_states in [false, true] {
+ for target in [WireFormat::OpenAiChat, WireFormat::OpenAiResponses, source] {
+ let mut decoder = StreamTranslationState::new(source, target);
+ let mut encoder = StreamTranslationState::new(source, target);
+ let mut events = Vec::new();
+ for event in &upstream {
+ let decoded =
+ engine.decode_stream_event(&mut decoder, source, event.clone())?;
+ if event["delta"]["partial_json"] == "{\"q\":" {
+ assert!(
+ decoded.normalized().iter().any(|chunk| matches!(chunk,
+ LlmResponseChunk::ToolCallDelta { arguments_delta: Some(delta), .. }
+ if delta == "{\"q\":")),
+ "nonempty input must stream immediately"
+ );
+ }
+ let state = if separate_states {
+ &mut encoder
+ } else {
+ &mut decoder
+ };
+ events.extend(engine.encode_stream_event(state, target, decoded)?);
+ }
+ let state = if separate_states {
+ &mut encoder
+ } else {
+ &mut decoder
+ };
+ events.extend(engine.finish_stream(state, target)?);
+ assert!(engine.finish_stream(state, target)?.is_empty());
+ if target == source {
+ assert_eq!(events, upstream);
+ continue;
+ }
+ let mut arguments = [String::new(), String::new()];
+ for event in &events {
+ if target == WireFormat::OpenAiChat {
+ if let Some(calls) = event["choices"][0]["delta"]["tool_calls"].as_array() {
+ for call in calls {
+ let index =
+ call["index"].as_u64().ok_or("missing tool index")? as usize;
+ if let Some(delta) = call["function"]["arguments"].as_str() {
+ arguments[index].push_str(delta);
+ }
+ }
+ }
+ } else if event["type"] == "response.function_call_arguments.delta" {
+ let index = event["output_index"]
+ .as_u64()
+ .ok_or("missing output index")?
+ as usize;
+ arguments[index]
+ .push_str(event["delta"].as_str().ok_or("missing argument delta")?);
+ }
+ }
+ if target == WireFormat::OpenAiResponses {
+ let completed = events.last().ok_or("missing completion")?;
+ assert_eq!(completed["type"], "response.completed");
+ for (index, argument) in arguments.iter().enumerate() {
+ assert_eq!(
+ completed["response"]["output"][index]["arguments"],
+ *argument
+ );
+ let done = events
+ .iter()
+ .find(|event| {
+ event["type"] == "response.output_item.done"
+ && event["output_index"] == index
+ })
+ .ok_or("missing completed tool")?;
+ assert_eq!(done["item"]["arguments"], *argument);
+ let done = events
+ .iter()
+ .find(|event| {
+ event["type"] == "response.function_call_arguments.done"
+ && event["output_index"] == index
+ })
+ .ok_or("missing completed arguments")?;
+ assert_eq!(done["arguments"], *argument);
+ }
+ }
+ observed.push(arguments);
+ }
+ }
+ assert_eq!(
+ observed,
+ vec![["{}".to_string(), r#"{"q":"rust"}"#.to_string()]; 4]
+ );
+ for arguments in observed {
+ assert_eq!(serde_json::from_str::<Value>(&arguments[0])?, json!({}));
+ assert_eq!(
+ serde_json::from_str::<Value>(&arguments[1])?,
+ json!({"q": "rust"})
+ );
+ }
+ }
+ // Initial values and partial inputs must not be replaced or completed with {}.
+ for (input, fragment, closes, expected) in [
+ (json!({"q": "rust"}), "", true, r#"{"q":"rust"}"#),
+ (json!({}), "{", true, "{"),
+ (json!({}), "", false, ""),
+ ] {
+ let mut state = StreamTranslationState::new(source, WireFormat::OpenAiChat);
+ let mut events = vec![
+ json!({"type": "content_block_start", "index": 0, "content_block": {
+ "type": "tool_use", "id": "call_control", "name": "lookup", "input": input}}),
+ json!({"type": "content_block_delta", "index": 0,
+ "delta": {"type": "input_json_delta", "partial_json": fragment}}),
+ ];
+ if closes {
+ events.push(json!({"type": "content_block_stop", "index": 0}));
+ }
+ events.push(json!({"type": "message_delta", "delta": {"stop_reason": "max_tokens"}}));
+ events.push(json!({"type": "message_stop"}));
+ let mut arguments = String::new();
+ for event in events {
+ for chunk in decode_stream_event(&mut state, source, &event) {
+ if let LlmResponseChunk::ToolCallDelta {
+ arguments_delta: Some(delta),
+ ..
+ } = chunk
+ {
+ arguments.push_str(&delta);
+ }
+ }
+ }
+ assert_eq!(arguments, expected);
+ }
+
+ Ok(())
+}
+
// An OpenAI-shaped error frame carries no `choices`, so it must decode to a stream error
// instead of a bare message start that silently drops the upstream message.
#[test]