Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): capture Anthropic Messages content in full mode (#17854)

Hassieb Pakzadce3ef136762b
diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.toml
index e6f95b2ea5..409ea68988 100644
--- a/ai-gateway/Cargo.toml
+++ b/ai-gateway/Cargo.toml
@@ -32,7 +32,7 @@ http-body = "1"
opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic-messages", "trace", "metrics"] }
opentelemetry_sdk = { version = "0.32", default-features = false, features = ["testing"] }
prost = "0.14"
-tokio = { version = "1.53.1", features = ["io-util"] }
+tokio = { version = "1.53.1", features = ["io-util", "test-util"] }
tower = { version = "0.5", features = ["util"] }
[lints.clippy]
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
index 97fcff2102..ae03501ac5 100644
--- a/ai-gateway/README.md
+++ b/ai-gateway/README.md
@@ -284,7 +284,7 @@ guardrails, not measured capacity: tune them independently using load tests for
instance resources, request sizes and stream durations. These limits bound work;
they do not guarantee fairness between clients or tenants.
-Other limits are 10 MiB request bodies in both namespaces, 10 seconds to read a request, 5 seconds to
+Other limits are 10 MiB request bodies in both namespaces, 30 seconds to read a request, 5 seconds to
connect, 120 seconds for provider response headers or an individual upstream read,
and 600 seconds overall from execution admission in both namespaces. Response size is not capped: a single task pumps chunks
through a one-slot channel, with chunks at most 64 KiB. It stops reading when that
@@ -534,10 +534,24 @@ non-empty `text`, `thinking` or `partial_json`; signatures, citations and pings
count. A mid-stream `error` event, or an HTTP error body, marks the generation failed;
its `type` and `message` are retained only in full mode. Unknown event types are ignored.
A `stop_reason` of `max_tokens` or `model_context_window_exceeded` sets a `WARNING`
-level. Only scalar model parameters (`max_tokens`, `temperature`, `top_p`, `top_k`,
-`stream`, `service_tier`, `speed`) are recorded; `speed` selects the fast-mode pricing
-tier. Request and response content are not captured
-in either mode yet, so full-mode output completeness is false and `input`/`output` are null.
+level. Scalar model parameters (`max_tokens`, `temperature`, `top_p`, `top_k`,
+`stream`, `service_tier`, `speed`) are recorded in both modes; `speed` selects the
+fast-mode pricing tier.
+
+In full mode the input is the native request with `model` and parameters projected
+out: `stop_sequences`, `thinking`, `tool_choice`, `context_management` and
+`output_config` become model parameters and `metadata` (where Claude Code stores its
+session identifiers) becomes `request.metadata`; `system`, `messages`, `tools` and
+unknown fields are kept as sent. The output is the native assistant message
+`{"role":"assistant","content":[...],"stop_reason":...}`. JSON responses contribute
+their `content` array at EOF. Streams rebuild each block from `content_block_start`,
+its deltas and `content_block_stop`: text, thinking and citations are appended, the
+opaque `signature` is kept, and `input_json_delta` fragments are parsed into the tool
+`input` once the block stops. Unlike OpenAI Responses, Anthropic has no completed-item
+event, so these deltas are stitched within the same retained-output budget. Only
+stopped blocks are recorded; an unfinished block, tool input that is not valid JSON, an
+unknown delta type or a block past the budget is left out and makes the output
+partial. Usage mode records neither input nor output.
For OpenAI Responses SSE, only `response.output_item.done` adds output. Terminal Responses events
provide model, service tier, status and usage; their repeated output is not copied.
diff --git a/ai-gateway/src/capture/anthropic_messages.rs b/ai-gateway/src/capture/anthropic_messages.rs
index e55cf698b3..00d66560d6 100644
--- a/ai-gateway/src/capture/anthropic_messages.rs
+++ b/ai-gateway/src/capture/anthropic_messages.rs
@@ -1,21 +1,209 @@
use super::{
- MAX_CAPTURE_BYTES, MAX_FACT_STRING, bounded_string, facts::ProviderFacts, identity_encoding,
- response::ResponseBody,
+ MAX_CAPTURE_BYTES, MAX_FACT_STRING, MAX_ITEMS, bounded_string, facts::ProviderFacts,
+ identity_encoding, response::ResponseBody,
};
use crate::resolution::IngestionMode;
use axum::http::HeaderMap;
-use serde_json::{Map, Value};
+use serde_json::{Map, Value, json};
+use std::collections::BTreeMap;
+
+const SCALAR_PARAMETERS: [&str; 7] = [
+ "max_tokens",
+ "temperature",
+ "top_p",
+ "top_k",
+ "stream",
+ "service_tier",
+ "speed",
+];
+/// Structured configuration projected out of the recorded input in full mode.
+const STRUCTURED_PARAMETERS: [&str; 5] = [
+ "stop_sequences",
+ "thinking",
+ "tool_choice",
+ "context_management",
+ "output_config",
+];
pub(super) struct AnthropicMessagesCapture {
facts: ProviderFacts,
mode: IngestionMode,
body: ResponseBody,
usage: Option<Map<String, Value>>,
+ /// Present only in full mode; usage mode never records output.
+ output: Option<AssistantMessage>,
terminal: bool,
request_complete: bool,
response_valid: bool,
}
+/// A capture gap: the recorded output no longer matches what was relayed.
+struct Gap;
+
+/// The assistant message rebuilt in full mode. Streamed blocks are assembled
+/// from their start, delta and stop events; only stopped blocks are recorded.
+#[derive(Default)]
+struct AssistantMessage {
+ stop: Map<String, Value>,
+ blocks: BTreeMap<u64, Block>,
+ bytes: usize,
+}
+
+struct Block {
+ fields: Map<String, Value>,
+ partial_json: String,
+ finished: bool,
+}
+
+impl AssistantMessage {
+ fn start(&mut self, index: u64, fields: Map<String, Value>) -> Result<(), Gap> {
+ if self.blocks.contains_key(&index) || self.blocks.len() >= MAX_ITEMS {
+ return Err(Gap);
+ }
+ let size = serde_json::to_string(&fields).map_or(usize::MAX, |json| json.len());
+ reserve(&mut self.bytes, size)?;
+ self.blocks.insert(
+ index,
+ Block {
+ fields,
+ partial_json: String::new(),
+ finished: false,
+ },
+ );
+ Ok(())
+ }
+
+ /// A block that cannot be rebuilt exactly is dropped rather than recorded wrong.
+ fn apply_delta(&mut self, index: u64, delta: &Map<String, Value>) -> Result<(), Gap> {
+ let applied = self.try_apply_delta(index, delta);
+ if applied.is_err() {
+ self.blocks.remove(&index);
+ }
+ applied
+ }
+
+ fn try_apply_delta(&mut self, index: u64, delta: &Map<String, Value>) -> Result<(), Gap> {
+ let block = self
+ .blocks
+ .get_mut(&index)
+ .filter(|block| !block.finished)
+ .ok_or(Gap)?;
+ let kind = delta.get("type").and_then(Value::as_str).ok_or(Gap)?;
+ if kind == "citations_delta" {
+ let citation = delta.get("citation").ok_or(Gap)?;
+ reserve(&mut self.bytes, citation.to_string().len())?;
+ match block.fields.get_mut("citations") {
+ Some(Value::Array(citations)) => citations.push(citation.clone()),
+ _ => {
+ block
+ .fields
+ .insert("citations".to_owned(), Value::Array(vec![citation.clone()]));
+ }
+ }
+ return Ok(());
+ }
+ // Every other delta appends a string fragment to one field of the block.
+ let (field, source) = match kind {
+ "text_delta" => ("text", "text"),
+ "thinking_delta" => ("thinking", "thinking"),
+ "signature_delta" => ("signature", "signature"),
+ "input_json_delta" => ("", "partial_json"),
+ _ => return Err(Gap),
+ };
+ let fragment = delta.get(source).and_then(Value::as_str).ok_or(Gap)?;
+ reserve(&mut self.bytes, fragment.len())?;
+ if field.is_empty() {
+ block.partial_json.push_str(fragment);
+ } else if let Some(Value::String(current)) = block.fields.get_mut(field) {
+ current.push_str(fragment);
+ } else {
+ block
+ .fields
+ .insert(field.to_owned(), Value::String(fragment.to_owned()));
+ }
+ Ok(())
+ }
+
+ fn stop_block(&mut self, index: u64) -> Result<(), Gap> {
+ let block = self
+ .blocks
+ .get_mut(&index)
+ .filter(|block| !block.finished)
+ .ok_or(Gap)?;
+ // Tool inputs stream as JSON fragments that only parse once complete.
+ if !block.partial_json.is_empty() {
+ let Ok(input) = serde_json::from_str::<Value>(&block.partial_json) else {
+ self.blocks.remove(&index);
+ return Err(Gap);
+ };
+ block.fields.insert("input".to_owned(), input);
+ block.partial_json = String::new();
+ }
+ block.finished = true;
+ Ok(())
+ }
+
+ /// A non-streamed response carries its complete content at once.
+ fn record_content(&mut self, content: Option<&Vec<Value>>) -> Result<(), Gap> {
+ let content = content
+ .filter(|content| content.len() <= MAX_ITEMS)
+ .ok_or(Gap)?;
+ for (index, fields) in content.iter().enumerate() {
+ let fields = fields.as_object().ok_or(Gap)?.clone();
+ self.blocks.insert(
+ index as u64,
+ Block {
+ fields,
+ partial_json: String::new(),
+ finished: true,
+ },
+ );
+ }
+ Ok(())
+ }
+
+ /// The renderer reads the finish reason from the recorded output.
+ fn record_stop(&mut self, source: &Map<String, Value>) {
+ for key in ["stop_reason", "stop_sequence"] {
+ if let Some(value) = source.get(key).filter(|value| !value.is_null()) {
+ self.stop.insert(key.to_owned(), value.clone());
+ }
+ }
+ }
+
+ fn is_complete(&self) -> bool {
+ self.blocks.values().all(|block| block.finished)
+ }
+
+ fn into_value(self) -> Value {
+ let content = self
+ .blocks
+ .into_values()
+ .filter(|block| block.finished)
+ .map(|block| Value::Object(block.fields))
+ .collect();
+ let mut message = Map::new();
+ message.insert("role".to_owned(), json!("assistant"));
+ message.insert("content".to_owned(), Value::Array(content));
+ message.extend(self.stop);
+ Value::Object(message)
+ }
+}
+
+fn reserve(used: &mut usize, bytes: usize) -> Result<(), Gap> {
+ match used.checked_add(bytes) {
+ Some(total) if total <= MAX_CAPTURE_BYTES => {
+ *used = total;
+ Ok(())
+ }
+ _ => Err(Gap),
+ }
+}
+
+fn block_index(event: &Map<String, Value>) -> Option<u64> {
+ event.get("index").and_then(Value::as_u64)
+}
+
impl AnthropicMessagesCapture {
pub fn new(headers: &HeaderMap, body: &[u8], mode: IngestionMode) -> Self {
let mut capture = Self {
@@ -23,6 +211,7 @@ impl AnthropicMessagesCapture {
mode,
body: ResponseBody::Unknown,
usage: None,
+ output: (mode == IngestionMode::Full).then(AssistantMessage::default),
terminal: false,
request_complete: false,
response_valid: true,
@@ -31,7 +220,7 @@ impl AnthropicMessagesCapture {
&& body.len() <= MAX_CAPTURE_BYTES
&& let Ok(Value::Object(request)) = serde_json::from_slice(body)
{
- capture.capture_request(&request);
+ capture.capture_request(request);
capture.request_complete = true;
}
capture
@@ -68,6 +257,7 @@ impl AnthropicMessagesCapture {
self.record_failure(&response);
} else {
self.capture_message(&response);
+ self.capture_json_content(&response);
self.terminal = true;
}
}
@@ -78,8 +268,12 @@ impl AnthropicMessagesCapture {
}
ResponseBody::Unknown | ResponseBody::Unavailable => {}
}
- self.facts.output_complete =
- self.terminal && self.response_valid && self.mode != IngestionMode::Full;
+ self.facts.output_complete = self.terminal
+ && self.response_valid
+ && self
+ .output
+ .as_ref()
+ .is_none_or(AssistantMessage::is_complete);
self.facts.capture_complete = self.request_complete && self.facts.output_complete;
}
@@ -88,33 +282,53 @@ impl AnthropicMessagesCapture {
self.end_body();
}
self.facts.usage_details = self.usage.take().map(Value::Object);
+ self.facts.output = self.output.take().map(AssistantMessage::into_value);
self.facts
}
- fn capture_request(&mut self, request: &Map<String, Value>) {
+ fn capture_request(&mut self, mut request: Map<String, Value>) {
self.facts.requested_model = request
- .get("model")
+ .remove("model")
+ .as_ref()
.and_then(Value::as_str)
.and_then(bounded_string);
self.facts.model.clone_from(&self.facts.requested_model);
- for key in [
- "max_tokens",
- "temperature",
- "top_p",
- "top_k",
- "stream",
- "service_tier",
- "speed",
- ] {
- if let Some(value) = request.get(key).filter(|v| {
+ for key in SCALAR_PARAMETERS {
+ if let Some(value) = request.remove(key).filter(|v| {
v.is_number()
|| v.is_boolean()
|| v.as_str().is_some_and(|s| s.len() <= MAX_FACT_STRING)
}) {
+ self.facts.model_parameters.insert(key.to_owned(), value);
+ }
+ }
+ if self.mode == IngestionMode::Full {
+ for key in STRUCTURED_PARAMETERS {
+ if let Some(value) = request.remove(key) {
+ self.facts.model_parameters.insert(key.to_owned(), value);
+ }
+ }
+ // Claude Code stores its session identifiers here; it is request
+ // metadata rather than prompt content.
+ if let Some(value) = request.remove("metadata") {
self.facts
- .model_parameters
- .insert(key.to_owned(), value.clone());
+ .request_metadata
+ .insert("metadata".to_owned(), value);
}
+ // `system`, `messages`, `tools` and unknown fields remain native.
+ self.facts.input = Some(Value::Object(request));
+ self.facts.input_complete = true;
+ }
+ }
+
+ fn capture_json_content(&mut self, message: &Map<String, Value>) {
+ let content = message.get("content").and_then(Value::as_array);
+ if self
+ .output
+ .as_mut()
+ .is_some_and(|output| output.record_content(content).is_err())
+ {
+ self.response_valid = false;
}
}
@@ -131,18 +345,40 @@ impl AnthropicMessagesCapture {
self.response_valid = false;
}
}
+ Some("content_block_start") => {
+ let block = event.get("content_block").and_then(Value::as_object);
+ if self.output.as_mut().is_some_and(|output| {
+ block_index(&event)
+ .zip(block)
+ .is_none_or(|(index, block)| output.start(index, block.clone()).is_err())
+ }) {
+ self.response_valid = false;
+ }
+ }
+ Some("content_block_stop") => {
+ if self.output.as_mut().is_some_and(|output| {
+ block_index(&event).is_none_or(|index| output.stop_block(index).is_err())
+ }) {
+ self.response_valid = false;
+ }
+ }
Some("content_block_delta") => {
- return event
- .get("delta")
- .and_then(Value::as_object)
- .is_some_and(|delta| {
- ["text", "thinking", "partial_json"].iter().any(|key| {
- delta
- .get(*key)
- .and_then(Value::as_str)
- .is_some_and(|content| !content.is_empty())
- })
- });
+ let delta = event.get("delta").and_then(Value::as_object);
+ if self.output.as_mut().is_some_and(|output| {
+ block_index(&event)
+ .zip(delta)
+ .is_none_or(|(index, delta)| output.apply_delta(index, delta).is_err())
+ }) {
+ self.response_valid = false;
+ }
+ return delta.is_some_and(|delta| {
+ ["text", "thinking", "partial_json"].iter().any(|key| {
+ delta
+ .get(*key)
+ .and_then(Value::as_str)
+ .is_some_and(|content| !content.is_empty())
+ })
+ });
}
Some("message_delta") => {
if let Some(delta) = event.get("delta").and_then(Value::as_object) {
@@ -182,6 +418,9 @@ impl AnthropicMessagesCapture {
let Some(stop_reason) = source.get("stop_reason").and_then(Value::as_str) else {
return;
};
+ if let Some(output) = &mut self.output {
+ output.record_stop(source);
+ }
self.facts.provider_status = Some(
match stop_reason {
"max_tokens" | "model_context_window_exceeded" => "incomplete",
diff --git a/ai-gateway/src/capture/anthropic_messages/tests.rs b/ai-gateway/src/capture/anthropic_messages/tests.rs
index a1c174487f..050abcbfd5 100644
--- a/ai-gateway/src/capture/anthropic_messages/tests.rs
+++ b/ai-gateway/src/capture/anthropic_messages/tests.rs
@@ -1,6 +1,6 @@
use super::*;
use crate::{
- capture::{ExecutionCapture, ProtocolCapture},
+ capture::{ExecutionCapture, ProtocolCapture, ProviderFacts},
providers::{ProviderLimits, ProviderTransport, Route},
resolution::ApiFormat,
test_support::{FakeServer, resolved_request_context_for},
@@ -166,9 +166,13 @@ async fn json_responses_capture_the_same_facts_and_error_bodies_mark_failure() {
.map(|usage| Value::Object(usage.clone())),
Some(json!({"input_tokens":10,"output_tokens":32000}))
);
- // Response content is not captured yet, so full mode cannot claim completeness.
- assert!(!facts.facts.output_complete);
- assert!(facts.facts.output.is_none());
+ assert!(facts.facts.output_complete);
+ assert_eq!(
+ finalized(&mut observer).output,
+ Some(
+ json!({"role":"assistant","content":[{"type":"text","text":"Hello"}],"stop_reason":"max_tokens"})
+ )
+ );
for (mode, expected_message) in [
(
@@ -236,28 +240,246 @@ async fn in_stream_errors_unknown_events_and_truncation_are_tolerated() {
);
}
+fn finalized(observer: &mut ExecutionCapture) -> ProviderFacts {
+ observer.protocol.take().unwrap().into_facts().1
+}
+
+fn block_events(index: u64, start: &Value, deltas: &[Value]) -> String {
+ let mut events = vec![event(
+ "content_block_start",
+ json!({"index":index,"content_block":start}),
+ )];
+ for delta in deltas {
+ events.push(event(
+ "content_block_delta",
+ json!({"index":index,"delta":delta}),
+ ));
+ }
+ events.push(event("content_block_stop", json!({"index":index})));
+ events.concat()
+}
+
+fn stop_events() -> String {
+ [
+ event(
+ "message_delta",
+ json!({"delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":20}}),
+ ),
+ event("message_stop", json!({})),
+ ]
+ .concat()
+}
+
+#[tokio::test]
+async fn request_configuration_and_metadata_are_projected_out_of_input_by_mode() {
+ let observer = new_observer("full").await;
+ let facts = &captured(&observer).facts;
+ assert_eq!(
+ facts.model_parameters,
+ json!({"max_tokens":32000,"temperature":1,"stream":true,"speed":"fast","thinking":{"type":"adaptive"},"tool_choice":{"type":"auto"}})
+ .as_object()
+ .cloned()
+ .unwrap()
+ );
+ assert_eq!(
+ facts.request_metadata,
+ json!({"metadata":{"user_id":"{\"session_id\":\"session-canary\"}"}})
+ .as_object()
+ .cloned()
+ .unwrap()
+ );
+ let input = facts.input.as_ref().unwrap();
+ assert_eq!(input["system"][0]["text"], json!("system-canary"));
+ assert_eq!(
+ input["system"][0]["cache_control"]["type"],
+ json!("ephemeral")
+ );
+ assert_eq!(input["messages"][0]["content"], json!("prompt-canary"));
+ assert_eq!(input["tools"][0]["name"], json!("Bash"));
+ assert_eq!(input["future_field"], json!({"kept":true}));
+ for projected in [
+ "model",
+ "max_tokens",
+ "speed",
+ "thinking",
+ "tool_choice",
+ "metadata",
+ ] {
+ assert!(input.get(projected).is_none(), "{projected} left in input");
+ }
+ assert!(facts.input_complete);
+
+ let mut observer = new_observer("usage").await;
+ record_response(&mut observer, 200, "text/event-stream");
+ observer.push_bytes(text_stream().as_bytes());
+ let facts = finalized(&mut observer);
+ assert_eq!(
+ facts.model_parameters,
+ json!({"max_tokens":32000,"temperature":1,"stream":true,"speed":"fast"})
+ .as_object()
+ .cloned()
+ .unwrap()
+ );
+ assert!(facts.request_metadata.is_empty());
+ assert!(facts.input.is_none());
+ assert!(facts.output.is_none());
+ let serialized = serde_json::to_string(&facts).unwrap();
+ let canaries = ["system-canary", "prompt-canary", "session-canary", "Hel"]; // codespell:ignore hel
+ for canary in canaries {
+ assert!(!serialized.contains(canary), "usage mode captured {canary}");
+ }
+}
+
+#[tokio::test]
+async fn full_mode_stitches_text_thinking_tool_and_citation_blocks_in_order() {
+ let mut observer = new_observer("full").await;
+ record_response(&mut observer, 200, "text/event-stream");
+ let stream = [
+ message_start(),
+ block_events(
+ 0,
+ &json!({"type":"thinking","thinking":"","signature":""}),
+ &[
+ json!({"type":"thinking_delta","thinking":"Let me "}),
+ json!({"type":"thinking_delta","thinking":"check."}),
+ json!({"type":"signature_delta","signature":"sig-"}),
+ json!({"type":"signature_delta","signature":"opaque"}),
+ ],
+ ),
+ event("ping", json!({})),
+ block_events(
+ 1,
+ &json!({"type":"text","text":""}),
+ &[
+ json!({"type":"text_delta","text":"See "}),
+ json!({"type":"citations_delta","citation":{"type":"char_location","cited_text":"x"}}),
+ json!({"type":"text_delta","text":"docs."}),
+ ],
+ ),
+ block_events(
+ 2,
+ &json!({"type":"tool_use","id":"toolu_1","name":"Bash","input":{}}),
+ &[
+ json!({"type":"input_json_delta","partial_json":"{\"command\":"}),
+ json!({"type":"input_json_delta","partial_json":"\"ls -la\"}"}),
+ ],
+ ),
+ block_events(3, &json!({"type":"redacted_thinking","data":"opaque"}), &[]),
+ stop_events(),
+ ]
+ .concat();
+ // Byte-at-a-time delivery must stitch identically to whole events.
+ for chunk in stream.as_bytes().chunks(7) {
+ observer.push_bytes(chunk);
+ }
+ observer.end_body();
+ assert!(captured(&observer).facts.output_complete);
+ assert_eq!(
+ finalized(&mut observer).output,
+ Some(json!({
+ "role": "assistant",
+ "content": [
+ {"type":"thinking","thinking":"Let me check.","signature":"sig-opaque"},
+ {"type":"text","text":"See docs.","citations":[{"type":"char_location","cited_text":"x"}]},
+ {"type":"tool_use","id":"toolu_1","name":"Bash","input":{"command":"ls -la"}},
+ {"type":"redacted_thinking","data":"opaque"}
+ ],
+ "stop_reason": "tool_use"
+ }))
+ );
+}
+
#[tokio::test]
-async fn only_scalar_parameters_are_recorded_and_content_is_never_captured() {
- for mode in ["usage", "full"] {
- let observer = new_observer(mode).await;
- let facts = &captured(&observer).facts;
- assert_eq!(
- facts.model_parameters,
- json!({"max_tokens":32000,"temperature":1,"stream":true,"speed":"fast"})
- .as_object()
- .cloned()
- .unwrap()
+async fn unfinished_malformed_or_unknown_blocks_are_not_recorded_and_mark_output_partial() {
+ let text = |index: u64, value: &str| {
+ block_events(
+ index,
+ &json!({"type":"text","text":""}),
+ &[json!({"type":"text_delta","text":value})],
+ )
+ };
+ let cases = [
+ // A disconnect midway through a block keeps completed blocks only.
+ [
+ message_start(),
+ text(0, "kept"),
+ event(
+ "content_block_start",
+ json!({"index":1,"content_block":{"type":"text","text":""}}),
+ ),
+ event(
+ "content_block_delta",
+ json!({"index":1,"delta":{"type":"text_delta","text":"lost"}}),
+ ),
+ ]
+ .concat(),
+ // Tool input that never forms valid JSON.
+ [
+ message_start(),
+ text(0, "kept"),
+ block_events(
+ 1,
+ &json!({"type":"tool_use","id":"t","name":"Bash","input":{}}),
+ &[json!({"type":"input_json_delta","partial_json":"{\"cut"})],
+ ),
+ stop_events(),
+ ]
+ .concat(),
+ // A delta type the adapter cannot reconstruct.
+ [
+ message_start(),
+ text(0, "kept"),
+ block_events(
+ 1,
+ &json!({"type":"text","text":""}),
+ &[json!({"type":"future_delta","value":"?"})],
+ ),
+ stop_events(),
+ ]
+ .concat(),
+ ];
+ for stream in cases {
+ let mut observer = new_observer("full").await;
+ record_response(&mut observer, 200, "text/event-stream");
+ observer.push_bytes(stream.as_bytes());
+ observer.end_body();
+ assert!(!captured(&observer).facts.output_complete);
+ let output = finalized(&mut observer).output.unwrap();
+ assert_eq!(output["content"], json!([{"type":"text","text":"kept"}]));
+ }
+}
+
+#[tokio::test]
+async fn retained_output_is_bounded_without_affecting_the_relay() {
+ let mut observer = new_observer("full").await;
+ record_response(&mut observer, 200, "text/event-stream");
+ observer.push_bytes(message_start().as_bytes());
+ observer.push_bytes(
+ event(
+ "content_block_start",
+ json!({"index":0,"content_block":{"type":"text","text":""}}),
+ )
+ .as_bytes(),
+ );
+ // Each delta fits the per-event inspection limit; together they exceed the
+ // retained-output budget.
+ let chunk = "x".repeat(200 * 1024);
+ for _ in 0..6 {
+ observer.push_bytes(
+ event(
+ "content_block_delta",
+ json!({"index":0,"delta":{"type":"text_delta","text":chunk}}),
+ )
+ .as_bytes(),
);
- assert!(facts.request_metadata.is_empty());
- assert!(facts.input.is_none());
- let serialized = serde_json::to_string(facts).unwrap();
- for canary in ["system-canary", "prompt-canary", "session-canary"] {
- assert!(
- !serialized.contains(canary),
- "{mode} mode captured {canary}"
- );
- }
}
+ observer.push_bytes(event("content_block_stop", json!({"index":0})).as_bytes());
+ observer.push_bytes(stop_events().as_bytes());
+ observer.end_body();
+ assert!(!captured(&observer).facts.output_complete);
+ let facts = finalized(&mut observer);
+ assert_eq!(facts.output.unwrap()["content"], json!([]));
+ assert_eq!(facts.usage_details.unwrap()["output_tokens"], json!(20));
}
#[tokio::test]
diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rs
index 0d21f1c52f..10e05f4c5f 100644
--- a/ai-gateway/src/http.rs
+++ b/ai-gateway/src/http.rs
@@ -19,7 +19,7 @@ use crate::{
};
const MAX_REQUEST_BYTES: usize = 10 * 1024 * 1024;
-const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(10);
+const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(30);
const MAX_GATEWAY_KEY_BYTES: usize = 8192;
#[derive(Clone)]
diff --git a/ai-gateway/src/http/tests.rs b/ai-gateway/src/http/tests.rs
index 3128f1e7be..baa8116980 100644
--- a/ai-gateway/src/http/tests.rs
+++ b/ai-gateway/src/http/tests.rs
@@ -464,11 +464,8 @@ async fn resolution_failures_are_sanitized_and_never_call_provider() {
}
#[tokio::test]
-async fn slow_request_body_and_resolution_have_bounded_waits() {
- let web = FakeServer::start(|request| async move {
- if request.headers()[header::AUTHORIZATION] != "Bearer slow-resolution" {
- return resolution_response("provider");
- }
+async fn slow_resolution_has_a_bounded_wait() {
+ let web = FakeServer::start(|_| async {
Response::new(Body::from_stream(stream::pending::<
Result<&'static str, Infallible>,
>()))
@@ -476,29 +473,31 @@ async fn slow_request_body_and_resolution_have_bounded_waits() {
.await;
let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
let gateway = Gateway::start(Some(inference(&web, &provider))).await;
- let slow_body = gateway
- .post()
- .bearer_auth("key")
- .body(reqwest::Body::wrap_stream(stream::pending::<
- Result<&'static str, Infallible>,
- >()))
- .send();
- let slow_resolution = gateway
- .post()
- .bearer_auth("slow-resolution")
- .body("{}")
- .send();
- let (body, resolution) = tokio::time::timeout(Duration::from_secs(12), async {
- tokio::join!(slow_body, slow_resolution)
- })
+ let resolution = tokio::time::timeout(
+ Duration::from_secs(12),
+ gateway
+ .post()
+ .bearer_auth("slow-resolution")
+ .body("{}")
+ .send(),
+ )
.await
.unwrap();
- assert_eq!(body.unwrap().status(), StatusCode::REQUEST_TIMEOUT);
assert_eq!(resolution.unwrap().status(), StatusCode::GATEWAY_TIMEOUT);
- assert_eq!(web.calls(), 2);
+ assert_eq!(web.calls(), 1);
assert_eq!(provider.calls(), 0);
}
+#[tokio::test(start_paused = true)]
+async fn a_stalled_request_body_times_out_at_the_read_deadline() {
+ let stalled = Body::from_stream(stream::pending::<Result<&'static str, Infallible>>());
+ let started = tokio::time::Instant::now();
+ let result = read_request_body(stalled).await;
+ assert!(matches!(result, Err(InferenceHttpError::RequestTimeout)));
+ assert_eq!(started.elapsed(), REQUEST_READ_TIMEOUT);
+ assert_eq!(REQUEST_READ_TIMEOUT, Duration::from_secs(30));
+}
+
#[tokio::test]
async fn pre_header_provider_failures_export_the_status_returned_to_the_caller() {
use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};