🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-gateway): capture Anthropic Messages content in full mode (#17854)
Hassieb Pakzadce3ef136762b
diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.tomlindex e6f95b2ea5..409ea68988 100644--- a/ai-gateway/Cargo.toml+++ b/ai-gateway/Cargo.toml@@ -32,7 +32,7 @@ http-body = "1"opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic-messages", "trace", "metrics"] }opentelemetry_sdk = { version = "0.32", default-features = false, features = ["testing"] }prost = "0.14"-tokio = { version = "1.53.1", features = ["io-util"] }+tokio = { version = "1.53.1", features = ["io-util", "test-util"] }tower = { version = "0.5", features = ["util"] }[lints.clippy]diff --git a/ai-gateway/README.md b/ai-gateway/README.mdindex 97fcff2102..ae03501ac5 100644--- a/ai-gateway/README.md+++ b/ai-gateway/README.md@@ -284,7 +284,7 @@ guardrails, not measured capacity: tune them independently using load tests forinstance resources, request sizes and stream durations. These limits bound work;they do not guarantee fairness between clients or tenants.-Other limits are 10 MiB request bodies in both namespaces, 10 seconds to read a request, 5 seconds to+Other limits are 10 MiB request bodies in both namespaces, 30 seconds to read a request, 5 seconds toconnect, 120 seconds for provider response headers or an individual upstream read,and 600 seconds overall from execution admission in both namespaces. Response size is not capped: a single task pumps chunksthrough a one-slot channel, with chunks at most 64 KiB. It stops reading when that@@ -534,10 +534,24 @@ non-empty `text`, `thinking` or `partial_json`; signatures, citations and pingscount. A mid-stream `error` event, or an HTTP error body, marks the generation failed;its `type` and `message` are retained only in full mode. Unknown event types are ignored.A `stop_reason` of `max_tokens` or `model_context_window_exceeded` sets a `WARNING`-level. Only scalar model parameters (`max_tokens`, `temperature`, `top_p`, `top_k`,-`stream`, `service_tier`, `speed`) are recorded; `speed` selects the fast-mode pricing-tier. Request and response content are not captured-in either mode yet, so full-mode output completeness is false and `input`/`output` are null.+level. Scalar model parameters (`max_tokens`, `temperature`, `top_p`, `top_k`,+`stream`, `service_tier`, `speed`) are recorded in both modes; `speed` selects the+fast-mode pricing tier.++In full mode the input is the native request with `model` and parameters projected+out: `stop_sequences`, `thinking`, `tool_choice`, `context_management` and+`output_config` become model parameters and `metadata` (where Claude Code stores its+session identifiers) becomes `request.metadata`; `system`, `messages`, `tools` and+unknown fields are kept as sent. The output is the native assistant message+`{"role":"assistant","content":[...],"stop_reason":...}`. JSON responses contribute+their `content` array at EOF. Streams rebuild each block from `content_block_start`,+its deltas and `content_block_stop`: text, thinking and citations are appended, the+opaque `signature` is kept, and `input_json_delta` fragments are parsed into the tool+`input` once the block stops. Unlike OpenAI Responses, Anthropic has no completed-item+event, so these deltas are stitched within the same retained-output budget. Only+stopped blocks are recorded; an unfinished block, tool input that is not valid JSON, an+unknown delta type or a block past the budget is left out and makes the output+partial. Usage mode records neither input nor output.For OpenAI Responses SSE, only `response.output_item.done` adds output. Terminal Responses eventsprovide model, service tier, status and usage; their repeated output is not copied.diff --git a/ai-gateway/src/capture/anthropic_messages.rs b/ai-gateway/src/capture/anthropic_messages.rsindex e55cf698b3..00d66560d6 100644--- a/ai-gateway/src/capture/anthropic_messages.rs+++ b/ai-gateway/src/capture/anthropic_messages.rs@@ -1,21 +1,209 @@use super::{- MAX_CAPTURE_BYTES, MAX_FACT_STRING, bounded_string, facts::ProviderFacts, identity_encoding,- response::ResponseBody,+ MAX_CAPTURE_BYTES, MAX_FACT_STRING, MAX_ITEMS, bounded_string, facts::ProviderFacts,+ identity_encoding, response::ResponseBody,};use crate::resolution::IngestionMode;use axum::http::HeaderMap;-use serde_json::{Map, Value};+use serde_json::{Map, Value, json};+use std::collections::BTreeMap;++const SCALAR_PARAMETERS: [&str; 7] = [+ "max_tokens",+ "temperature",+ "top_p",+ "top_k",+ "stream",+ "service_tier",+ "speed",+];+/// Structured configuration projected out of the recorded input in full mode.+const STRUCTURED_PARAMETERS: [&str; 5] = [+ "stop_sequences",+ "thinking",+ "tool_choice",+ "context_management",+ "output_config",+];pub(super) struct AnthropicMessagesCapture {facts: ProviderFacts,mode: IngestionMode,body: ResponseBody,usage: Option<Map<String, Value>>,+ /// Present only in full mode; usage mode never records output.+ output: Option<AssistantMessage>,terminal: bool,request_complete: bool,response_valid: bool,}+/// A capture gap: the recorded output no longer matches what was relayed.+struct Gap;++/// The assistant message rebuilt in full mode. Streamed blocks are assembled+/// from their start, delta and stop events; only stopped blocks are recorded.+#[derive(Default)]+struct AssistantMessage {+ stop: Map<String, Value>,+ blocks: BTreeMap<u64, Block>,+ bytes: usize,+}++struct Block {+ fields: Map<String, Value>,+ partial_json: String,+ finished: bool,+}++impl AssistantMessage {+ fn start(&mut self, index: u64, fields: Map<String, Value>) -> Result<(), Gap> {+ if self.blocks.contains_key(&index) || self.blocks.len() >= MAX_ITEMS {+ return Err(Gap);+ }+ let size = serde_json::to_string(&fields).map_or(usize::MAX, |json| json.len());+ reserve(&mut self.bytes, size)?;+ self.blocks.insert(+ index,+ Block {+ fields,+ partial_json: String::new(),+ finished: false,+ },+ );+ Ok(())+ }++ /// A block that cannot be rebuilt exactly is dropped rather than recorded wrong.+ fn apply_delta(&mut self, index: u64, delta: &Map<String, Value>) -> Result<(), Gap> {+ let applied = self.try_apply_delta(index, delta);+ if applied.is_err() {+ self.blocks.remove(&index);+ }+ applied+ }++ fn try_apply_delta(&mut self, index: u64, delta: &Map<String, Value>) -> Result<(), Gap> {+ let block = self+ .blocks+ .get_mut(&index)+ .filter(|block| !block.finished)+ .ok_or(Gap)?;+ let kind = delta.get("type").and_then(Value::as_str).ok_or(Gap)?;+ if kind == "citations_delta" {+ let citation = delta.get("citation").ok_or(Gap)?;+ reserve(&mut self.bytes, citation.to_string().len())?;+ match block.fields.get_mut("citations") {+ Some(Value::Array(citations)) => citations.push(citation.clone()),+ _ => {+ block+ .fields+ .insert("citations".to_owned(), Value::Array(vec![citation.clone()]));+ }+ }+ return Ok(());+ }+ // Every other delta appends a string fragment to one field of the block.+ let (field, source) = match kind {+ "text_delta" => ("text", "text"),+ "thinking_delta" => ("thinking", "thinking"),+ "signature_delta" => ("signature", "signature"),+ "input_json_delta" => ("", "partial_json"),+ _ => return Err(Gap),+ };+ let fragment = delta.get(source).and_then(Value::as_str).ok_or(Gap)?;+ reserve(&mut self.bytes, fragment.len())?;+ if field.is_empty() {+ block.partial_json.push_str(fragment);+ } else if let Some(Value::String(current)) = block.fields.get_mut(field) {+ current.push_str(fragment);+ } else {+ block+ .fields+ .insert(field.to_owned(), Value::String(fragment.to_owned()));+ }+ Ok(())+ }++ fn stop_block(&mut self, index: u64) -> Result<(), Gap> {+ let block = self+ .blocks+ .get_mut(&index)+ .filter(|block| !block.finished)+ .ok_or(Gap)?;+ // Tool inputs stream as JSON fragments that only parse once complete.+ if !block.partial_json.is_empty() {+ let Ok(input) = serde_json::from_str::<Value>(&block.partial_json) else {+ self.blocks.remove(&index);+ return Err(Gap);+ };+ block.fields.insert("input".to_owned(), input);+ block.partial_json = String::new();+ }+ block.finished = true;+ Ok(())+ }++ /// A non-streamed response carries its complete content at once.+ fn record_content(&mut self, content: Option<&Vec<Value>>) -> Result<(), Gap> {+ let content = content+ .filter(|content| content.len() <= MAX_ITEMS)+ .ok_or(Gap)?;+ for (index, fields) in content.iter().enumerate() {+ let fields = fields.as_object().ok_or(Gap)?.clone();+ self.blocks.insert(+ index as u64,+ Block {+ fields,+ partial_json: String::new(),+ finished: true,+ },+ );+ }+ Ok(())+ }++ /// The renderer reads the finish reason from the recorded output.+ fn record_stop(&mut self, source: &Map<String, Value>) {+ for key in ["stop_reason", "stop_sequence"] {+ if let Some(value) = source.get(key).filter(|value| !value.is_null()) {+ self.stop.insert(key.to_owned(), value.clone());+ }+ }+ }++ fn is_complete(&self) -> bool {+ self.blocks.values().all(|block| block.finished)+ }++ fn into_value(self) -> Value {+ let content = self+ .blocks+ .into_values()+ .filter(|block| block.finished)+ .map(|block| Value::Object(block.fields))+ .collect();+ let mut message = Map::new();+ message.insert("role".to_owned(), json!("assistant"));+ message.insert("content".to_owned(), Value::Array(content));+ message.extend(self.stop);+ Value::Object(message)+ }+}++fn reserve(used: &mut usize, bytes: usize) -> Result<(), Gap> {+ match used.checked_add(bytes) {+ Some(total) if total <= MAX_CAPTURE_BYTES => {+ *used = total;+ Ok(())+ }+ _ => Err(Gap),+ }+}++fn block_index(event: &Map<String, Value>) -> Option<u64> {+ event.get("index").and_then(Value::as_u64)+}+impl AnthropicMessagesCapture {pub fn new(headers: &HeaderMap, body: &[u8], mode: IngestionMode) -> Self {let mut capture = Self {@@ -23,6 +211,7 @@ impl AnthropicMessagesCapture {mode,body: ResponseBody::Unknown,usage: None,+ output: (mode == IngestionMode::Full).then(AssistantMessage::default),terminal: false,request_complete: false,response_valid: true,@@ -31,7 +220,7 @@ impl AnthropicMessagesCapture {&& body.len() <= MAX_CAPTURE_BYTES&& let Ok(Value::Object(request)) = serde_json::from_slice(body){- capture.capture_request(&request);+ capture.capture_request(request);capture.request_complete = true;}capture@@ -68,6 +257,7 @@ impl AnthropicMessagesCapture {self.record_failure(&response);} else {self.capture_message(&response);+ self.capture_json_content(&response);self.terminal = true;}}@@ -78,8 +268,12 @@ impl AnthropicMessagesCapture {}ResponseBody::Unknown | ResponseBody::Unavailable => {}}- self.facts.output_complete =- self.terminal && self.response_valid && self.mode != IngestionMode::Full;+ self.facts.output_complete = self.terminal+ && self.response_valid+ && self+ .output+ .as_ref()+ .is_none_or(AssistantMessage::is_complete);self.facts.capture_complete = self.request_complete && self.facts.output_complete;}@@ -88,33 +282,53 @@ impl AnthropicMessagesCapture {self.end_body();}self.facts.usage_details = self.usage.take().map(Value::Object);+ self.facts.output = self.output.take().map(AssistantMessage::into_value);self.facts}- fn capture_request(&mut self, request: &Map<String, Value>) {+ fn capture_request(&mut self, mut request: Map<String, Value>) {self.facts.requested_model = request- .get("model")+ .remove("model")+ .as_ref().and_then(Value::as_str).and_then(bounded_string);self.facts.model.clone_from(&self.facts.requested_model);- for key in [- "max_tokens",- "temperature",- "top_p",- "top_k",- "stream",- "service_tier",- "speed",- ] {- if let Some(value) = request.get(key).filter(|v| {+ for key in SCALAR_PARAMETERS {+ if let Some(value) = request.remove(key).filter(|v| {v.is_number()|| v.is_boolean()|| v.as_str().is_some_and(|s| s.len() <= MAX_FACT_STRING)}) {+ self.facts.model_parameters.insert(key.to_owned(), value);+ }+ }+ if self.mode == IngestionMode::Full {+ for key in STRUCTURED_PARAMETERS {+ if let Some(value) = request.remove(key) {+ self.facts.model_parameters.insert(key.to_owned(), value);+ }+ }+ // Claude Code stores its session identifiers here; it is request+ // metadata rather than prompt content.+ if let Some(value) = request.remove("metadata") {self.facts- .model_parameters- .insert(key.to_owned(), value.clone());+ .request_metadata+ .insert("metadata".to_owned(), value);}+ // `system`, `messages`, `tools` and unknown fields remain native.+ self.facts.input = Some(Value::Object(request));+ self.facts.input_complete = true;+ }+ }++ fn capture_json_content(&mut self, message: &Map<String, Value>) {+ let content = message.get("content").and_then(Value::as_array);+ if self+ .output+ .as_mut()+ .is_some_and(|output| output.record_content(content).is_err())+ {+ self.response_valid = false;}}@@ -131,18 +345,40 @@ impl AnthropicMessagesCapture {self.response_valid = false;}}+ Some("content_block_start") => {+ let block = event.get("content_block").and_then(Value::as_object);+ if self.output.as_mut().is_some_and(|output| {+ block_index(&event)+ .zip(block)+ .is_none_or(|(index, block)| output.start(index, block.clone()).is_err())+ }) {+ self.response_valid = false;+ }+ }+ Some("content_block_stop") => {+ if self.output.as_mut().is_some_and(|output| {+ block_index(&event).is_none_or(|index| output.stop_block(index).is_err())+ }) {+ self.response_valid = false;+ }+ }Some("content_block_delta") => {- return event- .get("delta")- .and_then(Value::as_object)- .is_some_and(|delta| {- ["text", "thinking", "partial_json"].iter().any(|key| {- delta- .get(*key)- .and_then(Value::as_str)- .is_some_and(|content| !content.is_empty())- })- });+ let delta = event.get("delta").and_then(Value::as_object);+ if self.output.as_mut().is_some_and(|output| {+ block_index(&event)+ .zip(delta)+ .is_none_or(|(index, delta)| output.apply_delta(index, delta).is_err())+ }) {+ self.response_valid = false;+ }+ return delta.is_some_and(|delta| {+ ["text", "thinking", "partial_json"].iter().any(|key| {+ delta+ .get(*key)+ .and_then(Value::as_str)+ .is_some_and(|content| !content.is_empty())+ })+ });}Some("message_delta") => {if let Some(delta) = event.get("delta").and_then(Value::as_object) {@@ -182,6 +418,9 @@ impl AnthropicMessagesCapture {let Some(stop_reason) = source.get("stop_reason").and_then(Value::as_str) else {return;};+ if let Some(output) = &mut self.output {+ output.record_stop(source);+ }self.facts.provider_status = Some(match stop_reason {"max_tokens" | "model_context_window_exceeded" => "incomplete",diff --git a/ai-gateway/src/capture/anthropic_messages/tests.rs b/ai-gateway/src/capture/anthropic_messages/tests.rsindex a1c174487f..050abcbfd5 100644--- a/ai-gateway/src/capture/anthropic_messages/tests.rs+++ b/ai-gateway/src/capture/anthropic_messages/tests.rs@@ -1,6 +1,6 @@use super::*;use crate::{- capture::{ExecutionCapture, ProtocolCapture},+ capture::{ExecutionCapture, ProtocolCapture, ProviderFacts},providers::{ProviderLimits, ProviderTransport, Route},resolution::ApiFormat,test_support::{FakeServer, resolved_request_context_for},@@ -166,9 +166,13 @@ async fn json_responses_capture_the_same_facts_and_error_bodies_mark_failure() {.map(|usage| Value::Object(usage.clone())),Some(json!({"input_tokens":10,"output_tokens":32000})));- // Response content is not captured yet, so full mode cannot claim completeness.- assert!(!facts.facts.output_complete);- assert!(facts.facts.output.is_none());+ assert!(facts.facts.output_complete);+ assert_eq!(+ finalized(&mut observer).output,+ Some(+ json!({"role":"assistant","content":[{"type":"text","text":"Hello"}],"stop_reason":"max_tokens"})+ )+ );for (mode, expected_message) in [(@@ -236,28 +240,246 @@ async fn in_stream_errors_unknown_events_and_truncation_are_tolerated() {);}+fn finalized(observer: &mut ExecutionCapture) -> ProviderFacts {+ observer.protocol.take().unwrap().into_facts().1+}++fn block_events(index: u64, start: &Value, deltas: &[Value]) -> String {+ let mut events = vec![event(+ "content_block_start",+ json!({"index":index,"content_block":start}),+ )];+ for delta in deltas {+ events.push(event(+ "content_block_delta",+ json!({"index":index,"delta":delta}),+ ));+ }+ events.push(event("content_block_stop", json!({"index":index})));+ events.concat()+}++fn stop_events() -> String {+ [+ event(+ "message_delta",+ json!({"delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":20}}),+ ),+ event("message_stop", json!({})),+ ]+ .concat()+}++#[tokio::test]+async fn request_configuration_and_metadata_are_projected_out_of_input_by_mode() {+ let observer = new_observer("full").await;+ let facts = &captured(&observer).facts;+ assert_eq!(+ facts.model_parameters,+ json!({"max_tokens":32000,"temperature":1,"stream":true,"speed":"fast","thinking":{"type":"adaptive"},"tool_choice":{"type":"auto"}})+ .as_object()+ .cloned()+ .unwrap()+ );+ assert_eq!(+ facts.request_metadata,+ json!({"metadata":{"user_id":"{\"session_id\":\"session-canary\"}"}})+ .as_object()+ .cloned()+ .unwrap()+ );+ let input = facts.input.as_ref().unwrap();+ assert_eq!(input["system"][0]["text"], json!("system-canary"));+ assert_eq!(+ input["system"][0]["cache_control"]["type"],+ json!("ephemeral")+ );+ assert_eq!(input["messages"][0]["content"], json!("prompt-canary"));+ assert_eq!(input["tools"][0]["name"], json!("Bash"));+ assert_eq!(input["future_field"], json!({"kept":true}));+ for projected in [+ "model",+ "max_tokens",+ "speed",+ "thinking",+ "tool_choice",+ "metadata",+ ] {+ assert!(input.get(projected).is_none(), "{projected} left in input");+ }+ assert!(facts.input_complete);++ let mut observer = new_observer("usage").await;+ record_response(&mut observer, 200, "text/event-stream");+ observer.push_bytes(text_stream().as_bytes());+ let facts = finalized(&mut observer);+ assert_eq!(+ facts.model_parameters,+ json!({"max_tokens":32000,"temperature":1,"stream":true,"speed":"fast"})+ .as_object()+ .cloned()+ .unwrap()+ );+ assert!(facts.request_metadata.is_empty());+ assert!(facts.input.is_none());+ assert!(facts.output.is_none());+ let serialized = serde_json::to_string(&facts).unwrap();+ let canaries = ["system-canary", "prompt-canary", "session-canary", "Hel"]; // codespell:ignore hel+ for canary in canaries {+ assert!(!serialized.contains(canary), "usage mode captured {canary}");+ }+}++#[tokio::test]+async fn full_mode_stitches_text_thinking_tool_and_citation_blocks_in_order() {+ let mut observer = new_observer("full").await;+ record_response(&mut observer, 200, "text/event-stream");+ let stream = [+ message_start(),+ block_events(+ 0,+ &json!({"type":"thinking","thinking":"","signature":""}),+ &[+ json!({"type":"thinking_delta","thinking":"Let me "}),+ json!({"type":"thinking_delta","thinking":"check."}),+ json!({"type":"signature_delta","signature":"sig-"}),+ json!({"type":"signature_delta","signature":"opaque"}),+ ],+ ),+ event("ping", json!({})),+ block_events(+ 1,+ &json!({"type":"text","text":""}),+ &[+ json!({"type":"text_delta","text":"See "}),+ json!({"type":"citations_delta","citation":{"type":"char_location","cited_text":"x"}}),+ json!({"type":"text_delta","text":"docs."}),+ ],+ ),+ block_events(+ 2,+ &json!({"type":"tool_use","id":"toolu_1","name":"Bash","input":{}}),+ &[+ json!({"type":"input_json_delta","partial_json":"{\"command\":"}),+ json!({"type":"input_json_delta","partial_json":"\"ls -la\"}"}),+ ],+ ),+ block_events(3, &json!({"type":"redacted_thinking","data":"opaque"}), &[]),+ stop_events(),+ ]+ .concat();+ // Byte-at-a-time delivery must stitch identically to whole events.+ for chunk in stream.as_bytes().chunks(7) {+ observer.push_bytes(chunk);+ }+ observer.end_body();+ assert!(captured(&observer).facts.output_complete);+ assert_eq!(+ finalized(&mut observer).output,+ Some(json!({+ "role": "assistant",+ "content": [+ {"type":"thinking","thinking":"Let me check.","signature":"sig-opaque"},+ {"type":"text","text":"See docs.","citations":[{"type":"char_location","cited_text":"x"}]},+ {"type":"tool_use","id":"toolu_1","name":"Bash","input":{"command":"ls -la"}},+ {"type":"redacted_thinking","data":"opaque"}+ ],+ "stop_reason": "tool_use"+ }))+ );+}+#[tokio::test]-async fn only_scalar_parameters_are_recorded_and_content_is_never_captured() {- for mode in ["usage", "full"] {- let observer = new_observer(mode).await;- let facts = &captured(&observer).facts;- assert_eq!(- facts.model_parameters,- json!({"max_tokens":32000,"temperature":1,"stream":true,"speed":"fast"})- .as_object()- .cloned()- .unwrap()+async fn unfinished_malformed_or_unknown_blocks_are_not_recorded_and_mark_output_partial() {+ let text = |index: u64, value: &str| {+ block_events(+ index,+ &json!({"type":"text","text":""}),+ &[json!({"type":"text_delta","text":value})],+ )+ };+ let cases = [+ // A disconnect midway through a block keeps completed blocks only.+ [+ message_start(),+ text(0, "kept"),+ event(+ "content_block_start",+ json!({"index":1,"content_block":{"type":"text","text":""}}),+ ),+ event(+ "content_block_delta",+ json!({"index":1,"delta":{"type":"text_delta","text":"lost"}}),+ ),+ ]+ .concat(),+ // Tool input that never forms valid JSON.+ [+ message_start(),+ text(0, "kept"),+ block_events(+ 1,+ &json!({"type":"tool_use","id":"t","name":"Bash","input":{}}),+ &[json!({"type":"input_json_delta","partial_json":"{\"cut"})],+ ),+ stop_events(),+ ]+ .concat(),+ // A delta type the adapter cannot reconstruct.+ [+ message_start(),+ text(0, "kept"),+ block_events(+ 1,+ &json!({"type":"text","text":""}),+ &[json!({"type":"future_delta","value":"?"})],+ ),+ stop_events(),+ ]+ .concat(),+ ];+ for stream in cases {+ let mut observer = new_observer("full").await;+ record_response(&mut observer, 200, "text/event-stream");+ observer.push_bytes(stream.as_bytes());+ observer.end_body();+ assert!(!captured(&observer).facts.output_complete);+ let output = finalized(&mut observer).output.unwrap();+ assert_eq!(output["content"], json!([{"type":"text","text":"kept"}]));+ }+}++#[tokio::test]+async fn retained_output_is_bounded_without_affecting_the_relay() {+ let mut observer = new_observer("full").await;+ record_response(&mut observer, 200, "text/event-stream");+ observer.push_bytes(message_start().as_bytes());+ observer.push_bytes(+ event(+ "content_block_start",+ json!({"index":0,"content_block":{"type":"text","text":""}}),+ )+ .as_bytes(),+ );+ // Each delta fits the per-event inspection limit; together they exceed the+ // retained-output budget.+ let chunk = "x".repeat(200 * 1024);+ for _ in 0..6 {+ observer.push_bytes(+ event(+ "content_block_delta",+ json!({"index":0,"delta":{"type":"text_delta","text":chunk}}),+ )+ .as_bytes(),);- assert!(facts.request_metadata.is_empty());- assert!(facts.input.is_none());- let serialized = serde_json::to_string(facts).unwrap();- for canary in ["system-canary", "prompt-canary", "session-canary"] {- assert!(- !serialized.contains(canary),- "{mode} mode captured {canary}"- );- }}+ observer.push_bytes(event("content_block_stop", json!({"index":0})).as_bytes());+ observer.push_bytes(stop_events().as_bytes());+ observer.end_body();+ assert!(!captured(&observer).facts.output_complete);+ let facts = finalized(&mut observer);+ assert_eq!(facts.output.unwrap()["content"], json!([]));+ assert_eq!(facts.usage_details.unwrap()["output_tokens"], json!(20));}#[tokio::test]diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rsindex 0d21f1c52f..10e05f4c5f 100644--- a/ai-gateway/src/http.rs+++ b/ai-gateway/src/http.rs@@ -19,7 +19,7 @@ use crate::{};const MAX_REQUEST_BYTES: usize = 10 * 1024 * 1024;-const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(10);+const REQUEST_READ_TIMEOUT: Duration = Duration::from_secs(30);const MAX_GATEWAY_KEY_BYTES: usize = 8192;#[derive(Clone)]diff --git a/ai-gateway/src/http/tests.rs b/ai-gateway/src/http/tests.rsindex 3128f1e7be..baa8116980 100644--- a/ai-gateway/src/http/tests.rs+++ b/ai-gateway/src/http/tests.rs@@ -464,11 +464,8 @@ async fn resolution_failures_are_sanitized_and_never_call_provider() {}#[tokio::test]-async fn slow_request_body_and_resolution_have_bounded_waits() {- let web = FakeServer::start(|request| async move {- if request.headers()[header::AUTHORIZATION] != "Bearer slow-resolution" {- return resolution_response("provider");- }+async fn slow_resolution_has_a_bounded_wait() {+ let web = FakeServer::start(|_| async {Response::new(Body::from_stream(stream::pending::<Result<&'static str, Infallible>,>()))@@ -476,29 +473,31 @@ async fn slow_request_body_and_resolution_have_bounded_waits() {.await;let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;let gateway = Gateway::start(Some(inference(&web, &provider))).await;- let slow_body = gateway- .post()- .bearer_auth("key")- .body(reqwest::Body::wrap_stream(stream::pending::<- Result<&'static str, Infallible>,- >()))- .send();- let slow_resolution = gateway- .post()- .bearer_auth("slow-resolution")- .body("{}")- .send();- let (body, resolution) = tokio::time::timeout(Duration::from_secs(12), async {- tokio::join!(slow_body, slow_resolution)- })+ let resolution = tokio::time::timeout(+ Duration::from_secs(12),+ gateway+ .post()+ .bearer_auth("slow-resolution")+ .body("{}")+ .send(),+ ).await.unwrap();- assert_eq!(body.unwrap().status(), StatusCode::REQUEST_TIMEOUT);assert_eq!(resolution.unwrap().status(), StatusCode::GATEWAY_TIMEOUT);- assert_eq!(web.calls(), 2);+ assert_eq!(web.calls(), 1);assert_eq!(provider.calls(), 0);}+#[tokio::test(start_paused = true)]+async fn a_stalled_request_body_times_out_at_the_read_deadline() {+ let stalled = Body::from_stream(stream::pending::<Result<&'static str, Infallible>>());+ let started = tokio::time::Instant::now();+ let result = read_request_body(stalled).await;+ assert!(matches!(result, Err(InferenceHttpError::RequestTimeout)));+ assert_eq!(started.elapsed(), REQUEST_READ_TIMEOUT);+ assert_eq!(REQUEST_READ_TIMEOUT, Duration::from_secs(30));+}+#[tokio::test]async fn pre_header_provider_failures_export_the_status_returned_to_the_caller() {use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};