🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(ai-gateway): preserve completion when clients close streams (#17633)
Hassieb Pakzad0ceeed99cbc6
diff --git a/ai-gateway/src/capture/openai_responses.rs b/ai-gateway/src/capture/openai_responses.rsindex e4b16a3b41..6f221f8365 100644--- a/ai-gateway/src/capture/openai_responses.rs+++ b/ai-gateway/src/capture/openai_responses.rs@@ -131,6 +131,9 @@ impl OpenAiResponsesCapture {}pub fn into_facts(mut self) -> ProviderFacts {+ if matches!(self.body, ResponseBody::Sse(_)) {+ self.end_body();+ }if self.mode == IngestionMode::Full {self.facts.output = Some(Value::Array(self.items.into_values().collect()));}diff --git a/ai-gateway/src/capture/openai_responses/tests.rs b/ai-gateway/src/capture/openai_responses/tests.rsindex c6c3a1ba05..ccc63f9ed9 100644--- a/ai-gateway/src/capture/openai_responses/tests.rs+++ b/ai-gateway/src/capture/openai_responses/tests.rs@@ -802,8 +802,13 @@ async fn capture_regression_preserves_native_request_and_usage() {#[tokio::test]async fn downstream_cancellation_does_not_erase_completed_capture() {- for streaming in [false, true] {- let mut execution = observer("full").await;+ for (streaming, upstream_eof, mode) in [+ (false, true, "full"),+ (true, true, "full"),+ (true, false, "full"),+ (true, false, "usage"),+ ] {+ let mut execution = observer(mode).await;record_response(&mut execution,if streaming {@@ -813,12 +818,14 @@ async fn downstream_cancellation_does_not_erase_completed_capture() {},);let body = if streaming {- terminal("completed", 0)+ format!("{}{}", completed_item(0, "hello"), terminal("completed", 1))} else {json!({"status":"completed","output":[]}).to_string()};execution.push_bytes(body.as_bytes());- execution.end_body();+ if upstream_eof {+ execution.end_body();+ }let writer = LogWriter::default();let subscriber = tracing_subscriber::fmt().json()diff --git a/ai-gateway/src/providers/openai/tests.rs b/ai-gateway/src/providers/openai/tests.rsindex bb6b10cf2e..72b0e65fb8 100644--- a/ai-gateway/src/providers/openai/tests.rs+++ b/ai-gateway/src/providers/openai/tests.rs@@ -359,17 +359,12 @@ async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {};use serde_json::Value;- for (content_type, native, sink_status) in [- (- "application/json",- r#"{"id":"resp-1","model":"actual","status":"completed","output":[],"usage":{"input_tokens":2,"output_tokens":1,"total_tokens":3}}"#,- 200,- ),- (- "text/event-stream",- "data: {\"type\":\"response.output_text.delta\",\"delta\":\"hello\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp-1\",\"model\":\"actual\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":2,\"output_tokens\":1,\"total_tokens\":3}}}\n\n",- 503,- ),+ const JSON: &str = r#"{"id":"resp-1","model":"actual","status":"completed","output":[],"usage":{"input_tokens":2,"output_tokens":1,"total_tokens":3}}"#;+ const SSE: &str = "data: {\"type\":\"response.output_text.delta\",\"delta\":\"hello\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp-1\",\"model\":\"actual\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":2,\"output_tokens\":1,\"total_tokens\":3}}}\n\n";+ for (content_type, native, sink_status, close_before_eof) in [+ ("application/json", JSON, 200, false),+ ("text/event-stream", SSE, 503, false),+ ("text/event-stream", SSE, 200, true),] {let (sent, mut received) = tokio::sync::mpsc::channel(2);let release = Arc::new(Notify::new());@@ -393,9 +388,17 @@ async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {let telemetry =Telemetry::new(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap()).unwrap();let upstream = FakeServer::start(move |_| async move {+ let body = if close_before_eof {+ Body::from_stream(+ stream::iter([Ok::<_, Infallible>(Bytes::from_static(native.as_bytes()))])+ .chain(stream::pending()),+ )+ } else {+ Body::from(native)+ };Response::builder().header("content-type", content_type)- .body(Body::from(native))+ .body(body).unwrap()}).await;@@ -425,13 +428,19 @@ async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {.await.unwrap();// Neither the last byte nor the provider permit waits for the sink response.- assert_eq!(- tokio::time::timeout(Duration::from_secs(1), to_bytes(response.into_body(), 4096))- .await- .unwrap()- .unwrap(),- native- );+ let mut body = response.into_body().into_data_stream();+ let mut forwarded = Vec::new();+ while let Some(chunk) = tokio::time::timeout(Duration::from_secs(1), body.next())+ .await+ .unwrap()+ {+ forwarded.extend_from_slice(&chunk.unwrap());+ if close_before_eof && forwarded.len() == native.len() {+ break;+ }+ }+ drop(body);+ assert_eq!(forwarded, native.as_bytes());assert!(relay.try_admit().is_ok());let payload = tokio::time::timeout(Duration::from_secs(1), received.recv()).await@@ -645,6 +654,14 @@ fn assert_completed_upload(payload: &serde_json::Value, streaming: bool) {);let span = &payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0];let attrs = span["attributes"].as_array().unwrap();+ assert!(attrs.iter().any(+ |a| a["key"] == "langfuse.observation.level" && a["value"]["stringValue"] == "DEFAULT"+ ));+ assert!(+ !attrs+ .iter()+ .any(|a| a["key"] == "langfuse.observation.status_message")+ );assert!(attrs.iter()diff --git a/ai-gateway/src/telemetry/mapping.rs b/ai-gateway/src/telemetry/mapping.rsindex 32f1af1ab9..a412a326af 100644--- a/ai-gateway/src/telemetry/mapping.rs+++ b/ai-gateway/src/telemetry/mapping.rs@@ -179,7 +179,9 @@ fn observation_status(facts: &InferenceFacts) -> (&'static str, Option<String>)message.push_str(error);}("ERROR", Some(message))- } else if facts.outcome == RelayOutcome::Cancelled {+ } else if facts.outcome == RelayOutcome::Cancelled+ && facts.inference.provider_status.as_deref() != Some("completed")+ {("WARNING", Some("Client cancelled the response".into()))} else if facts.inference.provider_status.as_deref() == Some("incomplete") {("WARNING", Some("Provider response incomplete".into()))@@ -550,6 +552,41 @@ mod tests {(None, None, RelayOutcome::TransportError, "ERROR", 2),(Some(200), None, RelayOutcome::Timeout, "ERROR", 2),(Some(200), None, RelayOutcome::Cancelled, "WARNING", 0),+ (+ Some(200),+ Some("completed"),+ RelayOutcome::Cancelled,+ "DEFAULT",+ 0,+ ),+ (+ Some(200),+ Some("failed"),+ RelayOutcome::Cancelled,+ "ERROR",+ 2,+ ),+ (+ Some(200),+ Some("incomplete"),+ RelayOutcome::Cancelled,+ "WARNING",+ 0,+ ),+ (+ Some(200),+ Some("completed"),+ RelayOutcome::Timeout,+ "ERROR",+ 2,+ ),+ (+ Some(200),+ Some("completed"),+ RelayOutcome::TransportError,+ "ERROR",+ 2,+ ),(Some(200),Some("incomplete"),@@ -566,6 +603,10 @@ mod tests {let attrs = attributes(&span);assert_eq!(attrs["langfuse.observation.level"], level);assert_eq!(span["status"]["code"], code);+ assert_eq!(+ attrs.contains_key("langfuse.observation.status_message"),+ level != "DEFAULT"+ );assert!(metadata(&attrs).get("relay_outcome").is_none());}}