Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(ai-gateway): preserve completion when clients close streams (#17633)

Hassieb Pakzad0ceeed99cbc6
diff --git a/ai-gateway/src/capture/openai_responses.rs b/ai-gateway/src/capture/openai_responses.rs
index e4b16a3b41..6f221f8365 100644
--- a/ai-gateway/src/capture/openai_responses.rs
+++ b/ai-gateway/src/capture/openai_responses.rs
@@ -131,6 +131,9 @@ impl OpenAiResponsesCapture {
}
pub fn into_facts(mut self) -> ProviderFacts {
+ if matches!(self.body, ResponseBody::Sse(_)) {
+ self.end_body();
+ }
if self.mode == IngestionMode::Full {
self.facts.output = Some(Value::Array(self.items.into_values().collect()));
}
diff --git a/ai-gateway/src/capture/openai_responses/tests.rs b/ai-gateway/src/capture/openai_responses/tests.rs
index c6c3a1ba05..ccc63f9ed9 100644
--- a/ai-gateway/src/capture/openai_responses/tests.rs
+++ b/ai-gateway/src/capture/openai_responses/tests.rs
@@ -802,8 +802,13 @@ async fn capture_regression_preserves_native_request_and_usage() {
#[tokio::test]
async fn downstream_cancellation_does_not_erase_completed_capture() {
- for streaming in [false, true] {
- let mut execution = observer("full").await;
+ for (streaming, upstream_eof, mode) in [
+ (false, true, "full"),
+ (true, true, "full"),
+ (true, false, "full"),
+ (true, false, "usage"),
+ ] {
+ let mut execution = observer(mode).await;
record_response(
&mut execution,
if streaming {
@@ -813,12 +818,14 @@ async fn downstream_cancellation_does_not_erase_completed_capture() {
},
);
let body = if streaming {
- terminal("completed", 0)
+ format!("{}{}", completed_item(0, "hello"), terminal("completed", 1))
} else {
json!({"status":"completed","output":[]}).to_string()
};
execution.push_bytes(body.as_bytes());
- execution.end_body();
+ if upstream_eof {
+ execution.end_body();
+ }
let writer = LogWriter::default();
let subscriber = tracing_subscriber::fmt()
.json()
diff --git a/ai-gateway/src/providers/openai/tests.rs b/ai-gateway/src/providers/openai/tests.rs
index bb6b10cf2e..72b0e65fb8 100644
--- a/ai-gateway/src/providers/openai/tests.rs
+++ b/ai-gateway/src/providers/openai/tests.rs
@@ -359,17 +359,12 @@ async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {
};
use serde_json::Value;
- for (content_type, native, sink_status) in [
- (
- "application/json",
- r#"{"id":"resp-1","model":"actual","status":"completed","output":[],"usage":{"input_tokens":2,"output_tokens":1,"total_tokens":3}}"#,
- 200,
- ),
- (
- "text/event-stream",
- "data: {\"type\":\"response.output_text.delta\",\"delta\":\"hello\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp-1\",\"model\":\"actual\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":2,\"output_tokens\":1,\"total_tokens\":3}}}\n\n",
- 503,
- ),
+ const JSON: &str = r#"{"id":"resp-1","model":"actual","status":"completed","output":[],"usage":{"input_tokens":2,"output_tokens":1,"total_tokens":3}}"#;
+ const SSE: &str = "data: {\"type\":\"response.output_text.delta\",\"delta\":\"hello\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp-1\",\"model\":\"actual\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":2,\"output_tokens\":1,\"total_tokens\":3}}}\n\n";
+ for (content_type, native, sink_status, close_before_eof) in [
+ ("application/json", JSON, 200, false),
+ ("text/event-stream", SSE, 503, false),
+ ("text/event-stream", SSE, 200, true),
] {
let (sent, mut received) = tokio::sync::mpsc::channel(2);
let release = Arc::new(Notify::new());
@@ -393,9 +388,17 @@ async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {
let telemetry =
Telemetry::new(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap()).unwrap();
let upstream = FakeServer::start(move |_| async move {
+ let body = if close_before_eof {
+ Body::from_stream(
+ stream::iter([Ok::<_, Infallible>(Bytes::from_static(native.as_bytes()))])
+ .chain(stream::pending()),
+ )
+ } else {
+ Body::from(native)
+ };
Response::builder()
.header("content-type", content_type)
- .body(Body::from(native))
+ .body(body)
.unwrap()
})
.await;
@@ -425,13 +428,19 @@ async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {
.await
.unwrap();
// Neither the last byte nor the provider permit waits for the sink response.
- assert_eq!(
- tokio::time::timeout(Duration::from_secs(1), to_bytes(response.into_body(), 4096))
- .await
- .unwrap()
- .unwrap(),
- native
- );
+ let mut body = response.into_body().into_data_stream();
+ let mut forwarded = Vec::new();
+ while let Some(chunk) = tokio::time::timeout(Duration::from_secs(1), body.next())
+ .await
+ .unwrap()
+ {
+ forwarded.extend_from_slice(&chunk.unwrap());
+ if close_before_eof && forwarded.len() == native.len() {
+ break;
+ }
+ }
+ drop(body);
+ assert_eq!(forwarded, native.as_bytes());
assert!(relay.try_admit().is_ok());
let payload = tokio::time::timeout(Duration::from_secs(1), received.recv())
.await
@@ -645,6 +654,14 @@ fn assert_completed_upload(payload: &serde_json::Value, streaming: bool) {
);
let span = &payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0];
let attrs = span["attributes"].as_array().unwrap();
+ assert!(attrs.iter().any(
+ |a| a["key"] == "langfuse.observation.level" && a["value"]["stringValue"] == "DEFAULT"
+ ));
+ assert!(
+ !attrs
+ .iter()
+ .any(|a| a["key"] == "langfuse.observation.status_message")
+ );
assert!(
attrs
.iter()
diff --git a/ai-gateway/src/telemetry/mapping.rs b/ai-gateway/src/telemetry/mapping.rs
index 32f1af1ab9..a412a326af 100644
--- a/ai-gateway/src/telemetry/mapping.rs
+++ b/ai-gateway/src/telemetry/mapping.rs
@@ -179,7 +179,9 @@ fn observation_status(facts: &InferenceFacts) -> (&'static str, Option<String>)
message.push_str(error);
}
("ERROR", Some(message))
- } else if facts.outcome == RelayOutcome::Cancelled {
+ } else if facts.outcome == RelayOutcome::Cancelled
+ && facts.inference.provider_status.as_deref() != Some("completed")
+ {
("WARNING", Some("Client cancelled the response".into()))
} else if facts.inference.provider_status.as_deref() == Some("incomplete") {
("WARNING", Some("Provider response incomplete".into()))
@@ -550,6 +552,41 @@ mod tests {
(None, None, RelayOutcome::TransportError, "ERROR", 2),
(Some(200), None, RelayOutcome::Timeout, "ERROR", 2),
(Some(200), None, RelayOutcome::Cancelled, "WARNING", 0),
+ (
+ Some(200),
+ Some("completed"),
+ RelayOutcome::Cancelled,
+ "DEFAULT",
+ 0,
+ ),
+ (
+ Some(200),
+ Some("failed"),
+ RelayOutcome::Cancelled,
+ "ERROR",
+ 2,
+ ),
+ (
+ Some(200),
+ Some("incomplete"),
+ RelayOutcome::Cancelled,
+ "WARNING",
+ 0,
+ ),
+ (
+ Some(200),
+ Some("completed"),
+ RelayOutcome::Timeout,
+ "ERROR",
+ 2,
+ ),
+ (
+ Some(200),
+ Some("completed"),
+ RelayOutcome::TransportError,
+ "ERROR",
+ 2,
+ ),
(
Some(200),
Some("incomplete"),
@@ -566,6 +603,10 @@ mod tests {
let attrs = attributes(&span);
assert_eq!(attrs["langfuse.observation.level"], level);
assert_eq!(span["status"]["code"], code);
+ assert_eq!(
+ attrs.contains_key("langfuse.observation.status_message"),
+ level != "DEFAULT"
+ );
assert!(metadata(&attrs).get("relay_outcome").is_none());
}
}