Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

Preserve transport errors in Google and OpenAI Responses streams (#65940)

Bennet Bo Fenner574df448ea9c
diff --git a/crates/google_ai/Cargo.toml b/crates/google_ai/Cargo.toml
index bd95a8e5346..b27e225a1ba 100644
--- a/crates/google_ai/Cargo.toml
+++ b/crates/google_ai/Cargo.toml
@@ -20,3 +20,6 @@ language_model_core.workspace = true
log.workspace = true
serde.workspace = true
serde_json.workspace = true
+
+[dev-dependencies]
+http_client = { workspace = true, features = ["test-support"] }
diff --git a/crates/google_ai/src/completion.rs b/crates/google_ai/src/completion.rs
index 6c87161c500..0bc68b828ce 100644
--- a/crates/google_ai/src/completion.rs
+++ b/crates/google_ai/src/completion.rs
@@ -1,12 +1,11 @@
use anyhow::Result;
-use futures::{Stream, StreamExt, TryStreamExt};
+use futures::{Stream, StreamExt, TryStreamExt, stream::BoxStream};
use language_model_core::{
GOOGLE_PROVIDER_NAME, LanguageModelCompletionError, LanguageModelCompletionEvent,
LanguageModelRequest, LanguageModelRequestToolInput, LanguageModelToolChoice,
LanguageModelToolUse, LanguageModelToolUseId, LanguageModelToolUseInput, MessageContent, Role,
StopReason, TokenUsage,
};
-use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{self, AtomicU64};
@@ -268,13 +267,13 @@ impl GoogleEventMapper {
pub fn map_stream(
self,
- events: Pin<Box<dyn Send + Stream<Item = Result<GenerateContentResponse>>>>,
+ events: BoxStream<'static, Result<GenerateContentResponse, LanguageModelCompletionError>>,
) -> impl Stream<Item = Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
{
futures::stream::try_unfold((self, events), |(mut mapper, mut events)| async move {
match events.next().await {
Some(event) => {
- let event = event.map_err(LanguageModelCompletionError::from)?;
+ let event = event?;
Ok(Some((
futures::stream::iter(mapper.map_event(event)),
(mapper, events),
@@ -564,31 +563,6 @@ mod tests {
}
}
- #[test]
- fn stream_read_error_is_not_followed_by_completion() {
- let responses = vec![
- Ok(serde_json::from_value(json!({"candidates": [{
- "content": {"role": "model", "parts": [{"text": "Partial answer"}]}
- }]}))
- .unwrap()),
- Err(std::io::Error::new(std::io::ErrorKind::ConnectionReset, "read failed").into()),
- ];
- let events = futures::executor::block_on(
- GoogleEventMapper::new()
- .map_stream(Box::pin(futures::stream::iter(responses)))
- .collect::<Vec<_>>(),
- );
- assert!(matches!(
- events.as_slice(),
- [
- Ok(LanguageModelCompletionEvent::Text(text)),
- Err(LanguageModelCompletionError::Other(error)),
- ] if text == "Partial answer"
- && error.downcast_ref::<std::io::Error>()
- .is_some_and(|error| error.kind() == std::io::ErrorKind::ConnectionReset)
- ));
- }
-
fn text_request() -> LanguageModelRequest {
LanguageModelRequest {
messages: vec![LanguageModelRequestMessage {
diff --git a/crates/google_ai/src/google_ai.rs b/crates/google_ai/src/google_ai.rs
index 22bb331ceb0..f909f7a6a03 100644
--- a/crates/google_ai/src/google_ai.rs
+++ b/crates/google_ai/src/google_ai.rs
@@ -1,6 +1,6 @@
use std::mem;
-use anyhow::{Result, anyhow, bail};
+use anyhow::{Result, bail};
use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};
use http_client::{
AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest, RequestBuilderExt,
@@ -198,7 +198,10 @@ pub async fn stream_generate_content(
api_key: &str,
mut request: GenerateContentRequest,
extra_headers: &CustomHeaders,
-) -> Result<BoxStream<'static, Result<GenerateContentResponse>>, LanguageModelCompletionError> {
+) -> Result<
+ BoxStream<'static, Result<GenerateContentResponse, LanguageModelCompletionError>>,
+ LanguageModelCompletionError,
+> {
let api_key = api_key.trim();
validate_generate_content_request(&request)?;
@@ -251,15 +254,21 @@ pub async fn stream_generate_content(
}
match serde_json::from_str(line) {
Ok(response) => Some(Ok(response)),
- Err(error) => Some(Err(anyhow!(format!(
- "Error parsing JSON: {error:?}\n{line:?}"
- )))),
+ Err(error) => {
+ Some(Err(LanguageModelCompletionError::DeserializeResponse {
+ provider: GOOGLE_PROVIDER_NAME,
+ error,
+ }))
+ }
}
} else {
None
}
}
- Err(error) => Some(Err(anyhow!(error))),
+ Err(error) => Some(Err(LanguageModelCompletionError::ApiReadResponseError {
+ provider: GOOGLE_PROVIDER_NAME,
+ error,
+ })),
}
})
.boxed())
@@ -956,7 +965,75 @@ struct ListModelEntry {
#[cfg(test)]
mod tests {
use super::*;
+ use completion::GoogleEventMapper;
+ use futures::{TryStreamExt, stream};
+ use http_client::{FakeHttpClient, Response};
+ use language_model_core::LanguageModelCompletionEvent;
use serde_json::json;
+ use std::io::{Error, ErrorKind};
+
+ #[test]
+ fn stream_body_failures_remain_retryable() {
+ for kind in [ErrorKind::TimedOut, ErrorKind::ConnectionReset] {
+ let error = completion_error(Some(kind));
+ assert!(error.is_transient());
+ assert!(matches!(
+ error,
+ LanguageModelCompletionError::ApiReadResponseError { provider, error }
+ if provider == GOOGLE_PROVIDER_NAME && error.kind() == kind
+ ));
+ }
+ }
+
+ #[test]
+ fn malformed_stream_data_remains_non_retryable() {
+ let error = completion_error(None);
+ assert!(!error.is_transient());
+ assert!(matches!(
+ error,
+ LanguageModelCompletionError::DeserializeResponse { provider, .. }
+ if provider == GOOGLE_PROVIDER_NAME
+ ));
+ }
+
+ fn completion_error(read_error: Option<ErrorKind>) -> LanguageModelCompletionError {
+ futures::executor::block_on(async {
+ let client = FakeHttpClient::create(move |_| async move {
+ let chunks = vec![
+ Ok(b"data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"Thinking\"}]}}]}\n\n".to_vec()),
+ match read_error {
+ Some(kind) => Err(Error::new(kind, "stream read failed")),
+ None => Ok(b"data: not json\n\n".to_vec()),
+ },
+ ];
+ Ok(Response::builder().body(AsyncBody::from_reader(
+ stream::iter(chunks).into_async_read(),
+ ))?)
+ });
+ let request = serde_json::from_value(json!({
+ "model": "models/gemini-3.8-flash",
+ "contents": [{"role": "user", "parts": [{"text": "Think"}]}]
+ }))
+ .unwrap();
+ let responses = stream_generate_content(
+ client.as_ref(),
+ API_URL,
+ "test-key",
+ request,
+ &CustomHeaders::default(),
+ )
+ .await
+ .unwrap();
+ let mut events = GoogleEventMapper::new().map_stream(responses).boxed();
+ assert!(matches!(
+ events.next().await,
+ Some(Ok(LanguageModelCompletionEvent::Text(text))) if text == "Thinking"
+ ));
+ let error = events.next().await.unwrap().unwrap_err();
+ assert!(events.next().await.is_none());
+ error
+ })
+ }
#[test]
fn test_lowest_thinking_level_per_model() {
diff --git a/crates/language_models/src/provider/google.rs b/crates/language_models/src/provider/google.rs
index 9d1eccb7ca1..e202f00e07b 100644
--- a/crates/language_models/src/provider/google.rs
+++ b/crates/language_models/src/provider/google.rs
@@ -246,7 +246,10 @@ impl GoogleLanguageModelProvider {
) -> BoxFuture<
'static,
Result<
- futures::stream::BoxStream<'static, Result<GenerateContentResponse>>,
+ futures::stream::BoxStream<
+ 'static,
+ Result<GenerateContentResponse, LanguageModelCompletionError>,
+ >,
LanguageModelCompletionError,
>,
> {
diff --git a/crates/language_models/src/provider/open_ai.rs b/crates/language_models/src/provider/open_ai.rs
index c6d429d4a1c..0542747c0c0 100644
--- a/crates/language_models/src/provider/open_ai.rs
+++ b/crates/language_models/src/provider/open_ai.rs
@@ -201,8 +201,15 @@ impl OpenAiLanguageModelProvider {
request_limiter: &RateLimiter,
request: ResponseRequest,
cx: &AsyncApp,
- ) -> BoxFuture<'static, Result<futures::stream::BoxStream<'static, Result<ResponsesStreamEvent>>>>
- {
+ ) -> BoxFuture<
+ 'static,
+ Result<
+ futures::stream::BoxStream<
+ 'static,
+ Result<ResponsesStreamEvent, open_ai::RequestError>,
+ >,
+ >,
+ > {
let http_client = self.http_client.clone();
let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {
diff --git a/crates/language_models/src/provider/open_ai_compatible.rs b/crates/language_models/src/provider/open_ai_compatible.rs
index 6ffb6520d07..b18388cc03c 100644
--- a/crates/language_models/src/provider/open_ai_compatible.rs
+++ b/crates/language_models/src/provider/open_ai_compatible.rs
@@ -177,8 +177,15 @@ impl OpenAiCompatibleLanguageModelProvider {
request_limiter: &RateLimiter,
request: ResponseRequest,
cx: &AsyncApp,
- ) -> BoxFuture<'static, Result<futures::stream::BoxStream<'static, Result<ResponsesStreamEvent>>>>
- {
+ ) -> BoxFuture<
+ 'static,
+ Result<
+ futures::stream::BoxStream<
+ 'static,
+ Result<ResponsesStreamEvent, open_ai::RequestError>,
+ >,
+ >,
+ > {
let http_client = self.http_client.clone();
let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, _cx| {
diff --git a/crates/open_ai/src/completion.rs b/crates/open_ai/src/completion.rs
index 69bb669ecba..e4d10fb5c7c 100644
--- a/crates/open_ai/src/completion.rs
+++ b/crates/open_ai/src/completion.rs
@@ -1,6 +1,6 @@
use anyhow::{Result, anyhow};
use collections::HashMap;
-use futures::{Stream, StreamExt};
+use futures::{Stream, StreamExt, stream::BoxStream};
use http_client::StatusCode;
use language_model_core::{
CompactedContext, CompactionUpdate, LanguageModelCompletionError, LanguageModelCompletionEvent,
@@ -11,7 +11,6 @@ use language_model_core::{
ProviderErrorCategory, Role, StopReason, TokenUsage, provider_name_for_id,
util::{fix_streamed_json, parse_tool_arguments},
};
-use std::pin::Pin;
use std::sync::Arc;
use crate::responses::{
@@ -790,15 +789,15 @@ impl OpenAiResponseEventMapper {
}
}
- pub fn map_stream(
+ pub fn map_stream<E: Into<LanguageModelCompletionError> + 'static>(
mut self,
- events: Pin<Box<dyn Send + Stream<Item = Result<ResponsesStreamEvent>>>>,
+ events: BoxStream<'static, Result<ResponsesStreamEvent, E>>,
) -> impl Stream<Item = Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
{
events.flat_map(move |event| {
futures::stream::iter(match event {
Ok(event) => self.map_event(event),
- Err(error) => vec![Err(LanguageModelCompletionError::from(anyhow!(error)))],
+ Err(error) => vec![Err(error.into())],
})
})
}
@@ -1592,7 +1591,9 @@ mod tests {
fn map_response_events(events: Vec<ResponsesStreamEvent>) -> Vec<LanguageModelCompletionEvent> {
block_on(async {
OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID)
- .map_stream(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
+ .map_stream(Box::pin(futures::stream::iter(
+ events.into_iter().map(Ok::<_, crate::RequestError>),
+ )))
.collect::<Vec<_>>()
.await
.into_iter()
diff --git a/crates/open_ai/src/responses.rs b/crates/open_ai/src/responses.rs
index 4037b40b72f..cc2293f205c 100644
--- a/crates/open_ai/src/responses.rs
+++ b/crates/open_ai/src/responses.rs
@@ -883,7 +883,7 @@ pub async fn stream_response(
api_key: &str,
request: Request,
extra_headers: &CustomHeaders,
-) -> Result<BoxStream<'static, Result<StreamEvent>>, RequestError> {
+) -> Result<BoxStream<'static, Result<StreamEvent, RequestError>>, RequestError> {
let uri = format!("{api_url}/responses");
let is_streaming = request.stream;
let request = HttpRequest::builder()
@@ -909,31 +909,41 @@ pub async fn stream_response(
if response.status().is_success() {
if is_streaming {
let reader = BufReader::new(response.into_body());
+ let provider_name = provider_name.to_owned();
Ok(reader
.lines()
- .filter_map(|line| async move {
- match line {
- Ok(line) => {
- let line = line
- .strip_prefix("data: ")
- .or_else(|| line.strip_prefix("data:"))?;
- if line == "[DONE]" || line.is_empty() {
- None
- } else {
- match decode_stream_event(line) {
- Ok(event) => Some(Ok(event)),
- Err(error) => {
- log::error!(
- "Failed to parse OpenAI responses stream event: `{}`\nResponse: `{}`",
- error,
- line,
- );
- Some(Err(anyhow!(error)))
+ .filter_map(move |line| {
+ let provider_name = provider_name.clone();
+ async move {
+ match line {
+ Ok(line) => {
+ let line = line
+ .strip_prefix("data: ")
+ .or_else(|| line.strip_prefix("data:"))?;
+ if line == "[DONE]" || line.is_empty() {
+ None
+ } else {
+ match decode_stream_event(line) {
+ Ok(event) => Some(Ok(event)),
+ Err(error) => {
+ log::error!(
+ "Failed to parse OpenAI responses stream event: `{}`\nResponse: `{}`",
+ error,
+ line,
+ );
+ Some(Err(RequestError::DeserializeResponse {
+ provider: provider_name,
+ error,
+ }))
+ }
}
}
}
+ Err(error) => Some(Err(RequestError::ReadResponse {
+ provider: provider_name,
+ error,
+ })),
}
- Err(error) => Some(Err(anyhow!(error))),
}
})
.boxed())
@@ -1094,12 +1104,30 @@ fn decode_stream_event(line: &str) -> serde_json::Result<StreamEvent> {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::completion::OpenAiResponseEventMapper;
use futures::executor::block_on;
+ use futures::{TryStreamExt, stream};
use http_client::FakeHttpClient;
- use language_model_core::OPEN_AI_PROVIDER_ID;
+ use language_model_core::{
+ LanguageModelCompletionError, LanguageModelCompletionEvent, OPEN_AI_PROVIDER_ID,
+ };
use serde_json::json;
+ use std::io::{Error, ErrorKind};
use std::sync::{Arc, Mutex};
+ #[test]
+ fn stream_body_failures_remain_retryable() {
+ for kind in [ErrorKind::TimedOut, ErrorKind::ConnectionReset] {
+ let error = stream_completion_error(Some(kind));
+ assert!(error.is_transient());
+ assert!(matches!(
+ error,
+ LanguageModelCompletionError::ApiReadResponseError { provider, error }
+ if provider.0.as_ref() == "Custom Responses" && error.kind() == kind
+ ));
+ }
+ }
+
#[test]
fn reasoning_content_accepts_unknown_parts() {
let content = json!([
@@ -1211,6 +1239,56 @@ mod tests {
}
}
+ #[test]
+ fn malformed_stream_data_remains_non_retryable() {
+ let error = stream_completion_error(None);
+ assert!(!error.is_transient());
+ assert!(matches!(
+ error,
+ LanguageModelCompletionError::DeserializeResponse { provider, .. }
+ if provider.0.as_ref() == "Custom Responses"
+ ));
+ }
+
+ fn stream_completion_error(read_error: Option<ErrorKind>) -> LanguageModelCompletionError {
+ block_on(async {
+ let client = FakeHttpClient::create(move |_| async move {
+ let chunks = vec![
+ Ok(b"data: {\"type\":\"response.output_text.delta\",\"item_id\":\"message\",\"output_index\":0,\"delta\":\"Partial answer\"}\n\n".to_vec()),
+ match read_error {
+ Some(kind) => Err(Error::new(kind, "stream read failed")),
+ None => Ok(b"data: not json\n\n".to_vec()),
+ },
+ ];
+ Ok(
+ http_client::Response::builder().body(AsyncBody::from_reader(
+ stream::iter(chunks).into_async_read(),
+ ))?,
+ )
+ });
+ let responses = stream_response(
+ client.as_ref(),
+ "Custom Responses",
+ "https://example.com/v1",
+ "test-key",
+ response_test_request(),
+ &CustomHeaders::default(),
+ )
+ .await
+ .unwrap();
+ let mut events = OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID)
+ .map_stream(responses)
+ .boxed();
+ assert!(matches!(
+ events.next().await,
+ Some(Ok(LanguageModelCompletionEvent::Text(text))) if text == "Partial answer"
+ ));
+ let error = events.next().await.unwrap().unwrap_err();
+ assert!(events.next().await.is_none());
+ error
+ })
+ }
+
#[test]
fn count_input_tokens_preserves_structured_input_without_generation_fields() {
let input = json!([{
diff --git a/crates/opencode/src/opencode.rs b/crates/opencode/src/opencode.rs
index 0b995993a38..41219c30775 100644
--- a/crates/opencode/src/opencode.rs
+++ b/crates/opencode/src/opencode.rs
@@ -8,7 +8,9 @@ use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::B
use http_client::{
AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest, RequestBuilderExt,
};
-use language_model_core::ReasoningEffort;
+use language_model_core::{
+ LanguageModelCompletionError, LanguageModelProviderName, ReasoningEffort,
+};
use serde::{Deserialize, Serialize};
pub mod catalog;
@@ -138,7 +140,9 @@ pub async fn stream_generate_content(
api_key: &str,
request: google_ai::GenerateContentRequest,
extra_headers: &CustomHeaders,
-) -> Result<BoxStream<'static, Result<google_ai::GenerateContentResponse>>> {
+) -> Result<
+ BoxStream<'static, Result<google_ai::GenerateContentResponse, LanguageModelCompletionError>>,
+> {
let api_key = api_key.trim();
let model_id = &request.model.model_id;
let uri = format!("{api_url}/v1/models/{model_id}:streamGenerateContent?alt=sse");
@@ -161,14 +165,20 @@ pub async fn stream_generate_content(
match serde_json::from_str(line) {
Ok(response) => Some(Ok(response)),
Err(error) => {
- Some(Err(anyhow!("Error parsing JSON: {error:?}\n{line:?}")))
+ Some(Err(LanguageModelCompletionError::DeserializeResponse {
+ provider: LanguageModelProviderName::new("OpenCode"),
+ error,
+ }))
}
}
} else {
None
}
}
- Err(error) => Some(Err(anyhow!(error))),
+ Err(error) => Some(Err(LanguageModelCompletionError::ApiReadResponseError {
+ provider: LanguageModelProviderName::new("OpenCode"),
+ error,
+ })),
}
})
.boxed())