Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

Move language model stream mapping to background executors (#63241)

Bennet Bo Fenner5ed5bf95e518
diff --git a/crates/copilot_chat/src/model.rs b/crates/copilot_chat/src/model.rs
index 3d03bbe0b7f..b447f0635cd 100644
--- a/crates/copilot_chat/src/model.rs
+++ b/crates/copilot_chat/src/model.rs
@@ -241,12 +241,16 @@ impl LanguageModel for CopilotChatLanguageModel {
anthropic_beta,
cx.clone(),
);
+ let executor = cx.background_executor().clone();
request_limiter
.stream(async move {
let events = stream.await?;
let mapper = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID);
- Ok(mapper.map_stream(events).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(events).boxed(),
+ executor,
+ ))
})
.await
});
@@ -269,11 +273,15 @@ impl LanguageModel for CopilotChatLanguageModel {
is_user_initiated,
cx.clone(),
);
+ let executor = cx.background_executor().clone();
request_limiter
.stream(async move {
let stream = request.await?;
let mapper = CopilotResponsesEventMapper::new();
- Ok(mapper.map_stream(stream).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(stream).boxed(),
+ executor,
+ ))
})
.await
});
diff --git a/crates/language_model/src/language_model.rs b/crates/language_model/src/language_model.rs
index 316b09c5a53..995b4818e95 100644
--- a/crates/language_model/src/language_model.rs
+++ b/crates/language_model/src/language_model.rs
@@ -8,9 +8,9 @@ pub mod fake_provider;
pub use language_model_core::*;
use anyhow::Result;
-use futures::FutureExt;
-use futures::{StreamExt, future::BoxFuture, stream::BoxStream};
-use gpui::{AnyView, App, AsyncApp, Task, Window};
+use futures::{FutureExt, SinkExt};
+use futures::{StreamExt, channel::mpsc, future::BoxFuture, stream::BoxStream};
+use gpui::{AnyView, App, AsyncApp, BackgroundExecutor, Task, Window};
use icons::IconName;
use parking_lot::Mutex;
use std::sync::Arc;
@@ -22,10 +22,36 @@ pub use crate::registry::*;
pub use crate::request::{LanguageModelImageExt, gpui_size_to_image_size, image_size_to_gpui};
pub use env_var::{EnvVar, env_var};
+const BACKGROUND_STREAM_BUFFER_SIZE: usize = 32;
+
pub fn init(cx: &mut App) {
registry::init(cx);
}
+pub fn stream_in_background<Output>(
+ mut events: BoxStream<'static, Output>,
+ executor: BackgroundExecutor,
+) -> BoxStream<'static, Output>
+where
+ Output: Send + 'static,
+{
+ let (mut sender, receiver) = mpsc::channel(BACKGROUND_STREAM_BUFFER_SIZE);
+ let task = executor.spawn(async move {
+ while let Some(event) = events.next().await {
+ if sender.send(event).await.is_err() {
+ return;
+ }
+ }
+ });
+
+ receiver
+ .map(move |event| {
+ let _task = &task;
+ event
+ })
+ .boxed()
+}
+
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DisabledReason(pub SharedString);
diff --git a/crates/language_models/src/provider.rs b/crates/language_models/src/provider.rs
index 65fc039b074..ec3a1bb0c7b 100644
--- a/crates/language_models/src/provider.rs
+++ b/crates/language_models/src/provider.rs
@@ -71,6 +71,20 @@ pub(crate) fn resolve_custom_headers(
#[cfg(test)]
mod tests {
use super::*;
+ use futures::{StreamExt, stream};
+ use language_model::stream_in_background;
+
+ #[gpui::test]
+ async fn test_stream_in_background(cx: &mut gpui::TestAppContext) {
+ let events = stream::iter([1, 2, 3])
+ .flat_map(|event| stream::iter([event, event * 10]))
+ .boxed();
+ let mapped_events = stream_in_background(events, cx.background_executor.clone())
+ .collect::<Vec<_>>()
+ .await;
+
+ assert_eq!(mapped_events, [1, 10, 2, 20, 3, 30]);
+ }
fn map(pairs: &[(&str, &str)]) -> HashMap<String, String> {
pairs
diff --git a/crates/language_models/src/provider/anthropic.rs b/crates/language_models/src/provider/anthropic.rs
index 5cf1504d491..93a2d79c6c5 100644
--- a/crates/language_models/src/provider/anthropic.rs
+++ b/crates/language_models/src/provider/anthropic.rs
@@ -827,9 +827,11 @@ impl LanguageModel for AnthropicModel {
request.speed = None;
}
let request = self.stream_completion(request, cx);
+ let executor = cx.background_executor().clone();
let future = self.request_limiter.run(async move {
let response = request.await?;
let stream = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID).map_stream(response);
+ let stream = language_model::stream_in_background(stream.boxed(), executor);
let (context, usage) = collect_compaction_result(stream.boxed(), PROVIDER_NAME).await?;
Ok(CompactionResult { context, usage })
});
@@ -906,9 +908,14 @@ impl LanguageModel for AnthropicModel {
request.speed = None;
}
let request = self.stream_completion(request, cx);
+ let executor = cx.background_executor().clone();
let future = self.request_limiter.stream(async move {
let response = request.await?;
- Ok(AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID).map_stream(response))
+ let events = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID).map_stream(response);
+ Ok(language_model::stream_in_background(
+ events.boxed(),
+ executor,
+ ))
});
async move { Ok(future.await?.boxed()) }.boxed()
}
diff --git a/crates/language_models/src/provider/anthropic_compatible.rs b/crates/language_models/src/provider/anthropic_compatible.rs
index c4970e699c1..0ad754a913b 100644
--- a/crates/language_models/src/provider/anthropic_compatible.rs
+++ b/crates/language_models/src/provider/anthropic_compatible.rs
@@ -457,9 +457,14 @@ impl LanguageModel for AnthropicCompatibleLanguageModel {
let completion_request = self.stream_completion(request, cx);
let provider_name = self.provider_name.clone();
let provider_id = self.provider_id.clone();
+ let executor = cx.background_executor().clone();
let future = self.request_limiter.stream(async move {
let response = completion_request.await?;
- Ok(AnthropicEventMapper::new(provider_name, provider_id).map_stream(response))
+ let events = AnthropicEventMapper::new(provider_name, provider_id).map_stream(response);
+ Ok(language_model::stream_in_background(
+ events.boxed(),
+ executor,
+ ))
});
async move { Ok(future.await?.boxed()) }.boxed()
}
diff --git a/crates/language_models/src/provider/bedrock.rs b/crates/language_models/src/provider/bedrock.rs
index e72880f3ae4..d8577fcae14 100644
--- a/crates/language_models/src/provider/bedrock.rs
+++ b/crates/language_models/src/provider/bedrock.rs
@@ -977,6 +977,7 @@ impl LanguageModel for BedrockModel {
let request = self.stream_completion(request, cx);
let display_name = self.model.display_name().to_string();
+ let executor = cx.background_executor().clone();
let future = self.request_limiter.stream(async move {
let response = request.await.map_err(|err| match err {
BedrockError::Validation(ref msg) => {
@@ -1043,7 +1044,10 @@ impl LanguageModel for BedrockModel {
}
other => LanguageModelCompletionError::Other(anyhow!(other)),
})?;
- let events = map_to_language_model_completion_events(response);
+ let events = language_model::stream_in_background(
+ map_to_language_model_completion_events(response).boxed(),
+ executor,
+ );
if deny_tool_calls {
Ok(deny_tool_use_events(events).boxed())
@@ -1945,9 +1949,13 @@ impl LanguageModel for BedrockMantleModel {
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let completions = self.stream_response(request, cx);
+ let executor = cx.background_executor().clone();
async move {
let mapper = MantleResponseEventMapper::new();
- Ok(mapper.map_stream(completions.await?).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
}
.boxed()
}
@@ -1967,9 +1975,13 @@ impl LanguageModel for BedrockMantleModel {
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let completions = self.stream_completion(request, cx);
+ let executor = cx.background_executor().clone();
async move {
let mapper = OpenAiEventMapper::new();
- Ok(mapper.map_stream(completions.await?).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
}
.boxed()
}
diff --git a/crates/language_models/src/provider/deepseek.rs b/crates/language_models/src/provider/deepseek.rs
index 2b1472fa6f8..b2365fbbc61 100644
--- a/crates/language_models/src/provider/deepseek.rs
+++ b/crates/language_models/src/provider/deepseek.rs
@@ -348,10 +348,14 @@ impl LanguageModel for DeepSeekLanguageModel {
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let stream = self.stream_completion(request, cx);
+ let executor = cx.background_executor().clone();
async move {
let mapper = DeepSeekEventMapper::new();
- Ok(mapper.map_stream(stream.await?).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(stream.await?).boxed(),
+ executor,
+ ))
}
.boxed()
}
diff --git a/crates/language_models/src/provider/mistral.rs b/crates/language_models/src/provider/mistral.rs
index c3fe10aeb49..d447cb12eee 100644
--- a/crates/language_models/src/provider/mistral.rs
+++ b/crates/language_models/src/provider/mistral.rs
@@ -348,11 +348,15 @@ impl LanguageModel for MistralLanguageModel {
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let stream = self.stream_completion(request, affinity, cx);
+ let executor = cx.background_executor().clone();
async move {
let stream = stream.await?;
let mapper = MistralEventMapper::new();
- Ok(mapper.map_stream(stream).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(stream).boxed(),
+ executor,
+ ))
}
.boxed()
}
diff --git a/crates/language_models/src/provider/open_ai.rs b/crates/language_models/src/provider/open_ai.rs
index 2e5137c4f7b..8c123999afe 100644
--- a/crates/language_models/src/provider/open_ai.rs
+++ b/crates/language_models/src/provider/open_ai.rs
@@ -10,7 +10,7 @@ use language_model::{
LanguageModelCompletionEvent, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice, OPEN_AI_PROVIDER_ID,
- OPEN_AI_PROVIDER_NAME, ProviderSettingsView, RateLimiter, env_var,
+ OPEN_AI_PROVIDER_NAME, ProviderSettingsView, RateLimiter, env_var, stream_in_background,
};
use open_ai::{
ResponseStreamEvent,
@@ -643,9 +643,13 @@ impl LanguageModel for OpenAiLanguageModel {
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let completions = self.stream_response(request, cx);
+ let executor = cx.background_executor().clone();
async move {
let mapper = OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID);
- Ok(mapper.map_stream(completions.await?).boxed())
+ Ok(stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
}
.boxed()
} else {
@@ -663,9 +667,13 @@ impl LanguageModel for OpenAiLanguageModel {
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let completions = self.stream_completion(request, cx);
+ let executor = cx.background_executor().clone();
async move {
let mapper = OpenAiEventMapper::new();
- Ok(mapper.map_stream(completions.await?).boxed())
+ Ok(stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
}
.boxed()
}
diff --git a/crates/language_models/src/provider/open_ai_compatible.rs b/crates/language_models/src/provider/open_ai_compatible.rs
index 6e9a30e91b7..f725ba981fe 100644
--- a/crates/language_models/src/provider/open_ai_compatible.rs
+++ b/crates/language_models/src/provider/open_ai_compatible.rs
@@ -438,9 +438,13 @@ impl LanguageModel for OpenAiCompatibleLanguageModel {
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let completions = self.stream_completion(request, cx);
+ let executor = cx.background_executor().clone();
async move {
let mapper = OpenAiEventMapper::new();
- Ok(mapper.map_stream(completions.await?).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
}
.boxed()
} else {
@@ -460,9 +464,13 @@ impl LanguageModel for OpenAiCompatibleLanguageModel {
};
let completions = self.stream_response(request, cx);
let compaction_state_owner = self.provider_id.clone();
+ let executor = cx.background_executor().clone();
async move {
let mapper = OpenAiResponseEventMapper::new(compaction_state_owner);
- Ok(mapper.map_stream(completions.await?).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
}
.boxed()
}
diff --git a/crates/language_models/src/provider/open_router.rs b/crates/language_models/src/provider/open_router.rs
index 8b8ce92e771..47bf634d9e9 100644
--- a/crates/language_models/src/provider/open_router.rs
+++ b/crates/language_models/src/provider/open_router.rs
@@ -407,9 +407,14 @@ impl LanguageModel for OpenRouterLanguageModel {
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let request = self.stream_completion(openrouter_request, cx);
+ let executor = cx.background_executor().clone();
let future = self.request_limiter.stream(async move {
let response = request.await?;
- Ok(OpenRouterEventMapper::new().map_stream(response))
+ let events = OpenRouterEventMapper::new().map_stream(response);
+ Ok(language_model::stream_in_background(
+ events.boxed(),
+ executor,
+ ))
});
async move { Ok(future.await?.boxed()) }.boxed()
}
diff --git a/crates/language_models/src/provider/opencode.rs b/crates/language_models/src/provider/opencode.rs
index f5281e019f7..ec0c5a0a90b 100644
--- a/crates/language_models/src/provider/opencode.rs
+++ b/crates/language_models/src/provider/opencode.rs
@@ -670,9 +670,13 @@ impl LanguageModel for OpenCodeLanguageModel {
};
let stream =
self.stream_anthropic(anthropic_request, http_client, extra_headers, cx);
+ let executor = cx.background_executor().clone();
async move {
let mapper = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID);
- Ok(mapper.map_stream(stream.await?).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(stream.await?).boxed(),
+ executor,
+ ))
}
.boxed()
}
@@ -700,9 +704,13 @@ impl LanguageModel for OpenCodeLanguageModel {
};
let stream =
self.stream_openai_chat(openai_request, http_client, extra_headers, cx);
+ let executor = cx.background_executor().clone();
async move {
let mapper = OpenAiEventMapper::new();
- Ok(mapper.map_stream(stream.await?).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(stream.await?).boxed(),
+ executor,
+ ))
}
.boxed()
}
@@ -726,9 +734,13 @@ impl LanguageModel for OpenCodeLanguageModel {
};
let stream =
self.stream_openai_response(response_request, http_client, extra_headers, cx);
+ let executor = cx.background_executor().clone();
async move {
let mapper = OpenAiResponseEventMapper::new(PROVIDER_ID);
- Ok(mapper.map_stream(stream.await?).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(stream.await?).boxed(),
+ executor,
+ ))
}
.boxed()
}
diff --git a/crates/language_models/src/provider/vercel_ai_gateway.rs b/crates/language_models/src/provider/vercel_ai_gateway.rs
index d47a9594d61..b13b022aa9d 100644
--- a/crates/language_models/src/provider/vercel_ai_gateway.rs
+++ b/crates/language_models/src/provider/vercel_ai_gateway.rs
@@ -465,9 +465,13 @@ impl LanguageModel for VercelAiGatewayLanguageModel {
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let completions = self.stream_open_ai(request, cx);
+ let executor = cx.background_executor().clone();
async move {
let mapper = crate::provider::open_ai::OpenAiEventMapper::new();
- Ok(mapper.map_stream(completions.await?).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
}
.boxed()
}
diff --git a/crates/language_models/src/provider/x_ai.rs b/crates/language_models/src/provider/x_ai.rs
index c5f44aab363..e7ba81ef706 100644
--- a/crates/language_models/src/provider/x_ai.rs
+++ b/crates/language_models/src/provider/x_ai.rs
@@ -437,9 +437,13 @@ impl LanguageModel for XAiLanguageModel {
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let completions = self.stream_completion(request, cx);
+ let executor = cx.background_executor().clone();
async move {
let mapper = crate::provider::open_ai::OpenAiEventMapper::new();
- Ok(mapper.map_stream(completions.await?).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(completions.await?).boxed(),
+ executor,
+ ))
}
.boxed()
}
diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rs
index c37d1144c82..8c5ce5104b7 100644
--- a/crates/language_models_cloud/src/language_models_cloud.rs
+++ b/crates/language_models_cloud/src/language_models_cloud.rs
@@ -274,6 +274,7 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudLanguageModel<TP> {
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
let auth_context = token_provider.auth_context(cx);
+ let executor = cx.background_executor().clone();
let future = self.request_limiter.run(async move {
let PerformLlmCompletionResponse {
response,
@@ -305,6 +306,7 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudLanguageModel<TP> {
&ANTHROPIC_PROVIDER_NAME,
move |event| mapper.map_event(event),
);
+ let stream = language_model::stream_in_background(stream.boxed(), executor);
let (context, usage) =
collect_compaction_result(stream, ANTHROPIC_PROVIDER_NAME).await?;
Ok(CompactionResult { context, usage })
@@ -768,6 +770,7 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
let auth_context = token_provider.auth_context(cx);
+ let executor = cx.background_executor().clone();
let future = self.request_limiter.stream(async move {
let PerformLlmCompletionResponse {
response,
@@ -794,10 +797,14 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
let mut mapper =
AnthropicEventMapper::new(provider_name.clone(), ANTHROPIC_PROVIDER_ID);
- Ok(map_cloud_completion_events(
+ let events = map_cloud_completion_events(
Box::pin(response_lines(response, includes_status_messages)),
&provider_name,
move |event| mapper.map_event(event),
+ );
+ Ok(language_model::stream_in_background(
+ events.boxed(),
+ executor,
))
});
async move { Ok(future.await?.boxed()) }.boxed()
@@ -838,6 +845,7 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
}
let auth_context = token_provider.auth_context(cx);
+ let executor = cx.background_executor().clone();
let future = self.request_limiter.stream(async move {
let PerformLlmCompletionResponse {
response,
@@ -863,10 +871,14 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
.await?;
let mut mapper = OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID);
- Ok(map_cloud_completion_events(
+ let events = map_cloud_completion_events(
Box::pin(response_lines(response, includes_status_messages)),
&provider_name,
move |event| mapper.map_event(event),
+ );
+ Ok(language_model::stream_in_background(
+ events.boxed(),
+ executor,
))
});
async move { Ok(future.await?.boxed()) }.boxed()
@@ -889,6 +901,7 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let auth_context = token_provider.auth_context(cx);
+ let executor = cx.background_executor().clone();
let future = self.request_limiter.stream(async move {
let PerformLlmCompletionResponse {
response,
@@ -914,10 +927,14 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
.await?;
let mut mapper = OpenAiEventMapper::new();
- Ok(map_cloud_completion_events(
+ let events = map_cloud_completion_events(
Box::pin(response_lines(response, includes_status_messages)),
&provider_name,
move |event| mapper.map_event(event),
+ );
+ Ok(language_model::stream_in_background(
+ events.boxed(),
+ executor,
))
});
async move { Ok(future.await?.boxed()) }.boxed()
diff --git a/crates/openai_subscribed/src/openai_subscribed.rs b/crates/openai_subscribed/src/openai_subscribed.rs
index 5cb36d36559..431bab98bd3 100644
--- a/crates/openai_subscribed/src/openai_subscribed.rs
+++ b/crates/openai_subscribed/src/openai_subscribed.rs
@@ -797,7 +797,10 @@ impl LanguageModel for OpenAiSubscribedLanguageModel {
})
.await?;
let mapper = OpenAiResponseEventMapper::new(PROVIDER_ID);
- let mut event_stream = mapper.map_stream(response_stream.boxed());
+ let mut event_stream = language_model::stream_in_background(
+ mapper.map_stream(response_stream.boxed()).boxed(),
+ cx.background_executor().clone(),
+ );
let mut compacted_context = None;
let mut usage = language_model::TokenUsage::default();
@@ -889,6 +892,7 @@ impl LanguageModel for OpenAiSubscribedLanguageModel {
let state = self.state.downgrade();
let http_client = self.http_client.clone();
let request_limiter = self.request_limiter.clone();
+ let executor = cx.background_executor().clone();
let future = cx.spawn(async move |cx| {
let creds = get_fresh_credentials(&state, &http_client, cx).await?;
@@ -914,7 +918,10 @@ impl LanguageModel for OpenAiSubscribedLanguageModel {
async move {
let mapper = OpenAiResponseEventMapper::new(PROVIDER_ID);
- Ok(mapper.map_stream(future.await?.boxed()).boxed())
+ Ok(language_model::stream_in_background(
+ mapper.map_stream(future.await?.boxed()).boxed(),
+ executor,
+ ))
}
.boxed()
}