Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
Move language model stream mapping to background executors (#63241)
Bennet Bo Fenner5ed5bf95e518
diff --git a/crates/copilot_chat/src/model.rs b/crates/copilot_chat/src/model.rsindex 3d03bbe0b7f..b447f0635cd 100644--- a/crates/copilot_chat/src/model.rs+++ b/crates/copilot_chat/src/model.rs@@ -241,12 +241,16 @@ impl LanguageModel for CopilotChatLanguageModel {anthropic_beta,cx.clone(),);+ let executor = cx.background_executor().clone();request_limiter.stream(async move {let events = stream.await?;let mapper = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID);- Ok(mapper.map_stream(events).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(events).boxed(),+ executor,+ ))}).await});@@ -269,11 +273,15 @@ impl LanguageModel for CopilotChatLanguageModel {is_user_initiated,cx.clone(),);+ let executor = cx.background_executor().clone();request_limiter.stream(async move {let stream = request.await?;let mapper = CopilotResponsesEventMapper::new();- Ok(mapper.map_stream(stream).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(stream).boxed(),+ executor,+ ))}).await});diff --git a/crates/language_model/src/language_model.rs b/crates/language_model/src/language_model.rsindex 316b09c5a53..995b4818e95 100644--- a/crates/language_model/src/language_model.rs+++ b/crates/language_model/src/language_model.rs@@ -8,9 +8,9 @@ pub mod fake_provider;pub use language_model_core::*;use anyhow::Result;-use futures::FutureExt;-use futures::{StreamExt, future::BoxFuture, stream::BoxStream};-use gpui::{AnyView, App, AsyncApp, Task, Window};+use futures::{FutureExt, SinkExt};+use futures::{StreamExt, channel::mpsc, future::BoxFuture, stream::BoxStream};+use gpui::{AnyView, App, AsyncApp, BackgroundExecutor, Task, Window};use icons::IconName;use parking_lot::Mutex;use std::sync::Arc;@@ -22,10 +22,36 @@ pub use crate::registry::*;pub use crate::request::{LanguageModelImageExt, gpui_size_to_image_size, image_size_to_gpui};pub use env_var::{EnvVar, env_var};+const BACKGROUND_STREAM_BUFFER_SIZE: usize = 32;+pub fn init(cx: &mut App) {registry::init(cx);}+pub fn stream_in_background<Output>(+ mut events: BoxStream<'static, Output>,+ executor: BackgroundExecutor,+) -> BoxStream<'static, Output>+where+ Output: Send + 'static,+{+ let (mut sender, receiver) = mpsc::channel(BACKGROUND_STREAM_BUFFER_SIZE);+ let task = executor.spawn(async move {+ while let Some(event) = events.next().await {+ if sender.send(event).await.is_err() {+ return;+ }+ }+ });++ receiver+ .map(move |event| {+ let _task = &task;+ event+ })+ .boxed()+}+#[derive(Debug, Clone, PartialEq, Eq)]pub struct DisabledReason(pub SharedString);diff --git a/crates/language_models/src/provider.rs b/crates/language_models/src/provider.rsindex 65fc039b074..ec3a1bb0c7b 100644--- a/crates/language_models/src/provider.rs+++ b/crates/language_models/src/provider.rs@@ -71,6 +71,20 @@ pub(crate) fn resolve_custom_headers(#[cfg(test)]mod tests {use super::*;+ use futures::{StreamExt, stream};+ use language_model::stream_in_background;++ #[gpui::test]+ async fn test_stream_in_background(cx: &mut gpui::TestAppContext) {+ let events = stream::iter([1, 2, 3])+ .flat_map(|event| stream::iter([event, event * 10]))+ .boxed();+ let mapped_events = stream_in_background(events, cx.background_executor.clone())+ .collect::<Vec<_>>()+ .await;++ assert_eq!(mapped_events, [1, 10, 2, 20, 3, 30]);+ }fn map(pairs: &[(&str, &str)]) -> HashMap<String, String> {pairsdiff --git a/crates/language_models/src/provider/anthropic.rs b/crates/language_models/src/provider/anthropic.rsindex 5cf1504d491..93a2d79c6c5 100644--- a/crates/language_models/src/provider/anthropic.rs+++ b/crates/language_models/src/provider/anthropic.rs@@ -827,9 +827,11 @@ impl LanguageModel for AnthropicModel {request.speed = None;}let request = self.stream_completion(request, cx);+ let executor = cx.background_executor().clone();let future = self.request_limiter.run(async move {let response = request.await?;let stream = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID).map_stream(response);+ let stream = language_model::stream_in_background(stream.boxed(), executor);let (context, usage) = collect_compaction_result(stream.boxed(), PROVIDER_NAME).await?;Ok(CompactionResult { context, usage })});@@ -906,9 +908,14 @@ impl LanguageModel for AnthropicModel {request.speed = None;}let request = self.stream_completion(request, cx);+ let executor = cx.background_executor().clone();let future = self.request_limiter.stream(async move {let response = request.await?;- Ok(AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID).map_stream(response))+ let events = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID).map_stream(response);+ Ok(language_model::stream_in_background(+ events.boxed(),+ executor,+ ))});async move { Ok(future.await?.boxed()) }.boxed()}diff --git a/crates/language_models/src/provider/anthropic_compatible.rs b/crates/language_models/src/provider/anthropic_compatible.rsindex c4970e699c1..0ad754a913b 100644--- a/crates/language_models/src/provider/anthropic_compatible.rs+++ b/crates/language_models/src/provider/anthropic_compatible.rs@@ -457,9 +457,14 @@ impl LanguageModel for AnthropicCompatibleLanguageModel {let completion_request = self.stream_completion(request, cx);let provider_name = self.provider_name.clone();let provider_id = self.provider_id.clone();+ let executor = cx.background_executor().clone();let future = self.request_limiter.stream(async move {let response = completion_request.await?;- Ok(AnthropicEventMapper::new(provider_name, provider_id).map_stream(response))+ let events = AnthropicEventMapper::new(provider_name, provider_id).map_stream(response);+ Ok(language_model::stream_in_background(+ events.boxed(),+ executor,+ ))});async move { Ok(future.await?.boxed()) }.boxed()}diff --git a/crates/language_models/src/provider/bedrock.rs b/crates/language_models/src/provider/bedrock.rsindex e72880f3ae4..d8577fcae14 100644--- a/crates/language_models/src/provider/bedrock.rs+++ b/crates/language_models/src/provider/bedrock.rs@@ -977,6 +977,7 @@ impl LanguageModel for BedrockModel {let request = self.stream_completion(request, cx);let display_name = self.model.display_name().to_string();+ let executor = cx.background_executor().clone();let future = self.request_limiter.stream(async move {let response = request.await.map_err(|err| match err {BedrockError::Validation(ref msg) => {@@ -1043,7 +1044,10 @@ impl LanguageModel for BedrockModel {}other => LanguageModelCompletionError::Other(anyhow!(other)),})?;- let events = map_to_language_model_completion_events(response);+ let events = language_model::stream_in_background(+ map_to_language_model_completion_events(response).boxed(),+ executor,+ );if deny_tool_calls {Ok(deny_tool_use_events(events).boxed())@@ -1945,9 +1949,13 @@ impl LanguageModel for BedrockMantleModel {Err(error) => return async move { Err(error.into()) }.boxed(),};let completions = self.stream_response(request, cx);+ let executor = cx.background_executor().clone();async move {let mapper = MantleResponseEventMapper::new();- Ok(mapper.map_stream(completions.await?).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(completions.await?).boxed(),+ executor,+ ))}.boxed()}@@ -1967,9 +1975,13 @@ impl LanguageModel for BedrockMantleModel {Err(error) => return async move { Err(error.into()) }.boxed(),};let completions = self.stream_completion(request, cx);+ let executor = cx.background_executor().clone();async move {let mapper = OpenAiEventMapper::new();- Ok(mapper.map_stream(completions.await?).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(completions.await?).boxed(),+ executor,+ ))}.boxed()}diff --git a/crates/language_models/src/provider/deepseek.rs b/crates/language_models/src/provider/deepseek.rsindex 2b1472fa6f8..b2365fbbc61 100644--- a/crates/language_models/src/provider/deepseek.rs+++ b/crates/language_models/src/provider/deepseek.rs@@ -348,10 +348,14 @@ impl LanguageModel for DeepSeekLanguageModel {Err(error) => return async move { Err(error.into()) }.boxed(),};let stream = self.stream_completion(request, cx);+ let executor = cx.background_executor().clone();async move {let mapper = DeepSeekEventMapper::new();- Ok(mapper.map_stream(stream.await?).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(stream.await?).boxed(),+ executor,+ ))}.boxed()}diff --git a/crates/language_models/src/provider/mistral.rs b/crates/language_models/src/provider/mistral.rsindex c3fe10aeb49..d447cb12eee 100644--- a/crates/language_models/src/provider/mistral.rs+++ b/crates/language_models/src/provider/mistral.rs@@ -348,11 +348,15 @@ impl LanguageModel for MistralLanguageModel {Err(error) => return async move { Err(error.into()) }.boxed(),};let stream = self.stream_completion(request, affinity, cx);+ let executor = cx.background_executor().clone();async move {let stream = stream.await?;let mapper = MistralEventMapper::new();- Ok(mapper.map_stream(stream).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(stream).boxed(),+ executor,+ ))}.boxed()}diff --git a/crates/language_models/src/provider/open_ai.rs b/crates/language_models/src/provider/open_ai.rsindex 2e5137c4f7b..8c123999afe 100644--- a/crates/language_models/src/provider/open_ai.rs+++ b/crates/language_models/src/provider/open_ai.rs@@ -10,7 +10,7 @@ use language_model::{LanguageModelCompletionEvent, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice, OPEN_AI_PROVIDER_ID,- OPEN_AI_PROVIDER_NAME, ProviderSettingsView, RateLimiter, env_var,+ OPEN_AI_PROVIDER_NAME, ProviderSettingsView, RateLimiter, env_var, stream_in_background,};use open_ai::{ResponseStreamEvent,@@ -643,9 +643,13 @@ impl LanguageModel for OpenAiLanguageModel {Err(error) => return async move { Err(error.into()) }.boxed(),};let completions = self.stream_response(request, cx);+ let executor = cx.background_executor().clone();async move {let mapper = OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID);- Ok(mapper.map_stream(completions.await?).boxed())+ Ok(stream_in_background(+ mapper.map_stream(completions.await?).boxed(),+ executor,+ ))}.boxed()} else {@@ -663,9 +667,13 @@ impl LanguageModel for OpenAiLanguageModel {Err(error) => return async move { Err(error.into()) }.boxed(),};let completions = self.stream_completion(request, cx);+ let executor = cx.background_executor().clone();async move {let mapper = OpenAiEventMapper::new();- Ok(mapper.map_stream(completions.await?).boxed())+ Ok(stream_in_background(+ mapper.map_stream(completions.await?).boxed(),+ executor,+ ))}.boxed()}diff --git a/crates/language_models/src/provider/open_ai_compatible.rs b/crates/language_models/src/provider/open_ai_compatible.rsindex 6e9a30e91b7..f725ba981fe 100644--- a/crates/language_models/src/provider/open_ai_compatible.rs+++ b/crates/language_models/src/provider/open_ai_compatible.rs@@ -438,9 +438,13 @@ impl LanguageModel for OpenAiCompatibleLanguageModel {Err(error) => return async move { Err(error.into()) }.boxed(),};let completions = self.stream_completion(request, cx);+ let executor = cx.background_executor().clone();async move {let mapper = OpenAiEventMapper::new();- Ok(mapper.map_stream(completions.await?).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(completions.await?).boxed(),+ executor,+ ))}.boxed()} else {@@ -460,9 +464,13 @@ impl LanguageModel for OpenAiCompatibleLanguageModel {};let completions = self.stream_response(request, cx);let compaction_state_owner = self.provider_id.clone();+ let executor = cx.background_executor().clone();async move {let mapper = OpenAiResponseEventMapper::new(compaction_state_owner);- Ok(mapper.map_stream(completions.await?).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(completions.await?).boxed(),+ executor,+ ))}.boxed()}diff --git a/crates/language_models/src/provider/open_router.rs b/crates/language_models/src/provider/open_router.rsindex 8b8ce92e771..47bf634d9e9 100644--- a/crates/language_models/src/provider/open_router.rs+++ b/crates/language_models/src/provider/open_router.rs@@ -407,9 +407,14 @@ impl LanguageModel for OpenRouterLanguageModel {Err(error) => return async move { Err(error.into()) }.boxed(),};let request = self.stream_completion(openrouter_request, cx);+ let executor = cx.background_executor().clone();let future = self.request_limiter.stream(async move {let response = request.await?;- Ok(OpenRouterEventMapper::new().map_stream(response))+ let events = OpenRouterEventMapper::new().map_stream(response);+ Ok(language_model::stream_in_background(+ events.boxed(),+ executor,+ ))});async move { Ok(future.await?.boxed()) }.boxed()}diff --git a/crates/language_models/src/provider/opencode.rs b/crates/language_models/src/provider/opencode.rsindex f5281e019f7..ec0c5a0a90b 100644--- a/crates/language_models/src/provider/opencode.rs+++ b/crates/language_models/src/provider/opencode.rs@@ -670,9 +670,13 @@ impl LanguageModel for OpenCodeLanguageModel {};let stream =self.stream_anthropic(anthropic_request, http_client, extra_headers, cx);+ let executor = cx.background_executor().clone();async move {let mapper = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID);- Ok(mapper.map_stream(stream.await?).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(stream.await?).boxed(),+ executor,+ ))}.boxed()}@@ -700,9 +704,13 @@ impl LanguageModel for OpenCodeLanguageModel {};let stream =self.stream_openai_chat(openai_request, http_client, extra_headers, cx);+ let executor = cx.background_executor().clone();async move {let mapper = OpenAiEventMapper::new();- Ok(mapper.map_stream(stream.await?).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(stream.await?).boxed(),+ executor,+ ))}.boxed()}@@ -726,9 +734,13 @@ impl LanguageModel for OpenCodeLanguageModel {};let stream =self.stream_openai_response(response_request, http_client, extra_headers, cx);+ let executor = cx.background_executor().clone();async move {let mapper = OpenAiResponseEventMapper::new(PROVIDER_ID);- Ok(mapper.map_stream(stream.await?).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(stream.await?).boxed(),+ executor,+ ))}.boxed()}diff --git a/crates/language_models/src/provider/vercel_ai_gateway.rs b/crates/language_models/src/provider/vercel_ai_gateway.rsindex d47a9594d61..b13b022aa9d 100644--- a/crates/language_models/src/provider/vercel_ai_gateway.rs+++ b/crates/language_models/src/provider/vercel_ai_gateway.rs@@ -465,9 +465,13 @@ impl LanguageModel for VercelAiGatewayLanguageModel {Err(error) => return async move { Err(error.into()) }.boxed(),};let completions = self.stream_open_ai(request, cx);+ let executor = cx.background_executor().clone();async move {let mapper = crate::provider::open_ai::OpenAiEventMapper::new();- Ok(mapper.map_stream(completions.await?).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(completions.await?).boxed(),+ executor,+ ))}.boxed()}diff --git a/crates/language_models/src/provider/x_ai.rs b/crates/language_models/src/provider/x_ai.rsindex c5f44aab363..e7ba81ef706 100644--- a/crates/language_models/src/provider/x_ai.rs+++ b/crates/language_models/src/provider/x_ai.rs@@ -437,9 +437,13 @@ impl LanguageModel for XAiLanguageModel {Err(error) => return async move { Err(error.into()) }.boxed(),};let completions = self.stream_completion(request, cx);+ let executor = cx.background_executor().clone();async move {let mapper = crate::provider::open_ai::OpenAiEventMapper::new();- Ok(mapper.map_stream(completions.await?).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(completions.await?).boxed(),+ executor,+ ))}.boxed()}diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rsindex c37d1144c82..8c5ce5104b7 100644--- a/crates/language_models_cloud/src/language_models_cloud.rs+++ b/crates/language_models_cloud/src/language_models_cloud.rs@@ -274,6 +274,7 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudLanguageModel<TP> {let http_client = self.http_client.clone();let token_provider = self.token_provider.clone();let auth_context = token_provider.auth_context(cx);+ let executor = cx.background_executor().clone();let future = self.request_limiter.run(async move {let PerformLlmCompletionResponse {response,@@ -305,6 +306,7 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudLanguageModel<TP> {&ANTHROPIC_PROVIDER_NAME,move |event| mapper.map_event(event),);+ let stream = language_model::stream_in_background(stream.boxed(), executor);let (context, usage) =collect_compaction_result(stream, ANTHROPIC_PROVIDER_NAME).await?;Ok(CompactionResult { context, usage })@@ -768,6 +770,7 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<Tlet http_client = self.http_client.clone();let token_provider = self.token_provider.clone();let auth_context = token_provider.auth_context(cx);+ let executor = cx.background_executor().clone();let future = self.request_limiter.stream(async move {let PerformLlmCompletionResponse {response,@@ -794,10 +797,14 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<Tlet mut mapper =AnthropicEventMapper::new(provider_name.clone(), ANTHROPIC_PROVIDER_ID);- Ok(map_cloud_completion_events(+ let events = map_cloud_completion_events(Box::pin(response_lines(response, includes_status_messages)),&provider_name,move |event| mapper.map_event(event),+ );+ Ok(language_model::stream_in_background(+ events.boxed(),+ executor,))});async move { Ok(future.await?.boxed()) }.boxed()@@ -838,6 +845,7 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T}let auth_context = token_provider.auth_context(cx);+ let executor = cx.background_executor().clone();let future = self.request_limiter.stream(async move {let PerformLlmCompletionResponse {response,@@ -863,10 +871,14 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T.await?;let mut mapper = OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID);- Ok(map_cloud_completion_events(+ let events = map_cloud_completion_events(Box::pin(response_lines(response, includes_status_messages)),&provider_name,move |event| mapper.map_event(event),+ );+ Ok(language_model::stream_in_background(+ events.boxed(),+ executor,))});async move { Ok(future.await?.boxed()) }.boxed()@@ -889,6 +901,7 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<TErr(error) => return async move { Err(error.into()) }.boxed(),};let auth_context = token_provider.auth_context(cx);+ let executor = cx.background_executor().clone();let future = self.request_limiter.stream(async move {let PerformLlmCompletionResponse {response,@@ -914,10 +927,14 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T.await?;let mut mapper = OpenAiEventMapper::new();- Ok(map_cloud_completion_events(+ let events = map_cloud_completion_events(Box::pin(response_lines(response, includes_status_messages)),&provider_name,move |event| mapper.map_event(event),+ );+ Ok(language_model::stream_in_background(+ events.boxed(),+ executor,))});async move { Ok(future.await?.boxed()) }.boxed()diff --git a/crates/openai_subscribed/src/openai_subscribed.rs b/crates/openai_subscribed/src/openai_subscribed.rsindex 5cb36d36559..431bab98bd3 100644--- a/crates/openai_subscribed/src/openai_subscribed.rs+++ b/crates/openai_subscribed/src/openai_subscribed.rs@@ -797,7 +797,10 @@ impl LanguageModel for OpenAiSubscribedLanguageModel {}).await?;let mapper = OpenAiResponseEventMapper::new(PROVIDER_ID);- let mut event_stream = mapper.map_stream(response_stream.boxed());+ let mut event_stream = language_model::stream_in_background(+ mapper.map_stream(response_stream.boxed()).boxed(),+ cx.background_executor().clone(),+ );let mut compacted_context = None;let mut usage = language_model::TokenUsage::default();@@ -889,6 +892,7 @@ impl LanguageModel for OpenAiSubscribedLanguageModel {let state = self.state.downgrade();let http_client = self.http_client.clone();let request_limiter = self.request_limiter.clone();+ let executor = cx.background_executor().clone();let future = cx.spawn(async move |cx| {let creds = get_fresh_credentials(&state, &http_client, cx).await?;@@ -914,7 +918,10 @@ impl LanguageModel for OpenAiSubscribedLanguageModel {async move {let mapper = OpenAiResponseEventMapper::new(PROVIDER_ID);- Ok(mapper.map_stream(future.await?.boxed()).boxed())+ Ok(language_model::stream_in_background(+ mapper.map_stream(future.await?.boxed()).boxed(),+ executor,+ ))}.boxed()}