Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
language_models_cloud: Support keepalives during explicit compaction (#66335)
Tom Houlédcf3f98c9a32
diff --git a/crates/cloud_llm_client/src/cloud_llm_client.rs b/crates/cloud_llm_client/src/cloud_llm_client.rsindex 2999d5638f7..3173463b8ff 100644--- a/crates/cloud_llm_client/src/cloud_llm_client.rs+++ b/crates/cloud_llm_client/src/cloud_llm_client.rs@@ -234,6 +234,7 @@ pub enum CompletionRequestStatus {position: usize,},Started,+ KeepAlive,Failed {code: String,message: String,diff --git a/crates/language_model_core/src/language_model_core.rs b/crates/language_model_core/src/language_model_core.rsindex 3e13f1dc51a..7f033a6c0cf 100644--- a/crates/language_model_core/src/language_model_core.rs+++ b/crates/language_model_core/src/language_model_core.rs@@ -92,7 +92,9 @@ impl LanguageModelCompletionEvent {Ok(Some(LanguageModelCompletionEvent::Queued { position }))}CompletionRequestStatus::Started => Ok(Some(LanguageModelCompletionEvent::Started)),- CompletionRequestStatus::Unknown | CompletionRequestStatus::StreamEnded => Ok(None),+ CompletionRequestStatus::KeepAlive+ | CompletionRequestStatus::Unknown+ | CompletionRequestStatus::StreamEnded => Ok(None),CompletionRequestStatus::Failed {code,message,diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rsindex 5c2c99bea46..af77b3c9954 100644--- a/crates/language_models_cloud/src/language_models_cloud.rs+++ b/crates/language_models_cloud/src/language_models_cloud.rs@@ -209,6 +209,11 @@ pub struct PerformLlmCompletionResponse {pub includes_status_messages: bool,}+enum StatusMessages {+ Request,+ Omit,+}+impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {pub async fn perform_llm_completion(http_client: &HttpClientWithUrl,@@ -219,7 +224,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {) -> Result<PerformLlmCompletionResponse, LanguageModelCompletionError> {Self::perform_llm_request("/completions",- true,+ StatusMessages::Request,http_client,token_provider,auth_context,@@ -238,7 +243,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {) -> Result<PerformLlmCompletionResponse, LanguageModelCompletionError> {Self::perform_llm_request("/completions/compact",- false,+ StatusMessages::Request,http_client,token_provider,auth_context,@@ -250,7 +255,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {async fn perform_llm_request(path: &str,- request_status_messages: bool,+ status_messages: StatusMessages,http_client: &HttpClientWithUrl,token_provider: &TP,auth_context: TP::AuthContext,@@ -277,7 +282,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {}).header("Content-Type", "application/json").header("Authorization", format!("Bearer {token}"));- if request_status_messages {+ if matches!(status_messages, StatusMessages::Request) {request = request.header(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true").header(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME, "true");@@ -289,7 +294,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {let status = response.status();if status.is_success() {- let includes_status_messages = request_status_messages+ let includes_status_messages = matches!(status_messages, StatusMessages::Request)&& response.headers().get(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME)@@ -859,6 +864,20 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudModelProvider<TP> {.map_err(LanguageModelCompletionError::Other)?;return Ok(CompactionResult { context, usage });}+ CompletionEvent::Status(CompletionRequestStatus::StreamEnded) => break,+ CompletionEvent::Status(CompletionRequestStatus::Failed {+ code,+ message,+ retry_after,+ ..+ }) => {+ return Err(LanguageModelCompletionError::from_cloud_failure(+ provider_name,+ code,+ message,+ retry_after.map(Duration::from_secs_f64),+ ));+ }CompletionEvent::Status(_) => {}}}@@ -955,7 +974,7 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudModelProvider<TP> {.run(async move {let mut response = Self::perform_llm_request("/count_tokens",- false,+ StatusMessages::Omit,&http_client,&*token_provider,auth_context,@@ -1372,6 +1391,7 @@ pub fn response_lines<T: DeserializeOwned>(#[cfg(test)]mod tests {use super::*;+ use futures::TryStreamExt as _;use http_client::FakeHttpClient;use http_client::http::{HeaderMap, StatusCode};use language_model::{@@ -1393,6 +1413,7 @@ mod tests {json!({"event": {"candidates": [{"content": {"role": "model", "parts": [{"text": "Hello"}]}}]}}),+ json!({"status": CompletionRequestStatus::KeepAlive}),json!({"event": {"candidates": [{"content": {"role": "model", "parts": [{"text": " world"}]},@@ -1435,12 +1456,15 @@ mod tests {cx: &mut gpui::TestAppContext,) {for stream_ended in [false, true] {- let mut events = vec![json!({"event": {"candidates": [{- "content": {"role": "model", "parts": [{- "functionCall": {"name": "list_directory", "args": {"path": "."}}- }]},- "finishReason": "STOP"- }]}})];+ let mut events = vec![+ json!({"event": {"candidates": [{+ "content": {"role": "model", "parts": [{+ "functionCall": {"name": "list_directory", "args": {"path": "."}}+ }]},+ "finishReason": "STOP"+ }]}}),+ json!({"status": CompletionRequestStatus::KeepAlive}),+ ];if stream_ended {events.push(json!({"status": "stream_ended"}));}@@ -1510,118 +1534,222 @@ mod tests {async fn cloud_explicit_compaction_forwards_supported_request_fields(cx: &mut gpui::TestAppContext,) {- let captured_request = Arc::new(Mutex::new(None));- let captured_request_for_handler = captured_request.clone();- let http_client = FakeHttpClient::create(move |request| {- let captured_request = captured_request_for_handler.clone();- async move {- let method = request.method().clone();- let uri = request.uri().to_string();- let authorization = request- .headers()- .get("Authorization")- .and_then(|value| value.to_str().ok())- .map(str::to_string);- let requested_status_messages = request- .headers()- .contains_key(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME);- let requested_stream_end = request- .headers()- .contains_key(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME);- let mut body = request.into_body();- let mut body_text = String::new();- body.read_to_string(&mut body_text).await?;- *captured_request.lock().unwrap() = Some((- method,- uri,- authorization,- requested_status_messages,- requested_stream_end,- body_text,- ));-- Ok(http_client::Response::builder()- .status(200)- .body(AsyncBody::from(format!(- "{}\n",- json!({- "id": "resp_compact",- "created_at": 1_700_000_000,- "object": "response.compaction",- "output": [{- "type": "compaction",- "id": "cmp_manual",- "encrypted_content": "opaque-state"- }],- "usage": {- "input_tokens": 100,- "input_tokens_details": {"cached_tokens": 20},- "output_tokens": 10,- "output_tokens_details": {"reasoning_tokens": 5},- "total_tokens": 110- }- })- )))?)- }- });let language_model = language_model(&cloud_test_config());assert!(language_model.supports_explicit_compaction());assert!(!language_model.supports_explicit_compaction_output_limit());++ for includes_status_messages in [false, true] {+ let captured_request = Arc::new(Mutex::new(None));+ let captured_request_for_handler = captured_request.clone();+ let http_client = FakeHttpClient::create(move |request| {+ let captured_request = captured_request_for_handler.clone();+ async move {+ let method = request.method().clone();+ let uri = request.uri().to_string();+ let authorization = request+ .headers()+ .get("Authorization")+ .and_then(|value| value.to_str().ok())+ .map(str::to_string);+ let requested_status_messages = request+ .headers()+ .contains_key(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME);+ let requested_stream_end = request+ .headers()+ .contains_key(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME);+ let mut body = request.into_body();+ let mut body_text = String::new();+ body.read_to_string(&mut body_text).await?;+ *captured_request.lock().unwrap() = Some((+ method,+ uri,+ authorization,+ requested_status_messages,+ requested_stream_end,+ body_text,+ ));++ let result = json!({+ "id": "resp_compact",+ "created_at": 1_700_000_000,+ "object": "response.compaction",+ "output": [{+ "type": "compaction",+ "id": "cmp_manual",+ "encrypted_content": "opaque-state"+ }],+ "usage": {+ "input_tokens": 100,+ "input_tokens_details": {"cached_tokens": 20},+ "output_tokens": 10,+ "output_tokens_details": {"reasoning_tokens": 5},+ "total_tokens": 110+ }+ });+ let mut response = http_client::Response::builder().status(200);+ let body = if includes_status_messages {+ response =+ response.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true");+ [+ json!({"status": "keep_alive"}),+ json!({"status": "future_transport_status"}),+ json!({"event": result}),+ json!({"status": "stream_ended"}),+ ]+ .into_iter()+ .map(|event| format!("{event}\n"))+ .collect::<String>()+ } else {+ format!("{result}\n")+ };+ Ok(response.body(AsyncBody::from(body))?)+ }+ });+ let (provider, model) = cloud_test_model(http_client);+ let request = compact_test_request();++ let result = cx+ .update(|cx| provider.compact(&model, request, cx))+ .await+ .unwrap();++ assert_eq!(+ result.usage,+ language_model::TokenUsage {+ input_tokens: 80,+ output_tokens: 10,+ cache_creation_input_tokens: 0,+ cache_read_input_tokens: 20,+ }+ );+ let language_model::CompactedContext::ProviderState(state) = result.context else {+ panic!("expected provider compaction state");+ };+ assert_eq!(+ open_ai::responses::provider_compaction_items(&state, &OPEN_AI_PROVIDER_ID)+ .unwrap(),+ Some(vec![json!({+ "type": "compaction",+ "id": "cmp_manual",+ "encrypted_content": "opaque-state"+ })])+ );+ let (method, uri, authorization, requested_status_messages, requested_stream_end, body) =+ captured_request.lock().unwrap().take().unwrap();+ assert_eq!(method, Method::POST);+ assert_eq!(uri, "http://test.example/completions/compact?");+ assert_eq!(authorization.as_deref(), Some("Bearer test-token"));+ assert!(requested_status_messages);+ assert!(requested_stream_end);+ let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();+ assert_eq!(body["thread_id"], "thread-123");+ assert_eq!(body["provider"], "open_ai");+ assert_eq!(body["model"], "gpt-5.4");+ assert_eq!(+ body["provider_request"],+ json!({+ "model": "gpt-5.4",+ "input": [{+ "type": "message",+ "role": "user",+ "content": [{+ "type": "input_text",+ "text": "Retain this context."+ }]+ }],+ "prompt_cache_key": "thread-123",+ "service_tier": "priority"+ })+ );+ }+ }++ #[gpui::test]+ async fn cloud_explicit_compaction_preserves_streamed_failure(cx: &mut gpui::TestAppContext) {+ let http_client = FakeHttpClient::create(|_| async move {+ Ok(Response::builder()+ .status(200)+ .header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")+ .body(AsyncBody::from(format!(+ "{}\n{}\n",+ json!({"status": "keep_alive"}),+ json!({"status": {"failed": {+ "code": "upstream_http_429",+ "message": "Compaction rate limit exceeded.",+ "request_id": "00000000-0000-0000-0000-000000000000",+ "retry_after": 1.5+ }}}),+ )))?)+ });let (provider, model) = cloud_test_model(http_client);- let request = compact_test_request();- let result = cx- .update(|cx| provider.compact(&model, request, cx))+ let error = cx+ .update(|cx| provider.compact(&model, compact_test_request(), cx)).await- .unwrap();+ .unwrap_err();- assert_eq!(- result.usage,- language_model::TokenUsage {- input_tokens: 80,- output_tokens: 10,- cache_creation_input_tokens: 0,- cache_read_input_tokens: 20,- }- );- let language_model::CompactedContext::ProviderState(state) = result.context else {- panic!("expected provider compaction state");+ let LanguageModelCompletionError::ProviderRejection {+ provider,+ status,+ code,+ message,+ retry_after,+ category,+ } = error+ else {+ panic!("expected the streamed provider rejection, got {error:?}");};- assert_eq!(- open_ai::responses::provider_compaction_items(&state, &OPEN_AI_PROVIDER_ID).unwrap(),- Some(vec![json!({- "type": "compaction",- "id": "cmp_manual",- "encrypted_content": "opaque-state"- })])- );- let (method, uri, authorization, requested_status_messages, requested_stream_end, body) =- captured_request.lock().unwrap().take().unwrap();- assert_eq!(method, Method::POST);- assert_eq!(uri, "http://test.example/completions/compact?");- assert_eq!(authorization.as_deref(), Some("Bearer test-token"));- assert!(!requested_status_messages);- assert!(!requested_stream_end);- let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();- assert_eq!(body["thread_id"], "thread-123");- assert_eq!(body["provider"], "open_ai");- assert_eq!(body["model"], "gpt-5.4");- assert_eq!(- body["provider_request"],- json!({- "model": "gpt-5.4",- "input": [{- "type": "message",- "role": "user",- "content": [{- "type": "input_text",- "text": "Retain this context."- }]- }],- "prompt_cache_key": "thread-123",- "service_tier": "priority"- })- );+ assert_eq!(provider, OPEN_AI_PROVIDER_NAME);+ assert_eq!(status, Some(StatusCode::TOO_MANY_REQUESTS));+ assert_eq!(code.as_deref(), Some("upstream_http_429"));+ assert_eq!(message, "Compaction rate limit exceeded.");+ assert_eq!(retry_after, Some(Duration::from_secs_f64(1.5)));+ assert_eq!(category, ProviderErrorCategory::RateLimit);+ }++ #[gpui::test]+ async fn cloud_explicit_compaction_rejects_stream_ending_without_result(+ cx: &mut gpui::TestAppContext,+ ) {+ for stream_ended in [false, true] {+ let http_client = FakeHttpClient::create(move |_| async move {+ let mut body = format!("{}\n", json!({"status": "keep_alive"}));+ if stream_ended {+ body.push_str(&format!("{}\n", json!({"status": "stream_ended"})));+ }+ let body = stream::iter([Ok::<_, std::io::Error>(body.into_bytes())])+ .chain(stream::poll_fn(move |_| {+ if stream_ended {+ Poll::Pending+ } else {+ Poll::Ready(None)+ }+ }))+ .into_async_read();+ Ok(Response::builder()+ .status(200)+ .header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")+ .body(AsyncBody::from_reader(body))?)+ });+ let (provider, model) = cloud_test_model(http_client);++ let error = cx+ .update(|cx| provider.compact(&model, compact_test_request(), cx))+ .now_or_never()+ .expect(+ "compaction should stop at EOF or StreamEnded without waiting for more bytes",+ )+ .unwrap_err();++ assert!(+ matches!(+ &error,+ LanguageModelCompletionError::StreamEndedUnexpectedly { provider }+ if provider == &OPEN_AI_PROVIDER_NAME+ ),+ "expected a missing compaction result error, got {error:?}"+ );+ }}#[gpui::test]