Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

language_models_cloud: Support keepalives during explicit compaction (#66335)

Tom Houlédcf3f98c9a32
diff --git a/crates/cloud_llm_client/src/cloud_llm_client.rs b/crates/cloud_llm_client/src/cloud_llm_client.rs
index 2999d5638f7..3173463b8ff 100644
--- a/crates/cloud_llm_client/src/cloud_llm_client.rs
+++ b/crates/cloud_llm_client/src/cloud_llm_client.rs
@@ -234,6 +234,7 @@ pub enum CompletionRequestStatus {
position: usize,
},
Started,
+ KeepAlive,
Failed {
code: String,
message: String,
diff --git a/crates/language_model_core/src/language_model_core.rs b/crates/language_model_core/src/language_model_core.rs
index 3e13f1dc51a..7f033a6c0cf 100644
--- a/crates/language_model_core/src/language_model_core.rs
+++ b/crates/language_model_core/src/language_model_core.rs
@@ -92,7 +92,9 @@ impl LanguageModelCompletionEvent {
Ok(Some(LanguageModelCompletionEvent::Queued { position }))
}
CompletionRequestStatus::Started => Ok(Some(LanguageModelCompletionEvent::Started)),
- CompletionRequestStatus::Unknown | CompletionRequestStatus::StreamEnded => Ok(None),
+ CompletionRequestStatus::KeepAlive
+ | CompletionRequestStatus::Unknown
+ | CompletionRequestStatus::StreamEnded => Ok(None),
CompletionRequestStatus::Failed {
code,
message,
diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rs
index 5c2c99bea46..af77b3c9954 100644
--- a/crates/language_models_cloud/src/language_models_cloud.rs
+++ b/crates/language_models_cloud/src/language_models_cloud.rs
@@ -209,6 +209,11 @@ pub struct PerformLlmCompletionResponse {
pub includes_status_messages: bool,
}
+enum StatusMessages {
+ Request,
+ Omit,
+}
+
impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
pub async fn perform_llm_completion(
http_client: &HttpClientWithUrl,
@@ -219,7 +224,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
) -> Result<PerformLlmCompletionResponse, LanguageModelCompletionError> {
Self::perform_llm_request(
"/completions",
- true,
+ StatusMessages::Request,
http_client,
token_provider,
auth_context,
@@ -238,7 +243,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
) -> Result<PerformLlmCompletionResponse, LanguageModelCompletionError> {
Self::perform_llm_request(
"/completions/compact",
- false,
+ StatusMessages::Request,
http_client,
token_provider,
auth_context,
@@ -250,7 +255,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
async fn perform_llm_request(
path: &str,
- request_status_messages: bool,
+ status_messages: StatusMessages,
http_client: &HttpClientWithUrl,
token_provider: &TP,
auth_context: TP::AuthContext,
@@ -277,7 +282,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
})
.header("Content-Type", "application/json")
.header("Authorization", format!("Bearer {token}"));
- if request_status_messages {
+ if matches!(status_messages, StatusMessages::Request) {
request = request
.header(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
.header(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME, "true");
@@ -289,7 +294,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
let status = response.status();
if status.is_success() {
- let includes_status_messages = request_status_messages
+ let includes_status_messages = matches!(status_messages, StatusMessages::Request)
&& response
.headers()
.get(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME)
@@ -859,6 +864,20 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudModelProvider<TP> {
.map_err(LanguageModelCompletionError::Other)?;
return Ok(CompactionResult { context, usage });
}
+ CompletionEvent::Status(CompletionRequestStatus::StreamEnded) => break,
+ CompletionEvent::Status(CompletionRequestStatus::Failed {
+ code,
+ message,
+ retry_after,
+ ..
+ }) => {
+ return Err(LanguageModelCompletionError::from_cloud_failure(
+ provider_name,
+ code,
+ message,
+ retry_after.map(Duration::from_secs_f64),
+ ));
+ }
CompletionEvent::Status(_) => {}
}
}
@@ -955,7 +974,7 @@ impl<TP: CloudLlmTokenProvider + 'static> CloudModelProvider<TP> {
.run(async move {
let mut response = Self::perform_llm_request(
"/count_tokens",
- false,
+ StatusMessages::Omit,
&http_client,
&*token_provider,
auth_context,
@@ -1372,6 +1391,7 @@ pub fn response_lines<T: DeserializeOwned>(
#[cfg(test)]
mod tests {
use super::*;
+ use futures::TryStreamExt as _;
use http_client::FakeHttpClient;
use http_client::http::{HeaderMap, StatusCode};
use language_model::{
@@ -1393,6 +1413,7 @@ mod tests {
json!({"event": {"candidates": [{
"content": {"role": "model", "parts": [{"text": "Hello"}]}
}]}}),
+ json!({"status": CompletionRequestStatus::KeepAlive}),
json!({"event": {
"candidates": [{
"content": {"role": "model", "parts": [{"text": " world"}]},
@@ -1435,12 +1456,15 @@ mod tests {
cx: &mut gpui::TestAppContext,
) {
for stream_ended in [false, true] {
- let mut events = vec![json!({"event": {"candidates": [{
- "content": {"role": "model", "parts": [{
- "functionCall": {"name": "list_directory", "args": {"path": "."}}
- }]},
- "finishReason": "STOP"
- }]}})];
+ let mut events = vec![
+ json!({"event": {"candidates": [{
+ "content": {"role": "model", "parts": [{
+ "functionCall": {"name": "list_directory", "args": {"path": "."}}
+ }]},
+ "finishReason": "STOP"
+ }]}}),
+ json!({"status": CompletionRequestStatus::KeepAlive}),
+ ];
if stream_ended {
events.push(json!({"status": "stream_ended"}));
}
@@ -1510,118 +1534,222 @@ mod tests {
async fn cloud_explicit_compaction_forwards_supported_request_fields(
cx: &mut gpui::TestAppContext,
) {
- let captured_request = Arc::new(Mutex::new(None));
- let captured_request_for_handler = captured_request.clone();
- let http_client = FakeHttpClient::create(move |request| {
- let captured_request = captured_request_for_handler.clone();
- async move {
- let method = request.method().clone();
- let uri = request.uri().to_string();
- let authorization = request
- .headers()
- .get("Authorization")
- .and_then(|value| value.to_str().ok())
- .map(str::to_string);
- let requested_status_messages = request
- .headers()
- .contains_key(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME);
- let requested_stream_end = request
- .headers()
- .contains_key(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME);
- let mut body = request.into_body();
- let mut body_text = String::new();
- body.read_to_string(&mut body_text).await?;
- *captured_request.lock().unwrap() = Some((
- method,
- uri,
- authorization,
- requested_status_messages,
- requested_stream_end,
- body_text,
- ));
-
- Ok(http_client::Response::builder()
- .status(200)
- .body(AsyncBody::from(format!(
- "{}\n",
- json!({
- "id": "resp_compact",
- "created_at": 1_700_000_000,
- "object": "response.compaction",
- "output": [{
- "type": "compaction",
- "id": "cmp_manual",
- "encrypted_content": "opaque-state"
- }],
- "usage": {
- "input_tokens": 100,
- "input_tokens_details": {"cached_tokens": 20},
- "output_tokens": 10,
- "output_tokens_details": {"reasoning_tokens": 5},
- "total_tokens": 110
- }
- })
- )))?)
- }
- });
let language_model = language_model(&cloud_test_config());
assert!(language_model.supports_explicit_compaction());
assert!(!language_model.supports_explicit_compaction_output_limit());
+
+ for includes_status_messages in [false, true] {
+ let captured_request = Arc::new(Mutex::new(None));
+ let captured_request_for_handler = captured_request.clone();
+ let http_client = FakeHttpClient::create(move |request| {
+ let captured_request = captured_request_for_handler.clone();
+ async move {
+ let method = request.method().clone();
+ let uri = request.uri().to_string();
+ let authorization = request
+ .headers()
+ .get("Authorization")
+ .and_then(|value| value.to_str().ok())
+ .map(str::to_string);
+ let requested_status_messages = request
+ .headers()
+ .contains_key(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME);
+ let requested_stream_end = request
+ .headers()
+ .contains_key(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME);
+ let mut body = request.into_body();
+ let mut body_text = String::new();
+ body.read_to_string(&mut body_text).await?;
+ *captured_request.lock().unwrap() = Some((
+ method,
+ uri,
+ authorization,
+ requested_status_messages,
+ requested_stream_end,
+ body_text,
+ ));
+
+ let result = json!({
+ "id": "resp_compact",
+ "created_at": 1_700_000_000,
+ "object": "response.compaction",
+ "output": [{
+ "type": "compaction",
+ "id": "cmp_manual",
+ "encrypted_content": "opaque-state"
+ }],
+ "usage": {
+ "input_tokens": 100,
+ "input_tokens_details": {"cached_tokens": 20},
+ "output_tokens": 10,
+ "output_tokens_details": {"reasoning_tokens": 5},
+ "total_tokens": 110
+ }
+ });
+ let mut response = http_client::Response::builder().status(200);
+ let body = if includes_status_messages {
+ response =
+ response.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true");
+ [
+ json!({"status": "keep_alive"}),
+ json!({"status": "future_transport_status"}),
+ json!({"event": result}),
+ json!({"status": "stream_ended"}),
+ ]
+ .into_iter()
+ .map(|event| format!("{event}\n"))
+ .collect::<String>()
+ } else {
+ format!("{result}\n")
+ };
+ Ok(response.body(AsyncBody::from(body))?)
+ }
+ });
+ let (provider, model) = cloud_test_model(http_client);
+ let request = compact_test_request();
+
+ let result = cx
+ .update(|cx| provider.compact(&model, request, cx))
+ .await
+ .unwrap();
+
+ assert_eq!(
+ result.usage,
+ language_model::TokenUsage {
+ input_tokens: 80,
+ output_tokens: 10,
+ cache_creation_input_tokens: 0,
+ cache_read_input_tokens: 20,
+ }
+ );
+ let language_model::CompactedContext::ProviderState(state) = result.context else {
+ panic!("expected provider compaction state");
+ };
+ assert_eq!(
+ open_ai::responses::provider_compaction_items(&state, &OPEN_AI_PROVIDER_ID)
+ .unwrap(),
+ Some(vec![json!({
+ "type": "compaction",
+ "id": "cmp_manual",
+ "encrypted_content": "opaque-state"
+ })])
+ );
+ let (method, uri, authorization, requested_status_messages, requested_stream_end, body) =
+ captured_request.lock().unwrap().take().unwrap();
+ assert_eq!(method, Method::POST);
+ assert_eq!(uri, "http://test.example/completions/compact?");
+ assert_eq!(authorization.as_deref(), Some("Bearer test-token"));
+ assert!(requested_status_messages);
+ assert!(requested_stream_end);
+ let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();
+ assert_eq!(body["thread_id"], "thread-123");
+ assert_eq!(body["provider"], "open_ai");
+ assert_eq!(body["model"], "gpt-5.4");
+ assert_eq!(
+ body["provider_request"],
+ json!({
+ "model": "gpt-5.4",
+ "input": [{
+ "type": "message",
+ "role": "user",
+ "content": [{
+ "type": "input_text",
+ "text": "Retain this context."
+ }]
+ }],
+ "prompt_cache_key": "thread-123",
+ "service_tier": "priority"
+ })
+ );
+ }
+ }
+
+ #[gpui::test]
+ async fn cloud_explicit_compaction_preserves_streamed_failure(cx: &mut gpui::TestAppContext) {
+ let http_client = FakeHttpClient::create(|_| async move {
+ Ok(Response::builder()
+ .status(200)
+ .header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
+ .body(AsyncBody::from(format!(
+ "{}\n{}\n",
+ json!({"status": "keep_alive"}),
+ json!({"status": {"failed": {
+ "code": "upstream_http_429",
+ "message": "Compaction rate limit exceeded.",
+ "request_id": "00000000-0000-0000-0000-000000000000",
+ "retry_after": 1.5
+ }}}),
+ )))?)
+ });
let (provider, model) = cloud_test_model(http_client);
- let request = compact_test_request();
- let result = cx
- .update(|cx| provider.compact(&model, request, cx))
+ let error = cx
+ .update(|cx| provider.compact(&model, compact_test_request(), cx))
.await
- .unwrap();
+ .unwrap_err();
- assert_eq!(
- result.usage,
- language_model::TokenUsage {
- input_tokens: 80,
- output_tokens: 10,
- cache_creation_input_tokens: 0,
- cache_read_input_tokens: 20,
- }
- );
- let language_model::CompactedContext::ProviderState(state) = result.context else {
- panic!("expected provider compaction state");
+ let LanguageModelCompletionError::ProviderRejection {
+ provider,
+ status,
+ code,
+ message,
+ retry_after,
+ category,
+ } = error
+ else {
+ panic!("expected the streamed provider rejection, got {error:?}");
};
- assert_eq!(
- open_ai::responses::provider_compaction_items(&state, &OPEN_AI_PROVIDER_ID).unwrap(),
- Some(vec![json!({
- "type": "compaction",
- "id": "cmp_manual",
- "encrypted_content": "opaque-state"
- })])
- );
- let (method, uri, authorization, requested_status_messages, requested_stream_end, body) =
- captured_request.lock().unwrap().take().unwrap();
- assert_eq!(method, Method::POST);
- assert_eq!(uri, "http://test.example/completions/compact?");
- assert_eq!(authorization.as_deref(), Some("Bearer test-token"));
- assert!(!requested_status_messages);
- assert!(!requested_stream_end);
- let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();
- assert_eq!(body["thread_id"], "thread-123");
- assert_eq!(body["provider"], "open_ai");
- assert_eq!(body["model"], "gpt-5.4");
- assert_eq!(
- body["provider_request"],
- json!({
- "model": "gpt-5.4",
- "input": [{
- "type": "message",
- "role": "user",
- "content": [{
- "type": "input_text",
- "text": "Retain this context."
- }]
- }],
- "prompt_cache_key": "thread-123",
- "service_tier": "priority"
- })
- );
+ assert_eq!(provider, OPEN_AI_PROVIDER_NAME);
+ assert_eq!(status, Some(StatusCode::TOO_MANY_REQUESTS));
+ assert_eq!(code.as_deref(), Some("upstream_http_429"));
+ assert_eq!(message, "Compaction rate limit exceeded.");
+ assert_eq!(retry_after, Some(Duration::from_secs_f64(1.5)));
+ assert_eq!(category, ProviderErrorCategory::RateLimit);
+ }
+
+ #[gpui::test]
+ async fn cloud_explicit_compaction_rejects_stream_ending_without_result(
+ cx: &mut gpui::TestAppContext,
+ ) {
+ for stream_ended in [false, true] {
+ let http_client = FakeHttpClient::create(move |_| async move {
+ let mut body = format!("{}\n", json!({"status": "keep_alive"}));
+ if stream_ended {
+ body.push_str(&format!("{}\n", json!({"status": "stream_ended"})));
+ }
+ let body = stream::iter([Ok::<_, std::io::Error>(body.into_bytes())])
+ .chain(stream::poll_fn(move |_| {
+ if stream_ended {
+ Poll::Pending
+ } else {
+ Poll::Ready(None)
+ }
+ }))
+ .into_async_read();
+ Ok(Response::builder()
+ .status(200)
+ .header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
+ .body(AsyncBody::from_reader(body))?)
+ });
+ let (provider, model) = cloud_test_model(http_client);
+
+ let error = cx
+ .update(|cx| provider.compact(&model, compact_test_request(), cx))
+ .now_or_never()
+ .expect(
+ "compaction should stop at EOF or StreamEnded without waiting for more bytes",
+ )
+ .unwrap_err();
+
+ assert!(
+ matches!(
+ &error,
+ LanguageModelCompletionError::StreamEndedUnexpectedly { provider }
+ if provider == &OPEN_AI_PROVIDER_NAME
+ ),
+ "expected a missing compaction result error, got {error:?}"
+ );
+ }
}
#[gpui::test]