Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

agent: Treat context window overflows as prompt-too-large errors (#60525)

Lukas Wirthd7c922c4a370
diff --git a/crates/agent/src/tests/mod.rs b/crates/agent/src/tests/mod.rs
index 73415f55af4..df466b88051 100644
--- a/crates/agent/src/tests/mod.rs
+++ b/crates/agent/src/tests/mod.rs
@@ -3359,6 +3359,82 @@ async fn test_latest_token_usage_counts_cached_input_tokens(cx: &mut TestAppCont
});
}
+#[gpui::test]
+async fn test_prompt_too_large_marks_token_usage_exceeded(cx: &mut TestAppContext) {
+ let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
+ let fake_model = model.as_fake();
+
+ thread
+ .update(cx, |thread, cx| {
+ thread.send(ClientUserMessageId::new(), ["Message 1"], cx)
+ })
+ .unwrap();
+ cx.run_until_parked();
+
+ fake_model.send_last_completion_stream_text_chunk("Response 1");
+ fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
+ language_model::TokenUsage {
+ input_tokens: 100,
+ output_tokens: 50,
+ ..Default::default()
+ },
+ ));
+ fake_model.end_last_completion_stream();
+ cx.run_until_parked();
+
+ thread.read_with(cx, |thread, _| {
+ assert_eq!(
+ thread.latest_token_usage().unwrap().ratio(),
+ acp_thread::TokenUsageRatio::Normal
+ );
+ });
+
+ thread
+ .update(cx, |thread, cx| {
+ thread.send(ClientUserMessageId::new(), ["Message 2"], cx)
+ })
+ .unwrap();
+ cx.run_until_parked();
+
+ fake_model.send_last_completion_stream_error(LanguageModelCompletionError::PromptTooLarge {
+ tokens: None,
+ });
+ fake_model.end_last_completion_stream();
+ cx.run_until_parked();
+
+ thread.read_with(cx, |thread, _| {
+ let usage = thread.latest_token_usage().unwrap();
+ assert_eq!(usage.used_tokens, 1_000_000);
+ assert_eq!(usage.max_tokens, 1_000_000);
+ assert_eq!(usage.ratio(), acp_thread::TokenUsageRatio::Exceeded);
+ });
+}
+
+#[gpui::test]
+async fn test_prompt_too_large_uses_reported_token_count(cx: &mut TestAppContext) {
+ let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
+ let fake_model = model.as_fake();
+
+ thread
+ .update(cx, |thread, cx| {
+ thread.send(ClientUserMessageId::new(), ["Message 1"], cx)
+ })
+ .unwrap();
+ cx.run_until_parked();
+
+ fake_model.send_last_completion_stream_error(LanguageModelCompletionError::PromptTooLarge {
+ tokens: Some(1_500_000),
+ });
+ fake_model.end_last_completion_stream();
+ cx.run_until_parked();
+
+ thread.read_with(cx, |thread, _| {
+ let usage = thread.latest_token_usage().unwrap();
+ assert_eq!(usage.used_tokens, 1_500_000);
+ assert_eq!(usage.ratio(), acp_thread::TokenUsageRatio::Exceeded);
+ });
+}
+
#[gpui::test]
async fn test_cumulative_token_usage(cx: &mut TestAppContext) {
let ThreadTest {
diff --git a/crates/agent/src/thread.rs b/crates/agent/src/thread.rs
index fe5c9f2004c..8dd22ec3feb 100644
--- a/crates/agent/src/thread.rs
+++ b/crates/agent/src/thread.rs
@@ -2314,6 +2314,31 @@ impl Thread {
cx.notify();
}
+ /// Records that the last request overflowed the model's context window so
+ /// the token usage indicator reports `Exceeded` instead of the stale usage
+ /// from the last successful request. Providers don't report usage for
+ /// failed requests, so we synthesize one from the reported overflow (when
+ /// available) or the model's context size.
+ fn mark_token_limit_exceeded(&mut self, tokens: Option<u64>, cx: &mut Context<Self>) {
+ let Some(model) = self.model() else {
+ return;
+ };
+ let input_tokens = tokens.unwrap_or(0).max(model.max_token_count());
+ let Some(last_user_message) = self.last_user_message() else {
+ return;
+ };
+
+ self.request_token_usage.insert(
+ last_user_message.id.clone(),
+ language_model::TokenUsage {
+ input_tokens,
+ ..Default::default()
+ },
+ );
+ cx.emit(TokenUsageUpdated(self.latest_token_usage()));
+ cx.notify();
+ }
+
pub fn truncate(
&mut self,
client_user_message_id: ClientUserMessageId,
@@ -3038,7 +3063,7 @@ impl Thread {
) -> Result<ControlFlow<()>> {
let retry = this.update(cx, |this, cx| {
let user_store = this.user_store.read(cx);
- this.handle_completion_error(error, attempt, user_store.plan())
+ this.handle_completion_error(error, attempt, user_store.plan(), cx)
})??;
let timer = cx.background_executor().timer(retry.duration);
event_stream.send_retry(retry);
@@ -3228,7 +3253,12 @@ impl Thread {
error: LanguageModelCompletionError,
attempt: u8,
plan: Option<Plan>,
+ cx: &mut Context<Self>,
) -> Result<acp_thread::RetryStatus> {
+ if let LanguageModelCompletionError::PromptTooLarge { tokens } = &error {
+ self.mark_token_limit_exceeded(*tokens, cx);
+ }
+
let Some(model) = self.model() else {
return Err(anyhow!(error));
};
diff --git a/crates/language_model_core/src/language_model_core.rs b/crates/language_model_core/src/language_model_core.rs
index 991046eab08..36b76cf7246 100644
--- a/crates/language_model_core/src/language_model_core.rs
+++ b/crates/language_model_core/src/language_model_core.rs
@@ -25,7 +25,10 @@ pub use crate::rate_limiter::*;
pub use crate::request::*;
pub use crate::role::*;
pub use crate::tool_schema::LanguageModelToolSchemaFormat;
-pub use crate::util::{fix_streamed_json, parse_prompt_too_long, parse_tool_arguments};
+pub use crate::util::{
+ fix_streamed_json, is_context_window_exceeded_message, parse_prompt_too_long,
+ parse_tool_arguments,
+};
pub use gpui_shared_string::SharedString;
/// A completion event from a language model.
@@ -241,7 +244,13 @@ impl LanguageModelCompletionError {
retry_after: Option<Duration>,
) -> Self {
match status_code {
- StatusCode::BAD_REQUEST => Self::BadRequestFormat { provider, message },
+ StatusCode::BAD_REQUEST => {
+ if is_context_window_exceeded_message(&message) {
+ Self::PromptTooLarge { tokens: None }
+ } else {
+ Self::BadRequestFormat { provider, message }
+ }
+ }
StatusCode::UNAUTHORIZED => Self::AuthenticationError { provider, message },
StatusCode::FORBIDDEN => Self::PermissionError { provider, message },
StatusCode::NOT_FOUND => Self::ApiEndpointNotFound { provider },
@@ -646,6 +655,33 @@ mod tests {
}
}
+ #[test]
+ fn test_from_http_status_maps_context_length_exceeded_to_prompt_too_large() {
+ let error = LanguageModelCompletionError::from_http_status(
+ String::from("OpenAI").into(),
+ StatusCode::BAD_REQUEST,
+ r#"{"error":{"type":"invalid_request_error","code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try again.","param":"input"}}"#.to_string(),
+ None,
+ );
+
+ assert!(matches!(
+ error,
+ LanguageModelCompletionError::PromptTooLarge { tokens: None }
+ ));
+
+ let error = LanguageModelCompletionError::from_http_status(
+ String::from("OpenAI").into(),
+ StatusCode::BAD_REQUEST,
+ "Invalid request.".to_string(),
+ None,
+ );
+
+ assert!(matches!(
+ error,
+ LanguageModelCompletionError::BadRequestFormat { .. }
+ ));
+ }
+
#[test]
fn test_from_cloud_failure_with_standard_format() {
let error = LanguageModelCompletionError::from_cloud_failure(
diff --git a/crates/language_model_core/src/util.rs b/crates/language_model_core/src/util.rs
index 3db2e0b76fd..3178deccbd0 100644
--- a/crates/language_model_core/src/util.rs
+++ b/crates/language_model_core/src/util.rs
@@ -48,6 +48,13 @@ pub fn parse_prompt_too_long(message: &str) -> Option<u64> {
.ok()
}
+/// Recognizes OpenAI-style context window overflow errors, which arrive either
+/// with the `context_length_exceeded` error code or a "Your input exceeds the
+/// context window of this model" message.
+pub fn is_context_window_exceeded_message(message: &str) -> bool {
+ message.contains("context_length_exceeded") || message.contains("exceeds the context window")
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/open_ai/src/completion.rs b/crates/open_ai/src/completion.rs
index a8541a4ee1e..18798a54b17 100644
--- a/crates/open_ai/src/completion.rs
+++ b/crates/open_ai/src/completion.rs
@@ -8,7 +8,7 @@ use language_model_core::{
LanguageModelToolChoice, LanguageModelToolResultContent, LanguageModelToolUse,
LanguageModelToolUseId, LanguageModelToolUseInput, MessageContent, Role, StopReason,
TokenUsage,
- util::{fix_streamed_json, parse_tool_arguments},
+ util::{fix_streamed_json, is_context_window_exceeded_message, parse_tool_arguments},
};
use std::pin::Pin;
use std::sync::Arc;
@@ -1080,20 +1080,18 @@ impl OpenAiResponseEventMapper {
events.push(Ok(LanguageModelCompletionEvent::Stop(stop_reason)));
events
}
- ResponsesStreamEvent::Failed { response } => {
- let message = response_failure_message(&response);
- vec![Err(LanguageModelCompletionError::Other(anyhow!(message)))]
- }
+ ResponsesStreamEvent::Failed { response } => match response.error.as_ref() {
+ Some(error) => vec![Err(completion_error_from_response_error(error))],
+ None => vec![Err(LanguageModelCompletionError::Other(anyhow!(
+ response_failure_message(&response)
+ )))],
+ },
ResponsesStreamEvent::Error { error } => {
- vec![Err(LanguageModelCompletionError::Other(anyhow!(
- response_error_message(&error)
- )))]
+ vec![Err(completion_error_from_response_error(&error))]
}
ResponsesStreamEvent::GenericError { error } => {
let error = error.into_response_error();
- vec![Err(LanguageModelCompletionError::Other(anyhow!(
- response_error_message(&error)
- )))]
+ vec![Err(completion_error_from_response_error(&error))]
}
ResponsesStreamEvent::ReasoningSummaryPartAdded { summary_index, .. } => {
if summary_index > 0 {
@@ -1386,6 +1384,15 @@ fn response_failure_message(response: &ResponsesSummary) -> String {
.unwrap_or_else(|| "response.failed".to_string())
}
+fn completion_error_from_response_error(error: &ResponseError) -> LanguageModelCompletionError {
+ let message = response_error_message(error);
+ if is_context_window_exceeded_message(&message) {
+ LanguageModelCompletionError::PromptTooLarge { tokens: None }
+ } else {
+ LanguageModelCompletionError::Other(anyhow!(message))
+ }
+}
+
fn response_error_message(error: &ResponseError) -> String {
let code = error.code.as_deref().filter(|code| !code.trim().is_empty());
let message = error.message.trim();
@@ -2846,8 +2853,8 @@ mod tests {
"type": "error",
"error": {
"type": "invalid_request_error",
- "code": "context_length_exceeded",
- "message": "Your input exceeds the context window of this model. Please adjust your input and try again.",
+ "code": "invalid_prompt",
+ "message": "Your prompt was flagged.",
"param": "input"
},
"sequence_number": 2
@@ -2861,10 +2868,58 @@ mod tests {
let error = mapped.into_iter().next().unwrap().unwrap_err();
assert_eq!(
error.to_string(),
- "context_length_exceeded: Your input exceeds the context window of this model. Please adjust your input and try again."
+ "invalid_prompt: Your prompt was flagged."
);
}
+ #[test]
+ fn responses_stream_maps_context_length_exceeded_to_prompt_too_large() {
+ let event = serde_json::from_value::<ResponsesStreamEvent>(json!({
+ "type": "error",
+ "error": {
+ "type": "invalid_request_error",
+ "code": "context_length_exceeded",
+ "message": "Your input exceeds the context window of this model. Please adjust your input and try again.",
+ "param": "input"
+ },
+ "sequence_number": 2
+ }))
+ .expect("nested error event");
+
+ let mut mapper = OpenAiResponseEventMapper::new();
+ let mapped = mapper.map_event(event);
+
+ assert_eq!(mapped.len(), 1);
+ let error = mapped.into_iter().next().unwrap().unwrap_err();
+ assert!(matches!(
+ error,
+ LanguageModelCompletionError::PromptTooLarge { tokens: None }
+ ));
+ }
+
+ #[test]
+ fn responses_stream_maps_failed_context_length_exceeded_to_prompt_too_large() {
+ let mut mapper = OpenAiResponseEventMapper::new();
+ let mapped = mapper.map_event(ResponsesStreamEvent::Failed {
+ response: ResponseSummary {
+ status: Some("failed".into()),
+ error: Some(ResponseError {
+ code: Some("context_length_exceeded".into()),
+ message: "Your input exceeds the context window of this model.".into(),
+ param: Some("input".into()),
+ }),
+ ..Default::default()
+ },
+ });
+
+ assert_eq!(mapped.len(), 1);
+ let error = mapped.into_iter().next().unwrap().unwrap_err();
+ assert!(matches!(
+ error,
+ LanguageModelCompletionError::PromptTooLarge { tokens: None }
+ ));
+ }
+
#[test]
fn responses_stream_deserializes_response_error_event() {
let event = serde_json::from_value::<ResponsesStreamEvent>(json!({