Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
agent: Treat context window overflows as prompt-too-large errors (#60525)
Lukas Wirthd7c922c4a370
diff --git a/crates/agent/src/tests/mod.rs b/crates/agent/src/tests/mod.rsindex 73415f55af4..df466b88051 100644--- a/crates/agent/src/tests/mod.rs+++ b/crates/agent/src/tests/mod.rs@@ -3359,6 +3359,82 @@ async fn test_latest_token_usage_counts_cached_input_tokens(cx: &mut TestAppCont});}+#[gpui::test]+async fn test_prompt_too_large_marks_token_usage_exceeded(cx: &mut TestAppContext) {+ let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;+ let fake_model = model.as_fake();++ thread+ .update(cx, |thread, cx| {+ thread.send(ClientUserMessageId::new(), ["Message 1"], cx)+ })+ .unwrap();+ cx.run_until_parked();++ fake_model.send_last_completion_stream_text_chunk("Response 1");+ fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(+ language_model::TokenUsage {+ input_tokens: 100,+ output_tokens: 50,+ ..Default::default()+ },+ ));+ fake_model.end_last_completion_stream();+ cx.run_until_parked();++ thread.read_with(cx, |thread, _| {+ assert_eq!(+ thread.latest_token_usage().unwrap().ratio(),+ acp_thread::TokenUsageRatio::Normal+ );+ });++ thread+ .update(cx, |thread, cx| {+ thread.send(ClientUserMessageId::new(), ["Message 2"], cx)+ })+ .unwrap();+ cx.run_until_parked();++ fake_model.send_last_completion_stream_error(LanguageModelCompletionError::PromptTooLarge {+ tokens: None,+ });+ fake_model.end_last_completion_stream();+ cx.run_until_parked();++ thread.read_with(cx, |thread, _| {+ let usage = thread.latest_token_usage().unwrap();+ assert_eq!(usage.used_tokens, 1_000_000);+ assert_eq!(usage.max_tokens, 1_000_000);+ assert_eq!(usage.ratio(), acp_thread::TokenUsageRatio::Exceeded);+ });+}++#[gpui::test]+async fn test_prompt_too_large_uses_reported_token_count(cx: &mut TestAppContext) {+ let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;+ let fake_model = model.as_fake();++ thread+ .update(cx, |thread, cx| {+ thread.send(ClientUserMessageId::new(), ["Message 1"], cx)+ })+ .unwrap();+ cx.run_until_parked();++ fake_model.send_last_completion_stream_error(LanguageModelCompletionError::PromptTooLarge {+ tokens: Some(1_500_000),+ });+ fake_model.end_last_completion_stream();+ cx.run_until_parked();++ thread.read_with(cx, |thread, _| {+ let usage = thread.latest_token_usage().unwrap();+ assert_eq!(usage.used_tokens, 1_500_000);+ assert_eq!(usage.ratio(), acp_thread::TokenUsageRatio::Exceeded);+ });+}+#[gpui::test]async fn test_cumulative_token_usage(cx: &mut TestAppContext) {let ThreadTest {diff --git a/crates/agent/src/thread.rs b/crates/agent/src/thread.rsindex fe5c9f2004c..8dd22ec3feb 100644--- a/crates/agent/src/thread.rs+++ b/crates/agent/src/thread.rs@@ -2314,6 +2314,31 @@ impl Thread {cx.notify();}+ /// Records that the last request overflowed the model's context window so+ /// the token usage indicator reports `Exceeded` instead of the stale usage+ /// from the last successful request. Providers don't report usage for+ /// failed requests, so we synthesize one from the reported overflow (when+ /// available) or the model's context size.+ fn mark_token_limit_exceeded(&mut self, tokens: Option<u64>, cx: &mut Context<Self>) {+ let Some(model) = self.model() else {+ return;+ };+ let input_tokens = tokens.unwrap_or(0).max(model.max_token_count());+ let Some(last_user_message) = self.last_user_message() else {+ return;+ };++ self.request_token_usage.insert(+ last_user_message.id.clone(),+ language_model::TokenUsage {+ input_tokens,+ ..Default::default()+ },+ );+ cx.emit(TokenUsageUpdated(self.latest_token_usage()));+ cx.notify();+ }+pub fn truncate(&mut self,client_user_message_id: ClientUserMessageId,@@ -3038,7 +3063,7 @@ impl Thread {) -> Result<ControlFlow<()>> {let retry = this.update(cx, |this, cx| {let user_store = this.user_store.read(cx);- this.handle_completion_error(error, attempt, user_store.plan())+ this.handle_completion_error(error, attempt, user_store.plan(), cx)})??;let timer = cx.background_executor().timer(retry.duration);event_stream.send_retry(retry);@@ -3228,7 +3253,12 @@ impl Thread {error: LanguageModelCompletionError,attempt: u8,plan: Option<Plan>,+ cx: &mut Context<Self>,) -> Result<acp_thread::RetryStatus> {+ if let LanguageModelCompletionError::PromptTooLarge { tokens } = &error {+ self.mark_token_limit_exceeded(*tokens, cx);+ }+let Some(model) = self.model() else {return Err(anyhow!(error));};diff --git a/crates/language_model_core/src/language_model_core.rs b/crates/language_model_core/src/language_model_core.rsindex 991046eab08..36b76cf7246 100644--- a/crates/language_model_core/src/language_model_core.rs+++ b/crates/language_model_core/src/language_model_core.rs@@ -25,7 +25,10 @@ pub use crate::rate_limiter::*;pub use crate::request::*;pub use crate::role::*;pub use crate::tool_schema::LanguageModelToolSchemaFormat;-pub use crate::util::{fix_streamed_json, parse_prompt_too_long, parse_tool_arguments};+pub use crate::util::{+ fix_streamed_json, is_context_window_exceeded_message, parse_prompt_too_long,+ parse_tool_arguments,+};pub use gpui_shared_string::SharedString;/// A completion event from a language model.@@ -241,7 +244,13 @@ impl LanguageModelCompletionError {retry_after: Option<Duration>,) -> Self {match status_code {- StatusCode::BAD_REQUEST => Self::BadRequestFormat { provider, message },+ StatusCode::BAD_REQUEST => {+ if is_context_window_exceeded_message(&message) {+ Self::PromptTooLarge { tokens: None }+ } else {+ Self::BadRequestFormat { provider, message }+ }+ }StatusCode::UNAUTHORIZED => Self::AuthenticationError { provider, message },StatusCode::FORBIDDEN => Self::PermissionError { provider, message },StatusCode::NOT_FOUND => Self::ApiEndpointNotFound { provider },@@ -646,6 +655,33 @@ mod tests {}}+ #[test]+ fn test_from_http_status_maps_context_length_exceeded_to_prompt_too_large() {+ let error = LanguageModelCompletionError::from_http_status(+ String::from("OpenAI").into(),+ StatusCode::BAD_REQUEST,+ r#"{"error":{"type":"invalid_request_error","code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try again.","param":"input"}}"#.to_string(),+ None,+ );++ assert!(matches!(+ error,+ LanguageModelCompletionError::PromptTooLarge { tokens: None }+ ));++ let error = LanguageModelCompletionError::from_http_status(+ String::from("OpenAI").into(),+ StatusCode::BAD_REQUEST,+ "Invalid request.".to_string(),+ None,+ );++ assert!(matches!(+ error,+ LanguageModelCompletionError::BadRequestFormat { .. }+ ));+ }+#[test]fn test_from_cloud_failure_with_standard_format() {let error = LanguageModelCompletionError::from_cloud_failure(diff --git a/crates/language_model_core/src/util.rs b/crates/language_model_core/src/util.rsindex 3db2e0b76fd..3178deccbd0 100644--- a/crates/language_model_core/src/util.rs+++ b/crates/language_model_core/src/util.rs@@ -48,6 +48,13 @@ pub fn parse_prompt_too_long(message: &str) -> Option<u64> {.ok()}+/// Recognizes OpenAI-style context window overflow errors, which arrive either+/// with the `context_length_exceeded` error code or a "Your input exceeds the+/// context window of this model" message.+pub fn is_context_window_exceeded_message(message: &str) -> bool {+ message.contains("context_length_exceeded") || message.contains("exceeds the context window")+}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/open_ai/src/completion.rs b/crates/open_ai/src/completion.rsindex a8541a4ee1e..18798a54b17 100644--- a/crates/open_ai/src/completion.rs+++ b/crates/open_ai/src/completion.rs@@ -8,7 +8,7 @@ use language_model_core::{LanguageModelToolChoice, LanguageModelToolResultContent, LanguageModelToolUse,LanguageModelToolUseId, LanguageModelToolUseInput, MessageContent, Role, StopReason,TokenUsage,- util::{fix_streamed_json, parse_tool_arguments},+ util::{fix_streamed_json, is_context_window_exceeded_message, parse_tool_arguments},};use std::pin::Pin;use std::sync::Arc;@@ -1080,20 +1080,18 @@ impl OpenAiResponseEventMapper {events.push(Ok(LanguageModelCompletionEvent::Stop(stop_reason)));events}- ResponsesStreamEvent::Failed { response } => {- let message = response_failure_message(&response);- vec![Err(LanguageModelCompletionError::Other(anyhow!(message)))]- }+ ResponsesStreamEvent::Failed { response } => match response.error.as_ref() {+ Some(error) => vec![Err(completion_error_from_response_error(error))],+ None => vec![Err(LanguageModelCompletionError::Other(anyhow!(+ response_failure_message(&response)+ )))],+ },ResponsesStreamEvent::Error { error } => {- vec![Err(LanguageModelCompletionError::Other(anyhow!(- response_error_message(&error)- )))]+ vec![Err(completion_error_from_response_error(&error))]}ResponsesStreamEvent::GenericError { error } => {let error = error.into_response_error();- vec![Err(LanguageModelCompletionError::Other(anyhow!(- response_error_message(&error)- )))]+ vec![Err(completion_error_from_response_error(&error))]}ResponsesStreamEvent::ReasoningSummaryPartAdded { summary_index, .. } => {if summary_index > 0 {@@ -1386,6 +1384,15 @@ fn response_failure_message(response: &ResponsesSummary) -> String {.unwrap_or_else(|| "response.failed".to_string())}+fn completion_error_from_response_error(error: &ResponseError) -> LanguageModelCompletionError {+ let message = response_error_message(error);+ if is_context_window_exceeded_message(&message) {+ LanguageModelCompletionError::PromptTooLarge { tokens: None }+ } else {+ LanguageModelCompletionError::Other(anyhow!(message))+ }+}+fn response_error_message(error: &ResponseError) -> String {let code = error.code.as_deref().filter(|code| !code.trim().is_empty());let message = error.message.trim();@@ -2846,8 +2853,8 @@ mod tests {"type": "error","error": {"type": "invalid_request_error",- "code": "context_length_exceeded",- "message": "Your input exceeds the context window of this model. Please adjust your input and try again.",+ "code": "invalid_prompt",+ "message": "Your prompt was flagged.","param": "input"},"sequence_number": 2@@ -2861,10 +2868,58 @@ mod tests {let error = mapped.into_iter().next().unwrap().unwrap_err();assert_eq!(error.to_string(),- "context_length_exceeded: Your input exceeds the context window of this model. Please adjust your input and try again."+ "invalid_prompt: Your prompt was flagged.");}+ #[test]+ fn responses_stream_maps_context_length_exceeded_to_prompt_too_large() {+ let event = serde_json::from_value::<ResponsesStreamEvent>(json!({+ "type": "error",+ "error": {+ "type": "invalid_request_error",+ "code": "context_length_exceeded",+ "message": "Your input exceeds the context window of this model. Please adjust your input and try again.",+ "param": "input"+ },+ "sequence_number": 2+ }))+ .expect("nested error event");++ let mut mapper = OpenAiResponseEventMapper::new();+ let mapped = mapper.map_event(event);++ assert_eq!(mapped.len(), 1);+ let error = mapped.into_iter().next().unwrap().unwrap_err();+ assert!(matches!(+ error,+ LanguageModelCompletionError::PromptTooLarge { tokens: None }+ ));+ }++ #[test]+ fn responses_stream_maps_failed_context_length_exceeded_to_prompt_too_large() {+ let mut mapper = OpenAiResponseEventMapper::new();+ let mapped = mapper.map_event(ResponsesStreamEvent::Failed {+ response: ResponseSummary {+ status: Some("failed".into()),+ error: Some(ResponseError {+ code: Some("context_length_exceeded".into()),+ message: "Your input exceeds the context window of this model.".into(),+ param: Some("input".into()),+ }),+ ..Default::default()+ },+ });++ assert_eq!(mapped.len(), 1);+ let error = mapped.into_iter().next().unwrap().unwrap_err();+ assert!(matches!(+ error,+ LanguageModelCompletionError::PromptTooLarge { tokens: None }+ ));+ }+#[test]fn responses_stream_deserializes_response_error_event() {let event = serde_json::from_value::<ResponsesStreamEvent>(json!({