Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

copilot: Fix cache control error (#56632)

Bennet Bo Fenner956dbea79a90
diff --git a/crates/anthropic/src/completion.rs b/crates/anthropic/src/completion.rs
index 889acfca84d..5cb864a088e 100644
--- a/crates/anthropic/src/completion.rs
+++ b/crates/anthropic/src/completion.rs
@@ -16,6 +16,51 @@ use crate::{
StringOrContents, Thinking, Tool, ToolChoice, ToolResultContent, ToolResultPart, Usage,
};
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum AnthropicPromptCacheMode {
+ Disabled,
+ Legacy,
+ #[default]
+ Automatic,
+}
+
+fn set_cache_control(content: &mut RequestContent, cache_control: Option<CacheControl>) -> bool {
+ match content {
+ RequestContent::RedactedThinking { .. } => false,
+ RequestContent::Text {
+ cache_control: target,
+ ..
+ }
+ | RequestContent::Thinking {
+ cache_control: target,
+ ..
+ }
+ | RequestContent::Image {
+ cache_control: target,
+ ..
+ }
+ | RequestContent::ToolUse {
+ cache_control: target,
+ ..
+ }
+ | RequestContent::ToolResult {
+ cache_control: target,
+ ..
+ } => {
+ *target = cache_control;
+ true
+ }
+ }
+}
+
+fn mark_last_cacheable_content(content: &mut [RequestContent], cache_control: CacheControl) {
+ for content in content.iter_mut().rev() {
+ if set_cache_control(content, Some(cache_control)) {
+ break;
+ }
+ }
+}
+
fn to_anthropic_content(content: MessageContent) -> Option<RequestContent> {
match content {
MessageContent::Text(text) => {
@@ -111,6 +156,7 @@ pub fn into_anthropic(
default_temperature: f32,
max_output_tokens: u64,
mode: AnthropicModelMode,
+ cache_mode: AnthropicPromptCacheMode,
) -> crate::Request {
let mut new_messages: Vec<Message> = Vec::new();
let mut system_message = String::new();
@@ -125,7 +171,7 @@ pub fn into_anthropic(
match message.role {
Role::User | Role::Assistant => {
- let anthropic_message_content: Vec<RequestContent> = message
+ let mut anthropic_message_content: Vec<RequestContent> = message
.content
.into_iter()
.filter_map(to_anthropic_content)
@@ -139,6 +185,16 @@ pub fn into_anthropic(
continue;
}
+ if cache_mode == AnthropicPromptCacheMode::Legacy && message.cache {
+ mark_last_cacheable_content(
+ &mut anthropic_message_content,
+ CacheControl {
+ cache_type: CacheControlType::Ephemeral,
+ ttl: None,
+ },
+ );
+ }
+
if let Some(last_message) = new_messages.last_mut()
&& last_message.role == anthropic_role
{
@@ -166,10 +222,12 @@ pub fn into_anthropic(
// requires that longer TTLs appear earlier in the prefix, and the prefix
// order is tools → system → messages, so long-TTL tools/system before a
// short-TTL conversation breakpoint is a valid mix.
- let long_lived_cache = any_message_wants_cache.then_some(CacheControl {
- cache_type: CacheControlType::Ephemeral,
- ttl: Some(CacheTtl::OneHour),
- });
+ let long_lived_cache = (cache_mode == AnthropicPromptCacheMode::Automatic
+ && any_message_wants_cache)
+ .then_some(CacheControl {
+ cache_type: CacheControlType::Ephemeral,
+ ttl: Some(CacheTtl::OneHour),
+ });
let system = if system_message.is_empty() {
None
@@ -208,10 +266,12 @@ pub fn into_anthropic(
// tail. Omitting `ttl` uses the default (short) TTL, which refreshes
// for free on every cache hit — ideal for the rapidly-changing
// conversation suffix.
- cache_control: any_message_wants_cache.then_some(CacheControl {
- cache_type: CacheControlType::Ephemeral,
- ttl: None,
- }),
+ cache_control: (cache_mode == AnthropicPromptCacheMode::Automatic
+ && any_message_wants_cache)
+ .then_some(CacheControl {
+ cache_type: CacheControlType::Ephemeral,
+ ttl: None,
+ }),
thinking: if request.thinking_allowed {
match mode {
AnthropicModelMode::Thinking { budget_tokens } => {
@@ -511,6 +571,7 @@ mod tests {
0.7,
4096,
AnthropicModelMode::Default,
+ AnthropicPromptCacheMode::Automatic,
);
// No message content block should carry cache_control anymore; the
@@ -571,6 +632,79 @@ mod tests {
));
}
+ #[test]
+ fn test_legacy_caching_marks_last_message_content_block() {
+ let request = LanguageModelRequest {
+ messages: vec![
+ LanguageModelRequestMessage {
+ role: Role::System,
+ content: vec![MessageContent::Text("You are helpful.".to_string())],
+ cache: false,
+ reasoning_details: None,
+ },
+ LanguageModelRequestMessage {
+ role: Role::User,
+ content: vec![
+ MessageContent::Text("Some prompt".to_string()),
+ MessageContent::Image(LanguageModelImage::empty()),
+ ],
+ cache: true,
+ reasoning_details: None,
+ },
+ ],
+ thread_id: None,
+ prompt_id: None,
+ intent: None,
+ stop: vec![],
+ temperature: None,
+ tools: vec![language_model_core::LanguageModelRequestTool {
+ name: "do_thing".into(),
+ description: "Does a thing.".into(),
+ input_schema: serde_json::json!({"type": "object"}),
+ use_input_streaming: false,
+ }],
+ tool_choice: None,
+ thinking_allowed: true,
+ thinking_effort: None,
+ speed: None,
+ };
+
+ let anthropic_request = into_anthropic(
+ request,
+ "claude-3-5-sonnet".to_string(),
+ 0.7,
+ 4096,
+ AnthropicModelMode::Default,
+ AnthropicPromptCacheMode::Legacy,
+ );
+
+ assert!(anthropic_request.cache_control.is_none());
+ assert!(matches!(
+ anthropic_request.system,
+ Some(StringOrContents::String(_))
+ ));
+ assert_eq!(anthropic_request.tools.len(), 1);
+ assert!(anthropic_request.tools[0].cache_control.is_none());
+ assert_eq!(anthropic_request.messages.len(), 1);
+ assert!(matches!(
+ anthropic_request.messages[0].content[0],
+ RequestContent::Text {
+ cache_control: None,
+ ..
+ }
+ ));
+ assert!(matches!(
+ anthropic_request.messages[0].content[1],
+ RequestContent::Image {
+ cache_control: Some(CacheControl {
+ cache_type: CacheControlType::Ephemeral,
+ ttl: None,
+ }),
+ ..
+ }
+ ));
+ }
+
#[test]
fn test_no_cache_control_when_caching_disabled() {
let request = LanguageModelRequest {
@@ -611,6 +745,7 @@ mod tests {
0.7,
4096,
AnthropicModelMode::Default,
+ AnthropicPromptCacheMode::Automatic,
);
assert!(anthropic_request.cache_control.is_none());
@@ -654,6 +789,7 @@ mod tests {
AnthropicModelMode::Thinking {
budget_tokens: Some(10000),
},
+ AnthropicPromptCacheMode::Automatic,
)
}
diff --git a/crates/language_models/src/provider/anthropic.rs b/crates/language_models/src/provider/anthropic.rs
index c4ea3dcfcde..5e5fa2ad85c 100644
--- a/crates/language_models/src/provider/anthropic.rs
+++ b/crates/language_models/src/provider/anthropic.rs
@@ -21,7 +21,7 @@ use ui::{ButtonLink, ConfiguredApiCard, List, ListBulletItem, prelude::*};
use ui_input::InputField;
use util::ResultExt;
-pub use anthropic::completion::{AnthropicEventMapper, into_anthropic};
+pub use anthropic::completion::{AnthropicEventMapper, AnthropicPromptCacheMode, into_anthropic};
pub use settings::AnthropicAvailableModel as AvailableModel;
const PROVIDER_ID: LanguageModelProviderId = ANTHROPIC_PROVIDER_ID;
@@ -495,6 +495,7 @@ impl LanguageModel for AnthropicModel {
self.model.default_temperature,
self.model.max_output_tokens,
self.model.mode.clone(),
+ AnthropicPromptCacheMode::Automatic,
);
if !self.model.supports_speed {
request.speed = None;
diff --git a/crates/language_models/src/provider/copilot_chat.rs b/crates/language_models/src/provider/copilot_chat.rs
index 1fc1dc3ce4a..cf8f11a5bb0 100644
--- a/crates/language_models/src/provider/copilot_chat.rs
+++ b/crates/language_models/src/provider/copilot_chat.rs
@@ -31,7 +31,7 @@ use settings::SettingsStore;
use ui::prelude::*;
use util::debug_panic;
-use crate::provider::anthropic::{AnthropicEventMapper, into_anthropic};
+use crate::provider::anthropic::{AnthropicEventMapper, AnthropicPromptCacheMode, into_anthropic};
use language_model::util::{fix_streamed_json, parse_tool_arguments};
const PROVIDER_ID: LanguageModelProviderId = LanguageModelProviderId::new("copilot_chat");
@@ -354,6 +354,7 @@ impl LanguageModel for CopilotChatLanguageModel {
} else {
AnthropicModelMode::Default
},
+ AnthropicPromptCacheMode::Legacy,
);
anthropic_request.temperature = None;
diff --git a/crates/language_models/src/provider/opencode.rs b/crates/language_models/src/provider/opencode.rs
index 1611694905f..64b3a6fcdbd 100644
--- a/crates/language_models/src/provider/opencode.rs
+++ b/crates/language_models/src/provider/opencode.rs
@@ -647,6 +647,7 @@ impl LanguageModel for OpenCodeLanguageModel {
1.0,
self.model.max_output_tokens().unwrap_or(8192),
mode,
+ anthropic::completion::AnthropicPromptCacheMode::Automatic,
);
let stream = self.stream_anthropic(anthropic_request, http_client, cx);
async move {
diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rs
index 928ec336489..17051ac874e 100644
--- a/crates/language_models_cloud/src/language_models_cloud.rs
+++ b/crates/language_models_cloud/src/language_models_cloud.rs
@@ -39,7 +39,7 @@ use std::task::Poll;
use std::time::Duration;
use thiserror::Error;
-use anthropic::completion::{AnthropicEventMapper, into_anthropic};
+use anthropic::completion::{AnthropicEventMapper, AnthropicPromptCacheMode, into_anthropic};
use google_ai::completion::{GoogleEventMapper, into_google};
use open_ai::completion::{
OpenAiEventMapper, OpenAiResponseEventMapper, into_open_ai, into_open_ai_response,
@@ -419,6 +419,7 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
} else {
AnthropicModelMode::Default
},
+ AnthropicPromptCacheMode::Automatic,
);
if enable_thinking && effort.is_some() {