Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
anthropic: Use automatic prompt caching with long-lived anchors on tools and system (#56472)
Richard Feldmanad916ca1af82
diff --git a/crates/anthropic/src/anthropic.rs b/crates/anthropic/src/anthropic.rsindex c7afbbfb6b0..183b9f107c4 100644--- a/crates/anthropic/src/anthropic.rs+++ b/crates/anthropic/src/anthropic.rs@@ -503,10 +503,27 @@ pub enum CacheControlType {Ephemeral,}+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]+pub enum CacheTtl {+ /// Anthropic's default ephemeral TTL (currently 5 minutes). Refreshes for+ /// free on every cache hit.+ #[serde(rename = "5m")]+ FiveMinutes,+ /// Anthropic's extended ephemeral TTL (currently 1 hour). Costs 2x base+ /// input tokens to write, but persists across longer idle gaps.+ #[serde(rename = "1h")]+ OneHour,+}+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]pub struct CacheControl {#[serde(rename = "type")]pub cache_type: CacheControlType,+ /// Omitted (None) means the API's default 5-minute TTL. Anthropic requires+ /// that cache entries with longer TTLs appear before shorter ones in the+ /// prefix order (tools → system → messages).+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub ttl: Option<CacheTtl>,}#[derive(Debug, Serialize, Deserialize)]@@ -614,6 +631,8 @@ pub struct Tool {pub input_schema: serde_json::Value,#[serde(default, skip_serializing_if = "is_false")]pub eager_input_streaming: bool,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub cache_control: Option<CacheControl>,}#[derive(Debug, Serialize, Deserialize)]@@ -680,6 +699,12 @@ pub struct Request {pub tool_choice: Option<ToolChoice>,#[serde(default, skip_serializing_if = "Option::is_none")]pub system: Option<StringOrContents>,+ /// Top-level cache_control opts into Anthropic's automatic prompt caching.+ /// When set, Anthropic places the cache breakpoint on the last cacheable block+ /// in the request (covering tools + system + the full conversation prefix), so+ /// we don't have to micromanage per-block breakpoints ourselves.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub cache_control: Option<CacheControl>,#[serde(default, skip_serializing_if = "Option::is_none")]pub metadata: Option<Metadata>,#[serde(default, skip_serializing_if = "Option::is_none")]diff --git a/crates/anthropic/src/completion.rs b/crates/anthropic/src/completion.rsindex 48eed580d68..889acfca84d 100644--- a/crates/anthropic/src/completion.rs+++ b/crates/anthropic/src/completion.rs@@ -12,8 +12,8 @@ use std::str::FromStr;use crate::{AdaptiveThinkingDisplay, AnthropicError, AnthropicModelMode, CacheControl, CacheControlType,- ContentDelta, Event, ImageSource, Message, RequestContent, ResponseContent, StringOrContents,- Thinking, Tool, ToolChoice, ToolResultContent, ToolResultPart, Usage,+ CacheTtl, ContentDelta, Event, ImageSource, Message, RequestContent, ResponseContent,+ StringOrContents, Thinking, Tool, ToolChoice, ToolResultContent, ToolResultPart, Usage,};fn to_anthropic_content(content: MessageContent) -> Option<RequestContent> {@@ -114,15 +114,18 @@ pub fn into_anthropic() -> crate::Request {let mut new_messages: Vec<Message> = Vec::new();let mut system_message = String::new();+ let mut any_message_wants_cache = false;for message in request.messages {if message.contents_empty() {continue;}+ any_message_wants_cache |= message.cache;+match message.role {Role::User | Role::Assistant => {- let mut anthropic_message_content: Vec<RequestContent> = message+ let anthropic_message_content: Vec<RequestContent> = message.content.into_iter().filter_map(to_anthropic_content)@@ -143,28 +146,6 @@ pub fn into_anthropic(continue;}- // Mark the last segment of the message as cached- if message.cache {- let cache_control_value = Some(CacheControl {- cache_type: CacheControlType::Ephemeral,- });- for message_content in anthropic_message_content.iter_mut().rev() {- match message_content {- RequestContent::RedactedThinking { .. } => {- // Caching is not possible, fallback to next message- }- RequestContent::Text { cache_control, .. }- | RequestContent::Thinking { cache_control, .. }- | RequestContent::Image { cache_control, .. }- | RequestContent::ToolUse { cache_control, .. }- | RequestContent::ToolResult { cache_control, .. } => {- *cache_control = cache_control_value;- break;- }- }- }- }-new_messages.push(Message {role: anthropic_role,content: anthropic_message_content,@@ -179,15 +160,58 @@ pub fn into_anthropic(}}+ // When caching is enabled, mark the static prefix (tools + system) with an+ // explicit long-TTL breakpoint, and let Anthropic's automatic top-level+ // cache_control handle the short-TTL conversation breakpoint. Anthropic+ // requires that longer TTLs appear earlier in the prefix, and the prefix+ // order is tools → system → messages, so long-TTL tools/system before a+ // short-TTL conversation breakpoint is a valid mix.+ let long_lived_cache = any_message_wants_cache.then_some(CacheControl {+ cache_type: CacheControlType::Ephemeral,+ ttl: Some(CacheTtl::OneHour),+ });++ let system = if system_message.is_empty() {+ None+ } else if let Some(cache_control) = long_lived_cache {+ Some(StringOrContents::Content(vec![RequestContent::Text {+ text: system_message,+ cache_control: Some(cache_control),+ }]))+ } else {+ Some(StringOrContents::String(system_message))+ };++ let mut tools: Vec<Tool> = request+ .tools+ .into_iter()+ .map(|tool| Tool {+ name: tool.name,+ description: tool.description,+ input_schema: tool.input_schema,+ eager_input_streaming: tool.use_input_streaming,+ cache_control: None,+ })+ .collect();+ if let Some(cache_control) = long_lived_cache+ && let Some(last_tool) = tools.last_mut()+ {+ last_tool.cache_control = Some(cache_control);+ }+crate::Request {model,messages: new_messages,max_tokens: max_output_tokens,- system: if system_message.is_empty() {- None- } else {- Some(StringOrContents::String(system_message))- },+ system,+ // Opt into Anthropic's automatic prompt caching for the conversation+ // tail. Omitting `ttl` uses the default (short) TTL, which refreshes+ // for free on every cache hit — ideal for the rapidly-changing+ // conversation suffix.+ cache_control: any_message_wants_cache.then_some(CacheControl {+ cache_type: CacheControlType::Ephemeral,+ ttl: None,+ }),thinking: if request.thinking_allowed {match mode {AnthropicModelMode::Thinking { budget_tokens } => {@@ -201,16 +225,7 @@ pub fn into_anthropic(} else {None},- tools: request- .tools- .into_iter()- .map(|tool| Tool {- name: tool.name,- description: tool.description,- input_schema: tool.input_schema,- eager_input_streaming: tool.use_input_streaming,- })- .collect(),+ tools,tool_choice: request.tool_choice.map(|choice| match choice {LanguageModelToolChoice::Auto => ToolChoice::Auto,LanguageModelToolChoice::Any => ToolChoice::Any,@@ -453,26 +468,37 @@ mod tests {use language_model_core::{LanguageModelImage, LanguageModelRequestMessage, MessageContent};#[test]- fn test_cache_control_only_on_last_segment() {+ fn test_caching_uses_top_level_auto_and_long_lived_prefix() {let request = LanguageModelRequest {- messages: vec![LanguageModelRequestMessage {- role: Role::User,- content: vec![- MessageContent::Text("Some prompt".to_string()),- MessageContent::Image(LanguageModelImage::empty()),- MessageContent::Image(LanguageModelImage::empty()),- MessageContent::Image(LanguageModelImage::empty()),- MessageContent::Image(LanguageModelImage::empty()),- ],- cache: true,- reasoning_details: None,- }],+ messages: vec![+ LanguageModelRequestMessage {+ role: Role::System,+ content: vec![MessageContent::Text("You are helpful.".to_string())],+ cache: false,+ reasoning_details: None,+ },+ LanguageModelRequestMessage {+ role: Role::User,+ content: vec![+ MessageContent::Text("Some prompt".to_string()),+ MessageContent::Image(LanguageModelImage::empty()),+ MessageContent::Image(LanguageModelImage::empty()),+ ],+ cache: true,+ reasoning_details: None,+ },+ ],thread_id: None,prompt_id: None,intent: None,stop: vec![],temperature: None,- tools: vec![],+ tools: vec![language_model_core::LanguageModelRequestTool {+ name: "do_thing".into(),+ description: "Does a thing.".into(),+ input_schema: serde_json::json!({"type": "object"}),+ use_input_streaming: false,+ }],tool_choice: None,thinking_allowed: true,thinking_effort: None,@@ -487,37 +513,112 @@ mod tests {AnthropicModelMode::Default,);+ // No message content block should carry cache_control anymore; the+ // conversation breakpoint is set via top-level automatic caching.assert_eq!(anthropic_request.messages.len(), 1);+ for block in &anthropic_request.messages[0].content {+ let cache_control = match block {+ RequestContent::Text { cache_control, .. }+ | RequestContent::Thinking { cache_control, .. }+ | RequestContent::Image { cache_control, .. }+ | RequestContent::ToolUse { cache_control, .. }+ | RequestContent::ToolResult { cache_control, .. } => *cache_control,+ RequestContent::RedactedThinking { .. } => None,+ };+ assert!(+ cache_control.is_none(),+ "message content blocks should no longer be individually marked",+ );+ }- let message = &anthropic_request.messages[0];- assert_eq!(message.content.len(), 5);-+ // Top-level cache_control opts into automatic caching with the default+ // 5-minute TTL for the conversation tail.assert!(matches!(- message.content[0],- RequestContent::Text {- cache_control: None,- ..- }+ anthropic_request.cache_control,+ Some(CacheControl {+ cache_type: CacheControlType::Ephemeral,+ ttl: None,+ })));- for i in 1..3 {- assert!(matches!(- message.content[i],- RequestContent::Image {- cache_control: None,- ..- }- ));++ // System prompt is emitted in array form with a long-TTL breakpoint on+ // the final text block.+ match anthropic_request.system {+ Some(StringOrContents::Content(ref blocks)) => {+ assert_eq!(blocks.len(), 1);+ assert!(matches!(+ blocks[0],+ RequestContent::Text {+ cache_control: Some(CacheControl {+ cache_type: CacheControlType::Ephemeral,+ ttl: Some(CacheTtl::OneHour),+ }),+ ..+ }+ ));+ }+ other => panic!("expected system content array, got {other:?}"),}+ // The last (and only) tool carries a long-TTL breakpoint.+ assert_eq!(anthropic_request.tools.len(), 1);assert!(matches!(- message.content[4],- RequestContent::Image {- cache_control: Some(CacheControl {- cache_type: CacheControlType::Ephemeral,- }),- ..- }+ anthropic_request.tools[0].cache_control,+ Some(CacheControl {+ cache_type: CacheControlType::Ephemeral,+ ttl: Some(CacheTtl::OneHour),+ })+ ));+ }++ #[test]+ fn test_no_cache_control_when_caching_disabled() {+ let request = LanguageModelRequest {+ messages: vec![+ LanguageModelRequestMessage {+ role: Role::System,+ content: vec![MessageContent::Text("You are helpful.".to_string())],+ cache: false,+ reasoning_details: None,+ },+ LanguageModelRequestMessage {+ role: Role::User,+ content: vec![MessageContent::Text("Hi".to_string())],+ cache: false,+ reasoning_details: None,+ },+ ],+ thread_id: None,+ prompt_id: None,+ intent: None,+ stop: vec![],+ temperature: None,+ tools: vec![language_model_core::LanguageModelRequestTool {+ name: "do_thing".into(),+ description: "Does a thing.".into(),+ input_schema: serde_json::json!({"type": "object"}),+ use_input_streaming: false,+ }],+ tool_choice: None,+ thinking_allowed: true,+ thinking_effort: None,+ speed: None,+ };++ let anthropic_request = into_anthropic(+ request,+ "claude-3-5-sonnet".to_string(),+ 0.7,+ 4096,+ AnthropicModelMode::Default,+ );++ assert!(anthropic_request.cache_control.is_none());+ assert!(matches!(+ anthropic_request.system,+ Some(StringOrContents::String(_))));+ assert!(anthropic_request.tools[0].cache_control.is_none());}fn request_with_assistant_content(assistant_content: Vec<MessageContent>) -> crate::Request {diff --git a/crates/edit_prediction_cli/src/anthropic_client.rs b/crates/edit_prediction_cli/src/anthropic_client.rsindex 7841e8a2cc1..c1706d6161b 100644--- a/crates/edit_prediction_cli/src/anthropic_client.rs+++ b/crates/edit_prediction_cli/src/anthropic_client.rs@@ -47,6 +47,7 @@ impl PlainLlmClient {thinking: None,tool_choice: None,system: None,+ cache_control: None,metadata: None,output_config: None,stop_sequences: Vec::new(),@@ -87,6 +88,7 @@ impl PlainLlmClient {thinking: None,tool_choice: None,system: None,+ cache_control: None,metadata: None,output_config: None,stop_sequences: Vec::new(),@@ -582,6 +584,7 @@ impl BatchingLlmClient {thinking: None,tool_choice: None,system: None,+ cache_control: None,metadata: None,output_config: None,stop_sequences: Vec::new(),