Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

anthropic: Use automatic prompt caching with long-lived anchors on tools and system (#56472)

Richard Feldmanad916ca1af82
diff --git a/crates/anthropic/src/anthropic.rs b/crates/anthropic/src/anthropic.rs
index c7afbbfb6b0..183b9f107c4 100644
--- a/crates/anthropic/src/anthropic.rs
+++ b/crates/anthropic/src/anthropic.rs
@@ -503,10 +503,27 @@ pub enum CacheControlType {
Ephemeral,
}
+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
+pub enum CacheTtl {
+ /// Anthropic's default ephemeral TTL (currently 5 minutes). Refreshes for
+ /// free on every cache hit.
+ #[serde(rename = "5m")]
+ FiveMinutes,
+ /// Anthropic's extended ephemeral TTL (currently 1 hour). Costs 2x base
+ /// input tokens to write, but persists across longer idle gaps.
+ #[serde(rename = "1h")]
+ OneHour,
+}
+
#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
pub struct CacheControl {
#[serde(rename = "type")]
pub cache_type: CacheControlType,
+ /// Omitted (None) means the API's default 5-minute TTL. Anthropic requires
+ /// that cache entries with longer TTLs appear before shorter ones in the
+ /// prefix order (tools → system → messages).
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub ttl: Option<CacheTtl>,
}
#[derive(Debug, Serialize, Deserialize)]
@@ -614,6 +631,8 @@ pub struct Tool {
pub input_schema: serde_json::Value,
#[serde(default, skip_serializing_if = "is_false")]
pub eager_input_streaming: bool,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub cache_control: Option<CacheControl>,
}
#[derive(Debug, Serialize, Deserialize)]
@@ -680,6 +699,12 @@ pub struct Request {
pub tool_choice: Option<ToolChoice>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub system: Option<StringOrContents>,
+ /// Top-level cache_control opts into Anthropic's automatic prompt caching.
+ /// When set, Anthropic places the cache breakpoint on the last cacheable block
+ /// in the request (covering tools + system + the full conversation prefix), so
+ /// we don't have to micromanage per-block breakpoints ourselves.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub cache_control: Option<CacheControl>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>,
#[serde(default, skip_serializing_if = "Option::is_none")]
diff --git a/crates/anthropic/src/completion.rs b/crates/anthropic/src/completion.rs
index 48eed580d68..889acfca84d 100644
--- a/crates/anthropic/src/completion.rs
+++ b/crates/anthropic/src/completion.rs
@@ -12,8 +12,8 @@ use std::str::FromStr;
use crate::{
AdaptiveThinkingDisplay, AnthropicError, AnthropicModelMode, CacheControl, CacheControlType,
- ContentDelta, Event, ImageSource, Message, RequestContent, ResponseContent, StringOrContents,
- Thinking, Tool, ToolChoice, ToolResultContent, ToolResultPart, Usage,
+ CacheTtl, ContentDelta, Event, ImageSource, Message, RequestContent, ResponseContent,
+ StringOrContents, Thinking, Tool, ToolChoice, ToolResultContent, ToolResultPart, Usage,
};
fn to_anthropic_content(content: MessageContent) -> Option<RequestContent> {
@@ -114,15 +114,18 @@ pub fn into_anthropic(
) -> crate::Request {
let mut new_messages: Vec<Message> = Vec::new();
let mut system_message = String::new();
+ let mut any_message_wants_cache = false;
for message in request.messages {
if message.contents_empty() {
continue;
}
+ any_message_wants_cache |= message.cache;
+
match message.role {
Role::User | Role::Assistant => {
- let mut anthropic_message_content: Vec<RequestContent> = message
+ let anthropic_message_content: Vec<RequestContent> = message
.content
.into_iter()
.filter_map(to_anthropic_content)
@@ -143,28 +146,6 @@ pub fn into_anthropic(
continue;
}
- // Mark the last segment of the message as cached
- if message.cache {
- let cache_control_value = Some(CacheControl {
- cache_type: CacheControlType::Ephemeral,
- });
- for message_content in anthropic_message_content.iter_mut().rev() {
- match message_content {
- RequestContent::RedactedThinking { .. } => {
- // Caching is not possible, fallback to next message
- }
- RequestContent::Text { cache_control, .. }
- | RequestContent::Thinking { cache_control, .. }
- | RequestContent::Image { cache_control, .. }
- | RequestContent::ToolUse { cache_control, .. }
- | RequestContent::ToolResult { cache_control, .. } => {
- *cache_control = cache_control_value;
- break;
- }
- }
- }
- }
-
new_messages.push(Message {
role: anthropic_role,
content: anthropic_message_content,
@@ -179,15 +160,58 @@ pub fn into_anthropic(
}
}
+ // When caching is enabled, mark the static prefix (tools + system) with an
+ // explicit long-TTL breakpoint, and let Anthropic's automatic top-level
+ // cache_control handle the short-TTL conversation breakpoint. Anthropic
+ // requires that longer TTLs appear earlier in the prefix, and the prefix
+ // order is tools → system → messages, so long-TTL tools/system before a
+ // short-TTL conversation breakpoint is a valid mix.
+ let long_lived_cache = any_message_wants_cache.then_some(CacheControl {
+ cache_type: CacheControlType::Ephemeral,
+ ttl: Some(CacheTtl::OneHour),
+ });
+
+ let system = if system_message.is_empty() {
+ None
+ } else if let Some(cache_control) = long_lived_cache {
+ Some(StringOrContents::Content(vec![RequestContent::Text {
+ text: system_message,
+ cache_control: Some(cache_control),
+ }]))
+ } else {
+ Some(StringOrContents::String(system_message))
+ };
+
+ let mut tools: Vec<Tool> = request
+ .tools
+ .into_iter()
+ .map(|tool| Tool {
+ name: tool.name,
+ description: tool.description,
+ input_schema: tool.input_schema,
+ eager_input_streaming: tool.use_input_streaming,
+ cache_control: None,
+ })
+ .collect();
+ if let Some(cache_control) = long_lived_cache
+ && let Some(last_tool) = tools.last_mut()
+ {
+ last_tool.cache_control = Some(cache_control);
+ }
+
crate::Request {
model,
messages: new_messages,
max_tokens: max_output_tokens,
- system: if system_message.is_empty() {
- None
- } else {
- Some(StringOrContents::String(system_message))
- },
+ system,
+ // Opt into Anthropic's automatic prompt caching for the conversation
+ // tail. Omitting `ttl` uses the default (short) TTL, which refreshes
+ // for free on every cache hit — ideal for the rapidly-changing
+ // conversation suffix.
+ cache_control: any_message_wants_cache.then_some(CacheControl {
+ cache_type: CacheControlType::Ephemeral,
+ ttl: None,
+ }),
thinking: if request.thinking_allowed {
match mode {
AnthropicModelMode::Thinking { budget_tokens } => {
@@ -201,16 +225,7 @@ pub fn into_anthropic(
} else {
None
},
- tools: request
- .tools
- .into_iter()
- .map(|tool| Tool {
- name: tool.name,
- description: tool.description,
- input_schema: tool.input_schema,
- eager_input_streaming: tool.use_input_streaming,
- })
- .collect(),
+ tools,
tool_choice: request.tool_choice.map(|choice| match choice {
LanguageModelToolChoice::Auto => ToolChoice::Auto,
LanguageModelToolChoice::Any => ToolChoice::Any,
@@ -453,26 +468,37 @@ mod tests {
use language_model_core::{LanguageModelImage, LanguageModelRequestMessage, MessageContent};
#[test]
- fn test_cache_control_only_on_last_segment() {
+ fn test_caching_uses_top_level_auto_and_long_lived_prefix() {
let request = LanguageModelRequest {
- messages: vec![LanguageModelRequestMessage {
- role: Role::User,
- content: vec![
- MessageContent::Text("Some prompt".to_string()),
- MessageContent::Image(LanguageModelImage::empty()),
- MessageContent::Image(LanguageModelImage::empty()),
- MessageContent::Image(LanguageModelImage::empty()),
- MessageContent::Image(LanguageModelImage::empty()),
- ],
- cache: true,
- reasoning_details: None,
- }],
+ messages: vec![
+ LanguageModelRequestMessage {
+ role: Role::System,
+ content: vec![MessageContent::Text("You are helpful.".to_string())],
+ cache: false,
+ reasoning_details: None,
+ },
+ LanguageModelRequestMessage {
+ role: Role::User,
+ content: vec![
+ MessageContent::Text("Some prompt".to_string()),
+ MessageContent::Image(LanguageModelImage::empty()),
+ MessageContent::Image(LanguageModelImage::empty()),
+ ],
+ cache: true,
+ reasoning_details: None,
+ },
+ ],
thread_id: None,
prompt_id: None,
intent: None,
stop: vec![],
temperature: None,
- tools: vec![],
+ tools: vec![language_model_core::LanguageModelRequestTool {
+ name: "do_thing".into(),
+ description: "Does a thing.".into(),
+ input_schema: serde_json::json!({"type": "object"}),
+ use_input_streaming: false,
+ }],
tool_choice: None,
thinking_allowed: true,
thinking_effort: None,
@@ -487,37 +513,112 @@ mod tests {
AnthropicModelMode::Default,
);
+ // No message content block should carry cache_control anymore; the
+ // conversation breakpoint is set via top-level automatic caching.
assert_eq!(anthropic_request.messages.len(), 1);
+ for block in &anthropic_request.messages[0].content {
+ let cache_control = match block {
+ RequestContent::Text { cache_control, .. }
+ | RequestContent::Thinking { cache_control, .. }
+ | RequestContent::Image { cache_control, .. }
+ | RequestContent::ToolUse { cache_control, .. }
+ | RequestContent::ToolResult { cache_control, .. } => *cache_control,
+ RequestContent::RedactedThinking { .. } => None,
+ };
+ assert!(
+ cache_control.is_none(),
+ "message content blocks should no longer be individually marked",
+ );
+ }
- let message = &anthropic_request.messages[0];
- assert_eq!(message.content.len(), 5);
-
+ // Top-level cache_control opts into automatic caching with the default
+ // 5-minute TTL for the conversation tail.
assert!(matches!(
- message.content[0],
- RequestContent::Text {
- cache_control: None,
- ..
- }
+ anthropic_request.cache_control,
+ Some(CacheControl {
+ cache_type: CacheControlType::Ephemeral,
+ ttl: None,
+ })
));
- for i in 1..3 {
- assert!(matches!(
- message.content[i],
- RequestContent::Image {
- cache_control: None,
- ..
- }
- ));
+
+ // System prompt is emitted in array form with a long-TTL breakpoint on
+ // the final text block.
+ match anthropic_request.system {
+ Some(StringOrContents::Content(ref blocks)) => {
+ assert_eq!(blocks.len(), 1);
+ assert!(matches!(
+ blocks[0],
+ RequestContent::Text {
+ cache_control: Some(CacheControl {
+ cache_type: CacheControlType::Ephemeral,
+ ttl: Some(CacheTtl::OneHour),
+ }),
+ ..
+ }
+ ));
+ }
+ other => panic!("expected system content array, got {other:?}"),
}
+ // The last (and only) tool carries a long-TTL breakpoint.
+ assert_eq!(anthropic_request.tools.len(), 1);
assert!(matches!(
- message.content[4],
- RequestContent::Image {
- cache_control: Some(CacheControl {
- cache_type: CacheControlType::Ephemeral,
- }),
- ..
- }
+ anthropic_request.tools[0].cache_control,
+ Some(CacheControl {
+ cache_type: CacheControlType::Ephemeral,
+ ttl: Some(CacheTtl::OneHour),
+ })
+ ));
+ }
+
+ #[test]
+ fn test_no_cache_control_when_caching_disabled() {
+ let request = LanguageModelRequest {
+ messages: vec![
+ LanguageModelRequestMessage {
+ role: Role::System,
+ content: vec![MessageContent::Text("You are helpful.".to_string())],
+ cache: false,
+ reasoning_details: None,
+ },
+ LanguageModelRequestMessage {
+ role: Role::User,
+ content: vec![MessageContent::Text("Hi".to_string())],
+ cache: false,
+ reasoning_details: None,
+ },
+ ],
+ thread_id: None,
+ prompt_id: None,
+ intent: None,
+ stop: vec![],
+ temperature: None,
+ tools: vec![language_model_core::LanguageModelRequestTool {
+ name: "do_thing".into(),
+ description: "Does a thing.".into(),
+ input_schema: serde_json::json!({"type": "object"}),
+ use_input_streaming: false,
+ }],
+ tool_choice: None,
+ thinking_allowed: true,
+ thinking_effort: None,
+ speed: None,
+ };
+
+ let anthropic_request = into_anthropic(
+ request,
+ "claude-3-5-sonnet".to_string(),
+ 0.7,
+ 4096,
+ AnthropicModelMode::Default,
+ );
+
+ assert!(anthropic_request.cache_control.is_none());
+ assert!(matches!(
+ anthropic_request.system,
+ Some(StringOrContents::String(_))
));
+ assert!(anthropic_request.tools[0].cache_control.is_none());
}
fn request_with_assistant_content(assistant_content: Vec<MessageContent>) -> crate::Request {
diff --git a/crates/edit_prediction_cli/src/anthropic_client.rs b/crates/edit_prediction_cli/src/anthropic_client.rs
index 7841e8a2cc1..c1706d6161b 100644
--- a/crates/edit_prediction_cli/src/anthropic_client.rs
+++ b/crates/edit_prediction_cli/src/anthropic_client.rs
@@ -47,6 +47,7 @@ impl PlainLlmClient {
thinking: None,
tool_choice: None,
system: None,
+ cache_control: None,
metadata: None,
output_config: None,
stop_sequences: Vec::new(),
@@ -87,6 +88,7 @@ impl PlainLlmClient {
thinking: None,
tool_choice: None,
system: None,
+ cache_control: None,
metadata: None,
output_config: None,
stop_sequences: Vec::new(),
@@ -582,6 +584,7 @@ impl BatchingLlmClient {
thinking: None,
tool_choice: None,
system: None,
+ cache_control: None,
metadata: None,
output_config: None,
stop_sequences: Vec::new(),