Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

open_router: Add prompt caching for Anthropic models (#57498)

James St-Pierref844c93c948d
diff --git a/crates/language_models/src/provider/open_router.rs b/crates/language_models/src/provider/open_router.rs
index 9fcb29b1fdf..979022a8878 100644
--- a/crates/language_models/src/provider/open_router.rs
+++ b/crates/language_models/src/provider/open_router.rs
@@ -30,6 +30,7 @@ const PROVIDER_NAME: LanguageModelProviderName = LanguageModelProviderName::new(
const API_KEY_ENV_VAR_NAME: &str = "OPENROUTER_API_KEY";
static API_KEY_ENV_VAR: LazyLock<EnvVar> = env_var!(API_KEY_ENV_VAR_NAME);
pub(crate) const RESERVED_HEADER_NAMES: &[&str] = &["HTTP-Referer", "X-Title"];
+const MAX_OPEN_ROUTER_SESSION_ID_LENGTH: usize = 256;
#[derive(Default, Clone, Debug, PartialEq)]
pub struct OpenRouterSettings {
@@ -449,23 +450,39 @@ pub fn into_open_router(
// If we ever have a more formal distionction between the models in the future,
// we should revise this to use that instead.
let is_anthropic_model = model.id().starts_with("anthropic/");
+ let session_id = open_router_session_id(request.thread_id);
let mut messages = Vec::new();
+ let mut any_message_wants_cache = false;
+ let mut last_cache_message_index: Option<usize> = None;
+
for message in request.messages {
+ let mut message_added_content = false;
let reasoning_details_for_message = if is_anthropic_model {
None
} else {
message.reasoning_details.clone()
};
+ let message_wants_cache = message.cache;
+ if message_wants_cache {
+ any_message_wants_cache = true;
+ }
+
for content in message.content {
match content {
- MessageContent::Text(text) => add_message_content_part(
- open_router::MessagePart::Text { text },
- message.role,
- &mut messages,
- reasoning_details_for_message.clone(),
- ),
+ MessageContent::Text(text) => {
+ add_message_content_part(
+ open_router::MessagePart::Text {
+ text,
+ cache_control: None,
+ },
+ message.role,
+ &mut messages,
+ reasoning_details_for_message.clone(),
+ );
+ message_added_content = true;
+ }
MessageContent::Thinking { .. } => {}
MessageContent::RedactedThinking(_) => {}
MessageContent::Compaction(_) => {}
@@ -478,6 +495,7 @@ pub fn into_open_router(
&mut messages,
reasoning_details_for_message.clone(),
);
+ message_added_content = true;
}
MessageContent::ToolUse(tool_use) => {
let tool_call = open_router::ToolCall {
@@ -503,6 +521,7 @@ pub fn into_open_router(
reasoning_details: reasoning_details_for_message.clone(),
});
}
+ message_added_content = true;
}
MessageContent::ToolResult(tool_result) => {
let content: Vec<open_router::MessagePart> = tool_result
@@ -512,6 +531,7 @@ pub fn into_open_router(
LanguageModelToolResultContent::Text(text) => {
open_router::MessagePart::Text {
text: text.to_string(),
+ cache_control: None,
}
}
LanguageModelToolResultContent::Image(image) => {
@@ -526,15 +546,42 @@ pub fn into_open_router(
content: content.into(),
tool_call_id: tool_result.tool_use_id.to_string(),
});
+ message_added_content = true;
}
}
}
+
+ if message_wants_cache && message_added_content {
+ last_cache_message_index = messages.len().checked_sub(1);
+ }
+ }
+
+ if is_anthropic_model && any_message_wants_cache {
+ // OpenRouter's top-level automatic cache_control restricts routing to
+ // Anthropic direct; explicit block breakpoints also work on Bedrock and Vertex.
+ if let Some(content) = last_cache_message_index
+ .and_then(|index| messages.get_mut(index))
+ .and_then(request_message_content_mut)
+ {
+ set_last_text_cache_control(content, cache_control(None));
+ }
+
+ if let Some(content) = messages.iter_mut().find_map(|message| match message {
+ open_router::RequestMessage::System { content } => Some(content),
+ _ => None,
+ }) {
+ set_last_text_cache_control(
+ content,
+ cache_control(Some(open_router::CacheTtl::OneHour)),
+ );
+ }
}
open_router::Request {
model: model.id().into(),
messages,
stream: true,
+ session_id,
stop: request.stop,
temperature: request.temperature.unwrap_or(0.4),
max_tokens: max_output_tokens,
@@ -576,6 +623,65 @@ pub fn into_open_router(
}
}
+fn open_router_session_id(thread_id: Option<String>) -> Option<String> {
+ thread_id.map(|thread_id| {
+ thread_id
+ .chars()
+ .take(MAX_OPEN_ROUTER_SESSION_ID_LENGTH)
+ .collect()
+ })
+}
+
+fn cache_control(ttl: Option<open_router::CacheTtl>) -> open_router::CacheControl {
+ open_router::CacheControl {
+ cache_type: open_router::CacheControlType::Ephemeral,
+ ttl,
+ }
+}
+
+fn request_message_content_mut(
+ message: &mut open_router::RequestMessage,
+) -> Option<&mut open_router::MessageContent> {
+ match message {
+ open_router::RequestMessage::User { content }
+ | open_router::RequestMessage::System { content }
+ | open_router::RequestMessage::Tool { content, .. } => Some(content),
+ open_router::RequestMessage::Assistant {
+ content: Some(content),
+ ..
+ } => Some(content),
+ open_router::RequestMessage::Assistant { content: None, .. } => None,
+ }
+}
+
+fn set_last_text_cache_control(
+ content: &mut open_router::MessageContent,
+ cache_control: open_router::CacheControl,
+) {
+ match content {
+ open_router::MessageContent::Plain(text) => {
+ let text = std::mem::take(text);
+ *content =
+ open_router::MessageContent::Multipart(vec![open_router::MessagePart::Text {
+ text,
+ cache_control: Some(cache_control),
+ }]);
+ }
+ open_router::MessageContent::Multipart(parts) => {
+ for part in parts.iter_mut().rev() {
+ if let open_router::MessagePart::Text {
+ cache_control: target,
+ ..
+ } = part
+ {
+ *target = Some(cache_control);
+ break;
+ }
+ }
+ }
+ }
+}
+
fn add_message_content_part(
new_part: open_router::MessagePart,
role: Role,
@@ -651,11 +757,23 @@ impl OpenRouterEventMapper {
let mut events = Vec::new();
if let Some(usage) = event.usage {
+ let cache_creation_input_tokens = usage
+ .prompt_tokens_details
+ .as_ref()
+ .map_or(0, |details| details.cache_write_tokens);
+ let cache_read_input_tokens = usage
+ .prompt_tokens_details
+ .as_ref()
+ .map_or(0, |details| details.cached_tokens);
+ let input_tokens = usage.prompt_tokens.saturating_sub(
+ cache_creation_input_tokens.saturating_add(cache_read_input_tokens),
+ );
+
events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
- input_tokens: usage.prompt_tokens,
+ input_tokens,
output_tokens: usage.completion_tokens,
- cache_creation_input_tokens: 0,
- cache_read_input_tokens: 0,
+ cache_creation_input_tokens,
+ cache_read_input_tokens,
})));
}
@@ -1114,19 +1232,57 @@ mod tests {
prompt_tokens: 12,
completion_tokens: 7,
total_tokens: 19,
+ prompt_tokens_details: Some(open_router::PromptTokensDetails {
+ cached_tokens: 5,
+ cache_write_tokens: 3,
+ }),
}),
});
assert_eq!(events.len(), 1);
- match events.into_iter().next().unwrap() {
- Ok(LanguageModelCompletionEvent::UsageUpdate(usage)) => {
- assert_eq!(usage.input_tokens, 12);
+ match events.into_iter().next() {
+ Some(Ok(LanguageModelCompletionEvent::UsageUpdate(usage))) => {
+ assert_eq!(usage.input_tokens, 4);
assert_eq!(usage.output_tokens, 7);
+ assert_eq!(usage.cache_creation_input_tokens, 3);
+ assert_eq!(usage.cache_read_input_tokens, 5);
+ assert_eq!(usage.total_tokens(), 19);
}
other => panic!("Expected usage update event, got: {other:?}"),
}
}
+ #[gpui::test]
+ async fn test_session_id_uses_thread_id() {
+ let model = open_router::Model::new(
+ "openai/gpt-4o",
+ Some("GPT-4o"),
+ Some(128000),
+ Some(true),
+ Some(false),
+ None,
+ None,
+ );
+ let expected_session_id = "a".repeat(MAX_OPEN_ROUTER_SESSION_ID_LENGTH);
+ let request = LanguageModelRequest {
+ thread_id: Some(format!("{expected_session_id}extra")),
+ messages: vec![language_model::LanguageModelRequestMessage {
+ role: Role::User,
+ content: vec![MessageContent::Text("Hello".to_string())],
+ cache: false,
+ reasoning_details: None,
+ }],
+ ..Default::default()
+ };
+
+ let result = into_open_router(request, &model, None);
+
+ assert_eq!(
+ result.session_id.as_deref(),
+ Some(expected_session_id.as_str())
+ );
+ }
+
#[gpui::test]
async fn test_agent_prevents_empty_reasoning_details_overwrite() {
// This test verifies that the agent layer prevents empty reasoning_details
@@ -1168,4 +1324,281 @@ mod tests {
panic!("Expected array");
}
}
+
+ #[gpui::test]
+ async fn test_anthropic_model_caching_two_tier() {
+ let model = open_router::Model::new(
+ "anthropic/claude-sonnet-4-5",
+ Some("Claude Sonnet"),
+ Some(200000),
+ Some(true),
+ Some(false),
+ None,
+ None,
+ );
+
+ let request = LanguageModelRequest {
+ messages: vec![
+ language_model::LanguageModelRequestMessage {
+ role: Role::System,
+ content: vec![MessageContent::Text("You are helpful.".to_string())],
+ cache: false,
+ reasoning_details: None,
+ },
+ language_model::LanguageModelRequestMessage {
+ role: Role::User,
+ content: vec![MessageContent::Text("Hello".to_string())],
+ cache: false,
+ reasoning_details: None,
+ },
+ language_model::LanguageModelRequestMessage {
+ role: Role::Assistant,
+ content: vec![MessageContent::Text("Hi there!".to_string())],
+ cache: false,
+ reasoning_details: None,
+ },
+ language_model::LanguageModelRequestMessage {
+ role: Role::User,
+ content: vec![MessageContent::Text("What is 2+2?".to_string())],
+ cache: true,
+ reasoning_details: None,
+ },
+ ],
+ stop: vec![],
+ temperature: None,
+ tools: vec![],
+ tool_choice: None,
+ thinking_allowed: false,
+ thinking_effort: None,
+ speed: None,
+ thread_id: None,
+ prompt_id: None,
+ intent: None,
+ compact_at_tokens: None,
+ };
+
+ let result = into_open_router(request, &model, None);
+
+ let system_cache = result.messages.iter().find_map(|m| {
+ if let open_router::RequestMessage::System { content } = m {
+ if let open_router::MessageContent::Multipart(parts) = content {
+ parts.iter().last().and_then(|p| {
+ if let open_router::MessagePart::Text { cache_control, .. } = p {
+ *cache_control
+ } else {
+ None
+ }
+ })
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ });
+ assert!(
+ matches!(
+ system_cache,
+ Some(open_router::CacheControl {
+ cache_type: open_router::CacheControlType::Ephemeral,
+ ttl: Some(open_router::CacheTtl::OneHour),
+ })
+ ),
+ "System message should have 1h cache_control, got: {system_cache:?}"
+ );
+
+ let tail_cache = result.messages.last().and_then(|last_message| {
+ if let open_router::RequestMessage::User { content } = last_message {
+ if let open_router::MessageContent::Multipart(parts) = content {
+ parts.iter().last().and_then(|part| {
+ if let open_router::MessagePart::Text { cache_control, .. } = part {
+ *cache_control
+ } else {
+ None
+ }
+ })
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ });
+ assert!(
+ matches!(
+ tail_cache,
+ Some(open_router::CacheControl {
+ cache_type: open_router::CacheControlType::Ephemeral,
+ ttl: None,
+ })
+ ),
+ "Last cache:true message should have 5min cache_control, got: {tail_cache:?}"
+ );
+
+ for (i, message) in result.messages.iter().enumerate() {
+ let is_system = matches!(message, open_router::RequestMessage::System { .. });
+ let is_last = i == result.messages.len() - 1;
+ if is_system || is_last {
+ continue;
+ }
+ let parts: Option<&Vec<open_router::MessagePart>> = match message {
+ open_router::RequestMessage::User { content }
+ | open_router::RequestMessage::System { content }
+ | open_router::RequestMessage::Tool { content, .. } => {
+ if let open_router::MessageContent::Multipart(parts) = content {
+ Some(parts)
+ } else {
+ None
+ }
+ }
+ open_router::RequestMessage::Assistant {
+ content: Some(content),
+ ..
+ } => {
+ if let open_router::MessageContent::Multipart(parts) = content {
+ Some(parts)
+ } else {
+ None
+ }
+ }
+ _ => None,
+ };
+ if let Some(parts) = parts {
+ for part in parts {
+ if let open_router::MessagePart::Text { cache_control, .. } = part {
+ assert!(
+ cache_control.is_none(),
+ "Message {i} should not have cache_control"
+ );
+ }
+ }
+ }
+ }
+ }
+
+ #[gpui::test]
+ async fn test_anthropic_model_no_cache_when_no_cache_flag() {
+ let model = open_router::Model::new(
+ "anthropic/claude-sonnet-4-5",
+ Some("Claude Sonnet"),
+ Some(200000),
+ Some(true),
+ Some(false),
+ None,
+ None,
+ );
+
+ let request = LanguageModelRequest {
+ messages: vec![
+ language_model::LanguageModelRequestMessage {
+ role: Role::System,
+ content: vec![MessageContent::Text("You are helpful.".to_string())],
+ cache: false,
+ reasoning_details: None,
+ },
+ language_model::LanguageModelRequestMessage {
+ role: Role::User,
+ content: vec![MessageContent::Text("Hello".to_string())],
+ cache: false,
+ reasoning_details: None,
+ },
+ ],
+ stop: vec![],
+ temperature: None,
+ tools: vec![],
+ tool_choice: None,
+ thinking_allowed: false,
+ thinking_effort: None,
+ speed: None,
+ thread_id: None,
+ prompt_id: None,
+ intent: None,
+ compact_at_tokens: None,
+ };
+
+ let result = into_open_router(request, &model, None);
+
+ for message in &result.messages {
+ let content = match message {
+ open_router::RequestMessage::User { content }
+ | open_router::RequestMessage::System { content } => Some(content),
+ _ => None,
+ };
+ if let Some(content) = content {
+ if let open_router::MessageContent::Multipart(parts) = content {
+ for part in parts {
+ if let open_router::MessagePart::Text { cache_control, .. } = part {
+ assert!(
+ cache_control.is_none(),
+ "No message should have cache_control when no cache:true flags"
+ );
+ }
+ }
+ }
+ }
+ }
+ }
+
+ #[gpui::test]
+ async fn test_non_anthropic_model_no_cache_control() {
+ let model = open_router::Model::new(
+ "openai/gpt-4o",
+ Some("GPT-4o"),
+ Some(128000),
+ Some(true),
+ Some(false),
+ None,
+ None,
+ );
+
+ let request = LanguageModelRequest {
+ messages: vec![
+ language_model::LanguageModelRequestMessage {
+ role: Role::System,
+ content: vec![MessageContent::Text("You are helpful.".to_string())],
+ cache: false,
+ reasoning_details: None,
+ },
+ language_model::LanguageModelRequestMessage {
+ role: Role::User,
+ content: vec![MessageContent::Text("Hello".to_string())],
+ cache: true,
+ reasoning_details: None,
+ },
+ ],
+ stop: vec![],
+ temperature: None,
+ tools: vec![],
+ tool_choice: None,
+ thinking_allowed: false,
+ thinking_effort: None,
+ speed: None,
+ thread_id: None,
+ prompt_id: None,
+ intent: None,
+ compact_at_tokens: None,
+ };
+
+ let result = into_open_router(request, &model, None);
+
+ for message in &result.messages {
+ let content = match message {
+ open_router::RequestMessage::User { content }
+ | open_router::RequestMessage::System { content } => Some(content),
+ _ => None,
+ };
+ if let Some(content) = content {
+ if let open_router::MessageContent::Multipart(parts) = content {
+ for part in parts {
+ if let open_router::MessagePart::Text { cache_control, .. } = part {
+ assert!(
+ cache_control.is_none(),
+ "Non-Anthropic model should never have cache_control"
+ );
+ }
+ }
+ }
+ }
+ }
+ }
}
diff --git a/crates/open_router/src/open_router.rs b/crates/open_router/src/open_router.rs
index 885e42fd9b5..006b1ffc97d 100644
--- a/crates/open_router/src/open_router.rs
+++ b/crates/open_router/src/open_router.rs
@@ -147,6 +147,8 @@ pub struct Request {
pub messages: Vec<RequestMessage>,
pub stream: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
+ pub session_id: Option<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub stop: Vec<String>,
@@ -248,6 +250,7 @@ impl MessageContent {
Self::Plain(text) => {
let text_part = MessagePart::Text {
text: std::mem::take(text),
+ cache_control: None,
};
*self = Self::Multipart(vec![text_part, part]);
}
@@ -259,7 +262,11 @@ impl MessageContent {
impl From<Vec<MessagePart>> for MessageContent {
fn from(parts: Vec<MessagePart>) -> Self {
if parts.len() == 1
- && let MessagePart::Text { text } = &parts[0]
+ && let MessagePart::Text {
+ text,
+ cache_control,
+ } = &parts[0]
+ && cache_control.is_none()
{
return Self::Plain(text.clone());
}
@@ -284,7 +291,7 @@ impl MessageContent {
match self {
Self::Plain(text) => Some(text),
Self::Multipart(parts) if parts.len() == 1 => {
- if let MessagePart::Text { text } = &parts[0] {
+ if let MessagePart::Text { text, .. } = &parts[0] {
Some(text)
} else {
None
@@ -300,7 +307,7 @@ impl MessageContent {
Self::Multipart(parts) => parts
.iter()
.filter_map(|part| {
- if let MessagePart::Text { text } = part {
+ if let MessagePart::Text { text, .. } = part {
Some(text.as_str())
} else {
None
@@ -312,16 +319,43 @@ impl MessageContent {
}
}
+#[derive(Debug, Serialize, Deserialize, Copy, Clone, Eq, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum CacheControlType {
+ Ephemeral,
+}
+
+#[derive(Debug, Serialize, Deserialize, Copy, Clone, Eq, PartialEq)]
+pub enum CacheTtl {
+ /// Anthropic's default ephemeral TTL (currently 5 minutes). Refreshes for
+ /// free on every cache hit.
+ #[serde(rename = "5m")]
+ FiveMinutes,
+ /// Anthropic's extended ephemeral TTL (currently 1 hour). Costs 2x base
+ /// input tokens to write, but persists across longer idle gaps.
+ #[serde(rename = "1h")]
+ OneHour,
+}
+
+#[derive(Debug, Serialize, Deserialize, Copy, Clone, Eq, PartialEq)]
+pub struct CacheControl {
+ #[serde(rename = "type")]
+ pub cache_type: CacheControlType,
+ /// Omitting this field uses the API's default 5-minute TTL.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub ttl: Option<CacheTtl>,
+}
+
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MessagePart {
Text {
text: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ cache_control: Option<CacheControl>,
},
#[serde(rename = "image_url")]
- Image {
- image_url: String,
- },
+ Image { image_url: String },
}
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
@@ -371,11 +405,21 @@ pub struct FunctionChunk {
pub thought_signature: Option<String>,
}
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct PromptTokensDetails {
+ #[serde(default)]
+ pub cached_tokens: u64,
+ #[serde(default)]
+ pub cache_write_tokens: u64,
+}
+
#[derive(Serialize, Deserialize, Debug)]
pub struct Usage {
pub prompt_tokens: u64,
pub completion_tokens: u64,
pub total_tokens: u64,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub prompt_tokens_details: Option<PromptTokensDetails>,
}
#[derive(Serialize, Deserialize, Debug)]