Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

language_model: Allow explicit prompt cache keys (#64460)

Agus Zubiagaa8535d86b7c4
diff --git a/crates/agent/src/thread.rs b/crates/agent/src/thread.rs
index 607aeb85b4f..c14739b2491 100644
--- a/crates/agent/src/thread.rs
+++ b/crates/agent/src/thread.rs
@@ -4148,6 +4148,7 @@ impl Thread {
let request = LanguageModelRequest {
thread_id: Some(self.id.to_string()),
+ prompt_cache_key: None,
prompt_id: Some(self.prompt_id.to_string()),
intent: Some(completion_intent),
messages,
diff --git a/crates/agent_ui/src/buffer_codegen.rs b/crates/agent_ui/src/buffer_codegen.rs
index e0d3b95277e..38798cadd99 100644
--- a/crates/agent_ui/src/buffer_codegen.rs
+++ b/crates/agent_ui/src/buffer_codegen.rs
@@ -546,6 +546,7 @@ impl CodegenAlternative {
LanguageModelRequest {
thread_id: Some(session_id),
+ prompt_cache_key: None,
prompt_id: None,
intent: Some(CompletionIntent::InlineAssist),
tools,
@@ -629,6 +630,7 @@ impl CodegenAlternative {
LanguageModelRequest {
thread_id: Some(session_id),
+ prompt_cache_key: None,
prompt_id: None,
intent: Some(CompletionIntent::InlineAssist),
tools: Vec::new(),
diff --git a/crates/agent_ui/src/terminal_inline_assistant.rs b/crates/agent_ui/src/terminal_inline_assistant.rs
index efbd1d24949..c7c9e7fb8e2 100644
--- a/crates/agent_ui/src/terminal_inline_assistant.rs
+++ b/crates/agent_ui/src/terminal_inline_assistant.rs
@@ -265,6 +265,7 @@ impl TerminalInlineAssistant {
LanguageModelRequest {
thread_id: Some(session_id),
+ prompt_cache_key: None,
prompt_id: None,
intent: Some(CompletionIntent::TerminalInlineAssist),
messages: vec![request_message],
diff --git a/crates/anthropic/src/completion.rs b/crates/anthropic/src/completion.rs
index ed909521ae5..7fdfeb319ec 100644
--- a/crates/anthropic/src/completion.rs
+++ b/crates/anthropic/src/completion.rs
@@ -895,6 +895,7 @@ mod tests {
},
],
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
stop: vec![],
@@ -1008,6 +1009,7 @@ mod tests {
},
],
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
stop: vec![],
@@ -1078,6 +1080,7 @@ mod tests {
reasoning_details: None,
}],
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
stop: vec![],
@@ -1130,6 +1133,7 @@ mod tests {
reasoning_details: None,
}],
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
stop: vec![],
@@ -1200,6 +1204,7 @@ mod tests {
reasoning_details: None,
}],
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
stop: vec![],
@@ -1262,6 +1267,7 @@ mod tests {
},
],
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
stop: vec![],
@@ -1313,6 +1319,7 @@ mod tests {
}],
thinking_effort: None,
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
stop: vec![],
diff --git a/crates/copilot_chat/src/model.rs b/crates/copilot_chat/src/model.rs
index 5d2ea914a53..e778a182583 100644
--- a/crates/copilot_chat/src/model.rs
+++ b/crates/copilot_chat/src/model.rs
@@ -1091,6 +1091,7 @@ fn into_copilot_responses(
.and_then(|_| request.effective_max_output_tokens(model.max_output_tokens()));
let LanguageModelRequest {
thread_id: _,
+ prompt_cache_key: _,
prompt_id: _,
intent: _,
messages,
diff --git a/crates/git_ui/src/git_panel.rs b/crates/git_ui/src/git_panel.rs
index 26126aeaa91..54582fcb0ec 100644
--- a/crates/git_ui/src/git_panel.rs
+++ b/crates/git_ui/src/git_panel.rs
@@ -4150,6 +4150,7 @@ impl GitPanel {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: Some(CompletionIntent::GenerateGitCommitMessage),
messages: vec![LanguageModelRequestMessage {
diff --git a/crates/language_model_core/src/request.rs b/crates/language_model_core/src/request.rs
index 6dedf9e93cd..0d9752e3e8c 100644
--- a/crates/language_model_core/src/request.rs
+++ b/crates/language_model_core/src/request.rs
@@ -469,6 +469,11 @@ pub enum CompletionIntent {
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
pub struct LanguageModelRequest {
pub thread_id: Option<String>,
+ /// Provider cache affinity, independent of conversation and transport session identity.
+ ///
+ /// Supporting providers fall back to `thread_id` when absent for legacy callers.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub prompt_cache_key: Option<String>,
pub prompt_id: Option<String>,
pub intent: Option<CompletionIntent>,
pub messages: Vec<LanguageModelRequestMessage>,
@@ -545,6 +550,33 @@ pub struct LanguageModelResponseMessage {
mod tests {
use super::*;
+ #[test]
+ fn request_prompt_cache_key_serialization() -> serde_json::Result<()> {
+ let request = LanguageModelRequest {
+ thread_id: Some("thread".into()),
+ ..Default::default()
+ };
+ let mut serialized = serde_json::to_value(&request)?;
+ assert!(serialized.get("prompt_cache_key").is_none());
+ assert_eq!(
+ serde_json::from_value::<LanguageModelRequest>(serialized.clone())?,
+ request
+ );
+
+ serialized["prompt_cache_key"] = serde_json::json!("cache-affinity");
+ let explicit: LanguageModelRequest = serde_json::from_value(serialized.clone())?;
+ assert_eq!(explicit.prompt_cache_key.as_deref(), Some("cache-affinity"));
+ assert_eq!(explicit.thread_id, request.thread_id);
+ assert_eq!(serde_json::to_value(explicit)?, serialized);
+
+ serialized["prompt_cache_key"] = serde_json::Value::Null;
+ assert_eq!(
+ serde_json::from_value::<LanguageModelRequest>(serialized)?,
+ request
+ );
+ Ok(())
+ }
+
#[test]
fn request_output_limit_serialization() -> serde_json::Result<()> {
let request = LanguageModelRequest::default();
diff --git a/crates/language_models/src/provider/mistral.rs b/crates/language_models/src/provider/mistral.rs
index 0ebb4a1b698..78841f097c7 100644
--- a/crates/language_models/src/provider/mistral.rs
+++ b/crates/language_models/src/provider/mistral.rs
@@ -877,6 +877,7 @@ mod tests {
tools: vec![],
tool_choice: None,
thread_id: Some("abcdef".into()),
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
stop: vec![],
@@ -925,6 +926,7 @@ mod tests {
tools: vec![],
tool_choice: None,
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
stop: vec![],
@@ -969,6 +971,7 @@ mod tests {
tool_choice: None,
temperature: None,
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
stop: vec![],
diff --git a/crates/language_models/src/provider/open_router.rs b/crates/language_models/src/provider/open_router.rs
index 368ff34c3d9..9a11aedce59 100644
--- a/crates/language_models/src/provider/open_router.rs
+++ b/crates/language_models/src/provider/open_router.rs
@@ -876,6 +876,7 @@ mod tests {
thinking_effort: None,
speed: None,
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
compact_at_tokens: None,
@@ -1019,6 +1020,7 @@ mod tests {
thinking_effort: None,
speed: None,
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
compact_at_tokens: None,
@@ -1086,6 +1088,7 @@ mod tests {
thinking_effort: None,
speed: None,
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
compact_at_tokens: None,
diff --git a/crates/open_ai/src/completion.rs b/crates/open_ai/src/completion.rs
index 7f0f1fc172d..359fe281bd5 100644
--- a/crates/open_ai/src/completion.rs
+++ b/crates/open_ai/src/completion.rs
@@ -216,7 +216,7 @@ pub fn into_open_ai(
None
},
prompt_cache_key: if supports_prompt_cache_key {
- request.thread_id
+ request.prompt_cache_key.or(request.thread_id)
} else {
None
},
@@ -268,6 +268,7 @@ pub fn into_open_ai_response(
let LanguageModelRequest {
thread_id,
+ prompt_cache_key,
prompt_id: _,
intent: _,
messages,
@@ -404,7 +405,7 @@ pub fn into_open_ai_response(
}),
tools,
prompt_cache_key: if supports_prompt_cache_key {
- thread_id
+ prompt_cache_key.or(thread_id)
} else {
None
},
@@ -1524,6 +1525,51 @@ mod tests {
use super::*;
+ #[test]
+ fn prompt_cache_key_respects_override_fallback_and_capability() -> Result<()> {
+ for (explicit, thread, supported, expected) in [
+ (Some("cache"), Some("thread"), true, Some("cache")),
+ (Some("cache"), None, true, Some("cache")),
+ (None, Some("thread"), true, Some("thread")),
+ (None, None, true, None),
+ (Some("cache"), Some("thread"), false, None),
+ (None, Some("thread"), false, None),
+ ] {
+ let request = LanguageModelRequest {
+ thread_id: thread.map(str::to_owned),
+ prompt_cache_key: explicit.map(str::to_owned),
+ ..Default::default()
+ };
+ let chat = into_open_ai(
+ request.clone(),
+ "gpt-5",
+ true,
+ supported,
+ None,
+ ChatCompletionMaxTokensParameter::MaxCompletionTokens,
+ None,
+ false,
+ )?;
+ let response = into_open_ai_response(
+ request,
+ "gpt-5",
+ true,
+ supported,
+ None,
+ None,
+ false,
+ &OPEN_AI_PROVIDER_ID,
+ )?;
+ assert_eq!(chat.prompt_cache_key.as_deref(), expected);
+ assert_eq!(response.prompt_cache_key.as_deref(), expected);
+ assert_eq!(
+ response.into_compact_request().prompt_cache_key.as_deref(),
+ expected
+ );
+ }
+ Ok(())
+ }
+
fn map_response_events(events: Vec<ResponsesStreamEvent>) -> Vec<LanguageModelCompletionEvent> {
block_on(async {
OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID)
@@ -1795,6 +1841,7 @@ mod tests {
let request = LanguageModelRequest {
thread_id: Some("thread-123".into()),
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![
@@ -2001,6 +2048,7 @@ mod tests {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![LanguageModelRequestMessage {
@@ -2095,6 +2143,7 @@ mod tests {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![LanguageModelRequestMessage {
@@ -2187,6 +2236,7 @@ mod tests {
fn into_open_ai_response_replays_reasoning_without_encrypted_content() {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![LanguageModelRequestMessage {
@@ -2268,6 +2318,7 @@ mod tests {
fn into_open_ai_response_omits_reasoning_when_thinking_is_disabled_and_none_is_unsupported() {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![LanguageModelRequestMessage {
@@ -2362,6 +2413,7 @@ mod tests {
] {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![LanguageModelRequestMessage {
@@ -2415,6 +2467,7 @@ mod tests {
] {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![LanguageModelRequestMessage {
@@ -2461,6 +2514,7 @@ mod tests {
fn into_open_ai_can_send_max_tokens_parameter() -> Result<()> {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![LanguageModelRequestMessage {
@@ -2538,6 +2592,7 @@ mod tests {
fn into_open_ai_response_sends_none_reasoning_when_thinking_is_disabled() -> Result<()> {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![LanguageModelRequestMessage {
@@ -2580,6 +2635,7 @@ mod tests {
fn into_open_ai_response_uses_default_effort_when_selected_effort_is_none() -> Result<()> {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![LanguageModelRequestMessage {
@@ -2624,6 +2680,7 @@ mod tests {
fn into_open_ai_response_replays_assistant_phase() {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![LanguageModelRequestMessage {
@@ -2719,6 +2776,7 @@ mod tests {
});
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![
@@ -2798,6 +2856,7 @@ mod tests {
fn into_open_ai_response_replays_reasoning_details_but_not_thinking_text() {
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![LanguageModelRequestMessage {
@@ -4098,6 +4157,7 @@ mod tests {
};
let request = LanguageModelRequest {
thread_id: None,
+ prompt_cache_key: None,
prompt_id: None,
intent: None,
messages: vec![