Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
bedrock: Add system-prompt cache anchor on caching-capable models (#56474)
Richard Feldman800a79554501
diff --git a/crates/bedrock/src/bedrock.rs b/crates/bedrock/src/bedrock.rsindex 75f5eddce9e..f3102b2d39c 100644--- a/crates/bedrock/src/bedrock.rs+++ b/crates/bedrock/src/bedrock.rs@@ -84,10 +84,8 @@ pub async fn stream_completion(response = response.inference_config(inference_config);- if let Some(system) = request.system {- if !system.is_empty() {- response = response.system(BedrockSystemContentBlock::Text(system));- }+ for system_block in request.system {+ response = response.system(system_block);}if let Some(guardrail_id) = &request.guardrail_identifier {@@ -207,7 +205,11 @@ pub struct Request {pub messages: Vec<BedrockMessage>,pub tools: Option<BedrockToolConfig>,pub thinking: Option<Thinking>,- pub system: Option<String>,+ /// System content blocks in prefix order. Typically `[Text(...)]` or, when+ /// the model supports prompt caching, `[Text(...), CachePoint(...)]` so the+ /// system prompt anchors its own cache prefix independent of tools and+ /// messages.+ pub system: Vec<BedrockSystemContentBlock>,pub metadata: Option<Metadata>,pub stop_sequences: Vec<String>,pub temperature: Option<f32>,diff --git a/crates/language_models/src/provider/bedrock.rs b/crates/language_models/src/provider/bedrock.rsindex 8dbad6d345d..7ff3aef5481 100644--- a/crates/language_models/src/provider/bedrock.rs+++ b/crates/language_models/src/provider/bedrock.rs@@ -7,6 +7,7 @@ use aws_config::stalled_stream_protection::StalledStreamProtectionConfig;use aws_config::{BehaviorVersion, Region};use aws_credential_types::{Credentials, Token};use aws_http_client::AwsHttpClient;+use bedrock::BedrockSystemContentBlock;use bedrock::bedrock_client::Client as BedrockClient;use bedrock::bedrock_client::config::timeout::TimeoutConfig;use bedrock::bedrock_client::types::{@@ -1104,11 +1105,24 @@ pub fn into_bedrock()};+ let mut system_blocks: Vec<BedrockSystemContentBlock> = Vec::new();+ if !system_message.is_empty() {+ system_blocks.push(BedrockSystemContentBlock::Text(system_message));+ if supports_caching {+ system_blocks.push(BedrockSystemContentBlock::CachePoint(+ CachePointBlock::builder()+ .r#type(CachePointType::Default)+ .build()+ .context("failed to build system cache point block")?,+ ));+ }+ }+Ok(bedrock::Request {model,messages: new_messages,max_tokens: max_output_tokens,- system: Some(system_message),+ system: system_blocks,tools: tool_config,thinking: if request.thinking_allowed {match thinking_mode {