Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

bedrock: Add system-prompt cache anchor on caching-capable models (#56474)

Richard Feldman800a79554501
diff --git a/crates/bedrock/src/bedrock.rs b/crates/bedrock/src/bedrock.rs
index 75f5eddce9e..f3102b2d39c 100644
--- a/crates/bedrock/src/bedrock.rs
+++ b/crates/bedrock/src/bedrock.rs
@@ -84,10 +84,8 @@ pub async fn stream_completion(
response = response.inference_config(inference_config);
- if let Some(system) = request.system {
- if !system.is_empty() {
- response = response.system(BedrockSystemContentBlock::Text(system));
- }
+ for system_block in request.system {
+ response = response.system(system_block);
}
if let Some(guardrail_id) = &request.guardrail_identifier {
@@ -207,7 +205,11 @@ pub struct Request {
pub messages: Vec<BedrockMessage>,
pub tools: Option<BedrockToolConfig>,
pub thinking: Option<Thinking>,
- pub system: Option<String>,
+ /// System content blocks in prefix order. Typically `[Text(...)]` or, when
+ /// the model supports prompt caching, `[Text(...), CachePoint(...)]` so the
+ /// system prompt anchors its own cache prefix independent of tools and
+ /// messages.
+ pub system: Vec<BedrockSystemContentBlock>,
pub metadata: Option<Metadata>,
pub stop_sequences: Vec<String>,
pub temperature: Option<f32>,
diff --git a/crates/language_models/src/provider/bedrock.rs b/crates/language_models/src/provider/bedrock.rs
index 8dbad6d345d..7ff3aef5481 100644
--- a/crates/language_models/src/provider/bedrock.rs
+++ b/crates/language_models/src/provider/bedrock.rs
@@ -7,6 +7,7 @@ use aws_config::stalled_stream_protection::StalledStreamProtectionConfig;
use aws_config::{BehaviorVersion, Region};
use aws_credential_types::{Credentials, Token};
use aws_http_client::AwsHttpClient;
+use bedrock::BedrockSystemContentBlock;
use bedrock::bedrock_client::Client as BedrockClient;
use bedrock::bedrock_client::config::timeout::TimeoutConfig;
use bedrock::bedrock_client::types::{
@@ -1104,11 +1105,24 @@ pub fn into_bedrock(
)
};
+ let mut system_blocks: Vec<BedrockSystemContentBlock> = Vec::new();
+ if !system_message.is_empty() {
+ system_blocks.push(BedrockSystemContentBlock::Text(system_message));
+ if supports_caching {
+ system_blocks.push(BedrockSystemContentBlock::CachePoint(
+ CachePointBlock::builder()
+ .r#type(CachePointType::Default)
+ .build()
+ .context("failed to build system cache point block")?,
+ ));
+ }
+ }
+
Ok(bedrock::Request {
model,
messages: new_messages,
max_tokens: max_output_tokens,
- system: Some(system_message),
+ system: system_blocks,
tools: tool_config,
thinking: if request.thinking_allowed {
match thinking_mode {