Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(agent): keep the Anthropic prompt cache across LoadTools and sessions (#7646)

Wolf Mermelstein528954a3045d
diff --git a/crates/agent/src/agent_loop.rs b/crates/agent/src/agent_loop.rs
index 62afff981a..78b90ce015 100644
--- a/crates/agent/src/agent_loop.rs
+++ b/crates/agent/src/agent_loop.rs
@@ -1,6 +1,6 @@
/// The main entry point: [`AgentLoop`] and [`Session`].
use crate::error::AgentError;
-use crate::hook::{BridgeInputs, RegisterFn, ToolRouter, UserToolFinisher};
+use crate::hook::{BridgeInputs, RegisterFn, ToolLoads, ToolRouter, UserToolFinisher};
use crate::model::metering::MeteringContext;
use crate::model::router::{ModelRouter, ProviderAgent};
use crate::model::{PredefinedModel, ReasoningEffort};
@@ -21,6 +21,77 @@ use tracing::Instrument as _;
const DEFAULT_MAX_TURNS: usize = 16;
const DEFAULT_MAX_TOKENS: u64 = 16_000;
+/// A session's system prompt, optionally split where the part every session
+/// of the agent repeats byte for byte ends.
+///
+/// Anthropic caches the shared part apart from the per-session rest
+/// (instructions, memory), so a session reads the shared part back even when
+/// its rest is new. Other providers receive the two joined.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SystemPrompt {
+ shared: Option<String>,
+ rest: String,
+}
+
+impl SystemPrompt {
+ /// A prompt whose leading `shared` part is the same for every session of
+ /// the agent and whose `rest` is the session's own.
+ pub fn split(shared: impl Into<String>, rest: impl Into<String>) -> Self {
+ Self {
+ shared: Some(shared.into()),
+ rest: rest.into(),
+ }
+ }
+
+ /// The shared part, when the prompt is split.
+ pub fn shared(&self) -> Option<&str> {
+ self.shared.as_deref()
+ }
+
+ /// The per-session part, or the whole prompt when it is not split.
+ pub fn rest(&self) -> &str {
+ &self.rest
+ }
+
+ /// The whole prompt as one string.
+ pub(crate) fn joined(&self) -> String {
+ self.to_string()
+ }
+
+ fn push_str(&mut self, text: &str) {
+ self.rest.push_str(text);
+ }
+}
+
+impl std::fmt::Display for SystemPrompt {
+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(
+ formatter,
+ "{}{}",
+ self.shared.as_deref().unwrap_or_default(),
+ self.rest
+ )
+ }
+}
+
+impl From<String> for SystemPrompt {
+ fn from(rest: String) -> Self {
+ Self { shared: None, rest }
+ }
+}
+
+impl From<&String> for SystemPrompt {
+ fn from(rest: &String) -> Self {
+ rest.clone().into()
+ }
+}
+
+impl From<&str> for SystemPrompt {
+ fn from(rest: &str) -> Self {
+ rest.to_owned().into()
+ }
+}
+
/// Factory for creating per-request agent sessions.
///
/// Routes each session to the provider serving the selected model id (see
@@ -153,7 +224,7 @@ impl AgentLoop {
&self,
toolset: Arc<dyn AiToolSet<Context> + Send + Sync>,
context: Arc<Context>,
- system_prompt: &str,
+ system_prompt: impl Into<SystemPrompt>,
usage_ctx: UsageContext,
) -> Session
where
@@ -167,7 +238,7 @@ impl AgentLoop {
context,
system_prompt,
usage_ctx,
- |handle, prompt, max_turns, max_tokens, telemetry| {
+ |handle, prompt, tool_search, max_turns, max_tokens, telemetry| {
let routed = ModelRouter::shared()
.expect("failed to initialize model router")
.route_or_default(&self.model);
@@ -176,6 +247,7 @@ impl AgentLoop {
self.reasoning_effort,
handle,
prompt,
+ tool_search,
max_turns,
max_tokens,
&telemetry,
@@ -194,9 +266,16 @@ impl AgentLoop {
&self,
toolset: Arc<dyn AiToolSet<Context> + Send + Sync>,
context: Arc<Context>,
- system_prompt: &str,
+ system_prompt: impl Into<SystemPrompt>,
usage_ctx: UsageContext,
- build: impl FnOnce(ToolServerHandle, &str, usize, u64, GenAiContext) -> ProviderAgent,
+ build: impl FnOnce(
+ ToolServerHandle,
+ &SystemPrompt,
+ &ToolSearch,
+ usize,
+ u64,
+ GenAiContext,
+ ) -> ProviderAgent,
) -> Session
where
Context: Clone + Send + Sync + 'static,
@@ -218,8 +297,13 @@ impl AgentLoop {
buffer.lock().expect("loaded_buffer poisoned").extend(tools)
})
};
+ let catalog = Arc::new(catalog);
+ let tool_search = ToolSearch {
+ catalog: catalog.clone(),
+ loads: ToolLoads::default(),
+ };
let request_context = RequestContext::new(usage_ctx.user.clone())
- .with_tool_search(Arc::new(catalog), loader)
+ .with_tool_search(catalog, loader)
.with_genai_telemetry(self.genai_telemetry);
// TODO this is cringe, make request context a RW lock newtype
let request_context_rw = Arc::new(RwLock::new(request_context.clone()));
@@ -291,12 +375,13 @@ impl AgentLoop {
// request. A model's training data predates its own release, so a
// newly released model doesn't recognize its own id and may fall back
// to identifying as a predecessor — tell it to trust the id.
- let mut system_prompt = format!(
- "{system_prompt}\n\nYou are the {} model. If this model id is unfamiliar, \
+ let mut system_prompt = system_prompt.into();
+ system_prompt.push_str(&format!(
+ "\n\nYou are the {} model. If this model id is unfamiliar, \
that is because it was released after your training data cutoff — trust \
this id over your training data when identifying yourself.",
self.model
- );
+ ));
// Tell the model which connected integrations it can reach via tool
// search. The prompt text lives in the `prompt` crate; the toolset names
// are the dynamic data injected here. Omitted when nothing is connected.
@@ -319,6 +404,7 @@ impl AgentLoop {
let agent = build(
handle,
&system_prompt,
+ &tool_search,
self.max_turns,
self.max_tokens,
telemetry.clone(),
@@ -333,6 +419,7 @@ impl AgentLoop {
routing,
loaded_buffer,
register_loaded,
+ tool_loads: tool_search.loads.clone(),
user_tool_finisher: self.user_tool_finisher.clone(),
},
recorder: self.recorder.clone(),
@@ -353,7 +440,7 @@ impl AgentLoop {
&self,
toolset: Arc<dyn AiToolSet<Context> + Send + Sync>,
context: Arc<Context>,
- system_prompt: &str,
+ system_prompt: impl Into<SystemPrompt>,
usage_ctx: UsageContext,
model: M,
) -> Session
@@ -366,12 +453,70 @@ impl AgentLoop {
context,
system_prompt,
usage_ctx,
- move |handle, prompt, max_turns, max_tokens, telemetry| {
- ProviderAgent::test(model, prompt, max_turns, max_tokens, handle, telemetry)
+ move |handle, prompt, _, max_turns, max_tokens, telemetry| {
+ ProviderAgent::test(
+ model,
+ &prompt.joined(),
+ max_turns,
+ max_tokens,
+ handle,
+ telemetry,
+ )
},
)
.await
}
+
+ /// Start a session routed through `router`, as production routes through
+ /// the shared one: the provider arms build the agent, over whatever
+ /// transport the router's clients were given.
+ #[cfg(test)]
+ pub(crate) async fn test_routed_session<Context, H>(
+ &self,
+ router: &ModelRouter<H>,
+ toolset: Arc<dyn AiToolSet<Context> + Send + Sync>,
+ context: Arc<Context>,
+ system_prompt: impl Into<SystemPrompt>,
+ usage_ctx: UsageContext,
+ ) -> Session
+ where
+ Context: Clone + Send + Sync + 'static,
+ H: rig_core::http_client::HttpClientExt
+ + Clone
+ + Default
+ + std::fmt::Debug
+ + Send
+ + Sync
+ + 'static,
+ {
+ self.session_with(
+ toolset,
+ context,
+ system_prompt,
+ usage_ctx,
+ |handle, prompt, tool_search, max_turns, max_tokens, telemetry| {
+ let routed = router.route_or_default(&self.model);
+ telemetry.set_model(routed.provider(), routed.model_name());
+ ProviderAgent::Test(Box::new(routed.into_agent(
+ self.reasoning_effort,
+ handle,
+ prompt,
+ tool_search,
+ max_turns,
+ max_tokens,
+ &telemetry,
+ )))
+ },
+ )
+ .await
+ }
+}
+
+/// A session's on-demand tools: the catalog the model loads from, and which
+/// result loaded which of them.
+pub(crate) struct ToolSearch {
+ pub(crate) catalog: Arc<Vec<SearchableTool>>,
+ pub(crate) loads: ToolLoads,
}
/// A single streaming conversation session.
diff --git a/crates/agent/src/hook.rs b/crates/agent/src/hook.rs
index fc19d6acc8..d75d00a8a8 100644
--- a/crates/agent/src/hook.rs
+++ b/crates/agent/src/hook.rs
@@ -68,6 +68,67 @@ pub type UserToolFinisher = Arc<
+ Sync,
>;
+/// Which call loaded which catalog tools this session, in load order.
+///
+/// The Anthropic request planner references a loaded tool from the result of
+/// the call that loaded it, so loading never changes the request's `tools`.
+#[derive(Clone, Default)]
+pub struct ToolLoads(Arc<Mutex<Vec<Load>>>);
+
+/// One call's loading of catalog tools.
+struct Load {
+ call: LoadingCall,
+ tools: Vec<String>,
+}
+
+/// A call as the history shows it. Rig hands a tool's result hook no provider
+/// call id, so an executed call is known by its name and arguments; an invalid
+/// one arrives with its id.
+#[derive(Clone, Debug, PartialEq)]
+enum LoadingCall {
+ Id(String),
+ Arguments {
+ tool_name: String,
+ arguments: serde_json::Value,
+ },
+}
+
+impl ToolLoads {
+ fn record(&self, call: LoadingCall, tools: &[SearchableTool]) {
+ self.0.lock().expect("tool_loads poisoned").push(Load {
+ call,
+ tools: tools.iter().map(|tool| tool.name.clone()).collect(),
+ });
+ }
+
+ /// The tools `call` loaded.
+ pub(crate) fn loaded_by(&self, call: &rig_core::message::ToolCall) -> Vec<String> {
+ self.0
+ .lock()
+ .expect("tool_loads poisoned")
+ .iter()
+ .filter(|load| match &load.call {
+ LoadingCall::Id(id) => *id == call.id,
+ LoadingCall::Arguments {
+ tool_name,
+ arguments,
+ } => *tool_name == call.function.name && *arguments == call.function.arguments,
+ })
+ .flat_map(|load| load.tools.iter().cloned())
+ .collect()
+ }
+
+ /// Every tool loaded this session.
+ pub(crate) fn loaded(&self) -> Vec<String> {
+ self.0
+ .lock()
+ .expect("tool_loads poisoned")
+ .iter()
+ .flat_map(|load| load.tools.iter().cloned())
+ .collect()
+ }
+}
+
/// Everything the session hands the stream bridge besides the request context.
#[derive(Clone)]
pub struct BridgeInputs {
@@ -77,6 +138,8 @@ pub struct BridgeInputs {
pub loaded_buffer: Arc<Mutex<Vec<SearchableTool>>>,
/// Registers loaded tools with the live tool server.
pub register_loaded: RegisterFn,
+ /// Records which result loaded which tools.
+ pub tool_loads: ToolLoads,
/// Finishes user tools mid-turn, when the host can.
pub user_tool_finisher: Option<UserToolFinisher>,
}
@@ -106,6 +169,8 @@ pub struct StreamBridge {
loaded_buffer: Arc<Mutex<Vec<SearchableTool>>>,
/// Registers drained tools with the live tool server.
register_loaded: RegisterFn,
+ /// Records which result loaded which tools.
+ tool_loads: ToolLoads,
/// The session's full searchable catalog. Read by
/// [`Self::on_invalid_tool_call`] to recover calls to tools the model
/// discovered but never loaded.
@@ -137,6 +202,7 @@ impl StreamBridge {
routing,
loaded_buffer,
register_loaded,
+ tool_loads,
user_tool_finisher,
} = inputs;
let (tx, rx) = mpsc::unbounded_channel();
@@ -146,6 +212,7 @@ impl StreamBridge {
routing,
loaded_buffer,
register_loaded,
+ tool_loads,
searchable_catalog,
user_tool_finisher,
cancel,
@@ -209,6 +276,7 @@ impl StreamBridge {
pub(crate) async fn handle_invalid_tool_call(
&self,
tool_name: &str,
+ tool_call_id: Option<&str>,
) -> Option<InvalidToolCallAction> {
match self
.searchable_catalog
@@ -216,7 +284,13 @@ impl StreamBridge {
.find(|tool| tool.name == tool_name)
{
Some(tool) => {
- (self.register_loaded)(vec![tool.clone()]).await;
+ let tools = vec![tool.clone()];
+ // The retry feedback answers this call, so it loads the tool.
+ if let Some(id) = tool_call_id {
+ self.tool_loads
+ .record(LoadingCall::Id(id.to_owned()), &tools);
+ }
+ (self.register_loaded)(tools).await;
tracing::info!(
tool = %tool_name,
"auto-loaded searchable tool the model called without loading"
@@ -288,6 +362,13 @@ impl StreamBridge {
std::mem::take(&mut *buf)
};
if !pending.is_empty() {
+ self.tool_loads.record(
+ LoadingCall::Arguments {
+ tool_name: tool_name.to_owned(),
+ arguments: serde_json::from_str(args).unwrap_or(serde_json::Value::Null),
+ },
+ &pending,
+ );
(self.register_loaded)(pending).await;
}
@@ -368,7 +449,8 @@ impl AgentHook for StreamBridge {
_ctx: &HookContext,
event: &InvalidToolCallContext,
) -> Option<InvalidToolCallAction> {
- self.handle_invalid_tool_call(&event.tool_name).await
+ self.handle_invalid_tool_call(&event.tool_name, event.tool_call_id.as_deref())
+ .await
}
async fn on_tool_call(
diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rs
index 219e76da1a..5a1898b6b3 100644
--- a/crates/agent/src/lib.rs
+++ b/crates/agent/src/lib.rs
@@ -20,7 +20,7 @@ mod test;
pub mod types;
pub use accumulator::StreamAccumulator;
-pub use agent_loop::{AgentLoop, Session};
+pub use agent_loop::{AgentLoop, Session, SystemPrompt};
pub use completion::{complete, complete_about_image, complete_with_history};
pub use convert::{merge_consecutive_parts, to_rig_messages};
pub use error::AgentError;
diff --git a/crates/agent/src/model/anthropic.rs b/crates/agent/src/model/anthropic.rs
index 1232e828e5..9ea9bc91a9 100644
--- a/crates/agent/src/model/anthropic.rs
+++ b/crates/agent/src/model/anthropic.rs
@@ -1,8 +1,33 @@
+use crate::hook::ToolLoads;
use crate::model::ReasoningEffort;
+use crate::model::anthropic_prompt_layout::{PROMPT_LAYOUT_KEY, PromptLayout, ToolReferences};
use crate::model::types::Model;
+use ai_toolset::SearchableTool;
+use rig_core::completion::{
+ CompletionError, CompletionModel, CompletionRequest, CompletionResponse,
+};
+use rig_core::message::{AssistantContent, Message, ToolResultContent, UserContent};
+use rig_core::streaming::StreamingCompletionResponse;
use rig_core::{client::CompletionClient, http_client::HttpClientExt, providers::anthropic};
+use std::collections::{HashMap, HashSet};
use std::sync::Arc;
+/// Models that load a `defer_loading` tool from a `tool_reference` in a tool
+/// result, read the whole cache prefix back after it, and call the tool with
+/// its schema. Each was probed against the API on 2026-10-06; none needs a
+/// beta header. The Fable models are documented to support it but were not
+/// reachable for this organization, so they keep the client-side path.
+const TOOL_REFERENCE_MODELS: &[&str] = &[
+ "claude-haiku-4-5",
+ "claude-opus-4-7",
+ "claude-opus-4-8",
+ "claude-opus-5",
+ "claude-opus-5-5",
+ "claude-sonnet-4-6",
+ "claude-sonnet-5",
+ "claude-sonnet-5-5",
+];
+
/// A Claude model bound to the native Anthropic client that serves it.
///
/// Carries the parsed [`Model`] id and a shared client. Which provider an id
@@ -24,6 +49,11 @@ impl<'a, H: HttpClientExt + Clone + Default + 'static> AnthropicModel<'a, H> {
&self.model
}
+ /// Whether this model loads deferred tools through `tool_reference`.
+ pub(crate) fn loads_tools_by_reference(&self) -> bool {
+ TOOL_REFERENCE_MODELS.contains(&self.model.name())
+ }
+
/// The rig completion model for this id. The id is passed verbatim to the
/// Anthropic API.
///
@@ -81,5 +111,219 @@ impl<'a, H: HttpClientExt + Clone + Default + 'static> AnthropicModel<'a, H> {
}
}
+/// How a session lays out its Anthropic requests for the prompt cache.
+#[derive(Clone, Default)]
+pub(crate) struct SessionLayout {
+ /// The first two system messages are the system prompt's shared part and
+ /// its per-session rest, each cached on its own.
+ pub(crate) shared_system: bool,
+ /// Catalog tools declared upfront and loaded by reference.
+ pub(crate) deferred: Option<DeferredTools>,
+}
+
+/// A session's catalog tools, declared with `defer_loading` on every request
+/// so loading one never changes `tools`, the front of the cache prefix.
+#[derive(Clone)]
+pub(crate) struct DeferredTools {
+ names: HashSet<String>,
+ /// Wire definitions in catalog order.
+ definitions: Arc<Vec<serde_json::Value>>,
+ loads: ToolLoads,
+}
+
+impl DeferredTools {
+ pub(crate) fn new(catalog: &[SearchableTool], loads: ToolLoads) -> Self {
+ let definitions = catalog
+ .iter()
+ .map(|tool| {
+ let definition = crate::tool_adapter::provider_definition(
+ tool.name.clone(),
+ tool.description.clone(),
+ &tool.schema,
+ );
+ serde_json::json!({
+ "name": definition.name,
+ "description": definition.description,
+ "input_schema": definition.parameters,
+ "defer_loading": true,
+ })
+ })
+ .collect();
+ Self {
+ names: catalog.iter().map(|tool| tool.name.clone()).collect(),
+ definitions: Arc::new(definitions),
+ loads,
+ }
+ }
+
+ /// Where each catalog tool's schema is loaded from: the result of the call
+ /// that loaded it this session, or, for a tool an earlier turn called, the
+ /// text result just before its first call. A tool is referenced once, and
+ /// the same history always places it in the same result, so the requests
+ /// of a run and of the turns after it share their prefix.
+ fn references(&self, history: &[Message]) -> Vec<ToolReferences> {
+ let loaded_this_session: HashSet<String> = self.loads.loaded().into_iter().collect();
+ let mut loaded_by_result: HashMap<String, Vec<String>> = HashMap::new();
+ let mut referenced = HashSet::new();
+ let mut references: Vec<ToolReferences> = Vec::new();
+ let mut place = |tool_use_id: &str, name: &String| {
+ if !self.names.contains(name) || !referenced.insert(name.clone()) {
+ return;
+ }
+ match references
+ .iter_mut()
+ .find(|references| references.tool_use_id == tool_use_id)
+ {
+ Some(references) => references.tool_names.push(name.clone()),
+ None => references.push(ToolReferences {
+ tool_use_id: tool_use_id.to_owned(),
+ tool_names: vec![name.clone()],
+ }),
+ }
+ };
+ let mut latest_text_result: Option<String> = None;
+ for message in history {
+ match message {
+ Message::User { content } => {
+ let results: Vec<_> = content
+ .iter()
+ .filter_map(|content| match content {
+ UserContent::ToolResult(result) => Some(result),
+ _ => None,
+ })
+ .filter(|result| {
+ result
+ .content
+ .iter()
+ .all(|content| !matches!(content, ToolResultContent::Image(_)))
+ })
+ .collect();
+ for result in &results {
+ for name in loaded_by_result.get(&result.id).into_iter().flatten() {
+ place(&result.id, name);
+ }
+ }
+ if let Some(first) = results.first() {
+ latest_text_result = Some(first.id.clone());
+ }
+ }
+ Message::Assistant { content, .. } => {
+ for content in content.iter() {
+ let AssistantContent::ToolCall(call) = content else {
+ continue;
+ };
+ loaded_by_result.insert(call.id.clone(), self.loads.loaded_by(call));
+ if !loaded_this_session.contains(&call.function.name)
+ && let Some(tool_use_id) = &latest_text_result
+ {
+ place(tool_use_id, &call.function.name);
+ }
+ }
+ }
+ Message::System { .. } => {}
+ }
+ }
+ references
+ }
+}
+
+/// An Anthropic completion model whose requests carry a [`SessionLayout`]:
+/// catalog tools move into `defer_loading` declarations and the rest of the
+/// layout is declared for
+/// [`AnthropicPromptLayout`](crate::model::anthropic_prompt_layout::AnthropicPromptLayout)
+/// to apply on the wire.
+#[derive(Clone)]
+pub(crate) struct LaidOutModel<M> {
+ inner: M,
+ layout: SessionLayout,
+}
+
+impl<M> LaidOutModel<M> {
+ pub(crate) fn new(inner: M, layout: SessionLayout) -> Self {
+ Self { inner, layout }
+ }
+
+ fn lay_out(
+ &self,
+ mut request: CompletionRequest,
+ ) -> Result<CompletionRequest, CompletionError> {
+ let history: Vec<Message> = request.chat_history.iter().cloned().collect();
+ let mut layout = PromptLayout::default();
+ if self.layout.shared_system {
+ if !matches!(
+ history.as_slice(),
+ [Message::System { .. }, Message::System { .. }, ..]
+ ) {
+ return Err(CompletionError::RequestError(
+ "the request must open with the shared and per-session system messages".into(),
+ ));
+ }
+ layout.cache_shared_system = true;
+ }
+ let mut params = match request.additional_params.take() {
+ None => serde_json::Map::new(),
+ Some(serde_json::Value::Object(params)) => params,
+ Some(_) => {
+ return Err(CompletionError::RequestError(
+ "additional_params must be an object".into(),
+ ));
+ }
+ };
+ if let Some(deferred) = &self.layout.deferred {
+ if params.contains_key("tools") {
+ return Err(CompletionError::RequestError(
+ "additional_params already declares tools".into(),
+ ));
+ }
+ request
+ .tools
+ .retain(|tool| !deferred.names.contains(&tool.name));
+ params.insert(
+ "tools".to_owned(),
+ serde_json::Value::Array(deferred.definitions.to_vec()),
+ );
+ layout.tool_references = deferred.references(&history);
+ }
+ if layout != PromptLayout::default() {
+ params.insert(
+ PROMPT_LAYOUT_KEY.to_owned(),
+ serde_json::to_value(&layout)
+ .map_err(|error| CompletionError::RequestError(error.into()))?,
+ );
+ }
+ request.additional_params =
+ (!params.is_empty()).then_some(serde_json::Value::Object(params));
+ Ok(request)
+ }
+}
+
+impl<M: CompletionModel> CompletionModel for LaidOutModel<M> {
+ type Response = M::Response;
+ type StreamingResponse = M::StreamingResponse;
+ type Client = M::Client;
+
+ fn make(client: &Self::Client, model: impl Into<String>) -> Self {
+ Self::new(M::make(client, model), SessionLayout::default())
+ }
+
+ async fn completion(
+ &self,
+ request: CompletionRequest,
+ ) -> Result<CompletionResponse<Self::Response>, CompletionError> {
+ self.inner.completion(self.lay_out(request)?).await
+ }
+
+ async fn stream(
+ &self,
+ request: CompletionRequest,
+ ) -> Result<StreamingCompletionResponse<Self::StreamingResponse>, CompletionError> {
+ self.inner.stream(self.lay_out(request)?).await
+ }
+
+ fn composes_native_output_with_tools(&self) -> bool {
+ self.inner.composes_native_output_with_tools()
+ }
+}
+
#[cfg(test)]
mod test;
diff --git a/crates/agent/src/model/anthropic/test.rs b/crates/agent/src/model/anthropic/test.rs
index 9221197122..c3661fdce2 100644
--- a/crates/agent/src/model/anthropic/test.rs
+++ b/crates/agent/src/model/anthropic/test.rs
@@ -142,3 +142,36 @@ async fn completion_caches_tools_system_and_conversation() {
async fn stream_caches_tools_system_and_conversation() {
assert_cached_wire_body(true).await;
}
+
+#[test]
+fn only_probed_models_load_tools_by_reference() {
+ let client = Arc::new(
+ anthropic::Client::builder()
+ .api_key("test-anthropic-key")
+ .build()
+ .unwrap(),
+ );
+ let loads_by_reference = |id: &str| {
+ AnthropicModel::new(Model::try_from(id).unwrap(), client.clone()).loads_tools_by_reference()
+ };
+ for id in [
+ "anthropic/claude-haiku-4-5",
+ "anthropic/claude-opus-4-7",
+ "anthropic/claude-opus-4-8",
+ "anthropic/claude-opus-5",
+ "anthropic/claude-opus-5-5",
+ "anthropic/claude-sonnet-4-6",
+ "anthropic/claude-sonnet-5",
+ "anthropic/claude-sonnet-5-5",
+ ] {
+ assert!(loads_by_reference(id), "{id}");
+ }
+ for id in [
+ "anthropic/claude-fable-5",
+ "anthropic/claude-fable-5-1",
+ "anthropic/claude-opus-4-6",
+ "anthropic/claude-opus-4-8-unreleased",
+ ] {
+ assert!(!loads_by_reference(id), "{id}");
+ }
+}
diff --git a/crates/agent/src/model/anthropic_prompt_layout.rs b/crates/agent/src/model/anthropic_prompt_layout.rs
new file mode 100644
index 0000000000..3af2b34911
--- /dev/null
+++ b/crates/agent/src/model/anthropic_prompt_layout.rs
@@ -0,0 +1,297 @@
+//! The two Anthropic prompt-cache shapes rig 0.41 cannot express, applied to
+//! the serialized request just before it leaves for the API.
+//!
+//! - A cache breakpoint on the *first* of two system blocks, so the part of the
+//! system prompt every session of an agent shares is cached apart from the
+//! per-session rest. Rig only marks the last system block.
+//! - `tool_reference` blocks in a tool result, which load a deferred tool's
+//! schema without touching `tools` (the front of the cache prefix). Rig's
+//! tool results carry only text and images.
+//!
+//! The request planner ([`super::anthropic::LaidOutModel`]) declares what to
+//! do in a [`PromptLayout`] under [`PROMPT_LAYOUT_KEY`]. This adapter applies
+//! it and strips it. A body without the key is sent byte for byte. A body
+//! whose shape does not match its layout fails the request: a layout that
+//! cannot be applied is a bug, not a request to send as it is.
+
+use bytes::Bytes;
+use rig_core::http_client::{
+ self, HttpClientExt, LazyBody, MultipartForm, Request, Response, StreamingResponse,
+};
+use serde::{Deserialize, Serialize};
+use serde_json::{Map, Value, json};
+
+#[cfg(test)]
+mod test;
+
+/// Where the request planner puts its [`PromptLayout`] in the request body.
+pub(crate) const PROMPT_LAYOUT_KEY: &str = "macro_prompt_layout";
+
+/// Anthropic's limit on `cache_control` markers per request, the top-level
+/// automatic one included.
+const MAX_CACHE_BREAKPOINTS: usize = 4;
+
+/// What to change in one request body.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub(crate) struct PromptLayout {
+ /// Mark the first of exactly two system blocks as a cache breakpoint.
+ pub(crate) cache_shared_system: bool,
+ /// Tool results that load deferred tools, in conversation order.
+ pub(crate) tool_references: Vec<ToolReferences>,
+}
+
+/// The deferred tools one tool result loads.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub(crate) struct ToolReferences {
+ /// The `tool_use_id` of the result that loads them.
+ pub(crate) tool_use_id: String,
+ /// Names of tools declared with `defer_loading`.
+ pub(crate) tool_names: Vec<String>,
+}
+
+/// Why a declared layout could not be applied.
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub(crate) enum PromptLayoutError {
+ #[error("prompt layout is malformed: {0}")]
+ Malformed(String),
+ #[error("expected exactly two system blocks, the first without a breakpoint")]
+ SharedSystem,
+ #[error("tool `{0}` is referenced but not declared with defer_loading")]
+ NotDeferred(String),
+ #[error("expected exactly one tool result `{0}`, found {1}")]
+ ToolResult(String, usize),
+ #[error("tool result `{0}` holds something other than text")]
+ ToolResultContent(String),
+ #[error("{0} cache breakpoints exceed Anthropic's limit of {MAX_CACHE_BREAKPOINTS}")]
+ Breakpoints(usize),
+}
+
+/// Applies the [`PromptLayout`] a body declares, then forwards it to `inner`.
+#[derive(Debug, Clone, Default)]
+pub struct AnthropicPromptLayout<H> {
+ inner: H,
+}
+
+impl<H> AnthropicPromptLayout<H> {
+ /// Shape requests on their way to `inner`.
+ pub fn new(inner: H) -> Self {
+ Self { inner }
+ }
+}
+
+impl<H: HttpClientExt + Clone + 'static> HttpClientExt for AnthropicPromptLayout<H> {
+ fn send<T, U>(
+ &self,
+ request: Request<T>,
+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static
+ where
+ T: Into<Bytes> + Send,
+ U: From<Bytes> + Send + 'static,
+ {
+ let shaped = shape_request(request.map(Into::into));
+ let inner = self.inner.clone();
+ async move { inner.send(shaped?).await }
+ }
+
+ fn send_multipart<U>(
+ &self,
+ request: Request<MultipartForm>,
+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static
+ where
+ U: From<Bytes> + Send + 'static,
+ {
+ self.inner.send_multipart(request)
+ }
+
+ async fn send_streaming<T>(&self, request: Request<T>) -> http_client::Result<StreamingResponse>
+ where
+ T: Into<Bytes> + Send,
+ {
+ self.inner
+ .send_streaming(shape_request(request.map(Into::into))?)
+ .await
+ }
+}
+
+fn shape_request(request: Request<Bytes>) -> http_client::Result<Request<Bytes>> {
+ let (mut parts, body) = request.into_parts();
+ let Some(shaped) =
+ shape_body(&body).map_err(|error| http_client::Error::Instance(error.into()))?
+ else {
+ return Ok(Request::from_parts(parts, body));
+ };
+ // The body changed length; let the transport compute it.
+ parts.headers.remove("content-length");
+ Ok(Request::from_parts(parts, shaped))
+}
+
+/// The body with its layout applied, or `None` when it declares none.
+fn shape_body(body: &[u8]) -> Result<Option<Bytes>, PromptLayoutError> {
+ let Ok(Value::Object(mut body)) = serde_json::from_slice::<Value>(body) else {
+ return Ok(None);
+ };
+ let Some(layout) = body.remove(PROMPT_LAYOUT_KEY) else {
+ return Ok(None);
+ };
+ let layout: PromptLayout = serde_json::from_value(layout)
+ .map_err(|error| PromptLayoutError::Malformed(error.to_string()))?;
+ apply(&mut body, &layout)?;
+ let body = serde_json::to_vec(&body)
+ .map_err(|error| PromptLayoutError::Malformed(error.to_string()))?;
+ Ok(Some(Bytes::from(body)))
+}
+
+fn apply(body: &mut Map<String, Value>, layout: &PromptLayout) -> Result<(), PromptLayoutError> {
+ if layout.cache_shared_system {
+ cache_shared_system(body)?;
+ }
+ for references in &layout.tool_references {
+ for name in &references.tool_names {
+ if !is_deferred(body, name) {
+ return Err(PromptLayoutError::NotDeferred(name.clone()));
+ }
+ }
+ reference_tools(body, references)?;
+ }
+ let breakpoints = count_breakpoints(body);
+ if breakpoints > MAX_CACHE_BREAKPOINTS {
+ return Err(PromptLayoutError::Breakpoints(breakpoints));
+ }
+ Ok(())
+}
+
+fn cache_shared_system(body: &mut Map<String, Value>) -> Result<(), PromptLayoutError> {
+ let Some(Value::Array(system)) = body.get_mut("system") else {
+ return Err(PromptLayoutError::SharedSystem);
+ };
+ let [Value::Object(shared), Value::Object(_)] = system.as_mut_slice() else {
+ return Err(PromptLayoutError::SharedSystem);
+ };
+ if shared.get("type") != Some(&json!("text")) || shared.contains_key("cache_control") {
+ return Err(PromptLayoutError::SharedSystem);
+ }
+ shared.insert("cache_control".to_owned(), json!({"type": "ephemeral"}));
+ Ok(())
+}
+
+fn is_deferred(body: &Map<String, Value>, name: &str) -> bool {
+ body.get("tools")
+ .and_then(Value::as_array)
+ .is_some_and(|tools| {
+ tools.iter().any(|tool| {
+ tool.get("name") == Some(&json!(name))
+ && tool.get("defer_loading") == Some(&json!(true))
+ })
+ })
+}
+
+/// Replace the result's content with `tool_reference` blocks, which cannot
+/// share a result with anything else, and keep what the tool said in a text
+/// block after the message's tool results.
+fn reference_tools(
+ body: &mut Map<String, Value>,
+ references: &ToolReferences,
+) -> Result<(), PromptLayoutError> {
+ let id = &references.tool_use_id;
+ let mut found = Vec::new();
+ if let Some(Value::Array(messages)) = body.get("messages") {
+ for (message_index, message) in messages.iter().enumerate() {
+ let Some(Value::Array(content)) = message.get("content") else {
+ continue;
+ };
+ for (block_index, block) in content.iter().enumerate() {
+ if block.get("type") == Some(&json!("tool_result"))
+ && block.get("tool_use_id") == Some(&json!(id))
+ {
+ found.push((message_index, block_index));
+ }
+ }
+ }
+ }
+ let [(message_index, block_index)] = found[..] else {
+ return Err(PromptLayoutError::ToolResult(id.clone(), found.len()));
+ };
+ let content = body["messages"][message_index]["content"]
+ .as_array_mut()
+ .expect("located above");
+ let said = result_text(&content[block_index])
+ .ok_or_else(|| PromptLayoutError::ToolResultContent(id.clone()))?;
+ content[block_index]["content"] = Value::Array(
+ references
+ .tool_names
+ .iter()
+ .map(|name| json!({"type": "tool_reference", "tool_name": name}))
+ .collect(),
+ );
+ let after_results = content
+ .iter()
+ .rposition(|block| block.get("type") == Some(&json!("tool_result")))
+ .expect("the referenced result is one")
+ + 1;
+ // After earlier results' moved text, so the order follows the results.
+ let position = content[after_results..]
+ .iter()
+ .take_while(|block| is_moved_result_text(block))
+ .count()
+ + after_results;
+ content.insert(
+ position,
+ json!({
+ "type": "text",
+ "text": format!("<tool_result_text tool_use_id=\"{id}\">{said}</tool_result_text>"),
+ }),
+ );
+ Ok(())
+}
+
+/// The text a tool result holds, as a string or text blocks.
+fn result_text(result: &Value) -> Option<String> {
+ match result.get("content")? {
+ Value::String(text) => Some(text.clone()),
+ Value::Array(blocks) => blocks
+ .iter()
+ .map(|block| {
+ (block.get("type") == Some(&json!("text")))
+ .then(|| block.get("text")?.as_str().map(str::to_owned))
+ .flatten()
+ })
+ .collect::<Option<Vec<_>>>()
+ .map(|texts| texts.concat()),
+ _ => None,
+ }
+}
+
+fn is_moved_result_text(block: &Value) -> bool {
+ block
+ .get("text")
+ .and_then(Value::as_str)
+ .is_some_and(|text| text.starts_with("<tool_result_text tool_use_id="))
+}
+
+/// Markers where Anthropic reads them: top level, tools, system blocks and
+/// message content blocks.
+fn count_breakpoints(body: &Map<String, Value>) -> usize {
+ let marked = |value: &Value| {
+ value
+ .get("cache_control")
+ .is_some_and(|marker| !marker.is_null())
+ };
+ let blocks = |key: &str| {
+ body.get(key).and_then(Value::as_array).map_or(0, |blocks| {
+ blocks.iter().filter(|block| marked(block)).count()
+ })
+ };
+ let message_blocks = body
+ .get("messages")
+ .and_then(Value::as_array)
+ .into_iter()
+ .flatten()
+ .filter_map(|message| message.get("content")?.as_array())
+ .flatten()
+ .filter(|block| marked(block))
+ .count();
+ usize::from(marked(&Value::Object(body.clone())))
+ + blocks("tools")
+ + blocks("system")
+ + message_blocks
+}
diff --git a/crates/agent/src/model/anthropic_prompt_layout/test.rs b/crates/agent/src/model/anthropic_prompt_layout/test.rs
new file mode 100644
index 0000000000..8398f8ce62
--- /dev/null
+++ b/crates/agent/src/model/anthropic_prompt_layout/test.rs
@@ -0,0 +1,308 @@
+use super::*;
+use std::sync::{Arc, Mutex};
+
+fn shaped(body: Value) -> Result<Option<Value>, PromptLayoutError> {
+ shape_body(&serde_json::to_vec(&body).unwrap())
+ .map(|shaped| shaped.map(|bytes| serde_json::from_slice(&bytes).unwrap()))
+}
+
+#[test]
+fn a_body_without_a_layout_is_left_alone() {
+ let body = json!({
+ "model": "claude-opus-4-8",
+ "system": [{"type": "text", "text": "You are Macro.", "cache_control": {"type": "ephemeral"}}],
+ "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}],
+ });
+ assert_eq!(shaped(body), Ok(None));
+ assert_eq!(shape_body(b"not json"), Ok(None));
+}
+
+#[test]
+fn shared_system_block_gets_its_own_breakpoint() {
+ let body = json!({
+ "system": [
+ {"type": "text", "text": "You are Macro."},
+ {"type": "text", "text": "<session_instructions>Task A</session_instructions>", "cache_control": {"type": "ephemeral"}},
+ ],
+ "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}],
+ "cache_control": {"type": "ephemeral"},
+ "macro_prompt_layout": {"cache_shared_system": true, "tool_references": []},
+ });
+ assert_eq!(
+ shaped(body),
+ Ok(Some(json!({
+ "system": [
+ {"type": "text", "text": "You are Macro.", "cache_control": {"type": "ephemeral"}},
+ {"type": "text", "text": "<session_instructions>Task A</session_instructions>", "cache_control": {"type": "ephemeral"}},
+ ],
+ "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}],
+ "cache_control": {"type": "ephemeral"},
+ })))
+ );
+}
+
+#[test]
+fn shared_system_needs_exactly_two_blocks_the_first_unmarked() {
+ let layout = json!({"cache_shared_system": true, "tool_references": []});
+ for system in [
+ json!([{"type": "text", "text": "You are Macro."}]),
+ json!([
+ {"type": "text", "text": "You are Macro.", "cache_control": {"type": "ephemeral"}},
+ {"type": "text", "text": "Task A"},
+ ]),
+ json!([
+ {"type": "text", "text": "You are Macro."},
+ {"type": "text", "text": "Task A"},
+ {"type": "text", "text": "Memory"},
+ ]),
+ ] {
+ let body = json!({"system": system, "messages": [], "macro_prompt_layout": layout});
+ assert_eq!(shaped(body), Err(PromptLayoutError::SharedSystem));
+ }
+}
+
+#[test]
+fn a_loading_result_references_its_tools_and_keeps_its_text_after_the_results() {
+ let body = json!({
+ "tools": [
+ {"name": "LoadTools", "description": "Load tools.", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}},
+ {"name": "EditPresentation", "description": "Edit a deck.", "input_schema": {"type": "object"}, "defer_loading": true},
+ {"name": "ReadPresentation", "description": "Read a deck.", "input_schema": {"type": "object"}, "defer_loading": true},
+ ],
+ "messages": [
+ {"role": "user", "content": [{"type": "text", "text": "edit my deck"}]},
+ {"role": "assistant", "content": [
+ {"type": "tool_use", "id": "toolu_load", "name": "LoadTools", "input": {"names": ["EditPresentation", "ReadPresentation"]}},
+ {"type": "tool_use", "id": "toolu_other", "name": "Search", "input": {}},
+ ]},
+ {"role": "user", "content": [
+ {"type": "tool_result", "tool_use_id": "toolu_load", "content": [{"type": "text", "text": "{\"loaded\":2}"}]},
+ {"type": "tool_result", "tool_use_id": "toolu_other", "content": "found nothing"},
+ ]},
+ ],
+ "macro_prompt_layout": {
+ "cache_shared_system": false,
+ "tool_references": [{"tool_use_id": "toolu_load", "tool_names": ["EditPresentation", "ReadPresentation"]}],
+ },
+ });
+ let shaped = shaped(body).unwrap().unwrap();
+ assert_eq!(
+ shaped["messages"][2],
+ json!({"role": "user", "content": [
+ {"type": "tool_result", "tool_use_id": "toolu_load", "content": [
+ {"type": "tool_reference", "tool_name": "EditPresentation"},
+ {"type": "tool_reference", "tool_name": "ReadPresentation"},
+ ]},
+ {"type": "tool_result", "tool_use_id": "toolu_other", "content": "found nothing"},
+ {"type": "text", "text": "<tool_result_text tool_use_id=\"toolu_load\">{\"loaded\":2}</tool_result_text>"},
+ ]})
+ );
+ assert_eq!(shaped.get("macro_prompt_layout"), None);
+}
+
+#[test]
+fn moved_texts_follow_the_order_their_results_were_referenced_in() {
+ let body = json!({
+ "tools": [
+ {"name": "A", "input_schema": {"type": "object"}, "defer_loading": true},
+ {"name": "B", "input_schema": {"type": "object"}, "defer_loading": true},
+ ],
+ "messages": [{"role": "user", "content": [
+ {"type": "tool_result", "tool_use_id": "toolu_1", "content": "one"},
+ {"type": "tool_result", "tool_use_id": "toolu_2", "content": "two"},
+ ]}],
+ "macro_prompt_layout": {
+ "cache_shared_system": false,
+ "tool_references": [
+ {"tool_use_id": "toolu_1", "tool_names": ["A"]},
+ {"tool_use_id": "toolu_2", "tool_names": ["B"]},
+ ],
+ },
+ });
+ assert_eq!(
+ shaped(body).unwrap().unwrap()["messages"][0]["content"],
+ json!([
+ {"type": "tool_result", "tool_use_id": "toolu_1", "content": [{"type": "tool_reference", "tool_name": "A"}]},
+ {"type": "tool_result", "tool_use_id": "toolu_2", "content": [{"type": "tool_reference", "tool_name": "B"}]},
+ {"type": "text", "text": "<tool_result_text tool_use_id=\"toolu_1\">one</tool_result_text>"},
+ {"type": "text", "text": "<tool_result_text tool_use_id=\"toolu_2\">two</tool_result_text>"},
+ ])
+ );
+}
+
+#[test]
+fn a_reference_must_name_a_deferred_tool() {
+ let body = json!({
+ "tools": [{"name": "EditPresentation", "input_schema": {"type": "object"}}],
+ "messages": [{"role": "user", "content": [
+ {"type": "tool_result", "tool_use_id": "toolu_load", "content": "loaded"},
+ ]}],
+ "macro_prompt_layout": {
+ "cache_shared_system": false,
+ "tool_references": [{"tool_use_id": "toolu_load", "tool_names": ["EditPresentation"]}],
+ },
+ });
+ assert_eq!(
+ shaped(body),
+ Err(PromptLayoutError::NotDeferred(
+ "EditPresentation".to_owned()
+ ))
+ );
+}
+
+#[test]
+fn a_reference_must_land_in_exactly_one_text_result() {
+ let tools = json!([{"name": "EditPresentation", "input_schema": {"type": "object"}, "defer_loading": true}]);
+ let layout = json!({
+ "cache_shared_system": false,
+ "tool_references": [{"tool_use_id": "toolu_load", "tool_names": ["EditPresentation"]}],
+ });
+ let missing = json!({"tools": tools, "messages": [], "macro_prompt_layout": layout});
+ assert_eq!(
+ shaped(missing),
+ Err(PromptLayoutError::ToolResult("toolu_load".to_owned(), 0))
+ );
+ let image = json!({
+ "tools": tools,
+ "messages": [{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_load", "content": [
+ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AA=="}},
+ ]}]}],
+ "macro_prompt_layout": layout,
+ });
+ assert_eq!(
+ shaped(image),
+ Err(PromptLayoutError::ToolResultContent(
+ "toolu_load".to_owned()
+ ))
+ );
+}
+
+#[test]
+fn a_layout_cannot_exceed_four_breakpoints() {
+ let body = json!({
+ "tools": [
+ {"name": "A", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}},
+ {"name": "B", "input_schema": {"type": "object", "properties": {"cache_control": {"type": "string"}}}, "cache_control": {"type": "ephemeral"}},
+ ],
+ "system": [
+ {"type": "text", "text": "You are Macro."},
+ {"type": "text", "text": "Task A", "cache_control": {"type": "ephemeral"}},
+ ],
+ "messages": [],
+ "cache_control": {"type": "ephemeral"},
+ "macro_prompt_layout": {"cache_shared_system": true, "tool_references": []},
+ });
+ assert_eq!(shaped(body), Err(PromptLayoutError::Breakpoints(5)));
+}
+
+#[test]
+fn a_malformed_layout_fails() {
+ let body = json!({"messages": [], "macro_prompt_layout": {"cache_shared_system": "yes"}});
+ assert!(matches!(shaped(body), Err(PromptLayoutError::Malformed(_))));
+}
+
+/// Records what reaches the transport under the layout adapter.
+#[derive(Clone, Debug, Default)]
+struct Recording(Arc<Mutex<Vec<(Option<String>, Bytes)>>>);
+
+impl HttpClientExt for Recording {
+ fn send<T, U>(
+ &self,
+ request: Request<T>,
+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static
+ where
+ T: Into<Bytes> + Send,
+ U: From<Bytes> + Send + 'static,
+ {
+ let length = request
+ .headers()
+ .get("content-length")
+ .map(|length| length.to_str().unwrap().to_owned());
+ self.0
+ .lock()
+ .unwrap()
+ .push((length, request.into_body().into()));
+ std::future::ready(Err(http_client::Error::StreamEnded))
+ }
+
+ fn send_multipart<U>(
+ &self,
+ _: Request<MultipartForm>,
+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static
+ where
+ U: From<Bytes> + Send + 'static,
+ {
+ std::future::ready(Err(http_client::Error::StreamEnded))
+ }
+
+ async fn send_streaming<T>(&self, request: Request<T>) -> http_client::Result<StreamingResponse>
+ where
+ T: Into<Bytes> + Send,
+ {
+ let _ = self.send::<T, Bytes>(request).await;
+ Err(http_client::Error::StreamEnded)
+ }
+}
+
+#[tokio::test]
+async fn the_adapter_forwards_plain_bodies_byte_for_byte_and_fails_bad_layouts() {
+ let recording = Recording::default();
+ let layout = AnthropicPromptLayout::new(recording.clone());
+ // Key order and spacing a re-serialization would not keep.
+ let plain = Bytes::from_static(br#"{"model":"claude-opus-4-8", "messages":[]}"#);
+ let request = Request::builder()
+ .header("content-length", plain.len())
+ .body(plain.clone())
+ .unwrap();
+ let _ = layout.send_streaming(request).await;
+ assert_eq!(
+ recording.0.lock().unwrap()[0],
+ (Some(plain.len().to_string()), plain)
+ );
+
+ let shaped = Bytes::from(
+ json!({
+ "system": [{"type": "text", "text": "Shared"}, {"type": "text", "text": "Rest"}],
+ "messages": [],
+ "macro_prompt_layout": {"cache_shared_system": true, "tool_references": []},
+ })
+ .to_string(),
+ );
+ let request = Request::builder()
+ .header("content-length", shaped.len())
+ .body(shaped)
+ .unwrap();
+ let _ = layout.send_streaming(request).await;
+ let (length, body) = recording.0.lock().unwrap()[1].clone();
+ assert_eq!(length, None, "the transport recomputes the changed length");
+ assert_eq!(
+ serde_json::from_slice::<Value>(&body).unwrap(),
+ json!({
+ "system": [
+ {"type": "text", "text": "Shared", "cache_control": {"type": "ephemeral"}},
+ {"type": "text", "text": "Rest"},
+ ],
+ "messages": [],
+ })
+ );
+
+ let bad = Bytes::from(
+ json!({"messages": [], "macro_prompt_layout": {"cache_shared_system": true, "tool_references": []}})
+ .to_string(),
+ );
+ let Err(error) = layout
+ .send_streaming(Request::builder().body(bad).unwrap())
+ .await
+ else {
+ panic!("a layout that cannot be applied must fail the request");
+ };
+ assert_eq!(
+ error.to_string(),
+ http_client::Error::Instance(PromptLayoutError::SharedSystem.into()).to_string()
+ );
+ assert_eq!(
+ recording.0.lock().unwrap().len(),
+ 2,
+ "a bad layout is never sent"
+ );
+}
diff --git a/crates/agent/src/model/metering_http/test.rs b/crates/agent/src/model/metering_http/test.rs
index 268f1574f6..1e8582e8c6 100644
--- a/crates/agent/src/model/metering_http/test.rs
+++ b/crates/agent/src/model/metering_http/test.rs
@@ -630,6 +630,7 @@ async fn rig_invalid_tool_retry_is_two_authorized_executions() {
#[tokio::test]
async fn routing_fallback_is_attributed_to_actual_provider_and_wire_model() {
+ use crate::model::anthropic_prompt_layout::AnthropicPromptLayout;
use crate::model::router::{ModelRouter, RoutedModel};
use rig_agent::agent::AgentBuilder;
use rig_agent::completion::Prompt;
@@ -639,11 +640,11 @@ async fn routing_fallback_is_attributed_to_actual_provider_and_wire_model() {
let transport = Transport::new(anthropic_response());
let anthropic = anthropic::Client::builder()
.api_key("test")
- .http_client(MeteredHttpClient::new(
+ .http_client(AnthropicPromptLayout::new(MeteredHttpClient::new(
transport.clone(),
"anthropic",
WireProtocol::Anthropic,
- ))
+ )))
.build()
.unwrap();
let openai = openai::Client::builder()
@@ -861,3 +862,70 @@ async fn provider_excess_is_not_clipped_to_the_authorized_budget() {
UsageEvidence::Reported(TrustedTokenUsage::from_disjoint(10, 1000, 0, 0, 0))
);
}
+
+#[tokio::test]
+async fn a_reshaped_anthropic_request_is_metered_with_its_cache_usage() {
+ use crate::model::anthropic_prompt_layout::AnthropicPromptLayout;
+
+ let journal = Arc::new(Journal::default());
+ let transport = Transport::new(concat!(
+ "data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg\",\"usage\":{\"input_tokens\":4,\"cache_read_input_tokens\":30000,\"cache_creation_input_tokens\":250}}}\n\n",
+ "data: {\"type\":\"message_delta\",\"usage\":{\"output_tokens\":7}}\n\n",
+ "data: {\"type\":\"message_stop\"}\n\n",
+ ));
+ let client = AnthropicPromptLayout::new(MeteredHttpClient::new(
+ transport.clone(),
+ "anthropic",
+ WireProtocol::Anthropic,
+ ));
+ let body = json!({
+ "model": "model",
+ "max_tokens": 50,
+ "tools": [
+ {"name": "LoadTools", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}},
+ {"name": "EditPresentation", "input_schema": {"type": "object"}, "defer_loading": true},
+ ],
+ "system": [
+ {"type": "text", "text": "Shared"},
+ {"type": "text", "text": "Task A", "cache_control": {"type": "ephemeral"}},
+ ],
+ "messages": [
+ {"role": "user", "content": [{"type": "text", "text": "edit my deck"}]},
+ {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_load", "name": "LoadTools", "input": {}}]},
+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_load", "content": "loaded"}]},
+ ],
+ "cache_control": {"type": "ephemeral"},
+ "macro_prompt_layout": {
+ "cache_shared_system": true,
+ "tool_references": [{"tool_use_id": "toolu_load", "tool_names": ["EditPresentation"]}],
+ },
+ });
+ context(
+ journal.clone(),
+ WireProtocol::Anthropic,
+ "anthropic",
+ "model",
+ )
+ .scope(async {
+ let request = Request::builder()
+ .uri("https://example.test/v1/messages")
+ .body(Bytes::from(body.to_string()))
+ .unwrap();
+ let mut stream = client.send_streaming(request).await.unwrap().into_body();
+ while stream.next().await.is_some() {}
+ })
+ .await;
+
+ let sent = transport.bodies.lock().unwrap()[0].clone();
+ assert_eq!(sent.get("macro_prompt_layout"), None);
+ assert_eq!(
+ sent["messages"][2]["content"][0]["content"],
+ json!([{"type": "tool_reference", "tool_name": "EditPresentation"}])
+ );
+ let begins = journal.begins.lock().unwrap();
+ assert_eq!(begins.len(), 1, "metering admits the reshaped request");
+ assert_eq!(
+ journal.finals.lock().unwrap()[0].usage,
+ UsageEvidence::Reported(TrustedTokenUsage::from_disjoint(4, 7, 30000, 250, 0))
+ );
+}
diff --git a/crates/agent/src/model/mod.rs b/crates/agent/src/model/mod.rs
index e48fc49230..bacdf46aaa 100644
--- a/crates/agent/src/model/mod.rs
+++ b/crates/agent/src/model/mod.rs
@@ -6,7 +6,8 @@ mod reasoning_effort;
pub(crate) mod types;
pub use predefined_model::*;
pub use reasoning_effort::*;
-mod anthropic;
+pub(crate) mod anthropic;
+pub mod anthropic_prompt_layout;
mod gemini;
pub mod metering;
pub mod metering_http;
diff --git a/crates/agent/src/model/router.rs b/crates/agent/src/model/router.rs
index 5e14b31dcb..4d7bf245e0 100644
--- a/crates/agent/src/model/router.rs
+++ b/crates/agent/src/model/router.rs
@@ -34,7 +34,8 @@ use rig_core::providers::{anthropic, gemini, openai};
use rig_core::streaming::StreamedAssistantContent;
use tracing::Instrument as _;
-use super::anthropic::AnthropicModel;
+use super::anthropic::{AnthropicModel, DeferredTools, LaidOutModel, SessionLayout};
+use super::anthropic_prompt_layout::AnthropicPromptLayout;
use super::gemini::{GeminiCompletionModel, GeminiModel};
use super::metering::{MeteringContext, WireProtocol};
use super::metering_http::MeteredHttpClient;
@@ -42,6 +43,7 @@ use super::openai::{OpenAiChatCompletionsModel, OpenAiResponsesModel};
use super::types::Model;
use super::usage_amount::usage_amount;
use super::{PredefinedModel, ReasoningEffort};
+use crate::agent_loop::{SystemPrompt, ToolSearch};
use crate::error::AgentError;
use crate::hook::{BridgeInputs, StreamBridge};
use crate::stream::{ChatCompletionStream, StreamPart};
@@ -78,7 +80,7 @@ const GOOGLE_PROVIDER: &str = "google";
/// A routed model id bound to the provider client that serves it.
pub(crate) enum RoutedModel<'a, H = MeteredHttpClient> {
/// A model on Anthropic's native API.
- Anthropic(AnthropicModel<'a, H>),
+ Anthropic(AnthropicModel<'a, AnthropicPromptLayout<H>>),
/// A model on Gemini's native GenerateContent API.
Gemini(GeminiModel<'a, H>),
/// A model on the OpenAI-compatible Chat Completions API.
@@ -120,27 +122,50 @@ impl<'a, H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> RoutedM
/// Build the rig agent for this model, applying provider-specific thinking
/// config. Pure construction — no model call is made here.
+ ///
+ /// Anthropic caches a split system prompt's shared part on its own, and a
+ /// model that loads tools by reference declares the whole catalog upfront.
+ /// Every other provider receives the prompt joined and loads tools by
+ /// registering them.
+ #[allow(clippy::too_many_arguments)]
pub(crate) fn into_agent(
self,
reasoning_effort: Option<ReasoningEffort>,
handle: ToolServerHandle,
- system_prompt: &str,
+ system_prompt: &SystemPrompt,
+ tool_search: &ToolSearch,
max_turns: usize,
max_tokens: u64,
telemetry: &GenAiContext,
) -> ProviderAgent<H> {
+ let joined = system_prompt.joined();
+ let system_prompt_text = joined.as_str();
match self {
RoutedModel::Anthropic(m) => {
let thinking = m.thinking_params(reasoning_effort);
- ProviderAgent::Anthropic(build_agent(
- m.completion(),
- thinking,
- handle,
- system_prompt,
- max_turns,
- max_tokens,
- telemetry,
- ))
+ let layout = SessionLayout {
+ shared_system: system_prompt.shared().is_some(),
+ deferred: (m.loads_tools_by_reference() && !tool_search.catalog.is_empty())
+ .then(|| {
+ DeferredTools::new(&tool_search.catalog, tool_search.loads.clone())
+ }),
+ };
+ let (preamble, session_system) = match system_prompt.shared() {
+ Some(shared) => (shared, Some(system_prompt.rest().to_owned())),
+ None => (system_prompt.rest(), None),
+ };
+ ProviderAgent::Anthropic(AnthropicAgent {
+ agent: build_agent(
+ LaidOutModel::new(m.completion(), layout),
+ thinking,
+ handle,
+ preamble,
+ max_turns,
+ max_tokens,
+ telemetry,
+ ),
+ session_system,
+ })
}
RoutedModel::Gemini(m) => {
let thinking = m.thinking_params();
@@ -148,7 +173,7 @@ impl<'a, H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> RoutedM
m.completion(),
thinking,
handle,
- system_prompt,
+ system_prompt_text,
max_turns,
max_tokens,
telemetry,
@@ -160,7 +185,7 @@ impl<'a, H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> RoutedM
m.completion(),
thinking,
handle,
- system_prompt,
+ system_prompt_text,
max_turns,
max_tokens,
telemetry,
@@ -172,7 +197,7 @@ impl<'a, H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> RoutedM
m.completion(),
thinking,
handle,
- system_prompt,
+ system_prompt_text,
max_turns,
max_tokens,
telemetry,
@@ -192,7 +217,7 @@ pub(crate) enum ProviderAgent<
H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static = MeteredHttpClient,
> {
/// An agent over Anthropic's native completion model.
- Anthropic(Agent<TracedModel<anthropic::completion::CompletionModel<H>>>),
+ Anthropic(AnthropicAgent<H>),
/// An agent over Gemini's native GenerateContent model.
Gemini(Agent<TracedModel<GeminiCompletionModel<H>>>),
/// An agent over the OpenAI Chat Completions model.
@@ -222,9 +247,17 @@ impl<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> ProviderAge
telemetry: GenAiContext,
) -> ChatCompletionStream<'static> {
match self {
- ProviderAgent::Anthropic(agent) => {
+ ProviderAgent::Anthropic(anthropic) => {
+ // After the agent's preamble, before the conversation: the
+ // second system block.
+ let history = anthropic
+ .session_system
+ .iter()
+ .map(Message::system)
+ .chain(history)
+ .collect();
drive_stream(
- agent,
+ &anthropic.agent,
WireProtocol::Anthropic,
prompt,
history,
@@ -306,6 +339,15 @@ impl<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> ProviderAge
}
}
+/// The Anthropic arm of [`ProviderAgent`]: the agent, whose preamble is the
+/// system prompt's shared part when it is split, and the per-session rest.
+pub(crate) struct AnthropicAgent<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> {
+ agent: Agent<
+ TracedModel<LaidOutModel<anthropic::completion::CompletionModel<AnthropicPromptLayout<H>>>>,
+ >,
+ session_system: Option<String>,
+}
+
/// Routes model api-id strings to the provider client that serves them.
///
/// Holds native Anthropic, Gemini, and OpenAI Responses clients plus a
@@ -314,7 +356,7 @@ impl<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> ProviderAge
/// compatible providers with `with_openai_provider` on the metered router.
#[derive(Clone)]
pub struct ModelRouter<H = ReqwestClient> {
- anthropic: Arc<anthropic::Client<H>>,
+ anthropic: Arc<anthropic::Client<AnthropicPromptLayout<H>>>,
openai: Arc<openai::Client<H>>,
gemini: Option<Arc<gemini::Client<H>>>,
openai_compatible: HashMap<String, Arc<openai::CompletionsClient<H>>>,
@@ -322,8 +364,12 @@ pub struct ModelRouter<H = ReqwestClient> {
impl<H: HttpClientExt + Clone + 'static> ModelRouter<H> {
/// Build a router over native Anthropic and OpenAI Responses clients, with
- /// no OpenAI-compatible Chat Completions providers registered yet.
- pub fn new(anthropic: anthropic::Client<H>, openai: openai::Client<H>) -> Self {
+ /// no OpenAI-compatible Chat Completions providers registered yet. The
+ /// Anthropic client sends through [`AnthropicPromptLayout`].
+ pub fn new(
+ anthropic: anthropic::Client<AnthropicPromptLayout<H>>,
+ openai: openai::Client<H>,
+ ) -> Self {
Self {
anthropic: Arc::new(anthropic),
openai: Arc::new(openai),
@@ -349,11 +395,11 @@ impl ModelRouter {
.map_err(|error| rig_core::http_client::Error::Instance(Box::new(error)))?;
let anthropic = anthropic::Client::builder()
.api_key(env.anthropic_api_key.to_string())
- .http_client(MeteredHttpClient::new(
+ .http_client(AnthropicPromptLayout::new(MeteredHttpClient::new(
http.clone(),
ANTHROPIC_PROVIDER,
WireProtocol::Anthropic,
- ))
+ )))
.build()?;
// Default base URL is api.openai.com; OpenAI's GPT models use
// Responses API so reasoning models get max_output_tokens.
@@ -862,6 +908,40 @@ where
}
}
+/// A routed agent over a test transport runs exactly as production's does.
+#[cfg(test)]
+impl<H> DynStreamAgent for ProviderAgent<H>
+where
+ H: HttpClientExt + Clone + Default + std::fmt::Debug + Send + Sync + 'static,
+{
+ fn run_stream_dyn<'a>(
+ &'a self,
+ prompt: Message,
+ history: Vec<Message>,
+ max_turns: usize,
+ inputs: BridgeInputs,
+ recorder: Arc<dyn UsageRecorder>,
+ usage_ctx: UsageContext,
+ model: String,
+ request_context: RequestContext,
+ telemetry: GenAiContext,
+ ) -> std::pin::Pin<
+ Box<dyn std::future::Future<Output = ChatCompletionStream<'static>> + Send + 'a>,
+ > {
+ Box::pin(self.run_stream(
+ prompt,
+ history,
+ max_turns,
+ inputs,
+ recorder,
+ usage_ctx,
+ model,
+ request_context,
+ telemetry,
+ ))
+ }
+}
+
#[cfg(test)]
impl ProviderAgent {
/// Build a test-only [`ProviderAgent`] backed by `model` (a fake completion
diff --git a/crates/agent/src/test/agent_loop/mod.rs b/crates/agent/src/test/agent_loop/mod.rs
index 6ae4f8b1a6..00a977ad56 100644
--- a/crates/agent/src/test/agent_loop/mod.rs
+++ b/crates/agent/src/test/agent_loop/mod.rs
@@ -7,6 +7,7 @@ mod util;
mod test_cooperative_cancellation;
mod test_eager_tools;
+mod test_prompt_cache;
mod test_telemetry;
mod test_thinking;
mod test_tool;
diff --git a/crates/agent/src/test/agent_loop/test_prompt_cache.rs b/crates/agent/src/test/agent_loop/test_prompt_cache.rs
new file mode 100644
index 0000000000..b3ffbdc0d9
--- /dev/null
+++ b/crates/agent/src/test/agent_loop/test_prompt_cache.rs
@@ -0,0 +1,453 @@
+//! The Anthropic cache prefix (`tools` → `system` → `messages`) across a run
+//! and across sessions, as the final HTTP bodies show it.
+//!
+//! A scripted transport stands in for the providers: it records each request
+//! body and answers with canned SSE. Sessions route through a real
+//! [`ModelRouter`], so the provider arms, the request planner and the wire
+//! layout build every byte the API would see.
+
+use super::util;
+use crate::SystemPrompt;
+use crate::model::anthropic_prompt_layout::AnthropicPromptLayout;
+use crate::model::router::ModelRouter;
+use crate::stream::ToolResponse;
+use ai_toolset::{
+ AsyncTool, AsyncToolCollection, RequestContext, RequestSchema, SearchableTool, ServiceContext,
+ ToolAnnotated, ToolAnnotations, ToolResult, ToolSet as AiToolSet, ToolSetError,
+};
+use async_trait::async_trait;
+use bytes::Bytes;
+use rig_core::http_client::HttpClientExt;
+use rig_core::http_client::{self, LazyBody, MultipartForm, Request, Response, StreamingResponse};
+use rig_core::providers::{anthropic, openai};
+use schemars::JsonSchema;
+use serde::Deserialize;
+use serde_json::{Value, json};
+use std::collections::VecDeque;
+use std::pin::Pin;
+use std::sync::{Arc, Mutex};
+
+/// Records request bodies and answers each with the next scripted SSE body,
+/// or fails the request once the script runs out.
+#[derive(Clone, Debug, Default)]
+struct ScriptedProvider {
+ bodies: Arc<Mutex<Vec<Value>>>,
+ responses: Arc<Mutex<VecDeque<String>>>,
+}
+
+impl ScriptedProvider {
+ fn new(responses: impl IntoIterator<Item = String>) -> Self {
+ Self {
+ bodies: Arc::default(),
+ responses: Arc::new(Mutex::new(responses.into_iter().collect())),
+ }
+ }
+
+ fn bodies(&self) -> Vec<Value> {
+ self.bodies.lock().unwrap().clone()
+ }
+}
+
+impl HttpClientExt for ScriptedProvider {
+ fn send<T, U>(
+ &self,
+ _: Request<T>,
+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static
+ where
+ T: Into<Bytes> + Send,
+ U: From<Bytes> + Send + 'static,
+ {
+ std::future::ready(Err(http_client::Error::StreamEnded))
+ }
+
+ fn send_multipart<U>(
+ &self,
+ _: Request<MultipartForm>,
+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static
+ where
+ U: From<Bytes> + Send + 'static,
+ {
+ std::future::ready(Err(http_client::Error::StreamEnded))
+ }
+
+ async fn send_streaming<T>(&self, request: Request<T>) -> http_client::Result<StreamingResponse>
+ where
+ T: Into<Bytes> + Send,
+ {
+ let body = serde_json::from_slice(&request.into_body().into()).unwrap();
+ self.bodies.lock().unwrap().push(body);
+ let Some(sse) = self.responses.lock().unwrap().pop_front() else {
+ return Err(http_client::Error::StreamEnded);
+ };
+ let stream = futures::stream::iter([Ok(Bytes::from(sse))]);
+ Ok(Response::builder()
+ .header("content-type", "text/event-stream")
+ .body(Box::pin(stream) as http_client::sse::BoxedStream)
+ .unwrap())
+ }
+}
+
+fn sse(events: &[Value]) -> String {
+ events
+ .iter()
+ .map(|event| {
+ format!(
+ "event: {}\ndata: {event}\n\n",
+ event["type"].as_str().unwrap()
+ )
+ })
+ .collect()
+}
+
+fn tool_use_response(id: &str, name: &str, input: Value) -> String {
+ sse(&[
+ json!({"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": "claude-opus-4-8", "content": [], "stop_reason": null, "usage": {"input_tokens": 1, "output_tokens": 0}}}),
+ json!({"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": id, "name": name, "input": {}}}),
+ json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": input.to_string()}}),
+ json!({"type": "content_block_stop", "index": 0}),
+ json!({"type": "message_delta", "delta": {"stop_reason": "tool_use"}, "usage": {"output_tokens": 1}}),
+ json!({"type": "message_stop"}),
+ ])
+}
+
+fn text_response(text: &str) -> String {
+ sse(&[
+ json!({"type": "message_start", "message": {"id": "msg_2", "type": "message", "role": "assistant", "model": "claude-opus-4-8", "content": [], "stop_reason": null, "usage": {"input_tokens": 1, "output_tokens": 0}}}),
+ json!({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),
+ json!({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}),
+ json!({"type": "content_block_stop", "index": 0}),
+ json!({"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}}),
+ json!({"type": "message_stop"}),
+ ])
+}
+
+/// Stand-in for `ai_tools::LoadTools`: hands the named catalog tools to the
+/// session's loader.
+#[derive(Deserialize, JsonSchema)]
+#[schemars(title = "LoadTools", description = "Load deferred tools by name.")]
+struct LoadToolsStandIn {
+ names: Vec<String>,
+}
+
+impl ToolAnnotated for LoadToolsStandIn {
+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Load tools");
+}
+
+#[async_trait]
+impl AsyncTool<()> for LoadToolsStandIn {
+ type Output = Value;
+
+ async fn call(
+ &self,
+ _service_context: ServiceContext<()>,
+ request_context: RequestContext,
+ ) -> ToolResult<Self::Output> {
+ let matched: Vec<SearchableTool> = request_context
+ .searchable_tools
+ .iter()
+ .filter(|tool| self.names.contains(&tool.name))
+ .cloned()
+ .collect();
+ if let Some(loader) = request_context.tool_loader.as_ref() {
+ loader.load(matched);
+ }
+ Ok(json!({ "loaded": self.names }))
+ }
+}
+
+/// `LoadTools` upfront, `EditPresentation` in the catalog.
+struct DeferringToolSet {
+ eager: AsyncToolCollection<()>,
+}
+
+impl AiToolSet<()> for DeferringToolSet {
+ fn dispatch_tool_call<'a>(
+ &'a self,
+ context: (),
+ request_context: RequestContext,
+ tool_name: &'a str,
+ json: &'a Value,
+ ) -> Pin<Box<dyn Future<Output = Result<ToolResult<Value>, ToolSetError>> + 'a + Send>> {
+ if tool_name == "EditPresentation" {
+ return Box::pin(async { Ok(Ok(json!({ "edited": true }))) });
+ }
+ self.eager
+ .dispatch_tool_call(context, request_context, tool_name, json)
+ }
+
+ fn request_schemas(&self) -> Option<Vec<RequestSchema>> {
+ self.eager.request_schemas()
+ }
+
+ fn searchable_catalog(&self) -> Vec<SearchableTool> {
+ vec![SearchableTool {
+ name: "EditPresentation".to_owned(),
+ description: "Edit a presentation.".to_owned(),
+ schema: serde_json::from_value(json!({
+ "type": "object",
+ "properties": {"id": {"type": "string"}}
+ }))
+ .unwrap(),
+ }]
+ }
+}
+
+fn router(transport: &ScriptedProvider) -> ModelRouter<ScriptedProvider> {
+ let anthropic = anthropic::Client::builder()
+ .api_key("test-anthropic-key")
+ .http_client(AnthropicPromptLayout::new(transport.clone()))
+ .build()
+ .unwrap();
+ let openai = openai::Client::builder()
+ .api_key("test-openai-key")
+ .http_client(transport.clone())
+ .build()
+ .unwrap();
+ ModelRouter::new(anthropic, openai)
+}
+
+/// Run one session of `model` over `transport`: what it sent, and what it
+/// streamed back.
+async fn run_session(
+ transport: &ScriptedProvider,
+ model: &str,
+ system_prompt: impl Into<SystemPrompt>,
+ prompt: &str,
+) -> (Vec<Value>, util::Collected) {
+ let toolset: Arc<dyn AiToolSet<()> + Send + Sync> = Arc::new(DeferringToolSet {
+ eager: AsyncToolCollection::<()>::new().add_tool::<LoadToolsStandIn, ()>(),
+ });
+ let before = transport.bodies().len();
+ let mut session = util::test_loop()
+ .with_model(model)
+ .test_routed_session(
+ &router(transport),
+ toolset,
+ Arc::new(()),
+ system_prompt,
+ util::usage_ctx(),
+ )
+ .await;
+ let collected = util::drive(&mut session, prompt).await;
+ (transport.bodies()[before..].to_vec(), collected)
+}
+
+/// Whether `EditPresentation` ran and answered.
+fn edited(collected: &util::Collected) -> bool {
+ collected.tool_responses().into_iter().any(|response| {
+ matches!(
+ response,
+ ToolResponse::Json { name, json, .. }
+ if name == "EditPresentation" && *json == json!({"edited": true})
+ )
+ })
+}
+
+const MODEL_LINE: &str = "\n\nYou are the anthropic/claude-opus-4-8 model. If this model id is \
+ unfamiliar, that is because it was released after your training data \
+ cutoff — trust this id over your training data when identifying yourself.";
+
+fn split_prompt(task: &str) -> SystemPrompt {
+ SystemPrompt::split(
+ "# Identity\nYou are Macro.\n",
+ format!("\n<session_instructions>\n{task}\n</session_instructions>"),
+ )
+}
+
+#[tokio::test]
+async fn loading_a_tool_keeps_every_request_prefix() {
+ let transport = ScriptedProvider::new([
+ tool_use_response(
+ "toolu_load",
+ "LoadTools",
+ json!({"names": ["EditPresentation"]}),
+ ),
+ tool_use_response("toolu_edit", "EditPresentation", json!({"id": "deck"})),
+ text_response("done"),
+ ]);
+ let (bodies, collected) = run_session(
+ &transport,
+ "anthropic/claude-opus-4-8",
+ split_prompt("task A"),
+ "edit my deck",
+ )
+ .await;
+ assert!(collected.error.is_none(), "{:?}", collected.error);
+ assert!(edited(&collected), "{:?}", collected.tool_responses());
+ assert_eq!(bodies.len(), 3);
+
+ // The catalog is declared deferred from the first request, so loading
+ // never changes `tools`.
+ let tools = json!([
+ {
+ "name": "LoadTools",
+ "description": "Load deferred tools by name.",
+ "input_schema": {
+ "title": "LoadTools",
+ "required": ["names"],
+ "type": "object",
+ "properties": {"names": {"type": "array", "items": {"type": "string"}}},
+ },
+ "cache_control": {"type": "ephemeral"},
+ },
+ {
+ "name": "EditPresentation",
+ "description": "Edit a presentation.",
+ "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}},
+ "defer_loading": true,
+ },
+ ]);
+ let system = json!([
+ {"type": "text", "text": "# Identity\nYou are Macro.\n", "cache_control": {"type": "ephemeral"}},
+ {
+ "type": "text",
+ "text": format!("\n<session_instructions>\ntask A\n</session_instructions>{MODEL_LINE}"),
+ "cache_control": {"type": "ephemeral"},
+ },
+ ]);
+ for body in &bodies {
+ assert_eq!(body["tools"], tools);
+ assert_eq!(body["system"], system);
+ assert_eq!(body["cache_control"], json!({"type": "ephemeral"}));
+ }
+ // Each request's conversation is the start of the next one's.
+ for pair in bodies.windows(2) {
+ let earlier = pair[0]["messages"].as_array().unwrap();
+ let later = pair[1]["messages"].as_array().unwrap();
+ assert_eq!(&later[..earlier.len()], earlier.as_slice());
+ }
+ assert_eq!(
+ bodies[1]["messages"][2],
+ json!({"role": "user", "content": [
+ {"type": "tool_result", "tool_use_id": "toolu_load", "content": [
+ {"type": "tool_reference", "tool_name": "EditPresentation"},
+ ]},
+ {"type": "text", "text": "<tool_result_text tool_use_id=\"toolu_load\">{\"loaded\":[\"EditPresentation\"]}</tool_result_text>"},
+ ]})
+ );
+}
+
+#[tokio::test]
+async fn sessions_with_different_instructions_share_the_shared_system_block() {
+ let transport = ScriptedProvider::new([text_response("one"), text_response("two")]);
+ let (first, _) = run_session(
+ &transport,
+ "anthropic/claude-opus-4-8",
+ split_prompt("task A"),
+ "hi",
+ )
+ .await;
+ let (second, _) = run_session(
+ &transport,
+ "anthropic/claude-opus-4-8",
+ split_prompt("task B"),
+ "hi",
+ )
+ .await;
+
+ assert_eq!(first[0]["tools"], second[0]["tools"]);
+ let shared = json!({
+ "type": "text",
+ "text": "# Identity\nYou are Macro.\n",
+ "cache_control": {"type": "ephemeral"},
+ });
+ assert_eq!(first[0]["system"][0], shared);
+ assert_eq!(second[0]["system"][0], shared);
+ assert_ne!(first[0]["system"][1], second[0]["system"][1]);
+}
+
+#[tokio::test]
+async fn a_deferred_tool_called_before_loading_is_loaded_from_its_retry() {
+ let transport = ScriptedProvider::new([
+ tool_use_response("toolu_early", "EditPresentation", json!({"id": "deck"})),
+ tool_use_response("toolu_edit", "EditPresentation", json!({"id": "deck"})),
+ text_response("done"),
+ ]);
+ let (bodies, collected) = run_session(
+ &transport,
+ "anthropic/claude-opus-4-8",
+ split_prompt("task A"),
+ "edit my deck",
+ )
+ .await;
+ assert!(collected.error.is_none(), "{:?}", collected.error);
+ assert!(edited(&collected), "{:?}", collected.tool_responses());
+
+ assert_eq!(bodies[0]["tools"], bodies[1]["tools"]);
+ let retry = &bodies[1]["messages"][2]["content"];
+ assert_eq!(
+ retry[0],
+ json!({"type": "tool_result", "tool_use_id": "toolu_early", "content": [
+ {"type": "tool_reference", "tool_name": "EditPresentation"},
+ ]})
+ );
+ assert!(
+ retry[1]["text"]
+ .as_str()
+ .unwrap()
+ .starts_with("<tool_result_text tool_use_id=\"toolu_early\">"),
+ "{retry}"
+ );
+}
+
+#[tokio::test]
+async fn a_model_without_tool_references_loads_by_registering() {
+ let transport = ScriptedProvider::new([
+ tool_use_response(
+ "toolu_load",
+ "LoadTools",
+ json!({"names": ["EditPresentation"]}),
+ ),
+ text_response("done"),
+ ]);
+ let (bodies, collected) = run_session(
+ &transport,
+ "anthropic/claude-fable-5-1",
+ split_prompt("task A"),
+ "edit my deck",
+ )
+ .await;
+ assert!(collected.error.is_none(), "{:?}", collected.error);
+
+ let names = |body: &Value| -> Vec<String> {
+ body["tools"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|tool| tool["name"].as_str().unwrap().to_owned())
+ .collect()
+ };
+ assert_eq!(names(&bodies[0]), ["LoadTools"]);
+ assert_eq!(names(&bodies[1]), ["LoadTools", "EditPresentation"]);
+ let wire = serde_json::to_string(&bodies).unwrap();
+ assert!(!wire.contains("defer_loading") && !wire.contains("tool_reference"));
+ // The system prompt is still cached in two parts.
+ assert_eq!(bodies[0]["system"].as_array().unwrap().len(), 2);
+}
+
+#[tokio::test]
+async fn other_providers_receive_the_prompt_joined_as_before() {
+ let split = ScriptedProvider::default();
+ let (split_bodies, _) =
+ run_session(&split, "openai/gpt-5.5", split_prompt("task A"), "hi").await;
+ let joined = ScriptedProvider::default();
+ let (joined_bodies, _) = run_session(
+ &joined,
+ "openai/gpt-5.5",
+ "# Identity\nYou are Macro.\n\n<session_instructions>\ntask A\n</session_instructions>",
+ "hi",
+ )
+ .await;
+
+ assert!(!split_bodies.is_empty());
+ assert_eq!(split_bodies, joined_bodies);
+ let wire = serde_json::to_string(&split_bodies).unwrap();
+ for anthropic_only in [
+ "defer_loading",
+ "tool_reference",
+ "macro_prompt_layout",
+ "cache_control",
+ ] {
+ assert!(!wire.contains(anthropic_only), "{anthropic_only} in {wire}");
+ }
+}
diff --git a/crates/agent/src/test/test_hook.rs b/crates/agent/src/test/test_hook.rs
index 3852d366ec..4c68bc1e7a 100644
--- a/crates/agent/src/test/test_hook.rs
+++ b/crates/agent/src/test/test_hook.rs
@@ -43,6 +43,7 @@ fn inputs(
routing: Arc::new(|_| None),
loaded_buffer,
register_loaded: register,
+ tool_loads: ToolLoads::default(),
user_tool_finisher,
}
}
@@ -195,18 +196,25 @@ async fn on_tool_call_coerces_non_object_args_to_empty_object() {
async fn invalid_call_to_searchable_tool_loads_it_and_retries() {
let (register, registered) = recording_register();
let catalog = Arc::new(vec![searchable("mcp__linear__create_issue")]);
- let (bridge, _rx) = StreamBridge::channel(
- inputs(Arc::new(Mutex::new(Vec::new())), register, None),
- catalog,
- CancellationToken::new(),
- );
+ let bridge_inputs = inputs(Arc::new(Mutex::new(Vec::new())), register, None);
+ let tool_loads = bridge_inputs.tool_loads.clone();
+ let (bridge, _rx) = StreamBridge::channel(bridge_inputs, catalog, CancellationToken::new());
let action = bridge
- .handle_invalid_tool_call("mcp__linear__create_issue")
+ .handle_invalid_tool_call("mcp__linear__create_issue", Some("toolu_1"))
.await;
- // The unloaded-but-searchable tool was registered and the turn retries.
+ // The unloaded-but-searchable tool was registered and the turn retries,
+ // the retry's result being the one that loaded it.
assert_eq!(&*registered.lock().unwrap(), &["mcp__linear__create_issue"]);
+ let call = rig_core::message::ToolCall::new(
+ "toolu_1".to_owned(),
+ rig_core::message::ToolFunction::new(
+ "mcp__linear__create_issue".to_owned(),
+ serde_json::json!({}),
+ ),
+ );
+ assert_eq!(tool_loads.loaded_by(&call), ["mcp__linear__create_issue"]);
let Some(InvalidToolCallAction::Retry { feedback }) = action else {
panic!("expected retry, got {action:?}");
};
@@ -224,7 +232,7 @@ async fn invalid_call_to_unknown_tool_retries_with_feedback_without_loading() {
);
let action = bridge
- .handle_invalid_tool_call("mcp__nope__hallucinated")
+ .handle_invalid_tool_call("mcp__nope__hallucinated", Some("toolu_1"))
.await;
// Nothing exists to load; the model gets corrective feedback instead of
diff --git a/crates/agent/src/test/test_router.rs b/crates/agent/src/test/test_router.rs
index cfba169433..fcdd34a698 100644
--- a/crates/agent/src/test/test_router.rs
+++ b/crates/agent/src/test/test_router.rs
@@ -1,3 +1,4 @@
+use crate::model::anthropic_prompt_layout::AnthropicPromptLayout;
use crate::model::router::*;
use crate::model::types::Model;
use crate::model::{PredefinedModel, ReasoningEffort};
@@ -6,6 +7,7 @@ use rig_core::providers::{anthropic, gemini, openai};
fn test_router() -> ModelRouter {
let anthropic = anthropic::Client::builder()
.api_key("test-anthropic-key")
+ .http_client(AnthropicPromptLayout::default())
.build()
.unwrap();
let openai = openai::Client::builder()
diff --git a/crates/agent/src/tool_adapter.rs b/crates/agent/src/tool_adapter.rs
index 66481264f8..609f723762 100644
--- a/crates/agent/src/tool_adapter.rs
+++ b/crates/agent/src/tool_adapter.rs
@@ -73,6 +73,23 @@ pub(crate) fn take_description(
.unwrap_or_else(|| name.to_owned())
}
+/// A tool's definition as providers receive it, from its schema.
+pub(crate) fn provider_definition(
+ name: String,
+ description: String,
+ schema: &schemars::Schema,
+) -> rig_core::completion::ToolDefinition {
+ let mut parameters =
+ serde_json::to_value(schema).unwrap_or(serde_json::Value::Object(serde_json::Map::new()));
+ normalize_request_schema(&mut parameters);
+ let description = take_description(&name, description, &mut parameters);
+ rig_core::completion::ToolDefinition {
+ name,
+ description,
+ parameters,
+ }
+}
+
type Deserializer<Context> = Arc<
dyn Fn(
&serde_json::Value,
@@ -205,12 +222,11 @@ impl DynToolSetAdapter {
description,
schema,
}| {
- let schema_json = serde_json::to_value(&schema)
- .unwrap_or(serde_json::Value::Object(serde_json::Map::new()));
+ let definition = provider_definition(name, description, &schema);
Self::build(
- name,
- description,
- schema_json,
+ definition.name,
+ definition.description,
+ definition.parameters,
toolset.clone(),
context.clone(),
request_context.clone(),
@@ -237,12 +253,11 @@ impl DynToolSetAdapter {
where
Context: Clone + Send + Sync + 'static,
{
- let schema_json = serde_json::to_value(&schema)
- .unwrap_or(serde_json::Value::Object(serde_json::Map::new()));
+ let definition = provider_definition(name, description, &schema);
Self::build(
- name,
- description,
- schema_json,
+ definition.name,
+ definition.description,
+ definition.parameters,
toolset,
context,
request_context,
@@ -252,7 +267,7 @@ impl DynToolSetAdapter {
fn build<Context>(
name: String,
description: String,
- mut schema: serde_json::Value,
+ schema: serde_json::Value,
toolset: Arc<dyn AiToolSet<Context> + Send + Sync>,
context: Arc<Context>,
request_context: Arc<RwLock<RequestContext>>,
@@ -260,8 +275,6 @@ impl DynToolSetAdapter {
where
Context: Clone + Send + Sync + 'static,
{
- normalize_request_schema(&mut schema);
- let description = take_description(&name, description, &mut schema);
let tool_name = name.clone();
DynamicTool::new(
name,
diff --git a/crates/agent_inmem/src/outbound/rig_engine/measure_prompt_cache.rs b/crates/agent_inmem/src/outbound/rig_engine/measure_prompt_cache.rs
new file mode 100644
index 0000000000..4d77b95154
--- /dev/null
+++ b/crates/agent_inmem/src/outbound/rig_engine/measure_prompt_cache.rs
@@ -0,0 +1,188 @@
+//! Measures Anthropic prompt-cache reads and writes per model call, against
+//! the real API, with Macro's real tools and system prompt routed through the
+//! production agent loop. Tool calls other than `LoadTools` answer `ok`
+//! without running, so nothing touches a workspace.
+//!
+//! ```text
+//! doppler run --project agent-harness-service --config dev -- \
+//! cargo test -p agent_inmem measure_prompt_cache -- --ignored --nocapture
+//! ```
+
+use super::*;
+use ai_toolset::{RequestSchema, SearchableTool, ToolResult, ToolSetError};
+use ai_usage::financial::{FinalizeInvocation, UsageEvidence, WriteDisposition};
+use ai_usage::{AiFeature, FinancialFuture, TrackedInvocation, UsageContext, UsageTracking};
+use std::pin::Pin;
+use std::sync::Mutex;
+
+const MODEL: &str = "anthropic/claude-sonnet-5-5";
+
+/// Every model call's reported usage, in order.
+#[derive(Default)]
+struct RecordedUsage(Mutex<Vec<UsageEvidence>>);
+
+impl UsageTracking for RecordedUsage {
+ fn begin(&self, _: TrackedInvocation) -> FinancialFuture<'_, WriteDisposition> {
+ Box::pin(async { Ok(WriteDisposition::Inserted) })
+ }
+
+ fn finalize(&self, evidence: FinalizeInvocation) -> FinancialFuture<'_, WriteDisposition> {
+ self.0.lock().unwrap().push(evidence.usage);
+ Box::pin(async { Ok(WriteDisposition::Inserted) })
+ }
+}
+
+/// The agent session's tools as production defers them; `LoadTools` loads,
+/// every other call answers without running.
+struct InertTools(Arc<dyn AiToolSet<InMemToolContext> + Send + Sync>);
+
+impl AiToolSet<()> for InertTools {
+ fn dispatch_tool_call<'a>(
+ &'a self,
+ _: (),
+ request_context: ai_toolset::RequestContext,
+ tool_name: &'a str,
+ json: &'a serde_json::Value,
+ ) -> Pin<
+ Box<dyn Future<Output = Result<ToolResult<serde_json::Value>, ToolSetError>> + 'a + Send>,
+ > {
+ Box::pin(async move {
+ if tool_name == "LoadTools" {
+ let names: Vec<String> =
+ serde_json::from_value(json["names"].clone()).unwrap_or_default();
+ let loaded: Vec<SearchableTool> = request_context
+ .searchable_tools
+ .iter()
+ .filter(|tool| names.contains(&tool.name))
+ .cloned()
+ .collect();
+ if let Some(loader) = &request_context.tool_loader {
+ loader.load(loaded);
+ }
+ return Ok(Ok(serde_json::json!({ "loaded": names })));
+ }
+ Ok(Ok(serde_json::json!({ "ok": true })))
+ })
+ }
+
+ fn request_schemas(&self) -> Option<Vec<RequestSchema>> {
+ self.0.request_schemas()
+ }
+
+ fn searchable_catalog(&self) -> Vec<SearchableTool> {
+ self.0.searchable_catalog()
+ }
+
+ fn searchable_toolset_names(&self) -> Vec<String> {
+ self.0.searchable_toolset_names()
+ }
+}
+
+/// One session's turn: each model call's (input, cache read, cache write).
+async fn turn(
+ native: &NativeTools,
+ run: &str,
+ instructions: &str,
+ prompt: &str,
+) -> Vec<(u64, u64, u64)> {
+ let tools: Arc<dyn AiToolSet<InMemToolContext> + Send + Sync> = Arc::new(DeferredToolSet::new(
+ native.for_turn(false),
+ Arc::clone(&native.deferred),
+ ));
+ let identity = AgentIdentity {
+ bot: bot_id::BotId::TEST_A,
+ // Unique per run, so every run starts from a cold cache.
+ name: format!("Macro {run}"),
+ handle: "macro".to_owned(),
+ };
+ let system_prompt = system_prompt(&native.prompt, Some(&identity), Some(instructions), None);
+ let user = MacroUserIdStr::try_from_email("[email protected]").unwrap();
+ let usage = UsageContext::new(AiFeature::AgentSession, user);
+ let recorded = Arc::new(RecordedUsage::default());
+ let metering = agent::MeteringContext::tracking_only(recorded.clone(), usage.clone());
+ metering
+ .scope(async {
+ let mut session = AgentLoop::new(Arc::new(ai_usage::NoOpUsageRecorder))
+ .with_model(MODEL)
+ .session(
+ Arc::new(InertTools(tools)),
+ Arc::new(()),
+ system_prompt,
+ usage,
+ )
+ .await;
+ let mut stream = session
+ .send_message(vec![agent::Message::user(prompt)])
+ .await
+ .unwrap();
+ while let Some(part) = stream.next().await {
+ part.unwrap();
+ }
+ })
+ .await;
+ recorded
+ .0
+ .lock()
+ .unwrap()
+ .iter()
+ .map(|evidence| match evidence {
+ UsageEvidence::Reported(usage) => {
+ (usage.input(), usage.cache_read(), usage.cache_write())
+ }
+ other => panic!("unreported usage: {other:?}"),
+ })
+ .collect()
+}
+
+fn print(label: &str, calls: &[(u64, u64, u64)]) {
+ for (index, (input, read, write)) in calls.iter().enumerate() {
+ println!(
+ "{label} call {}: input {input}, cache read {read}, cache write {write}",
+ index + 1
+ );
+ }
+}
+
+#[tokio::test]
+#[ignore = "calls the Anthropic API; run under doppler with --nocapture"]
+async fn measure_prompt_cache() {
+ let native = NativeTools::new();
+ let run = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos()
+ .to_string();
+ let task = |id: &str| {
+ format!(
+ "You've been assigned the original Macro task linked below. Carry out its requested work.\n\
+ <m-document-mention>{{\"documentId\":\"{id}\",\"documentName\":\"Original assigned task\"}}</m-document-mention>"
+ )
+ };
+
+ let load = turn(
+ &native,
+ &format!("{run}-load"),
+ &task("01a11170-ddae-7975-8fa5-9cf207fd77a6"),
+ "Call LoadTools to load ListReminders. Do not call ListReminders or any other tool. \
+ After LoadTools answers, reply with the single word done.",
+ )
+ .await;
+ print("LoadTools run", &load);
+
+ let first = turn(
+ &native,
+ &format!("{run}-sessions"),
+ &task("01a11170-ddae-7975-8fa5-9cf207fd77a6"),
+ "Reply with the single word hi. Do not call any tool.",
+ )
+ .await;
+ let second = turn(
+ &native,
+ &format!("{run}-sessions"),
+ &task("01a11171-1ca6-7a6b-9594-99b4f2b56c34"),
+ "Reply with the single word hi. Do not call any tool.",
+ )
+ .await;
+ print("session 1", &first);
+ print("session 2", &second);
+}
diff --git a/crates/agent_inmem/src/outbound/rig_engine/test.rs b/crates/agent_inmem/src/outbound/rig_engine/test.rs
index 7d65fe1634..49453eac88 100644
--- a/crates/agent_inmem/src/outbound/rig_engine/test.rs
+++ b/crates/agent_inmem/src/outbound/rig_engine/test.rs
@@ -128,7 +128,7 @@ async fn shared_definitions_keep_the_callers_context_and_check_each_sessions_gat
#[test]
fn instructions_are_a_delimited_section_after_the_standing_prompt() {
- let prompt = system_prompt(&TOOLS, None, Some("be terse"), None);
+ let prompt = system_prompt(&TOOLS, None, Some("be terse"), None).to_string();
assert!(
prompt.starts_with(&prompt::agent_session::PROMPT.to_string()),
@@ -152,7 +152,7 @@ fn identity_precedes_the_standing_prompt_and_does_not_need_instructions() {
name: "Grunk".to_owned(),
handle: "grunk".to_owned(),
};
- let prompt = system_prompt(&TOOLS, Some(&identity), None, None);
+ let prompt = system_prompt(&TOOLS, Some(&identity), None, None).to_string();
let identity_section = prompt::agent_identity::render("Grunk", "grunk");
assert!(
@@ -168,7 +168,7 @@ fn identity_precedes_the_standing_prompt_and_does_not_need_instructions() {
/// as an instruction.
#[test]
fn memory_follows_instructions_rather_than_preceding_them() {
- let prompt = system_prompt(&TOOLS, None, Some("be terse"), Some("prefers Rust"));
+ let prompt = system_prompt(&TOOLS, None, Some("be terse"), Some("prefers Rust")).to_string();
let instructions = prompt
.find("<session_instructions>")
@@ -179,11 +179,45 @@ fn memory_follows_instructions_rather_than_preceding_them() {
assert!(instructions < memory);
}
+/// Two sessions of one agent differ only after the static Macro prompt, so
+/// the part before it is cached once for all of them.
+#[test]
+fn sessions_of_an_agent_share_everything_before_their_instructions() {
+ let identity = AgentIdentity {
+ bot: bot_id::BotId::TEST_A,
+ name: "Grunk".to_owned(),
+ handle: "grunk".to_owned(),
+ };
+ let first = system_prompt(
+ &TOOLS,
+ Some(&identity),
+ Some("task A"),
+ Some("prefers Rust"),
+ );
+ let second = system_prompt(&TOOLS, Some(&identity), Some("task B"), None);
+
+ let shared = format!(
+ "{}\n{}\n{TOOLS}",
+ prompt::agent_identity::render("Grunk", "grunk"),
+ prompt::agent_session::PROMPT
+ );
+ assert_eq!(first.shared(), Some(shared.as_str()));
+ assert_eq!(second.shared(), Some(shared.as_str()));
+ assert_eq!(
+ first.rest(),
+ "\n<session_instructions>\ntask A\n</session_instructions>\n<user_memory>\nprefers Rust\n</user_memory>"
+ );
+ assert_eq!(
+ second.rest(),
+ "\n<session_instructions>\ntask B\n</session_instructions>"
+ );
+}
+
/// Absent instructions add no section at all, rather than an empty one the
/// model would have to interpret.
#[test]
fn no_instructions_means_no_section() {
- let prompt = system_prompt(&TOOLS, None, None, Some("prefers Rust"));
+ let prompt = system_prompt(&TOOLS, None, None, Some("prefers Rust")).to_string();
assert!(!prompt.contains("session_instructions"));
assert!(prompt.contains("<user_memory>\nprefers Rust\n</user_memory>"));
@@ -193,7 +227,7 @@ fn no_instructions_means_no_section() {
/// not become a blank delimited section.
#[test]
fn empty_instructions_add_no_section() {
- let prompt = system_prompt(&TOOLS, None, Some(""), None);
+ let prompt = system_prompt(&TOOLS, None, Some(""), None).to_string();
assert!(!prompt.contains("session_instructions"));
}
diff --git a/crates/agent_inmem/src/rig_engine.rs b/crates/agent_inmem/src/rig_engine.rs
index 1811b2d97c..4f26b0ec48 100644
--- a/crates/agent_inmem/src/rig_engine.rs
+++ b/crates/agent_inmem/src/rig_engine.rs
@@ -23,7 +23,7 @@
use std::sync::Arc;
-use agent::{AgentError, AgentLoop, StreamPart};
+use agent::{AgentError, AgentLoop, StreamPart, SystemPrompt};
use ai_tools::user_tool_review::user_tool_finisher;
use ai_tools::{AiHost, DeferredToolSet, ToolServiceContext, tools_for};
use ai_toolset::{AsyncToolCollection, ToolSet as AiToolSet};
@@ -46,6 +46,10 @@ use crate::inbound::ask_user::{AskUser, AskUserContext};
#[path = "outbound/rig_engine/test.rs"]
mod test;
+#[cfg(test)]
+#[path = "outbound/rig_engine/measure_prompt_cache.rs"]
+mod measure_prompt_cache;
+
/// How many stream parts may sit unread before the engine pauses; keeps a
/// slow consumer from buffering a whole turn.
const PART_BUFFER: usize = 256;
@@ -344,7 +348,7 @@ async fn drive_turn(
awaiting,
});
let session = agent_loop
- .session(toolset, Arc::new(tool_context), &system_prompt, usage_ctx)
+ .session(toolset, Arc::new(tool_context), system_prompt, usage_ctx)
.await;
let (mut session, loop_cancel) = session.cancellable();
@@ -388,37 +392,42 @@ async fn drive_turn(
/// the model reads as it takes in the caller's word — the same reason DCS
/// puts `additional_instructions` after the standing prompt. Memory stays
/// last so a remembered fact is never read as an instruction.
+///
+/// The prompt is split after the static Macro prompt: everything before is
+/// the same for every session of the agent and is cached on its own, so a
+/// session whose instructions name its own task still reads it back.
fn system_prompt(
tools_prompt: &impl std::fmt::Display,
identity: Option<&AgentIdentity>,
instructions: Option<&str>,
user_memory: Option<&str>,
-) -> String {
- let mut prompt = String::new();
+) -> SystemPrompt {
+ let mut shared = String::new();
if let Some(identity) = identity {
- prompt.push_str(&prompt::agent_identity::render(
+ shared.push_str(&prompt::agent_identity::render(
&identity.name,
&identity.handle,
));
- prompt.push('\n');
+ shared.push('\n');
}
- prompt.push_str(&prompt::agent_session::PROMPT.to_string());
- prompt.push('\n');
- prompt.push_str(&tools_prompt.to_string());
+ shared.push_str(&prompt::agent_session::PROMPT.to_string());
+ shared.push('\n');
+ shared.push_str(&tools_prompt.to_string());
+ let mut rest = String::new();
// Blank instructions are "none" stated clumsily. A delimited section with
// nothing in it is worse than no section: the model has to decide what an
// empty instruction means.
if let Some(instructions) = instructions.filter(|text| !text.trim().is_empty()) {
- prompt.push_str("\n<session_instructions>\n");
- prompt.push_str(instructions);
- prompt.push_str("\n</session_instructions>");
+ rest.push_str("\n<session_instructions>\n");
+ rest.push_str(instructions);
+ rest.push_str("\n</session_instructions>");
}
if let Some(memory) = user_memory {
- prompt.push_str("\n<user_memory>\n");
- prompt.push_str(memory);
- prompt.push_str("\n</user_memory>");
+ rest.push_str("\n<user_memory>\n");
+ rest.push_str(memory);
+ rest.push_str("\n</user_memory>");
}
- prompt
+ SystemPrompt::split(shared, rest)
}
/// The owner's memory block, or `None` when it is missing or failed to load.
diff --git a/crates/ai_projections/src/outbound/agent_generator.rs b/crates/ai_projections/src/outbound/agent_generator.rs
index 392e8264a6..3e397c98d6 100644
--- a/crates/ai_projections/src/outbound/agent_generator.rs
+++ b/crates/ai_projections/src/outbound/agent_generator.rs
@@ -73,7 +73,7 @@ impl AgentProjectionGenerator {
.session(
toolset,
Arc::new(tool_context),
- &self.system_prompt,
+ &*self.system_prompt,
usage_ctx,
)
.await;