Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(agent): keep the Anthropic prompt cache across LoadTools and sessions (#7646)
Wolf Mermelstein528954a3045d
diff --git a/crates/agent/src/agent_loop.rs b/crates/agent/src/agent_loop.rsindex 62afff981a..78b90ce015 100644--- a/crates/agent/src/agent_loop.rs+++ b/crates/agent/src/agent_loop.rs@@ -1,6 +1,6 @@/// The main entry point: [`AgentLoop`] and [`Session`].use crate::error::AgentError;-use crate::hook::{BridgeInputs, RegisterFn, ToolRouter, UserToolFinisher};+use crate::hook::{BridgeInputs, RegisterFn, ToolLoads, ToolRouter, UserToolFinisher};use crate::model::metering::MeteringContext;use crate::model::router::{ModelRouter, ProviderAgent};use crate::model::{PredefinedModel, ReasoningEffort};@@ -21,6 +21,77 @@ use tracing::Instrument as _;const DEFAULT_MAX_TURNS: usize = 16;const DEFAULT_MAX_TOKENS: u64 = 16_000;+/// A session's system prompt, optionally split where the part every session+/// of the agent repeats byte for byte ends.+///+/// Anthropic caches the shared part apart from the per-session rest+/// (instructions, memory), so a session reads the shared part back even when+/// its rest is new. Other providers receive the two joined.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct SystemPrompt {+ shared: Option<String>,+ rest: String,+}++impl SystemPrompt {+ /// A prompt whose leading `shared` part is the same for every session of+ /// the agent and whose `rest` is the session's own.+ pub fn split(shared: impl Into<String>, rest: impl Into<String>) -> Self {+ Self {+ shared: Some(shared.into()),+ rest: rest.into(),+ }+ }++ /// The shared part, when the prompt is split.+ pub fn shared(&self) -> Option<&str> {+ self.shared.as_deref()+ }++ /// The per-session part, or the whole prompt when it is not split.+ pub fn rest(&self) -> &str {+ &self.rest+ }++ /// The whole prompt as one string.+ pub(crate) fn joined(&self) -> String {+ self.to_string()+ }++ fn push_str(&mut self, text: &str) {+ self.rest.push_str(text);+ }+}++impl std::fmt::Display for SystemPrompt {+ fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(+ formatter,+ "{}{}",+ self.shared.as_deref().unwrap_or_default(),+ self.rest+ )+ }+}++impl From<String> for SystemPrompt {+ fn from(rest: String) -> Self {+ Self { shared: None, rest }+ }+}++impl From<&String> for SystemPrompt {+ fn from(rest: &String) -> Self {+ rest.clone().into()+ }+}++impl From<&str> for SystemPrompt {+ fn from(rest: &str) -> Self {+ rest.to_owned().into()+ }+}+/// Factory for creating per-request agent sessions.////// Routes each session to the provider serving the selected model id (see@@ -153,7 +224,7 @@ impl AgentLoop {&self,toolset: Arc<dyn AiToolSet<Context> + Send + Sync>,context: Arc<Context>,- system_prompt: &str,+ system_prompt: impl Into<SystemPrompt>,usage_ctx: UsageContext,) -> Sessionwhere@@ -167,7 +238,7 @@ impl AgentLoop {context,system_prompt,usage_ctx,- |handle, prompt, max_turns, max_tokens, telemetry| {+ |handle, prompt, tool_search, max_turns, max_tokens, telemetry| {let routed = ModelRouter::shared().expect("failed to initialize model router").route_or_default(&self.model);@@ -176,6 +247,7 @@ impl AgentLoop {self.reasoning_effort,handle,prompt,+ tool_search,max_turns,max_tokens,&telemetry,@@ -194,9 +266,16 @@ impl AgentLoop {&self,toolset: Arc<dyn AiToolSet<Context> + Send + Sync>,context: Arc<Context>,- system_prompt: &str,+ system_prompt: impl Into<SystemPrompt>,usage_ctx: UsageContext,- build: impl FnOnce(ToolServerHandle, &str, usize, u64, GenAiContext) -> ProviderAgent,+ build: impl FnOnce(+ ToolServerHandle,+ &SystemPrompt,+ &ToolSearch,+ usize,+ u64,+ GenAiContext,+ ) -> ProviderAgent,) -> SessionwhereContext: Clone + Send + Sync + 'static,@@ -218,8 +297,13 @@ impl AgentLoop {buffer.lock().expect("loaded_buffer poisoned").extend(tools)})};+ let catalog = Arc::new(catalog);+ let tool_search = ToolSearch {+ catalog: catalog.clone(),+ loads: ToolLoads::default(),+ };let request_context = RequestContext::new(usage_ctx.user.clone())- .with_tool_search(Arc::new(catalog), loader)+ .with_tool_search(catalog, loader).with_genai_telemetry(self.genai_telemetry);// TODO this is cringe, make request context a RW lock newtypelet request_context_rw = Arc::new(RwLock::new(request_context.clone()));@@ -291,12 +375,13 @@ impl AgentLoop {// request. A model's training data predates its own release, so a// newly released model doesn't recognize its own id and may fall back// to identifying as a predecessor — tell it to trust the id.- let mut system_prompt = format!(- "{system_prompt}\n\nYou are the {} model. If this model id is unfamiliar, \+ let mut system_prompt = system_prompt.into();+ system_prompt.push_str(&format!(+ "\n\nYou are the {} model. If this model id is unfamiliar, \that is because it was released after your training data cutoff — trust \this id over your training data when identifying yourself.",self.model- );+ ));// Tell the model which connected integrations it can reach via tool// search. The prompt text lives in the `prompt` crate; the toolset names// are the dynamic data injected here. Omitted when nothing is connected.@@ -319,6 +404,7 @@ impl AgentLoop {let agent = build(handle,&system_prompt,+ &tool_search,self.max_turns,self.max_tokens,telemetry.clone(),@@ -333,6 +419,7 @@ impl AgentLoop {routing,loaded_buffer,register_loaded,+ tool_loads: tool_search.loads.clone(),user_tool_finisher: self.user_tool_finisher.clone(),},recorder: self.recorder.clone(),@@ -353,7 +440,7 @@ impl AgentLoop {&self,toolset: Arc<dyn AiToolSet<Context> + Send + Sync>,context: Arc<Context>,- system_prompt: &str,+ system_prompt: impl Into<SystemPrompt>,usage_ctx: UsageContext,model: M,) -> Session@@ -366,12 +453,70 @@ impl AgentLoop {context,system_prompt,usage_ctx,- move |handle, prompt, max_turns, max_tokens, telemetry| {- ProviderAgent::test(model, prompt, max_turns, max_tokens, handle, telemetry)+ move |handle, prompt, _, max_turns, max_tokens, telemetry| {+ ProviderAgent::test(+ model,+ &prompt.joined(),+ max_turns,+ max_tokens,+ handle,+ telemetry,+ )},).await}++ /// Start a session routed through `router`, as production routes through+ /// the shared one: the provider arms build the agent, over whatever+ /// transport the router's clients were given.+ #[cfg(test)]+ pub(crate) async fn test_routed_session<Context, H>(+ &self,+ router: &ModelRouter<H>,+ toolset: Arc<dyn AiToolSet<Context> + Send + Sync>,+ context: Arc<Context>,+ system_prompt: impl Into<SystemPrompt>,+ usage_ctx: UsageContext,+ ) -> Session+ where+ Context: Clone + Send + Sync + 'static,+ H: rig_core::http_client::HttpClientExt+ + Clone+ + Default+ + std::fmt::Debug+ + Send+ + Sync+ + 'static,+ {+ self.session_with(+ toolset,+ context,+ system_prompt,+ usage_ctx,+ |handle, prompt, tool_search, max_turns, max_tokens, telemetry| {+ let routed = router.route_or_default(&self.model);+ telemetry.set_model(routed.provider(), routed.model_name());+ ProviderAgent::Test(Box::new(routed.into_agent(+ self.reasoning_effort,+ handle,+ prompt,+ tool_search,+ max_turns,+ max_tokens,+ &telemetry,+ )))+ },+ )+ .await+ }+}++/// A session's on-demand tools: the catalog the model loads from, and which+/// result loaded which of them.+pub(crate) struct ToolSearch {+ pub(crate) catalog: Arc<Vec<SearchableTool>>,+ pub(crate) loads: ToolLoads,}/// A single streaming conversation session.diff --git a/crates/agent/src/hook.rs b/crates/agent/src/hook.rsindex fc19d6acc8..d75d00a8a8 100644--- a/crates/agent/src/hook.rs+++ b/crates/agent/src/hook.rs@@ -68,6 +68,67 @@ pub type UserToolFinisher = Arc<+ Sync,>;+/// Which call loaded which catalog tools this session, in load order.+///+/// The Anthropic request planner references a loaded tool from the result of+/// the call that loaded it, so loading never changes the request's `tools`.+#[derive(Clone, Default)]+pub struct ToolLoads(Arc<Mutex<Vec<Load>>>);++/// One call's loading of catalog tools.+struct Load {+ call: LoadingCall,+ tools: Vec<String>,+}++/// A call as the history shows it. Rig hands a tool's result hook no provider+/// call id, so an executed call is known by its name and arguments; an invalid+/// one arrives with its id.+#[derive(Clone, Debug, PartialEq)]+enum LoadingCall {+ Id(String),+ Arguments {+ tool_name: String,+ arguments: serde_json::Value,+ },+}++impl ToolLoads {+ fn record(&self, call: LoadingCall, tools: &[SearchableTool]) {+ self.0.lock().expect("tool_loads poisoned").push(Load {+ call,+ tools: tools.iter().map(|tool| tool.name.clone()).collect(),+ });+ }++ /// The tools `call` loaded.+ pub(crate) fn loaded_by(&self, call: &rig_core::message::ToolCall) -> Vec<String> {+ self.0+ .lock()+ .expect("tool_loads poisoned")+ .iter()+ .filter(|load| match &load.call {+ LoadingCall::Id(id) => *id == call.id,+ LoadingCall::Arguments {+ tool_name,+ arguments,+ } => *tool_name == call.function.name && *arguments == call.function.arguments,+ })+ .flat_map(|load| load.tools.iter().cloned())+ .collect()+ }++ /// Every tool loaded this session.+ pub(crate) fn loaded(&self) -> Vec<String> {+ self.0+ .lock()+ .expect("tool_loads poisoned")+ .iter()+ .flat_map(|load| load.tools.iter().cloned())+ .collect()+ }+}+/// Everything the session hands the stream bridge besides the request context.#[derive(Clone)]pub struct BridgeInputs {@@ -77,6 +138,8 @@ pub struct BridgeInputs {pub loaded_buffer: Arc<Mutex<Vec<SearchableTool>>>,/// Registers loaded tools with the live tool server.pub register_loaded: RegisterFn,+ /// Records which result loaded which tools.+ pub tool_loads: ToolLoads,/// Finishes user tools mid-turn, when the host can.pub user_tool_finisher: Option<UserToolFinisher>,}@@ -106,6 +169,8 @@ pub struct StreamBridge {loaded_buffer: Arc<Mutex<Vec<SearchableTool>>>,/// Registers drained tools with the live tool server.register_loaded: RegisterFn,+ /// Records which result loaded which tools.+ tool_loads: ToolLoads,/// The session's full searchable catalog. Read by/// [`Self::on_invalid_tool_call`] to recover calls to tools the model/// discovered but never loaded.@@ -137,6 +202,7 @@ impl StreamBridge {routing,loaded_buffer,register_loaded,+ tool_loads,user_tool_finisher,} = inputs;let (tx, rx) = mpsc::unbounded_channel();@@ -146,6 +212,7 @@ impl StreamBridge {routing,loaded_buffer,register_loaded,+ tool_loads,searchable_catalog,user_tool_finisher,cancel,@@ -209,6 +276,7 @@ impl StreamBridge {pub(crate) async fn handle_invalid_tool_call(&self,tool_name: &str,+ tool_call_id: Option<&str>,) -> Option<InvalidToolCallAction> {match self.searchable_catalog@@ -216,7 +284,13 @@ impl StreamBridge {.find(|tool| tool.name == tool_name){Some(tool) => {- (self.register_loaded)(vec![tool.clone()]).await;+ let tools = vec![tool.clone()];+ // The retry feedback answers this call, so it loads the tool.+ if let Some(id) = tool_call_id {+ self.tool_loads+ .record(LoadingCall::Id(id.to_owned()), &tools);+ }+ (self.register_loaded)(tools).await;tracing::info!(tool = %tool_name,"auto-loaded searchable tool the model called without loading"@@ -288,6 +362,13 @@ impl StreamBridge {std::mem::take(&mut *buf)};if !pending.is_empty() {+ self.tool_loads.record(+ LoadingCall::Arguments {+ tool_name: tool_name.to_owned(),+ arguments: serde_json::from_str(args).unwrap_or(serde_json::Value::Null),+ },+ &pending,+ );(self.register_loaded)(pending).await;}@@ -368,7 +449,8 @@ impl AgentHook for StreamBridge {_ctx: &HookContext,event: &InvalidToolCallContext,) -> Option<InvalidToolCallAction> {- self.handle_invalid_tool_call(&event.tool_name).await+ self.handle_invalid_tool_call(&event.tool_name, event.tool_call_id.as_deref())+ .await}async fn on_tool_call(diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rsindex 219e76da1a..5a1898b6b3 100644--- a/crates/agent/src/lib.rs+++ b/crates/agent/src/lib.rs@@ -20,7 +20,7 @@ mod test;pub mod types;pub use accumulator::StreamAccumulator;-pub use agent_loop::{AgentLoop, Session};+pub use agent_loop::{AgentLoop, Session, SystemPrompt};pub use completion::{complete, complete_about_image, complete_with_history};pub use convert::{merge_consecutive_parts, to_rig_messages};pub use error::AgentError;diff --git a/crates/agent/src/model/anthropic.rs b/crates/agent/src/model/anthropic.rsindex 1232e828e5..9ea9bc91a9 100644--- a/crates/agent/src/model/anthropic.rs+++ b/crates/agent/src/model/anthropic.rs@@ -1,8 +1,33 @@+use crate::hook::ToolLoads;use crate::model::ReasoningEffort;+use crate::model::anthropic_prompt_layout::{PROMPT_LAYOUT_KEY, PromptLayout, ToolReferences};use crate::model::types::Model;+use ai_toolset::SearchableTool;+use rig_core::completion::{+ CompletionError, CompletionModel, CompletionRequest, CompletionResponse,+};+use rig_core::message::{AssistantContent, Message, ToolResultContent, UserContent};+use rig_core::streaming::StreamingCompletionResponse;use rig_core::{client::CompletionClient, http_client::HttpClientExt, providers::anthropic};+use std::collections::{HashMap, HashSet};use std::sync::Arc;+/// Models that load a `defer_loading` tool from a `tool_reference` in a tool+/// result, read the whole cache prefix back after it, and call the tool with+/// its schema. Each was probed against the API on 2026-10-06; none needs a+/// beta header. The Fable models are documented to support it but were not+/// reachable for this organization, so they keep the client-side path.+const TOOL_REFERENCE_MODELS: &[&str] = &[+ "claude-haiku-4-5",+ "claude-opus-4-7",+ "claude-opus-4-8",+ "claude-opus-5",+ "claude-opus-5-5",+ "claude-sonnet-4-6",+ "claude-sonnet-5",+ "claude-sonnet-5-5",+];+/// A Claude model bound to the native Anthropic client that serves it.////// Carries the parsed [`Model`] id and a shared client. Which provider an id@@ -24,6 +49,11 @@ impl<'a, H: HttpClientExt + Clone + Default + 'static> AnthropicModel<'a, H> {&self.model}+ /// Whether this model loads deferred tools through `tool_reference`.+ pub(crate) fn loads_tools_by_reference(&self) -> bool {+ TOOL_REFERENCE_MODELS.contains(&self.model.name())+ }+/// The rig completion model for this id. The id is passed verbatim to the/// Anthropic API.///@@ -81,5 +111,219 @@ impl<'a, H: HttpClientExt + Clone + Default + 'static> AnthropicModel<'a, H> {}}+/// How a session lays out its Anthropic requests for the prompt cache.+#[derive(Clone, Default)]+pub(crate) struct SessionLayout {+ /// The first two system messages are the system prompt's shared part and+ /// its per-session rest, each cached on its own.+ pub(crate) shared_system: bool,+ /// Catalog tools declared upfront and loaded by reference.+ pub(crate) deferred: Option<DeferredTools>,+}++/// A session's catalog tools, declared with `defer_loading` on every request+/// so loading one never changes `tools`, the front of the cache prefix.+#[derive(Clone)]+pub(crate) struct DeferredTools {+ names: HashSet<String>,+ /// Wire definitions in catalog order.+ definitions: Arc<Vec<serde_json::Value>>,+ loads: ToolLoads,+}++impl DeferredTools {+ pub(crate) fn new(catalog: &[SearchableTool], loads: ToolLoads) -> Self {+ let definitions = catalog+ .iter()+ .map(|tool| {+ let definition = crate::tool_adapter::provider_definition(+ tool.name.clone(),+ tool.description.clone(),+ &tool.schema,+ );+ serde_json::json!({+ "name": definition.name,+ "description": definition.description,+ "input_schema": definition.parameters,+ "defer_loading": true,+ })+ })+ .collect();+ Self {+ names: catalog.iter().map(|tool| tool.name.clone()).collect(),+ definitions: Arc::new(definitions),+ loads,+ }+ }++ /// Where each catalog tool's schema is loaded from: the result of the call+ /// that loaded it this session, or, for a tool an earlier turn called, the+ /// text result just before its first call. A tool is referenced once, and+ /// the same history always places it in the same result, so the requests+ /// of a run and of the turns after it share their prefix.+ fn references(&self, history: &[Message]) -> Vec<ToolReferences> {+ let loaded_this_session: HashSet<String> = self.loads.loaded().into_iter().collect();+ let mut loaded_by_result: HashMap<String, Vec<String>> = HashMap::new();+ let mut referenced = HashSet::new();+ let mut references: Vec<ToolReferences> = Vec::new();+ let mut place = |tool_use_id: &str, name: &String| {+ if !self.names.contains(name) || !referenced.insert(name.clone()) {+ return;+ }+ match references+ .iter_mut()+ .find(|references| references.tool_use_id == tool_use_id)+ {+ Some(references) => references.tool_names.push(name.clone()),+ None => references.push(ToolReferences {+ tool_use_id: tool_use_id.to_owned(),+ tool_names: vec![name.clone()],+ }),+ }+ };+ let mut latest_text_result: Option<String> = None;+ for message in history {+ match message {+ Message::User { content } => {+ let results: Vec<_> = content+ .iter()+ .filter_map(|content| match content {+ UserContent::ToolResult(result) => Some(result),+ _ => None,+ })+ .filter(|result| {+ result+ .content+ .iter()+ .all(|content| !matches!(content, ToolResultContent::Image(_)))+ })+ .collect();+ for result in &results {+ for name in loaded_by_result.get(&result.id).into_iter().flatten() {+ place(&result.id, name);+ }+ }+ if let Some(first) = results.first() {+ latest_text_result = Some(first.id.clone());+ }+ }+ Message::Assistant { content, .. } => {+ for content in content.iter() {+ let AssistantContent::ToolCall(call) = content else {+ continue;+ };+ loaded_by_result.insert(call.id.clone(), self.loads.loaded_by(call));+ if !loaded_this_session.contains(&call.function.name)+ && let Some(tool_use_id) = &latest_text_result+ {+ place(tool_use_id, &call.function.name);+ }+ }+ }+ Message::System { .. } => {}+ }+ }+ references+ }+}++/// An Anthropic completion model whose requests carry a [`SessionLayout`]:+/// catalog tools move into `defer_loading` declarations and the rest of the+/// layout is declared for+/// [`AnthropicPromptLayout`](crate::model::anthropic_prompt_layout::AnthropicPromptLayout)+/// to apply on the wire.+#[derive(Clone)]+pub(crate) struct LaidOutModel<M> {+ inner: M,+ layout: SessionLayout,+}++impl<M> LaidOutModel<M> {+ pub(crate) fn new(inner: M, layout: SessionLayout) -> Self {+ Self { inner, layout }+ }++ fn lay_out(+ &self,+ mut request: CompletionRequest,+ ) -> Result<CompletionRequest, CompletionError> {+ let history: Vec<Message> = request.chat_history.iter().cloned().collect();+ let mut layout = PromptLayout::default();+ if self.layout.shared_system {+ if !matches!(+ history.as_slice(),+ [Message::System { .. }, Message::System { .. }, ..]+ ) {+ return Err(CompletionError::RequestError(+ "the request must open with the shared and per-session system messages".into(),+ ));+ }+ layout.cache_shared_system = true;+ }+ let mut params = match request.additional_params.take() {+ None => serde_json::Map::new(),+ Some(serde_json::Value::Object(params)) => params,+ Some(_) => {+ return Err(CompletionError::RequestError(+ "additional_params must be an object".into(),+ ));+ }+ };+ if let Some(deferred) = &self.layout.deferred {+ if params.contains_key("tools") {+ return Err(CompletionError::RequestError(+ "additional_params already declares tools".into(),+ ));+ }+ request+ .tools+ .retain(|tool| !deferred.names.contains(&tool.name));+ params.insert(+ "tools".to_owned(),+ serde_json::Value::Array(deferred.definitions.to_vec()),+ );+ layout.tool_references = deferred.references(&history);+ }+ if layout != PromptLayout::default() {+ params.insert(+ PROMPT_LAYOUT_KEY.to_owned(),+ serde_json::to_value(&layout)+ .map_err(|error| CompletionError::RequestError(error.into()))?,+ );+ }+ request.additional_params =+ (!params.is_empty()).then_some(serde_json::Value::Object(params));+ Ok(request)+ }+}++impl<M: CompletionModel> CompletionModel for LaidOutModel<M> {+ type Response = M::Response;+ type StreamingResponse = M::StreamingResponse;+ type Client = M::Client;++ fn make(client: &Self::Client, model: impl Into<String>) -> Self {+ Self::new(M::make(client, model), SessionLayout::default())+ }++ async fn completion(+ &self,+ request: CompletionRequest,+ ) -> Result<CompletionResponse<Self::Response>, CompletionError> {+ self.inner.completion(self.lay_out(request)?).await+ }++ async fn stream(+ &self,+ request: CompletionRequest,+ ) -> Result<StreamingCompletionResponse<Self::StreamingResponse>, CompletionError> {+ self.inner.stream(self.lay_out(request)?).await+ }++ fn composes_native_output_with_tools(&self) -> bool {+ self.inner.composes_native_output_with_tools()+ }+}+#[cfg(test)]mod test;diff --git a/crates/agent/src/model/anthropic/test.rs b/crates/agent/src/model/anthropic/test.rsindex 9221197122..c3661fdce2 100644--- a/crates/agent/src/model/anthropic/test.rs+++ b/crates/agent/src/model/anthropic/test.rs@@ -142,3 +142,36 @@ async fn completion_caches_tools_system_and_conversation() {async fn stream_caches_tools_system_and_conversation() {assert_cached_wire_body(true).await;}++#[test]+fn only_probed_models_load_tools_by_reference() {+ let client = Arc::new(+ anthropic::Client::builder()+ .api_key("test-anthropic-key")+ .build()+ .unwrap(),+ );+ let loads_by_reference = |id: &str| {+ AnthropicModel::new(Model::try_from(id).unwrap(), client.clone()).loads_tools_by_reference()+ };+ for id in [+ "anthropic/claude-haiku-4-5",+ "anthropic/claude-opus-4-7",+ "anthropic/claude-opus-4-8",+ "anthropic/claude-opus-5",+ "anthropic/claude-opus-5-5",+ "anthropic/claude-sonnet-4-6",+ "anthropic/claude-sonnet-5",+ "anthropic/claude-sonnet-5-5",+ ] {+ assert!(loads_by_reference(id), "{id}");+ }+ for id in [+ "anthropic/claude-fable-5",+ "anthropic/claude-fable-5-1",+ "anthropic/claude-opus-4-6",+ "anthropic/claude-opus-4-8-unreleased",+ ] {+ assert!(!loads_by_reference(id), "{id}");+ }+}diff --git a/crates/agent/src/model/anthropic_prompt_layout.rs b/crates/agent/src/model/anthropic_prompt_layout.rsnew file mode 100644index 0000000000..3af2b34911--- /dev/null+++ b/crates/agent/src/model/anthropic_prompt_layout.rs@@ -0,0 +1,297 @@+//! The two Anthropic prompt-cache shapes rig 0.41 cannot express, applied to+//! the serialized request just before it leaves for the API.+//!+//! - A cache breakpoint on the *first* of two system blocks, so the part of the+//! system prompt every session of an agent shares is cached apart from the+//! per-session rest. Rig only marks the last system block.+//! - `tool_reference` blocks in a tool result, which load a deferred tool's+//! schema without touching `tools` (the front of the cache prefix). Rig's+//! tool results carry only text and images.+//!+//! The request planner ([`super::anthropic::LaidOutModel`]) declares what to+//! do in a [`PromptLayout`] under [`PROMPT_LAYOUT_KEY`]. This adapter applies+//! it and strips it. A body without the key is sent byte for byte. A body+//! whose shape does not match its layout fails the request: a layout that+//! cannot be applied is a bug, not a request to send as it is.++use bytes::Bytes;+use rig_core::http_client::{+ self, HttpClientExt, LazyBody, MultipartForm, Request, Response, StreamingResponse,+};+use serde::{Deserialize, Serialize};+use serde_json::{Map, Value, json};++#[cfg(test)]+mod test;++/// Where the request planner puts its [`PromptLayout`] in the request body.+pub(crate) const PROMPT_LAYOUT_KEY: &str = "macro_prompt_layout";++/// Anthropic's limit on `cache_control` markers per request, the top-level+/// automatic one included.+const MAX_CACHE_BREAKPOINTS: usize = 4;++/// What to change in one request body.+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]+pub(crate) struct PromptLayout {+ /// Mark the first of exactly two system blocks as a cache breakpoint.+ pub(crate) cache_shared_system: bool,+ /// Tool results that load deferred tools, in conversation order.+ pub(crate) tool_references: Vec<ToolReferences>,+}++/// The deferred tools one tool result loads.+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub(crate) struct ToolReferences {+ /// The `tool_use_id` of the result that loads them.+ pub(crate) tool_use_id: String,+ /// Names of tools declared with `defer_loading`.+ pub(crate) tool_names: Vec<String>,+}++/// Why a declared layout could not be applied.+#[derive(Debug, thiserror::Error, PartialEq, Eq)]+pub(crate) enum PromptLayoutError {+ #[error("prompt layout is malformed: {0}")]+ Malformed(String),+ #[error("expected exactly two system blocks, the first without a breakpoint")]+ SharedSystem,+ #[error("tool `{0}` is referenced but not declared with defer_loading")]+ NotDeferred(String),+ #[error("expected exactly one tool result `{0}`, found {1}")]+ ToolResult(String, usize),+ #[error("tool result `{0}` holds something other than text")]+ ToolResultContent(String),+ #[error("{0} cache breakpoints exceed Anthropic's limit of {MAX_CACHE_BREAKPOINTS}")]+ Breakpoints(usize),+}++/// Applies the [`PromptLayout`] a body declares, then forwards it to `inner`.+#[derive(Debug, Clone, Default)]+pub struct AnthropicPromptLayout<H> {+ inner: H,+}++impl<H> AnthropicPromptLayout<H> {+ /// Shape requests on their way to `inner`.+ pub fn new(inner: H) -> Self {+ Self { inner }+ }+}++impl<H: HttpClientExt + Clone + 'static> HttpClientExt for AnthropicPromptLayout<H> {+ fn send<T, U>(+ &self,+ request: Request<T>,+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static+ where+ T: Into<Bytes> + Send,+ U: From<Bytes> + Send + 'static,+ {+ let shaped = shape_request(request.map(Into::into));+ let inner = self.inner.clone();+ async move { inner.send(shaped?).await }+ }++ fn send_multipart<U>(+ &self,+ request: Request<MultipartForm>,+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static+ where+ U: From<Bytes> + Send + 'static,+ {+ self.inner.send_multipart(request)+ }++ async fn send_streaming<T>(&self, request: Request<T>) -> http_client::Result<StreamingResponse>+ where+ T: Into<Bytes> + Send,+ {+ self.inner+ .send_streaming(shape_request(request.map(Into::into))?)+ .await+ }+}++fn shape_request(request: Request<Bytes>) -> http_client::Result<Request<Bytes>> {+ let (mut parts, body) = request.into_parts();+ let Some(shaped) =+ shape_body(&body).map_err(|error| http_client::Error::Instance(error.into()))?+ else {+ return Ok(Request::from_parts(parts, body));+ };+ // The body changed length; let the transport compute it.+ parts.headers.remove("content-length");+ Ok(Request::from_parts(parts, shaped))+}++/// The body with its layout applied, or `None` when it declares none.+fn shape_body(body: &[u8]) -> Result<Option<Bytes>, PromptLayoutError> {+ let Ok(Value::Object(mut body)) = serde_json::from_slice::<Value>(body) else {+ return Ok(None);+ };+ let Some(layout) = body.remove(PROMPT_LAYOUT_KEY) else {+ return Ok(None);+ };+ let layout: PromptLayout = serde_json::from_value(layout)+ .map_err(|error| PromptLayoutError::Malformed(error.to_string()))?;+ apply(&mut body, &layout)?;+ let body = serde_json::to_vec(&body)+ .map_err(|error| PromptLayoutError::Malformed(error.to_string()))?;+ Ok(Some(Bytes::from(body)))+}++fn apply(body: &mut Map<String, Value>, layout: &PromptLayout) -> Result<(), PromptLayoutError> {+ if layout.cache_shared_system {+ cache_shared_system(body)?;+ }+ for references in &layout.tool_references {+ for name in &references.tool_names {+ if !is_deferred(body, name) {+ return Err(PromptLayoutError::NotDeferred(name.clone()));+ }+ }+ reference_tools(body, references)?;+ }+ let breakpoints = count_breakpoints(body);+ if breakpoints > MAX_CACHE_BREAKPOINTS {+ return Err(PromptLayoutError::Breakpoints(breakpoints));+ }+ Ok(())+}++fn cache_shared_system(body: &mut Map<String, Value>) -> Result<(), PromptLayoutError> {+ let Some(Value::Array(system)) = body.get_mut("system") else {+ return Err(PromptLayoutError::SharedSystem);+ };+ let [Value::Object(shared), Value::Object(_)] = system.as_mut_slice() else {+ return Err(PromptLayoutError::SharedSystem);+ };+ if shared.get("type") != Some(&json!("text")) || shared.contains_key("cache_control") {+ return Err(PromptLayoutError::SharedSystem);+ }+ shared.insert("cache_control".to_owned(), json!({"type": "ephemeral"}));+ Ok(())+}++fn is_deferred(body: &Map<String, Value>, name: &str) -> bool {+ body.get("tools")+ .and_then(Value::as_array)+ .is_some_and(|tools| {+ tools.iter().any(|tool| {+ tool.get("name") == Some(&json!(name))+ && tool.get("defer_loading") == Some(&json!(true))+ })+ })+}++/// Replace the result's content with `tool_reference` blocks, which cannot+/// share a result with anything else, and keep what the tool said in a text+/// block after the message's tool results.+fn reference_tools(+ body: &mut Map<String, Value>,+ references: &ToolReferences,+) -> Result<(), PromptLayoutError> {+ let id = &references.tool_use_id;+ let mut found = Vec::new();+ if let Some(Value::Array(messages)) = body.get("messages") {+ for (message_index, message) in messages.iter().enumerate() {+ let Some(Value::Array(content)) = message.get("content") else {+ continue;+ };+ for (block_index, block) in content.iter().enumerate() {+ if block.get("type") == Some(&json!("tool_result"))+ && block.get("tool_use_id") == Some(&json!(id))+ {+ found.push((message_index, block_index));+ }+ }+ }+ }+ let [(message_index, block_index)] = found[..] else {+ return Err(PromptLayoutError::ToolResult(id.clone(), found.len()));+ };+ let content = body["messages"][message_index]["content"]+ .as_array_mut()+ .expect("located above");+ let said = result_text(&content[block_index])+ .ok_or_else(|| PromptLayoutError::ToolResultContent(id.clone()))?;+ content[block_index]["content"] = Value::Array(+ references+ .tool_names+ .iter()+ .map(|name| json!({"type": "tool_reference", "tool_name": name}))+ .collect(),+ );+ let after_results = content+ .iter()+ .rposition(|block| block.get("type") == Some(&json!("tool_result")))+ .expect("the referenced result is one")+ + 1;+ // After earlier results' moved text, so the order follows the results.+ let position = content[after_results..]+ .iter()+ .take_while(|block| is_moved_result_text(block))+ .count()+ + after_results;+ content.insert(+ position,+ json!({+ "type": "text",+ "text": format!("<tool_result_text tool_use_id=\"{id}\">{said}</tool_result_text>"),+ }),+ );+ Ok(())+}++/// The text a tool result holds, as a string or text blocks.+fn result_text(result: &Value) -> Option<String> {+ match result.get("content")? {+ Value::String(text) => Some(text.clone()),+ Value::Array(blocks) => blocks+ .iter()+ .map(|block| {+ (block.get("type") == Some(&json!("text")))+ .then(|| block.get("text")?.as_str().map(str::to_owned))+ .flatten()+ })+ .collect::<Option<Vec<_>>>()+ .map(|texts| texts.concat()),+ _ => None,+ }+}++fn is_moved_result_text(block: &Value) -> bool {+ block+ .get("text")+ .and_then(Value::as_str)+ .is_some_and(|text| text.starts_with("<tool_result_text tool_use_id="))+}++/// Markers where Anthropic reads them: top level, tools, system blocks and+/// message content blocks.+fn count_breakpoints(body: &Map<String, Value>) -> usize {+ let marked = |value: &Value| {+ value+ .get("cache_control")+ .is_some_and(|marker| !marker.is_null())+ };+ let blocks = |key: &str| {+ body.get(key).and_then(Value::as_array).map_or(0, |blocks| {+ blocks.iter().filter(|block| marked(block)).count()+ })+ };+ let message_blocks = body+ .get("messages")+ .and_then(Value::as_array)+ .into_iter()+ .flatten()+ .filter_map(|message| message.get("content")?.as_array())+ .flatten()+ .filter(|block| marked(block))+ .count();+ usize::from(marked(&Value::Object(body.clone())))+ + blocks("tools")+ + blocks("system")+ + message_blocks+}diff --git a/crates/agent/src/model/anthropic_prompt_layout/test.rs b/crates/agent/src/model/anthropic_prompt_layout/test.rsnew file mode 100644index 0000000000..8398f8ce62--- /dev/null+++ b/crates/agent/src/model/anthropic_prompt_layout/test.rs@@ -0,0 +1,308 @@+use super::*;+use std::sync::{Arc, Mutex};++fn shaped(body: Value) -> Result<Option<Value>, PromptLayoutError> {+ shape_body(&serde_json::to_vec(&body).unwrap())+ .map(|shaped| shaped.map(|bytes| serde_json::from_slice(&bytes).unwrap()))+}++#[test]+fn a_body_without_a_layout_is_left_alone() {+ let body = json!({+ "model": "claude-opus-4-8",+ "system": [{"type": "text", "text": "You are Macro.", "cache_control": {"type": "ephemeral"}}],+ "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}],+ });+ assert_eq!(shaped(body), Ok(None));+ assert_eq!(shape_body(b"not json"), Ok(None));+}++#[test]+fn shared_system_block_gets_its_own_breakpoint() {+ let body = json!({+ "system": [+ {"type": "text", "text": "You are Macro."},+ {"type": "text", "text": "<session_instructions>Task A</session_instructions>", "cache_control": {"type": "ephemeral"}},+ ],+ "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}],+ "cache_control": {"type": "ephemeral"},+ "macro_prompt_layout": {"cache_shared_system": true, "tool_references": []},+ });+ assert_eq!(+ shaped(body),+ Ok(Some(json!({+ "system": [+ {"type": "text", "text": "You are Macro.", "cache_control": {"type": "ephemeral"}},+ {"type": "text", "text": "<session_instructions>Task A</session_instructions>", "cache_control": {"type": "ephemeral"}},+ ],+ "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}],+ "cache_control": {"type": "ephemeral"},+ })))+ );+}++#[test]+fn shared_system_needs_exactly_two_blocks_the_first_unmarked() {+ let layout = json!({"cache_shared_system": true, "tool_references": []});+ for system in [+ json!([{"type": "text", "text": "You are Macro."}]),+ json!([+ {"type": "text", "text": "You are Macro.", "cache_control": {"type": "ephemeral"}},+ {"type": "text", "text": "Task A"},+ ]),+ json!([+ {"type": "text", "text": "You are Macro."},+ {"type": "text", "text": "Task A"},+ {"type": "text", "text": "Memory"},+ ]),+ ] {+ let body = json!({"system": system, "messages": [], "macro_prompt_layout": layout});+ assert_eq!(shaped(body), Err(PromptLayoutError::SharedSystem));+ }+}++#[test]+fn a_loading_result_references_its_tools_and_keeps_its_text_after_the_results() {+ let body = json!({+ "tools": [+ {"name": "LoadTools", "description": "Load tools.", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}},+ {"name": "EditPresentation", "description": "Edit a deck.", "input_schema": {"type": "object"}, "defer_loading": true},+ {"name": "ReadPresentation", "description": "Read a deck.", "input_schema": {"type": "object"}, "defer_loading": true},+ ],+ "messages": [+ {"role": "user", "content": [{"type": "text", "text": "edit my deck"}]},+ {"role": "assistant", "content": [+ {"type": "tool_use", "id": "toolu_load", "name": "LoadTools", "input": {"names": ["EditPresentation", "ReadPresentation"]}},+ {"type": "tool_use", "id": "toolu_other", "name": "Search", "input": {}},+ ]},+ {"role": "user", "content": [+ {"type": "tool_result", "tool_use_id": "toolu_load", "content": [{"type": "text", "text": "{\"loaded\":2}"}]},+ {"type": "tool_result", "tool_use_id": "toolu_other", "content": "found nothing"},+ ]},+ ],+ "macro_prompt_layout": {+ "cache_shared_system": false,+ "tool_references": [{"tool_use_id": "toolu_load", "tool_names": ["EditPresentation", "ReadPresentation"]}],+ },+ });+ let shaped = shaped(body).unwrap().unwrap();+ assert_eq!(+ shaped["messages"][2],+ json!({"role": "user", "content": [+ {"type": "tool_result", "tool_use_id": "toolu_load", "content": [+ {"type": "tool_reference", "tool_name": "EditPresentation"},+ {"type": "tool_reference", "tool_name": "ReadPresentation"},+ ]},+ {"type": "tool_result", "tool_use_id": "toolu_other", "content": "found nothing"},+ {"type": "text", "text": "<tool_result_text tool_use_id=\"toolu_load\">{\"loaded\":2}</tool_result_text>"},+ ]})+ );+ assert_eq!(shaped.get("macro_prompt_layout"), None);+}++#[test]+fn moved_texts_follow_the_order_their_results_were_referenced_in() {+ let body = json!({+ "tools": [+ {"name": "A", "input_schema": {"type": "object"}, "defer_loading": true},+ {"name": "B", "input_schema": {"type": "object"}, "defer_loading": true},+ ],+ "messages": [{"role": "user", "content": [+ {"type": "tool_result", "tool_use_id": "toolu_1", "content": "one"},+ {"type": "tool_result", "tool_use_id": "toolu_2", "content": "two"},+ ]}],+ "macro_prompt_layout": {+ "cache_shared_system": false,+ "tool_references": [+ {"tool_use_id": "toolu_1", "tool_names": ["A"]},+ {"tool_use_id": "toolu_2", "tool_names": ["B"]},+ ],+ },+ });+ assert_eq!(+ shaped(body).unwrap().unwrap()["messages"][0]["content"],+ json!([+ {"type": "tool_result", "tool_use_id": "toolu_1", "content": [{"type": "tool_reference", "tool_name": "A"}]},+ {"type": "tool_result", "tool_use_id": "toolu_2", "content": [{"type": "tool_reference", "tool_name": "B"}]},+ {"type": "text", "text": "<tool_result_text tool_use_id=\"toolu_1\">one</tool_result_text>"},+ {"type": "text", "text": "<tool_result_text tool_use_id=\"toolu_2\">two</tool_result_text>"},+ ])+ );+}++#[test]+fn a_reference_must_name_a_deferred_tool() {+ let body = json!({+ "tools": [{"name": "EditPresentation", "input_schema": {"type": "object"}}],+ "messages": [{"role": "user", "content": [+ {"type": "tool_result", "tool_use_id": "toolu_load", "content": "loaded"},+ ]}],+ "macro_prompt_layout": {+ "cache_shared_system": false,+ "tool_references": [{"tool_use_id": "toolu_load", "tool_names": ["EditPresentation"]}],+ },+ });+ assert_eq!(+ shaped(body),+ Err(PromptLayoutError::NotDeferred(+ "EditPresentation".to_owned()+ ))+ );+}++#[test]+fn a_reference_must_land_in_exactly_one_text_result() {+ let tools = json!([{"name": "EditPresentation", "input_schema": {"type": "object"}, "defer_loading": true}]);+ let layout = json!({+ "cache_shared_system": false,+ "tool_references": [{"tool_use_id": "toolu_load", "tool_names": ["EditPresentation"]}],+ });+ let missing = json!({"tools": tools, "messages": [], "macro_prompt_layout": layout});+ assert_eq!(+ shaped(missing),+ Err(PromptLayoutError::ToolResult("toolu_load".to_owned(), 0))+ );+ let image = json!({+ "tools": tools,+ "messages": [{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_load", "content": [+ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AA=="}},+ ]}]}],+ "macro_prompt_layout": layout,+ });+ assert_eq!(+ shaped(image),+ Err(PromptLayoutError::ToolResultContent(+ "toolu_load".to_owned()+ ))+ );+}++#[test]+fn a_layout_cannot_exceed_four_breakpoints() {+ let body = json!({+ "tools": [+ {"name": "A", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}},+ {"name": "B", "input_schema": {"type": "object", "properties": {"cache_control": {"type": "string"}}}, "cache_control": {"type": "ephemeral"}},+ ],+ "system": [+ {"type": "text", "text": "You are Macro."},+ {"type": "text", "text": "Task A", "cache_control": {"type": "ephemeral"}},+ ],+ "messages": [],+ "cache_control": {"type": "ephemeral"},+ "macro_prompt_layout": {"cache_shared_system": true, "tool_references": []},+ });+ assert_eq!(shaped(body), Err(PromptLayoutError::Breakpoints(5)));+}++#[test]+fn a_malformed_layout_fails() {+ let body = json!({"messages": [], "macro_prompt_layout": {"cache_shared_system": "yes"}});+ assert!(matches!(shaped(body), Err(PromptLayoutError::Malformed(_))));+}++/// Records what reaches the transport under the layout adapter.+#[derive(Clone, Debug, Default)]+struct Recording(Arc<Mutex<Vec<(Option<String>, Bytes)>>>);++impl HttpClientExt for Recording {+ fn send<T, U>(+ &self,+ request: Request<T>,+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static+ where+ T: Into<Bytes> + Send,+ U: From<Bytes> + Send + 'static,+ {+ let length = request+ .headers()+ .get("content-length")+ .map(|length| length.to_str().unwrap().to_owned());+ self.0+ .lock()+ .unwrap()+ .push((length, request.into_body().into()));+ std::future::ready(Err(http_client::Error::StreamEnded))+ }++ fn send_multipart<U>(+ &self,+ _: Request<MultipartForm>,+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static+ where+ U: From<Bytes> + Send + 'static,+ {+ std::future::ready(Err(http_client::Error::StreamEnded))+ }++ async fn send_streaming<T>(&self, request: Request<T>) -> http_client::Result<StreamingResponse>+ where+ T: Into<Bytes> + Send,+ {+ let _ = self.send::<T, Bytes>(request).await;+ Err(http_client::Error::StreamEnded)+ }+}++#[tokio::test]+async fn the_adapter_forwards_plain_bodies_byte_for_byte_and_fails_bad_layouts() {+ let recording = Recording::default();+ let layout = AnthropicPromptLayout::new(recording.clone());+ // Key order and spacing a re-serialization would not keep.+ let plain = Bytes::from_static(br#"{"model":"claude-opus-4-8", "messages":[]}"#);+ let request = Request::builder()+ .header("content-length", plain.len())+ .body(plain.clone())+ .unwrap();+ let _ = layout.send_streaming(request).await;+ assert_eq!(+ recording.0.lock().unwrap()[0],+ (Some(plain.len().to_string()), plain)+ );++ let shaped = Bytes::from(+ json!({+ "system": [{"type": "text", "text": "Shared"}, {"type": "text", "text": "Rest"}],+ "messages": [],+ "macro_prompt_layout": {"cache_shared_system": true, "tool_references": []},+ })+ .to_string(),+ );+ let request = Request::builder()+ .header("content-length", shaped.len())+ .body(shaped)+ .unwrap();+ let _ = layout.send_streaming(request).await;+ let (length, body) = recording.0.lock().unwrap()[1].clone();+ assert_eq!(length, None, "the transport recomputes the changed length");+ assert_eq!(+ serde_json::from_slice::<Value>(&body).unwrap(),+ json!({+ "system": [+ {"type": "text", "text": "Shared", "cache_control": {"type": "ephemeral"}},+ {"type": "text", "text": "Rest"},+ ],+ "messages": [],+ })+ );++ let bad = Bytes::from(+ json!({"messages": [], "macro_prompt_layout": {"cache_shared_system": true, "tool_references": []}})+ .to_string(),+ );+ let Err(error) = layout+ .send_streaming(Request::builder().body(bad).unwrap())+ .await+ else {+ panic!("a layout that cannot be applied must fail the request");+ };+ assert_eq!(+ error.to_string(),+ http_client::Error::Instance(PromptLayoutError::SharedSystem.into()).to_string()+ );+ assert_eq!(+ recording.0.lock().unwrap().len(),+ 2,+ "a bad layout is never sent"+ );+}diff --git a/crates/agent/src/model/metering_http/test.rs b/crates/agent/src/model/metering_http/test.rsindex 268f1574f6..1e8582e8c6 100644--- a/crates/agent/src/model/metering_http/test.rs+++ b/crates/agent/src/model/metering_http/test.rs@@ -630,6 +630,7 @@ async fn rig_invalid_tool_retry_is_two_authorized_executions() {#[tokio::test]async fn routing_fallback_is_attributed_to_actual_provider_and_wire_model() {+ use crate::model::anthropic_prompt_layout::AnthropicPromptLayout;use crate::model::router::{ModelRouter, RoutedModel};use rig_agent::agent::AgentBuilder;use rig_agent::completion::Prompt;@@ -639,11 +640,11 @@ async fn routing_fallback_is_attributed_to_actual_provider_and_wire_model() {let transport = Transport::new(anthropic_response());let anthropic = anthropic::Client::builder().api_key("test")- .http_client(MeteredHttpClient::new(+ .http_client(AnthropicPromptLayout::new(MeteredHttpClient::new(transport.clone(),"anthropic",WireProtocol::Anthropic,- ))+ ))).build().unwrap();let openai = openai::Client::builder()@@ -861,3 +862,70 @@ async fn provider_excess_is_not_clipped_to_the_authorized_budget() {UsageEvidence::Reported(TrustedTokenUsage::from_disjoint(10, 1000, 0, 0, 0)));}++#[tokio::test]+async fn a_reshaped_anthropic_request_is_metered_with_its_cache_usage() {+ use crate::model::anthropic_prompt_layout::AnthropicPromptLayout;++ let journal = Arc::new(Journal::default());+ let transport = Transport::new(concat!(+ "data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg\",\"usage\":{\"input_tokens\":4,\"cache_read_input_tokens\":30000,\"cache_creation_input_tokens\":250}}}\n\n",+ "data: {\"type\":\"message_delta\",\"usage\":{\"output_tokens\":7}}\n\n",+ "data: {\"type\":\"message_stop\"}\n\n",+ ));+ let client = AnthropicPromptLayout::new(MeteredHttpClient::new(+ transport.clone(),+ "anthropic",+ WireProtocol::Anthropic,+ ));+ let body = json!({+ "model": "model",+ "max_tokens": 50,+ "tools": [+ {"name": "LoadTools", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}},+ {"name": "EditPresentation", "input_schema": {"type": "object"}, "defer_loading": true},+ ],+ "system": [+ {"type": "text", "text": "Shared"},+ {"type": "text", "text": "Task A", "cache_control": {"type": "ephemeral"}},+ ],+ "messages": [+ {"role": "user", "content": [{"type": "text", "text": "edit my deck"}]},+ {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_load", "name": "LoadTools", "input": {}}]},+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_load", "content": "loaded"}]},+ ],+ "cache_control": {"type": "ephemeral"},+ "macro_prompt_layout": {+ "cache_shared_system": true,+ "tool_references": [{"tool_use_id": "toolu_load", "tool_names": ["EditPresentation"]}],+ },+ });+ context(+ journal.clone(),+ WireProtocol::Anthropic,+ "anthropic",+ "model",+ )+ .scope(async {+ let request = Request::builder()+ .uri("https://example.test/v1/messages")+ .body(Bytes::from(body.to_string()))+ .unwrap();+ let mut stream = client.send_streaming(request).await.unwrap().into_body();+ while stream.next().await.is_some() {}+ })+ .await;++ let sent = transport.bodies.lock().unwrap()[0].clone();+ assert_eq!(sent.get("macro_prompt_layout"), None);+ assert_eq!(+ sent["messages"][2]["content"][0]["content"],+ json!([{"type": "tool_reference", "tool_name": "EditPresentation"}])+ );+ let begins = journal.begins.lock().unwrap();+ assert_eq!(begins.len(), 1, "metering admits the reshaped request");+ assert_eq!(+ journal.finals.lock().unwrap()[0].usage,+ UsageEvidence::Reported(TrustedTokenUsage::from_disjoint(4, 7, 30000, 250, 0))+ );+}diff --git a/crates/agent/src/model/mod.rs b/crates/agent/src/model/mod.rsindex e48fc49230..bacdf46aaa 100644--- a/crates/agent/src/model/mod.rs+++ b/crates/agent/src/model/mod.rs@@ -6,7 +6,8 @@ mod reasoning_effort;pub(crate) mod types;pub use predefined_model::*;pub use reasoning_effort::*;-mod anthropic;+pub(crate) mod anthropic;+pub mod anthropic_prompt_layout;mod gemini;pub mod metering;pub mod metering_http;diff --git a/crates/agent/src/model/router.rs b/crates/agent/src/model/router.rsindex 5e14b31dcb..4d7bf245e0 100644--- a/crates/agent/src/model/router.rs+++ b/crates/agent/src/model/router.rs@@ -34,7 +34,8 @@ use rig_core::providers::{anthropic, gemini, openai};use rig_core::streaming::StreamedAssistantContent;use tracing::Instrument as _;-use super::anthropic::AnthropicModel;+use super::anthropic::{AnthropicModel, DeferredTools, LaidOutModel, SessionLayout};+use super::anthropic_prompt_layout::AnthropicPromptLayout;use super::gemini::{GeminiCompletionModel, GeminiModel};use super::metering::{MeteringContext, WireProtocol};use super::metering_http::MeteredHttpClient;@@ -42,6 +43,7 @@ use super::openai::{OpenAiChatCompletionsModel, OpenAiResponsesModel};use super::types::Model;use super::usage_amount::usage_amount;use super::{PredefinedModel, ReasoningEffort};+use crate::agent_loop::{SystemPrompt, ToolSearch};use crate::error::AgentError;use crate::hook::{BridgeInputs, StreamBridge};use crate::stream::{ChatCompletionStream, StreamPart};@@ -78,7 +80,7 @@ const GOOGLE_PROVIDER: &str = "google";/// A routed model id bound to the provider client that serves it.pub(crate) enum RoutedModel<'a, H = MeteredHttpClient> {/// A model on Anthropic's native API.- Anthropic(AnthropicModel<'a, H>),+ Anthropic(AnthropicModel<'a, AnthropicPromptLayout<H>>),/// A model on Gemini's native GenerateContent API.Gemini(GeminiModel<'a, H>),/// A model on the OpenAI-compatible Chat Completions API.@@ -120,27 +122,50 @@ impl<'a, H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> RoutedM/// Build the rig agent for this model, applying provider-specific thinking/// config. Pure construction — no model call is made here.+ ///+ /// Anthropic caches a split system prompt's shared part on its own, and a+ /// model that loads tools by reference declares the whole catalog upfront.+ /// Every other provider receives the prompt joined and loads tools by+ /// registering them.+ #[allow(clippy::too_many_arguments)]pub(crate) fn into_agent(self,reasoning_effort: Option<ReasoningEffort>,handle: ToolServerHandle,- system_prompt: &str,+ system_prompt: &SystemPrompt,+ tool_search: &ToolSearch,max_turns: usize,max_tokens: u64,telemetry: &GenAiContext,) -> ProviderAgent<H> {+ let joined = system_prompt.joined();+ let system_prompt_text = joined.as_str();match self {RoutedModel::Anthropic(m) => {let thinking = m.thinking_params(reasoning_effort);- ProviderAgent::Anthropic(build_agent(- m.completion(),- thinking,- handle,- system_prompt,- max_turns,- max_tokens,- telemetry,- ))+ let layout = SessionLayout {+ shared_system: system_prompt.shared().is_some(),+ deferred: (m.loads_tools_by_reference() && !tool_search.catalog.is_empty())+ .then(|| {+ DeferredTools::new(&tool_search.catalog, tool_search.loads.clone())+ }),+ };+ let (preamble, session_system) = match system_prompt.shared() {+ Some(shared) => (shared, Some(system_prompt.rest().to_owned())),+ None => (system_prompt.rest(), None),+ };+ ProviderAgent::Anthropic(AnthropicAgent {+ agent: build_agent(+ LaidOutModel::new(m.completion(), layout),+ thinking,+ handle,+ preamble,+ max_turns,+ max_tokens,+ telemetry,+ ),+ session_system,+ })}RoutedModel::Gemini(m) => {let thinking = m.thinking_params();@@ -148,7 +173,7 @@ impl<'a, H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> RoutedMm.completion(),thinking,handle,- system_prompt,+ system_prompt_text,max_turns,max_tokens,telemetry,@@ -160,7 +185,7 @@ impl<'a, H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> RoutedMm.completion(),thinking,handle,- system_prompt,+ system_prompt_text,max_turns,max_tokens,telemetry,@@ -172,7 +197,7 @@ impl<'a, H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> RoutedMm.completion(),thinking,handle,- system_prompt,+ system_prompt_text,max_turns,max_tokens,telemetry,@@ -192,7 +217,7 @@ pub(crate) enum ProviderAgent<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static = MeteredHttpClient,> {/// An agent over Anthropic's native completion model.- Anthropic(Agent<TracedModel<anthropic::completion::CompletionModel<H>>>),+ Anthropic(AnthropicAgent<H>),/// An agent over Gemini's native GenerateContent model.Gemini(Agent<TracedModel<GeminiCompletionModel<H>>>),/// An agent over the OpenAI Chat Completions model.@@ -222,9 +247,17 @@ impl<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> ProviderAgetelemetry: GenAiContext,) -> ChatCompletionStream<'static> {match self {- ProviderAgent::Anthropic(agent) => {+ ProviderAgent::Anthropic(anthropic) => {+ // After the agent's preamble, before the conversation: the+ // second system block.+ let history = anthropic+ .session_system+ .iter()+ .map(Message::system)+ .chain(history)+ .collect();drive_stream(- agent,+ &anthropic.agent,WireProtocol::Anthropic,prompt,history,@@ -306,6 +339,15 @@ impl<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> ProviderAge}}+/// The Anthropic arm of [`ProviderAgent`]: the agent, whose preamble is the+/// system prompt's shared part when it is split, and the per-session rest.+pub(crate) struct AnthropicAgent<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> {+ agent: Agent<+ TracedModel<LaidOutModel<anthropic::completion::CompletionModel<AnthropicPromptLayout<H>>>>,+ >,+ session_system: Option<String>,+}+/// Routes model api-id strings to the provider client that serves them.////// Holds native Anthropic, Gemini, and OpenAI Responses clients plus a@@ -314,7 +356,7 @@ impl<H: HttpClientExt + Clone + Default + std::fmt::Debug + 'static> ProviderAge/// compatible providers with `with_openai_provider` on the metered router.#[derive(Clone)]pub struct ModelRouter<H = ReqwestClient> {- anthropic: Arc<anthropic::Client<H>>,+ anthropic: Arc<anthropic::Client<AnthropicPromptLayout<H>>>,openai: Arc<openai::Client<H>>,gemini: Option<Arc<gemini::Client<H>>>,openai_compatible: HashMap<String, Arc<openai::CompletionsClient<H>>>,@@ -322,8 +364,12 @@ pub struct ModelRouter<H = ReqwestClient> {impl<H: HttpClientExt + Clone + 'static> ModelRouter<H> {/// Build a router over native Anthropic and OpenAI Responses clients, with- /// no OpenAI-compatible Chat Completions providers registered yet.- pub fn new(anthropic: anthropic::Client<H>, openai: openai::Client<H>) -> Self {+ /// no OpenAI-compatible Chat Completions providers registered yet. The+ /// Anthropic client sends through [`AnthropicPromptLayout`].+ pub fn new(+ anthropic: anthropic::Client<AnthropicPromptLayout<H>>,+ openai: openai::Client<H>,+ ) -> Self {Self {anthropic: Arc::new(anthropic),openai: Arc::new(openai),@@ -349,11 +395,11 @@ impl ModelRouter {.map_err(|error| rig_core::http_client::Error::Instance(Box::new(error)))?;let anthropic = anthropic::Client::builder().api_key(env.anthropic_api_key.to_string())- .http_client(MeteredHttpClient::new(+ .http_client(AnthropicPromptLayout::new(MeteredHttpClient::new(http.clone(),ANTHROPIC_PROVIDER,WireProtocol::Anthropic,- ))+ ))).build()?;// Default base URL is api.openai.com; OpenAI's GPT models use// Responses API so reasoning models get max_output_tokens.@@ -862,6 +908,40 @@ where}}+/// A routed agent over a test transport runs exactly as production's does.+#[cfg(test)]+impl<H> DynStreamAgent for ProviderAgent<H>+where+ H: HttpClientExt + Clone + Default + std::fmt::Debug + Send + Sync + 'static,+{+ fn run_stream_dyn<'a>(+ &'a self,+ prompt: Message,+ history: Vec<Message>,+ max_turns: usize,+ inputs: BridgeInputs,+ recorder: Arc<dyn UsageRecorder>,+ usage_ctx: UsageContext,+ model: String,+ request_context: RequestContext,+ telemetry: GenAiContext,+ ) -> std::pin::Pin<+ Box<dyn std::future::Future<Output = ChatCompletionStream<'static>> + Send + 'a>,+ > {+ Box::pin(self.run_stream(+ prompt,+ history,+ max_turns,+ inputs,+ recorder,+ usage_ctx,+ model,+ request_context,+ telemetry,+ ))+ }+}+#[cfg(test)]impl ProviderAgent {/// Build a test-only [`ProviderAgent`] backed by `model` (a fake completiondiff --git a/crates/agent/src/test/agent_loop/mod.rs b/crates/agent/src/test/agent_loop/mod.rsindex 6ae4f8b1a6..00a977ad56 100644--- a/crates/agent/src/test/agent_loop/mod.rs+++ b/crates/agent/src/test/agent_loop/mod.rs@@ -7,6 +7,7 @@ mod util;mod test_cooperative_cancellation;mod test_eager_tools;+mod test_prompt_cache;mod test_telemetry;mod test_thinking;mod test_tool;diff --git a/crates/agent/src/test/agent_loop/test_prompt_cache.rs b/crates/agent/src/test/agent_loop/test_prompt_cache.rsnew file mode 100644index 0000000000..b3ffbdc0d9--- /dev/null+++ b/crates/agent/src/test/agent_loop/test_prompt_cache.rs@@ -0,0 +1,453 @@+//! The Anthropic cache prefix (`tools` → `system` → `messages`) across a run+//! and across sessions, as the final HTTP bodies show it.+//!+//! A scripted transport stands in for the providers: it records each request+//! body and answers with canned SSE. Sessions route through a real+//! [`ModelRouter`], so the provider arms, the request planner and the wire+//! layout build every byte the API would see.++use super::util;+use crate::SystemPrompt;+use crate::model::anthropic_prompt_layout::AnthropicPromptLayout;+use crate::model::router::ModelRouter;+use crate::stream::ToolResponse;+use ai_toolset::{+ AsyncTool, AsyncToolCollection, RequestContext, RequestSchema, SearchableTool, ServiceContext,+ ToolAnnotated, ToolAnnotations, ToolResult, ToolSet as AiToolSet, ToolSetError,+};+use async_trait::async_trait;+use bytes::Bytes;+use rig_core::http_client::HttpClientExt;+use rig_core::http_client::{self, LazyBody, MultipartForm, Request, Response, StreamingResponse};+use rig_core::providers::{anthropic, openai};+use schemars::JsonSchema;+use serde::Deserialize;+use serde_json::{Value, json};+use std::collections::VecDeque;+use std::pin::Pin;+use std::sync::{Arc, Mutex};++/// Records request bodies and answers each with the next scripted SSE body,+/// or fails the request once the script runs out.+#[derive(Clone, Debug, Default)]+struct ScriptedProvider {+ bodies: Arc<Mutex<Vec<Value>>>,+ responses: Arc<Mutex<VecDeque<String>>>,+}++impl ScriptedProvider {+ fn new(responses: impl IntoIterator<Item = String>) -> Self {+ Self {+ bodies: Arc::default(),+ responses: Arc::new(Mutex::new(responses.into_iter().collect())),+ }+ }++ fn bodies(&self) -> Vec<Value> {+ self.bodies.lock().unwrap().clone()+ }+}++impl HttpClientExt for ScriptedProvider {+ fn send<T, U>(+ &self,+ _: Request<T>,+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static+ where+ T: Into<Bytes> + Send,+ U: From<Bytes> + Send + 'static,+ {+ std::future::ready(Err(http_client::Error::StreamEnded))+ }++ fn send_multipart<U>(+ &self,+ _: Request<MultipartForm>,+ ) -> impl Future<Output = http_client::Result<Response<LazyBody<U>>>> + Send + 'static+ where+ U: From<Bytes> + Send + 'static,+ {+ std::future::ready(Err(http_client::Error::StreamEnded))+ }++ async fn send_streaming<T>(&self, request: Request<T>) -> http_client::Result<StreamingResponse>+ where+ T: Into<Bytes> + Send,+ {+ let body = serde_json::from_slice(&request.into_body().into()).unwrap();+ self.bodies.lock().unwrap().push(body);+ let Some(sse) = self.responses.lock().unwrap().pop_front() else {+ return Err(http_client::Error::StreamEnded);+ };+ let stream = futures::stream::iter([Ok(Bytes::from(sse))]);+ Ok(Response::builder()+ .header("content-type", "text/event-stream")+ .body(Box::pin(stream) as http_client::sse::BoxedStream)+ .unwrap())+ }+}++fn sse(events: &[Value]) -> String {+ events+ .iter()+ .map(|event| {+ format!(+ "event: {}\ndata: {event}\n\n",+ event["type"].as_str().unwrap()+ )+ })+ .collect()+}++fn tool_use_response(id: &str, name: &str, input: Value) -> String {+ sse(&[+ json!({"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": "claude-opus-4-8", "content": [], "stop_reason": null, "usage": {"input_tokens": 1, "output_tokens": 0}}}),+ json!({"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": id, "name": name, "input": {}}}),+ json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": input.to_string()}}),+ json!({"type": "content_block_stop", "index": 0}),+ json!({"type": "message_delta", "delta": {"stop_reason": "tool_use"}, "usage": {"output_tokens": 1}}),+ json!({"type": "message_stop"}),+ ])+}++fn text_response(text: &str) -> String {+ sse(&[+ json!({"type": "message_start", "message": {"id": "msg_2", "type": "message", "role": "assistant", "model": "claude-opus-4-8", "content": [], "stop_reason": null, "usage": {"input_tokens": 1, "output_tokens": 0}}}),+ json!({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),+ json!({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}),+ json!({"type": "content_block_stop", "index": 0}),+ json!({"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}}),+ json!({"type": "message_stop"}),+ ])+}++/// Stand-in for `ai_tools::LoadTools`: hands the named catalog tools to the+/// session's loader.+#[derive(Deserialize, JsonSchema)]+#[schemars(title = "LoadTools", description = "Load deferred tools by name.")]+struct LoadToolsStandIn {+ names: Vec<String>,+}++impl ToolAnnotated for LoadToolsStandIn {+ const ANNOTATIONS: ToolAnnotations = ToolAnnotations::read_only("Load tools");+}++#[async_trait]+impl AsyncTool<()> for LoadToolsStandIn {+ type Output = Value;++ async fn call(+ &self,+ _service_context: ServiceContext<()>,+ request_context: RequestContext,+ ) -> ToolResult<Self::Output> {+ let matched: Vec<SearchableTool> = request_context+ .searchable_tools+ .iter()+ .filter(|tool| self.names.contains(&tool.name))+ .cloned()+ .collect();+ if let Some(loader) = request_context.tool_loader.as_ref() {+ loader.load(matched);+ }+ Ok(json!({ "loaded": self.names }))+ }+}++/// `LoadTools` upfront, `EditPresentation` in the catalog.+struct DeferringToolSet {+ eager: AsyncToolCollection<()>,+}++impl AiToolSet<()> for DeferringToolSet {+ fn dispatch_tool_call<'a>(+ &'a self,+ context: (),+ request_context: RequestContext,+ tool_name: &'a str,+ json: &'a Value,+ ) -> Pin<Box<dyn Future<Output = Result<ToolResult<Value>, ToolSetError>> + 'a + Send>> {+ if tool_name == "EditPresentation" {+ return Box::pin(async { Ok(Ok(json!({ "edited": true }))) });+ }+ self.eager+ .dispatch_tool_call(context, request_context, tool_name, json)+ }++ fn request_schemas(&self) -> Option<Vec<RequestSchema>> {+ self.eager.request_schemas()+ }++ fn searchable_catalog(&self) -> Vec<SearchableTool> {+ vec![SearchableTool {+ name: "EditPresentation".to_owned(),+ description: "Edit a presentation.".to_owned(),+ schema: serde_json::from_value(json!({+ "type": "object",+ "properties": {"id": {"type": "string"}}+ }))+ .unwrap(),+ }]+ }+}++fn router(transport: &ScriptedProvider) -> ModelRouter<ScriptedProvider> {+ let anthropic = anthropic::Client::builder()+ .api_key("test-anthropic-key")+ .http_client(AnthropicPromptLayout::new(transport.clone()))+ .build()+ .unwrap();+ let openai = openai::Client::builder()+ .api_key("test-openai-key")+ .http_client(transport.clone())+ .build()+ .unwrap();+ ModelRouter::new(anthropic, openai)+}++/// Run one session of `model` over `transport`: what it sent, and what it+/// streamed back.+async fn run_session(+ transport: &ScriptedProvider,+ model: &str,+ system_prompt: impl Into<SystemPrompt>,+ prompt: &str,+) -> (Vec<Value>, util::Collected) {+ let toolset: Arc<dyn AiToolSet<()> + Send + Sync> = Arc::new(DeferringToolSet {+ eager: AsyncToolCollection::<()>::new().add_tool::<LoadToolsStandIn, ()>(),+ });+ let before = transport.bodies().len();+ let mut session = util::test_loop()+ .with_model(model)+ .test_routed_session(+ &router(transport),+ toolset,+ Arc::new(()),+ system_prompt,+ util::usage_ctx(),+ )+ .await;+ let collected = util::drive(&mut session, prompt).await;+ (transport.bodies()[before..].to_vec(), collected)+}++/// Whether `EditPresentation` ran and answered.+fn edited(collected: &util::Collected) -> bool {+ collected.tool_responses().into_iter().any(|response| {+ matches!(+ response,+ ToolResponse::Json { name, json, .. }+ if name == "EditPresentation" && *json == json!({"edited": true})+ )+ })+}++const MODEL_LINE: &str = "\n\nYou are the anthropic/claude-opus-4-8 model. If this model id is \+ unfamiliar, that is because it was released after your training data \+ cutoff — trust this id over your training data when identifying yourself.";++fn split_prompt(task: &str) -> SystemPrompt {+ SystemPrompt::split(+ "# Identity\nYou are Macro.\n",+ format!("\n<session_instructions>\n{task}\n</session_instructions>"),+ )+}++#[tokio::test]+async fn loading_a_tool_keeps_every_request_prefix() {+ let transport = ScriptedProvider::new([+ tool_use_response(+ "toolu_load",+ "LoadTools",+ json!({"names": ["EditPresentation"]}),+ ),+ tool_use_response("toolu_edit", "EditPresentation", json!({"id": "deck"})),+ text_response("done"),+ ]);+ let (bodies, collected) = run_session(+ &transport,+ "anthropic/claude-opus-4-8",+ split_prompt("task A"),+ "edit my deck",+ )+ .await;+ assert!(collected.error.is_none(), "{:?}", collected.error);+ assert!(edited(&collected), "{:?}", collected.tool_responses());+ assert_eq!(bodies.len(), 3);++ // The catalog is declared deferred from the first request, so loading+ // never changes `tools`.+ let tools = json!([+ {+ "name": "LoadTools",+ "description": "Load deferred tools by name.",+ "input_schema": {+ "title": "LoadTools",+ "required": ["names"],+ "type": "object",+ "properties": {"names": {"type": "array", "items": {"type": "string"}}},+ },+ "cache_control": {"type": "ephemeral"},+ },+ {+ "name": "EditPresentation",+ "description": "Edit a presentation.",+ "input_schema": {"type": "object", "properties": {"id": {"type": "string"}}},+ "defer_loading": true,+ },+ ]);+ let system = json!([+ {"type": "text", "text": "# Identity\nYou are Macro.\n", "cache_control": {"type": "ephemeral"}},+ {+ "type": "text",+ "text": format!("\n<session_instructions>\ntask A\n</session_instructions>{MODEL_LINE}"),+ "cache_control": {"type": "ephemeral"},+ },+ ]);+ for body in &bodies {+ assert_eq!(body["tools"], tools);+ assert_eq!(body["system"], system);+ assert_eq!(body["cache_control"], json!({"type": "ephemeral"}));+ }+ // Each request's conversation is the start of the next one's.+ for pair in bodies.windows(2) {+ let earlier = pair[0]["messages"].as_array().unwrap();+ let later = pair[1]["messages"].as_array().unwrap();+ assert_eq!(&later[..earlier.len()], earlier.as_slice());+ }+ assert_eq!(+ bodies[1]["messages"][2],+ json!({"role": "user", "content": [+ {"type": "tool_result", "tool_use_id": "toolu_load", "content": [+ {"type": "tool_reference", "tool_name": "EditPresentation"},+ ]},+ {"type": "text", "text": "<tool_result_text tool_use_id=\"toolu_load\">{\"loaded\":[\"EditPresentation\"]}</tool_result_text>"},+ ]})+ );+}++#[tokio::test]+async fn sessions_with_different_instructions_share_the_shared_system_block() {+ let transport = ScriptedProvider::new([text_response("one"), text_response("two")]);+ let (first, _) = run_session(+ &transport,+ "anthropic/claude-opus-4-8",+ split_prompt("task A"),+ "hi",+ )+ .await;+ let (second, _) = run_session(+ &transport,+ "anthropic/claude-opus-4-8",+ split_prompt("task B"),+ "hi",+ )+ .await;++ assert_eq!(first[0]["tools"], second[0]["tools"]);+ let shared = json!({+ "type": "text",+ "text": "# Identity\nYou are Macro.\n",+ "cache_control": {"type": "ephemeral"},+ });+ assert_eq!(first[0]["system"][0], shared);+ assert_eq!(second[0]["system"][0], shared);+ assert_ne!(first[0]["system"][1], second[0]["system"][1]);+}++#[tokio::test]+async fn a_deferred_tool_called_before_loading_is_loaded_from_its_retry() {+ let transport = ScriptedProvider::new([+ tool_use_response("toolu_early", "EditPresentation", json!({"id": "deck"})),+ tool_use_response("toolu_edit", "EditPresentation", json!({"id": "deck"})),+ text_response("done"),+ ]);+ let (bodies, collected) = run_session(+ &transport,+ "anthropic/claude-opus-4-8",+ split_prompt("task A"),+ "edit my deck",+ )+ .await;+ assert!(collected.error.is_none(), "{:?}", collected.error);+ assert!(edited(&collected), "{:?}", collected.tool_responses());++ assert_eq!(bodies[0]["tools"], bodies[1]["tools"]);+ let retry = &bodies[1]["messages"][2]["content"];+ assert_eq!(+ retry[0],+ json!({"type": "tool_result", "tool_use_id": "toolu_early", "content": [+ {"type": "tool_reference", "tool_name": "EditPresentation"},+ ]})+ );+ assert!(+ retry[1]["text"]+ .as_str()+ .unwrap()+ .starts_with("<tool_result_text tool_use_id=\"toolu_early\">"),+ "{retry}"+ );+}++#[tokio::test]+async fn a_model_without_tool_references_loads_by_registering() {+ let transport = ScriptedProvider::new([+ tool_use_response(+ "toolu_load",+ "LoadTools",+ json!({"names": ["EditPresentation"]}),+ ),+ text_response("done"),+ ]);+ let (bodies, collected) = run_session(+ &transport,+ "anthropic/claude-fable-5-1",+ split_prompt("task A"),+ "edit my deck",+ )+ .await;+ assert!(collected.error.is_none(), "{:?}", collected.error);++ let names = |body: &Value| -> Vec<String> {+ body["tools"]+ .as_array()+ .unwrap()+ .iter()+ .map(|tool| tool["name"].as_str().unwrap().to_owned())+ .collect()+ };+ assert_eq!(names(&bodies[0]), ["LoadTools"]);+ assert_eq!(names(&bodies[1]), ["LoadTools", "EditPresentation"]);+ let wire = serde_json::to_string(&bodies).unwrap();+ assert!(!wire.contains("defer_loading") && !wire.contains("tool_reference"));+ // The system prompt is still cached in two parts.+ assert_eq!(bodies[0]["system"].as_array().unwrap().len(), 2);+}++#[tokio::test]+async fn other_providers_receive_the_prompt_joined_as_before() {+ let split = ScriptedProvider::default();+ let (split_bodies, _) =+ run_session(&split, "openai/gpt-5.5", split_prompt("task A"), "hi").await;+ let joined = ScriptedProvider::default();+ let (joined_bodies, _) = run_session(+ &joined,+ "openai/gpt-5.5",+ "# Identity\nYou are Macro.\n\n<session_instructions>\ntask A\n</session_instructions>",+ "hi",+ )+ .await;++ assert!(!split_bodies.is_empty());+ assert_eq!(split_bodies, joined_bodies);+ let wire = serde_json::to_string(&split_bodies).unwrap();+ for anthropic_only in [+ "defer_loading",+ "tool_reference",+ "macro_prompt_layout",+ "cache_control",+ ] {+ assert!(!wire.contains(anthropic_only), "{anthropic_only} in {wire}");+ }+}diff --git a/crates/agent/src/test/test_hook.rs b/crates/agent/src/test/test_hook.rsindex 3852d366ec..4c68bc1e7a 100644--- a/crates/agent/src/test/test_hook.rs+++ b/crates/agent/src/test/test_hook.rs@@ -43,6 +43,7 @@ fn inputs(routing: Arc::new(|_| None),loaded_buffer,register_loaded: register,+ tool_loads: ToolLoads::default(),user_tool_finisher,}}@@ -195,18 +196,25 @@ async fn on_tool_call_coerces_non_object_args_to_empty_object() {async fn invalid_call_to_searchable_tool_loads_it_and_retries() {let (register, registered) = recording_register();let catalog = Arc::new(vec![searchable("mcp__linear__create_issue")]);- let (bridge, _rx) = StreamBridge::channel(- inputs(Arc::new(Mutex::new(Vec::new())), register, None),- catalog,- CancellationToken::new(),- );+ let bridge_inputs = inputs(Arc::new(Mutex::new(Vec::new())), register, None);+ let tool_loads = bridge_inputs.tool_loads.clone();+ let (bridge, _rx) = StreamBridge::channel(bridge_inputs, catalog, CancellationToken::new());let action = bridge- .handle_invalid_tool_call("mcp__linear__create_issue")+ .handle_invalid_tool_call("mcp__linear__create_issue", Some("toolu_1")).await;- // The unloaded-but-searchable tool was registered and the turn retries.+ // The unloaded-but-searchable tool was registered and the turn retries,+ // the retry's result being the one that loaded it.assert_eq!(&*registered.lock().unwrap(), &["mcp__linear__create_issue"]);+ let call = rig_core::message::ToolCall::new(+ "toolu_1".to_owned(),+ rig_core::message::ToolFunction::new(+ "mcp__linear__create_issue".to_owned(),+ serde_json::json!({}),+ ),+ );+ assert_eq!(tool_loads.loaded_by(&call), ["mcp__linear__create_issue"]);let Some(InvalidToolCallAction::Retry { feedback }) = action else {panic!("expected retry, got {action:?}");};@@ -224,7 +232,7 @@ async fn invalid_call_to_unknown_tool_retries_with_feedback_without_loading() {);let action = bridge- .handle_invalid_tool_call("mcp__nope__hallucinated")+ .handle_invalid_tool_call("mcp__nope__hallucinated", Some("toolu_1")).await;// Nothing exists to load; the model gets corrective feedback instead ofdiff --git a/crates/agent/src/test/test_router.rs b/crates/agent/src/test/test_router.rsindex cfba169433..fcdd34a698 100644--- a/crates/agent/src/test/test_router.rs+++ b/crates/agent/src/test/test_router.rs@@ -1,3 +1,4 @@+use crate::model::anthropic_prompt_layout::AnthropicPromptLayout;use crate::model::router::*;use crate::model::types::Model;use crate::model::{PredefinedModel, ReasoningEffort};@@ -6,6 +7,7 @@ use rig_core::providers::{anthropic, gemini, openai};fn test_router() -> ModelRouter {let anthropic = anthropic::Client::builder().api_key("test-anthropic-key")+ .http_client(AnthropicPromptLayout::default()).build().unwrap();let openai = openai::Client::builder()diff --git a/crates/agent/src/tool_adapter.rs b/crates/agent/src/tool_adapter.rsindex 66481264f8..609f723762 100644--- a/crates/agent/src/tool_adapter.rs+++ b/crates/agent/src/tool_adapter.rs@@ -73,6 +73,23 @@ pub(crate) fn take_description(.unwrap_or_else(|| name.to_owned())}+/// A tool's definition as providers receive it, from its schema.+pub(crate) fn provider_definition(+ name: String,+ description: String,+ schema: &schemars::Schema,+) -> rig_core::completion::ToolDefinition {+ let mut parameters =+ serde_json::to_value(schema).unwrap_or(serde_json::Value::Object(serde_json::Map::new()));+ normalize_request_schema(&mut parameters);+ let description = take_description(&name, description, &mut parameters);+ rig_core::completion::ToolDefinition {+ name,+ description,+ parameters,+ }+}+type Deserializer<Context> = Arc<dyn Fn(&serde_json::Value,@@ -205,12 +222,11 @@ impl DynToolSetAdapter {description,schema,}| {- let schema_json = serde_json::to_value(&schema)- .unwrap_or(serde_json::Value::Object(serde_json::Map::new()));+ let definition = provider_definition(name, description, &schema);Self::build(- name,- description,- schema_json,+ definition.name,+ definition.description,+ definition.parameters,toolset.clone(),context.clone(),request_context.clone(),@@ -237,12 +253,11 @@ impl DynToolSetAdapter {whereContext: Clone + Send + Sync + 'static,{- let schema_json = serde_json::to_value(&schema)- .unwrap_or(serde_json::Value::Object(serde_json::Map::new()));+ let definition = provider_definition(name, description, &schema);Self::build(- name,- description,- schema_json,+ definition.name,+ definition.description,+ definition.parameters,toolset,context,request_context,@@ -252,7 +267,7 @@ impl DynToolSetAdapter {fn build<Context>(name: String,description: String,- mut schema: serde_json::Value,+ schema: serde_json::Value,toolset: Arc<dyn AiToolSet<Context> + Send + Sync>,context: Arc<Context>,request_context: Arc<RwLock<RequestContext>>,@@ -260,8 +275,6 @@ impl DynToolSetAdapter {whereContext: Clone + Send + Sync + 'static,{- normalize_request_schema(&mut schema);- let description = take_description(&name, description, &mut schema);let tool_name = name.clone();DynamicTool::new(name,diff --git a/crates/agent_inmem/src/outbound/rig_engine/measure_prompt_cache.rs b/crates/agent_inmem/src/outbound/rig_engine/measure_prompt_cache.rsnew file mode 100644index 0000000000..4d77b95154--- /dev/null+++ b/crates/agent_inmem/src/outbound/rig_engine/measure_prompt_cache.rs@@ -0,0 +1,188 @@+//! Measures Anthropic prompt-cache reads and writes per model call, against+//! the real API, with Macro's real tools and system prompt routed through the+//! production agent loop. Tool calls other than `LoadTools` answer `ok`+//! without running, so nothing touches a workspace.+//!+//! ```text+//! doppler run --project agent-harness-service --config dev -- \+//! cargo test -p agent_inmem measure_prompt_cache -- --ignored --nocapture+//! ```++use super::*;+use ai_toolset::{RequestSchema, SearchableTool, ToolResult, ToolSetError};+use ai_usage::financial::{FinalizeInvocation, UsageEvidence, WriteDisposition};+use ai_usage::{AiFeature, FinancialFuture, TrackedInvocation, UsageContext, UsageTracking};+use std::pin::Pin;+use std::sync::Mutex;++const MODEL: &str = "anthropic/claude-sonnet-5-5";++/// Every model call's reported usage, in order.+#[derive(Default)]+struct RecordedUsage(Mutex<Vec<UsageEvidence>>);++impl UsageTracking for RecordedUsage {+ fn begin(&self, _: TrackedInvocation) -> FinancialFuture<'_, WriteDisposition> {+ Box::pin(async { Ok(WriteDisposition::Inserted) })+ }++ fn finalize(&self, evidence: FinalizeInvocation) -> FinancialFuture<'_, WriteDisposition> {+ self.0.lock().unwrap().push(evidence.usage);+ Box::pin(async { Ok(WriteDisposition::Inserted) })+ }+}++/// The agent session's tools as production defers them; `LoadTools` loads,+/// every other call answers without running.+struct InertTools(Arc<dyn AiToolSet<InMemToolContext> + Send + Sync>);++impl AiToolSet<()> for InertTools {+ fn dispatch_tool_call<'a>(+ &'a self,+ _: (),+ request_context: ai_toolset::RequestContext,+ tool_name: &'a str,+ json: &'a serde_json::Value,+ ) -> Pin<+ Box<dyn Future<Output = Result<ToolResult<serde_json::Value>, ToolSetError>> + 'a + Send>,+ > {+ Box::pin(async move {+ if tool_name == "LoadTools" {+ let names: Vec<String> =+ serde_json::from_value(json["names"].clone()).unwrap_or_default();+ let loaded: Vec<SearchableTool> = request_context+ .searchable_tools+ .iter()+ .filter(|tool| names.contains(&tool.name))+ .cloned()+ .collect();+ if let Some(loader) = &request_context.tool_loader {+ loader.load(loaded);+ }+ return Ok(Ok(serde_json::json!({ "loaded": names })));+ }+ Ok(Ok(serde_json::json!({ "ok": true })))+ })+ }++ fn request_schemas(&self) -> Option<Vec<RequestSchema>> {+ self.0.request_schemas()+ }++ fn searchable_catalog(&self) -> Vec<SearchableTool> {+ self.0.searchable_catalog()+ }++ fn searchable_toolset_names(&self) -> Vec<String> {+ self.0.searchable_toolset_names()+ }+}++/// One session's turn: each model call's (input, cache read, cache write).+async fn turn(+ native: &NativeTools,+ run: &str,+ instructions: &str,+ prompt: &str,+) -> Vec<(u64, u64, u64)> {+ let tools: Arc<dyn AiToolSet<InMemToolContext> + Send + Sync> = Arc::new(DeferredToolSet::new(+ native.for_turn(false),+ Arc::clone(&native.deferred),+ ));+ let identity = AgentIdentity {+ bot: bot_id::BotId::TEST_A,+ // Unique per run, so every run starts from a cold cache.+ name: format!("Macro {run}"),+ handle: "macro".to_owned(),+ };+ let system_prompt = system_prompt(&native.prompt, Some(&identity), Some(instructions), None);+ let user = MacroUserIdStr::try_from_email("[email protected]").unwrap();+ let usage = UsageContext::new(AiFeature::AgentSession, user);+ let recorded = Arc::new(RecordedUsage::default());+ let metering = agent::MeteringContext::tracking_only(recorded.clone(), usage.clone());+ metering+ .scope(async {+ let mut session = AgentLoop::new(Arc::new(ai_usage::NoOpUsageRecorder))+ .with_model(MODEL)+ .session(+ Arc::new(InertTools(tools)),+ Arc::new(()),+ system_prompt,+ usage,+ )+ .await;+ let mut stream = session+ .send_message(vec![agent::Message::user(prompt)])+ .await+ .unwrap();+ while let Some(part) = stream.next().await {+ part.unwrap();+ }+ })+ .await;+ recorded+ .0+ .lock()+ .unwrap()+ .iter()+ .map(|evidence| match evidence {+ UsageEvidence::Reported(usage) => {+ (usage.input(), usage.cache_read(), usage.cache_write())+ }+ other => panic!("unreported usage: {other:?}"),+ })+ .collect()+}++fn print(label: &str, calls: &[(u64, u64, u64)]) {+ for (index, (input, read, write)) in calls.iter().enumerate() {+ println!(+ "{label} call {}: input {input}, cache read {read}, cache write {write}",+ index + 1+ );+ }+}++#[tokio::test]+#[ignore = "calls the Anthropic API; run under doppler with --nocapture"]+async fn measure_prompt_cache() {+ let native = NativeTools::new();+ let run = std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap()+ .as_nanos()+ .to_string();+ let task = |id: &str| {+ format!(+ "You've been assigned the original Macro task linked below. Carry out its requested work.\n\+ <m-document-mention>{{\"documentId\":\"{id}\",\"documentName\":\"Original assigned task\"}}</m-document-mention>"+ )+ };++ let load = turn(+ &native,+ &format!("{run}-load"),+ &task("01a11170-ddae-7975-8fa5-9cf207fd77a6"),+ "Call LoadTools to load ListReminders. Do not call ListReminders or any other tool. \+ After LoadTools answers, reply with the single word done.",+ )+ .await;+ print("LoadTools run", &load);++ let first = turn(+ &native,+ &format!("{run}-sessions"),+ &task("01a11170-ddae-7975-8fa5-9cf207fd77a6"),+ "Reply with the single word hi. Do not call any tool.",+ )+ .await;+ let second = turn(+ &native,+ &format!("{run}-sessions"),+ &task("01a11171-1ca6-7a6b-9594-99b4f2b56c34"),+ "Reply with the single word hi. Do not call any tool.",+ )+ .await;+ print("session 1", &first);+ print("session 2", &second);+}diff --git a/crates/agent_inmem/src/outbound/rig_engine/test.rs b/crates/agent_inmem/src/outbound/rig_engine/test.rsindex 7d65fe1634..49453eac88 100644--- a/crates/agent_inmem/src/outbound/rig_engine/test.rs+++ b/crates/agent_inmem/src/outbound/rig_engine/test.rs@@ -128,7 +128,7 @@ async fn shared_definitions_keep_the_callers_context_and_check_each_sessions_gat#[test]fn instructions_are_a_delimited_section_after_the_standing_prompt() {- let prompt = system_prompt(&TOOLS, None, Some("be terse"), None);+ let prompt = system_prompt(&TOOLS, None, Some("be terse"), None).to_string();assert!(prompt.starts_with(&prompt::agent_session::PROMPT.to_string()),@@ -152,7 +152,7 @@ fn identity_precedes_the_standing_prompt_and_does_not_need_instructions() {name: "Grunk".to_owned(),handle: "grunk".to_owned(),};- let prompt = system_prompt(&TOOLS, Some(&identity), None, None);+ let prompt = system_prompt(&TOOLS, Some(&identity), None, None).to_string();let identity_section = prompt::agent_identity::render("Grunk", "grunk");assert!(@@ -168,7 +168,7 @@ fn identity_precedes_the_standing_prompt_and_does_not_need_instructions() {/// as an instruction.#[test]fn memory_follows_instructions_rather_than_preceding_them() {- let prompt = system_prompt(&TOOLS, None, Some("be terse"), Some("prefers Rust"));+ let prompt = system_prompt(&TOOLS, None, Some("be terse"), Some("prefers Rust")).to_string();let instructions = prompt.find("<session_instructions>")@@ -179,11 +179,45 @@ fn memory_follows_instructions_rather_than_preceding_them() {assert!(instructions < memory);}+/// Two sessions of one agent differ only after the static Macro prompt, so+/// the part before it is cached once for all of them.+#[test]+fn sessions_of_an_agent_share_everything_before_their_instructions() {+ let identity = AgentIdentity {+ bot: bot_id::BotId::TEST_A,+ name: "Grunk".to_owned(),+ handle: "grunk".to_owned(),+ };+ let first = system_prompt(+ &TOOLS,+ Some(&identity),+ Some("task A"),+ Some("prefers Rust"),+ );+ let second = system_prompt(&TOOLS, Some(&identity), Some("task B"), None);++ let shared = format!(+ "{}\n{}\n{TOOLS}",+ prompt::agent_identity::render("Grunk", "grunk"),+ prompt::agent_session::PROMPT+ );+ assert_eq!(first.shared(), Some(shared.as_str()));+ assert_eq!(second.shared(), Some(shared.as_str()));+ assert_eq!(+ first.rest(),+ "\n<session_instructions>\ntask A\n</session_instructions>\n<user_memory>\nprefers Rust\n</user_memory>"+ );+ assert_eq!(+ second.rest(),+ "\n<session_instructions>\ntask B\n</session_instructions>"+ );+}+/// Absent instructions add no section at all, rather than an empty one the/// model would have to interpret.#[test]fn no_instructions_means_no_section() {- let prompt = system_prompt(&TOOLS, None, None, Some("prefers Rust"));+ let prompt = system_prompt(&TOOLS, None, None, Some("prefers Rust")).to_string();assert!(!prompt.contains("session_instructions"));assert!(prompt.contains("<user_memory>\nprefers Rust\n</user_memory>"));@@ -193,7 +227,7 @@ fn no_instructions_means_no_section() {/// not become a blank delimited section.#[test]fn empty_instructions_add_no_section() {- let prompt = system_prompt(&TOOLS, None, Some(""), None);+ let prompt = system_prompt(&TOOLS, None, Some(""), None).to_string();assert!(!prompt.contains("session_instructions"));}diff --git a/crates/agent_inmem/src/rig_engine.rs b/crates/agent_inmem/src/rig_engine.rsindex 1811b2d97c..4f26b0ec48 100644--- a/crates/agent_inmem/src/rig_engine.rs+++ b/crates/agent_inmem/src/rig_engine.rs@@ -23,7 +23,7 @@use std::sync::Arc;-use agent::{AgentError, AgentLoop, StreamPart};+use agent::{AgentError, AgentLoop, StreamPart, SystemPrompt};use ai_tools::user_tool_review::user_tool_finisher;use ai_tools::{AiHost, DeferredToolSet, ToolServiceContext, tools_for};use ai_toolset::{AsyncToolCollection, ToolSet as AiToolSet};@@ -46,6 +46,10 @@ use crate::inbound::ask_user::{AskUser, AskUserContext};#[path = "outbound/rig_engine/test.rs"]mod test;+#[cfg(test)]+#[path = "outbound/rig_engine/measure_prompt_cache.rs"]+mod measure_prompt_cache;+/// How many stream parts may sit unread before the engine pauses; keeps a/// slow consumer from buffering a whole turn.const PART_BUFFER: usize = 256;@@ -344,7 +348,7 @@ async fn drive_turn(awaiting,});let session = agent_loop- .session(toolset, Arc::new(tool_context), &system_prompt, usage_ctx)+ .session(toolset, Arc::new(tool_context), system_prompt, usage_ctx).await;let (mut session, loop_cancel) = session.cancellable();@@ -388,37 +392,42 @@ async fn drive_turn(/// the model reads as it takes in the caller's word — the same reason DCS/// puts `additional_instructions` after the standing prompt. Memory stays/// last so a remembered fact is never read as an instruction.+///+/// The prompt is split after the static Macro prompt: everything before is+/// the same for every session of the agent and is cached on its own, so a+/// session whose instructions name its own task still reads it back.fn system_prompt(tools_prompt: &impl std::fmt::Display,identity: Option<&AgentIdentity>,instructions: Option<&str>,user_memory: Option<&str>,-) -> String {- let mut prompt = String::new();+) -> SystemPrompt {+ let mut shared = String::new();if let Some(identity) = identity {- prompt.push_str(&prompt::agent_identity::render(+ shared.push_str(&prompt::agent_identity::render(&identity.name,&identity.handle,));- prompt.push('\n');+ shared.push('\n');}- prompt.push_str(&prompt::agent_session::PROMPT.to_string());- prompt.push('\n');- prompt.push_str(&tools_prompt.to_string());+ shared.push_str(&prompt::agent_session::PROMPT.to_string());+ shared.push('\n');+ shared.push_str(&tools_prompt.to_string());+ let mut rest = String::new();// Blank instructions are "none" stated clumsily. A delimited section with// nothing in it is worse than no section: the model has to decide what an// empty instruction means.if let Some(instructions) = instructions.filter(|text| !text.trim().is_empty()) {- prompt.push_str("\n<session_instructions>\n");- prompt.push_str(instructions);- prompt.push_str("\n</session_instructions>");+ rest.push_str("\n<session_instructions>\n");+ rest.push_str(instructions);+ rest.push_str("\n</session_instructions>");}if let Some(memory) = user_memory {- prompt.push_str("\n<user_memory>\n");- prompt.push_str(memory);- prompt.push_str("\n</user_memory>");+ rest.push_str("\n<user_memory>\n");+ rest.push_str(memory);+ rest.push_str("\n</user_memory>");}- prompt+ SystemPrompt::split(shared, rest)}/// The owner's memory block, or `None` when it is missing or failed to load.diff --git a/crates/ai_projections/src/outbound/agent_generator.rs b/crates/ai_projections/src/outbound/agent_generator.rsindex 392e8264a6..3e397c98d6 100644--- a/crates/ai_projections/src/outbound/agent_generator.rs+++ b/crates/ai_projections/src/outbound/agent_generator.rs@@ -73,7 +73,7 @@ impl AgentProjectionGenerator {.session(toolset,Arc::new(tool_context),- &self.system_prompt,+ &*self.system_prompt,usage_ctx,).await;