The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(anthropic): AnthropicPromptCachingMiddleware: apply explicit caching to system message and tool definitions (#35969)
ccurme043ef0721ad9
diff --git a/libs/partners/anthropic/langchain_anthropic/middleware/prompt_caching.py b/libs/partners/anthropic/langchain_anthropic/middleware/prompt_caching.pyindex f27285c7f0..eb35b6b974 100644--- a/libs/partners/anthropic/langchain_anthropic/middleware/prompt_caching.py+++ b/libs/partners/anthropic/langchain_anthropic/middleware/prompt_caching.py@@ -5,10 +5,15 @@ Requires:- `langchain-anthropic`: For `ChatAnthropic` model (already a dependency)"""+from __future__ import annotations+from collections.abc import Awaitable, Callable-from typing import Literal+from typing import Any, Literalfrom warnings import warn+from langchain_core.messages import SystemMessage+from langchain_core.tools import BaseTool+from langchain_anthropic.chat_models import ChatAnthropictry:@@ -34,6 +39,15 @@ class AnthropicPromptCachingMiddleware(AgentMiddleware):Requires both `langchain` and `langchain-anthropic` packages to be installed.+ Applies cache control breakpoints to:++ - **System message**: Tags the last content block of the system message+ with `cache_control` so static system prompt content is cached.+ - **Tools**: Tags all tool definitions with `cache_control` so tool+ schemas are cached across turns.+ - **Last cacheable block**: Tags last cacheable block of message sequence using+ Anthropic's automatic caching feature.+Learn more about Anthropic prompt caching[here](https://platform.claude.com/docs/en/build-with-claude/prompt-caching)."""@@ -68,6 +82,10 @@ class AnthropicPromptCachingMiddleware(AgentMiddleware):self.min_messages_to_cache = min_messages_to_cacheself.unsupported_model_behavior = unsupported_model_behavior+ @property+ def _cache_control(self) -> dict[str, str]:+ return {"type": self.type, "ttl": self.ttl}+def _should_apply_caching(self, request: ModelRequest) -> bool:"""Check if caching should be applied to the request.@@ -98,6 +116,33 @@ class AnthropicPromptCachingMiddleware(AgentMiddleware):)return messages_count >= self.min_messages_to_cache+ def _apply_caching(self, request: ModelRequest) -> ModelRequest:+ """Apply cache control to system message, tools, and model settings.++ Args:+ request: The model request to modify.++ Returns:+ New request with cache control applied.+ """+ overrides: dict[str, Any] = {}+ cache_control = self._cache_control++ overrides["model_settings"] = {+ **request.model_settings,+ "cache_control": cache_control,+ }++ system_message = _tag_system_message(request.system_message, cache_control)+ if system_message is not request.system_message:+ overrides["system_message"] = system_message++ tools = _tag_tools(request.tools, cache_control)+ if tools is not request.tools:+ overrides["tools"] = tools++ return request.override(**overrides)+def wrap_model_call(self,request: ModelRequest,@@ -115,12 +160,7 @@ class AnthropicPromptCachingMiddleware(AgentMiddleware):if not self._should_apply_caching(request):return handler(request)- model_settings = request.model_settings- new_model_settings = {- **model_settings,- "cache_control": {"type": self.type, "ttl": self.ttl},- }- return handler(request.override(model_settings=new_model_settings))+ return handler(self._apply_caching(request))async def awrap_model_call(self,@@ -139,9 +179,77 @@ class AnthropicPromptCachingMiddleware(AgentMiddleware):if not self._should_apply_caching(request):return await handler(request)- model_settings = request.model_settings- new_model_settings = {- **model_settings,- "cache_control": {"type": self.type, "ttl": self.ttl},- }- return await handler(request.override(model_settings=new_model_settings))+ return await handler(self._apply_caching(request))+++def _tag_system_message(+ system_message: Any,+ cache_control: dict[str, str],+) -> Any:+ """Tag the last content block of a system message with cache_control.++ Returns the original system_message unchanged if there are no blocks+ to tag.++ Args:+ system_message: The system message to tag.+ cache_control: The cache control dict to apply.++ Returns:+ A new SystemMessage with cache_control on the last block, or the+ original if no modification was needed.+ """+ if system_message is None:+ return system_message++ content = system_message.content+ if isinstance(content, str):+ if not content:+ return system_message+ new_content: list[str | dict[str, Any]] = [+ {"type": "text", "text": content, "cache_control": cache_control}+ ]+ elif isinstance(content, list):+ if not content:+ return system_message+ new_content = list(content)+ last = new_content[-1]+ base = last if isinstance(last, dict) else {}+ new_content[-1] = {**base, "cache_control": cache_control}+ else:+ return system_message++ return SystemMessage(content=new_content)+++def _tag_tools(+ tools: list[Any] | None,+ cache_control: dict[str, str],+) -> list[Any] | None:+ """Tag the last tool with cache_control via its extras dict.++ Only the last tool is tagged to minimize the number of explicit cache+ breakpoints (Anthropic limits these to 4 per request). Since tool+ definitions are sent as a contiguous block, a single breakpoint on the+ last tool caches the entire set.++ Creates a copy of the last tool with cache_control added to extras,+ without mutating the original.++ Args:+ tools: The list of tools to tag.+ cache_control: The cache control dict to apply.++ Returns:+ A new list with cache_control on the last tool's extras, or the+ original if no tools are present.+ """+ if not tools:+ return tools++ last = tools[-1]+ if not isinstance(last, BaseTool):+ return tools++ new_extras = {**(last.extras or {}), "cache_control": cache_control}+ return [*tools[:-1], last.model_copy(update={"extras": new_extras})]diff --git a/libs/partners/anthropic/tests/unit_tests/middleware/test_prompt_caching.py b/libs/partners/anthropic/tests/unit_tests/middleware/test_prompt_caching.pyindex 634d8ff91a..d8221cfca1 100644--- a/libs/partners/anthropic/tests/unit_tests/middleware/test_prompt_caching.py+++ b/libs/partners/anthropic/tests/unit_tests/middleware/test_prompt_caching.py@@ -11,8 +11,9 @@ from langchain_core.callbacks import (CallbackManagerForLLMRun,)from langchain_core.language_models import BaseChatModel-from langchain_core.messages import AIMessage, BaseMessage, HumanMessage+from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessagefrom langchain_core.outputs import ChatGeneration, ChatResult+from langchain_core.tools import BaseTool, toolfrom langgraph.runtime import Runtimefrom langchain_anthropic.chat_models import ChatAnthropic@@ -334,3 +335,240 @@ async def test_anthropic_prompt_caching_middleware_async_default_values() -> Nonassert modified_request.model_settings == {"cache_control": {"type": "ephemeral", "ttl": "5m"}}+++class TestSystemMessageCaching:+ """Tests for system message cache_control tagging."""++ def _make_request(+ self,+ system_message: SystemMessage | None = None,+ **kwargs: Any,+ ) -> ModelRequest:+ mock_model = MagicMock(spec=ChatAnthropic)+ defaults: dict[str, Any] = {+ "model": mock_model,+ "messages": [HumanMessage("Hello")],+ "system_message": system_message,+ "tool_choice": None,+ "tools": [],+ "response_format": None,+ "state": {"messages": [HumanMessage("Hello")]},+ "runtime": cast(Runtime, object()),+ "model_settings": {},+ }+ defaults.update(kwargs)+ return ModelRequest(**defaults)++ def _run(self, request: ModelRequest) -> ModelRequest:+ middleware = AnthropicPromptCachingMiddleware()+ captured: ModelRequest | None = None++ def handler(req: ModelRequest) -> ModelResponse:+ nonlocal captured+ captured = req+ return ModelResponse(result=[AIMessage(content="ok")])++ middleware.wrap_model_call(request, handler)+ assert captured is not None+ return captured++ def _get_content_blocks(self, result: ModelRequest) -> list[dict[str, Any]]:+ assert result.system_message is not None+ content = result.system_message.content+ assert isinstance(content, list)+ return cast("list[dict[str, Any]]", content)++ def test_tags_last_block_of_string_system_message(self) -> None:+ result = self._run(self._make_request(SystemMessage("Base prompt")))+ blocks = self._get_content_blocks(result)+ assert len(blocks) == 1+ assert blocks[0]["text"] == "Base prompt"+ assert blocks[0]["cache_control"] == {"type": "ephemeral", "ttl": "5m"}++ def test_tags_only_last_block_of_multi_block_system_message(self) -> None:+ msg = SystemMessage(+ content=[+ {"type": "text", "text": "Block 1"},+ {"type": "text", "text": "Block 2"},+ {"type": "text", "text": "Block 3"},+ ]+ )+ blocks = self._get_content_blocks(self._run(self._make_request(msg)))+ assert len(blocks) == 3+ assert "cache_control" not in blocks[0]+ assert "cache_control" not in blocks[1]+ assert blocks[2]["cache_control"] == {"type": "ephemeral", "ttl": "5m"}++ def test_does_not_mutate_original_system_message(self) -> None:+ original_content: list[str | dict[str, str]] = [+ {"type": "text", "text": "Block 1"},+ {"type": "text", "text": "Block 2"},+ ]+ msg = SystemMessage(content=original_content)+ self._run(self._make_request(msg))+ assert "cache_control" not in original_content[1]++ def test_passes_through_when_no_system_message(self) -> None:+ result = self._run(self._make_request(system_message=None))+ assert result.system_message is None++ def test_passes_through_when_system_message_has_empty_string(self) -> None:+ msg = SystemMessage(content="")+ result = self._run(self._make_request(msg))+ assert result.system_message is not None+ assert result.system_message.content == ""++ def test_passes_through_when_system_message_has_empty_list(self) -> None:+ msg = SystemMessage(content=[])+ result = self._run(self._make_request(msg))+ assert result.system_message is not None+ assert result.system_message.content == []++ def test_preserves_non_text_block_types(self) -> None:+ msg = SystemMessage(+ content=[+ {"type": "text", "text": "Prompt"},+ {"type": "custom_type", "data": "value"},+ ]+ )+ blocks = self._get_content_blocks(self._run(self._make_request(msg)))+ assert blocks[0] == {"type": "text", "text": "Prompt"}+ assert blocks[1]["type"] == "custom_type"+ assert blocks[1]["data"] == "value"+ assert blocks[1]["cache_control"] == {"type": "ephemeral", "ttl": "5m"}++ def test_respects_custom_ttl(self) -> None:+ middleware = AnthropicPromptCachingMiddleware(ttl="1h")+ request = self._make_request(SystemMessage("Prompt"))+ captured: ModelRequest | None = None++ def handler(req: ModelRequest) -> ModelResponse:+ nonlocal captured+ captured = req+ return ModelResponse(result=[AIMessage(content="ok")])++ middleware.wrap_model_call(request, handler)+ assert captured is not None+ blocks = self._get_content_blocks(captured)+ assert blocks[0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}+++class TestToolCaching:+ """Tests for tool definition cache_control tagging."""++ def _make_request(+ self,+ tools: list[Any] | None = None,+ **kwargs: Any,+ ) -> ModelRequest:+ mock_model = MagicMock(spec=ChatAnthropic)+ defaults: dict[str, Any] = {+ "model": mock_model,+ "messages": [HumanMessage("Hello")],+ "system_message": None,+ "tool_choice": None,+ "tools": tools or [],+ "response_format": None,+ "state": {"messages": [HumanMessage("Hello")]},+ "runtime": cast(Runtime, object()),+ "model_settings": {},+ }+ defaults.update(kwargs)+ return ModelRequest(**defaults)++ def _run(self, request: ModelRequest) -> ModelRequest:+ middleware = AnthropicPromptCachingMiddleware()+ captured: ModelRequest | None = None++ def handler(req: ModelRequest) -> ModelResponse:+ nonlocal captured+ captured = req+ return ModelResponse(result=[AIMessage(content="ok")])++ middleware.wrap_model_call(request, handler)+ assert captured is not None+ return captured++ def test_tags_only_last_tool_with_cache_control(self) -> None:+ @tool+ def get_weather(location: str) -> str:+ """Get weather for a location."""+ return "sunny"++ @tool+ def get_time(timezone: str) -> str:+ """Get time in a timezone."""+ return "12:00"++ result = self._run(self._make_request(tools=[get_weather, get_time]))+ assert result.tools is not None+ assert len(result.tools) == 2+ first = result.tools[0]+ assert isinstance(first, BaseTool)+ assert first.extras is None or "cache_control" not in first.extras+ last = result.tools[1]+ assert isinstance(last, BaseTool)+ assert last.extras is not None+ assert last.extras["cache_control"] == {"type": "ephemeral", "ttl": "5m"}++ def test_does_not_mutate_original_tools(self) -> None:+ @tool+ def my_tool(x: str) -> str:+ """A tool."""+ return x++ original_extras = my_tool.extras+ self._run(self._make_request(tools=[my_tool]))+ assert my_tool.extras is original_extras++ def test_preserves_existing_extras(self) -> None:+ @tool(extras={"defer_loading": True})+ def my_tool(x: str) -> str:+ """A tool."""+ return x++ result = self._run(self._make_request(tools=[my_tool]))+ assert result.tools is not None+ t = result.tools[0]+ assert isinstance(t, BaseTool)+ assert t.extras is not None+ assert t.extras["defer_loading"] is True+ assert t.extras["cache_control"] == {+ "type": "ephemeral",+ "ttl": "5m",+ }++ def test_passes_through_empty_tools(self) -> None:+ result = self._run(self._make_request(tools=[]))+ assert result.tools == []++ def test_passes_through_none_tools(self) -> None:+ result = self._run(self._make_request(tools=None))+ assert result.tools == []++ def test_respects_custom_ttl(self) -> None:+ @tool+ def my_tool(x: str) -> str:+ """A tool."""+ return x++ middleware = AnthropicPromptCachingMiddleware(ttl="1h")+ request = self._make_request(tools=[my_tool])+ captured: ModelRequest | None = None++ def handler(req: ModelRequest) -> ModelResponse:+ nonlocal captured+ captured = req+ return ModelResponse(result=[AIMessage(content="ok")])++ middleware.wrap_model_call(request, handler)+ assert captured is not None+ assert captured.tools is not None+ t = captured.tools[0]+ assert isinstance(t, BaseTool)+ assert t.extras is not None+ assert t.extras["cache_control"] == {+ "type": "ephemeral",+ "ttl": "1h",+ }