The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
fix(langchain): sanitize cache settings for fallback models (#40886)
Mason Daugherty2ade674ffccf
diff --git a/libs/langchain_v1/langchain/agents/middleware/model_fallback.py b/libs/langchain_v1/langchain/agents/middleware/model_fallback.pyindex 6f5fa29d05..2bd58aa323 100644--- a/libs/langchain_v1/langchain/agents/middleware/model_fallback.py+++ b/libs/langchain_v1/langchain/agents/middleware/model_fallback.py@@ -1,21 +1,19 @@"""Model fallback middleware for agents.-When a caching middleware such as `AnthropicPromptCachingMiddleware` wraps this-middleware from the outside, it applies Anthropic `cache_control` markers to the-request *before* the fallback loop runs. Those markers are provider-specific and-cause API errors on non-Anthropic fallback models, so this middleware strips them-from fallback attempts — but only when the fallback model itself cannot accept-Anthropic cache markers. When the fallback is another Anthropic model the markers-are valid and preserve prompt caching, so they are left intact.--The knowledge of the `cache_control` marker is duplicated here (rather than owned-solely by the Anthropic partner package) because an outer caching middleware-never re-runs during fallback and therefore cannot clean up after itself.+When an outer caching middleware modifies a request, those changes happen before+the fallback loop runs. Provider-specific cache settings can cause API errors if+the loop reuses them with a different provider. This middleware therefore strips+unsupported Anthropic and Fireworks cache settings from each fallback attempt,+while preserving settings accepted by the selected fallback.++This provider knowledge lives here because an outer caching middleware never+re-runs during fallback and therefore cannot clean up after itself."""from __future__ import annotationsimport logging+from collections.abc import Mappingfrom typing import TYPE_CHECKING, Anyfrom langchain_core.tools import BaseTool@@ -39,6 +37,9 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)+_FIREWORKS_LLM_TYPE = "fireworks-chat"+_FIREWORKS_SESSION_AFFINITY_HEADER = "x-session-affinity"+def _sanitize_content_blocks(content: str | list[str | dict[str, Any]],@@ -109,25 +110,48 @@ def _sanitize_tools(return sanitized_tools if changed else tools-def _sanitize_request_for_fallback(request: ModelRequest[ContextT]) -> ModelRequest[ContextT]:- """Sanitize provider-specific Anthropic cache markers before fallback attempts."""+def _sanitize_request_for_fallback(+ request: ModelRequest[ContextT],+ fallback_model: BaseChatModel | None = None,+) -> ModelRequest[ContextT]:+ """Sanitize provider-specific cache settings before a fallback attempt."""overrides: dict[str, Any] = {}+ supports_anthropic_cache = _supports_anthropic_cache_control(fallback_model)++ model_settings = request.model_settings+ model_settings_changed = False++ if not supports_anthropic_cache:+ model_settings, cache_control_changed = _without_cache_control(model_settings)+ model_settings_changed = model_settings_changed or cache_control_changed++ if not _supports_fireworks_prompt_cache(fallback_model):+ model_settings, fireworks_cache_changed = _without_fireworks_session_affinity(+ model_settings+ )+ model_settings_changed = model_settings_changed or fireworks_cache_changed++ if not _supports_prompt_cache_key(fallback_model) and "prompt_cache_key" in model_settings:+ model_settings = {+ key: value for key, value in model_settings.items() if key != "prompt_cache_key"+ }+ model_settings_changed = True- model_settings, model_settings_changed = _without_cache_control(request.model_settings)if model_settings_changed:overrides["model_settings"] = model_settings- system_message = _sanitize_system_message(request.system_message)- if system_message is not request.system_message:- overrides["system_message"] = system_message+ if not supports_anthropic_cache:+ system_message = _sanitize_system_message(request.system_message)+ if system_message is not request.system_message:+ overrides["system_message"] = system_message- messages = _sanitize_messages(request.messages)- if messages is not request.messages:- overrides["messages"] = messages+ messages = _sanitize_messages(request.messages)+ if messages is not request.messages:+ overrides["messages"] = messages- tools = _sanitize_tools(request.tools)- if tools is not request.tools:- overrides["tools"] = tools+ tools = _sanitize_tools(request.tools)+ if tools is not request.tools:+ overrides["tools"] = toolsif not overrides:return request@@ -135,7 +159,7 @@ def _sanitize_request_for_fallback(request: ModelRequest[ContextT]) -> ModelRequ# Log only the field names that changed, never request content (may contain# prompt data or PII).logger.debug(- "Stripped Anthropic cache_control markers from %s before fallback attempt",+ "Stripped provider-specific cache settings from %s before fallback attempt",sorted(overrides),)@@ -208,6 +232,30 @@ def _without_cache_control(payload: dict[str, Any]) -> tuple[dict[str, Any], boo)+def _without_fireworks_session_affinity(+ model_settings: dict[str, Any],+) -> tuple[dict[str, Any], bool]:+ """Return model settings without the Fireworks-specific affinity header."""+ extra_headers = model_settings.get("extra_headers")+ if not isinstance(extra_headers, Mapping):+ return model_settings, False++ sanitized_headers = {+ key: value+ for key, value in extra_headers.items()+ if not (isinstance(key, str) and key.lower() == _FIREWORKS_SESSION_AFFINITY_HEADER)+ }+ if len(sanitized_headers) == len(extra_headers):+ return model_settings, False++ sanitized_settings = dict(model_settings)+ if sanitized_headers:+ sanitized_settings["extra_headers"] = sanitized_headers+ else:+ sanitized_settings.pop("extra_headers")+ return sanitized_settings, True++def _without_cache_control_from_content_block(block: dict[str, Any],) -> tuple[dict[str, Any], bool]:@@ -264,7 +312,7 @@ _ANTHROPIC_LLM_TYPES: frozenset[str] = frozenset()-def _supports_anthropic_cache_control(model: BaseChatModel) -> bool:+def _supports_anthropic_cache_control(model: BaseChatModel | None) -> bool:"""Return whether `model` accepts Anthropic `cache_control` markers.Checked via `_llm_type` so the decision is provider-based rather than@@ -276,6 +324,21 @@ def _supports_anthropic_cache_control(model: BaseChatModel) -> bool:return isinstance(llm_type, str) and llm_type in _ANTHROPIC_LLM_TYPES+def _supports_fireworks_prompt_cache(model: BaseChatModel | None) -> bool:+ """Return whether `model` accepts the Fireworks session-affinity header."""+ return getattr(model, "_llm_type", None) == _FIREWORKS_LLM_TYPE+++def _supports_prompt_cache_key(model: BaseChatModel | None) -> bool:+ """Return whether `model` accepts the shared `prompt_cache_key` parameter."""+ llm_type = getattr(model, "_llm_type", None)+ return isinstance(llm_type, str) and llm_type in {+ _FIREWORKS_LLM_TYPE,+ "openai-chat",+ "azure-openai-chat",+ }++class ModelFallbackMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, ResponseT]):"""Automatic fallback to alternative models on errors.@@ -350,16 +413,12 @@ class ModelFallbackMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Rexcept Exception as e:last_exception = e- # Try fallback models — sanitize cache markers only when the fallback- # model cannot accept them (i.e. is not an Anthropic-compatible model).+ # Try fallback models — sanitize provider-specific cache settings that+ # the selected fallback cannot accept.# The request is derived outside the try so a sanitizer or `_llm_type`# bug surfaces directly instead of being masked as a model failure.for fallback_model in self.models:- fallback_request = (- request- if _supports_anthropic_cache_control(fallback_model)- else _sanitize_request_for_fallback(request)- )+ fallback_request = _sanitize_request_for_fallback(request, fallback_model)try:return handler(fallback_request.override(model=fallback_model))except GraphBubbleUp:@@ -396,16 +455,12 @@ class ModelFallbackMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Rexcept Exception as e:last_exception = e- # Try fallback models — sanitize cache markers only when the fallback- # model cannot accept them (i.e. is not an Anthropic-compatible model).+ # Try fallback models — sanitize provider-specific cache settings that+ # the selected fallback cannot accept.# The request is derived outside the try so a sanitizer or `_llm_type`# bug surfaces directly instead of being masked as a model failure.for fallback_model in self.models:- fallback_request = (- request- if _supports_anthropic_cache_control(fallback_model)- else _sanitize_request_for_fallback(request)- )+ fallback_request = _sanitize_request_for_fallback(request, fallback_model)try:return await handler(fallback_request.override(model=fallback_model))except GraphBubbleUp:diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_fallback.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_fallback.pyindex 5de942978b..320352cf90 100644--- a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_fallback.py+++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_fallback.py@@ -11,6 +11,7 @@ from langchain_core.language_models.fake_chat_models import GenericFakeChatModelfrom langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessagefrom langchain_core.outputs import ChatGeneration, ChatResultfrom langchain_core.tools import BaseTool, tool+from langchain_openai import AzureChatOpenAI, ChatOpenAIfrom langgraph.errors import GraphInterruptfrom typing_extensions import override@@ -128,6 +129,31 @@ def _make_request_with_cache_markers(primary_model: BaseChatModel) -> ModelReque)+def _make_request_with_fireworks_cache_settings(+ primary_model: BaseChatModel,+) -> ModelRequest:+ """Create a request with Fireworks prompt-cache affinity settings."""+ return _make_request().override(+ model=primary_model,+ model_settings={+ "temperature": 0.3,+ "prompt_cache_key": "thread-123",+ "extra_headers": {+ "X-Request-ID": "request-123",+ "X-Session-Affinity": "thread-123",+ },+ },+ )+++def _assert_fireworks_cache_settings_removed(request: ModelRequest) -> None:+ """Assert Fireworks affinity is removed while unrelated settings remain."""+ assert request.model_settings == {+ "temperature": 0.3,+ "extra_headers": {"X-Request-ID": "request-123"},+ }++def _assert_request_has_cache_markers(request: ModelRequest) -> None:"""Assert request still contains Anthropic-style cache markers."""assert "cache_control" in request.model_settings@@ -313,6 +339,54 @@ async def test_fallback_sanitizes_cache_markers_async() -> None:_assert_request_has_cache_markers(request)+def test_fallback_removes_fireworks_cache_settings_sync() -> None:+ """A non-Fireworks fallback should not receive Fireworks cache settings."""+ primary_model = GenericFakeChatModel(messages=iter([]))+ fallback_model = GenericFakeChatModel(messages=iter([]))+ middleware = ModelFallbackMiddleware(fallback_model)+ request = _make_request_with_fireworks_cache_settings(primary_model)+ attempts: list[ModelRequest] = []++ def mock_handler(req: ModelRequest) -> ModelResponse:+ attempts.append(req)+ if len(attempts) == 1:+ msg = "Primary model failed"+ raise ValueError(msg)++ assert req.model is fallback_model+ _assert_fireworks_cache_settings_removed(req)+ return ModelResponse(result=[AIMessage(content="fallback response")])++ middleware.wrap_model_call(request, mock_handler)++ assert len(attempts) == 2+ assert request.model_settings["prompt_cache_key"] == "thread-123"+++async def test_fallback_removes_fireworks_cache_settings_async() -> None:+ """Async non-Fireworks fallback should not receive Fireworks cache settings."""+ primary_model = GenericFakeChatModel(messages=iter([]))+ fallback_model = GenericFakeChatModel(messages=iter([]))+ middleware = ModelFallbackMiddleware(fallback_model)+ request = _make_request_with_fireworks_cache_settings(primary_model)+ attempts: list[ModelRequest] = []++ async def mock_handler(req: ModelRequest) -> ModelResponse:+ attempts.append(req)+ if len(attempts) == 1:+ msg = "Primary model failed"+ raise ValueError(msg)++ assert req.model is fallback_model+ _assert_fireworks_cache_settings_removed(req)+ return ModelResponse(result=[AIMessage(content="fallback response")])++ await middleware.awrap_model_call(request, mock_handler)++ assert len(attempts) == 2+ assert request.model_settings["prompt_cache_key"] == "thread-123"++def test_sanitize_collapses_emptied_extras_to_none() -> None:"""Stripping the only `extras` key (`cache_control`) resets `extras` to None."""@@ -810,6 +884,14 @@ class _FakeNonStringLlmTypeModel(GenericFakeChatModel):return ["anthropic-chat"] # type: ignore[return-value]+class _FakeFireworksModel(GenericFakeChatModel):+ """Fake model that reports the `ChatFireworks` `_llm_type`."""++ @property+ def _llm_type(self) -> str:+ return "fireworks-chat"++_ANTHROPIC_COMPATIBLE_FAKES = [_FakeAnthropicModel,_FakeBedrockAnthropicModel,@@ -828,6 +910,85 @@ def test_supports_anthropic_cache_control() -> None:assert not _supports_anthropic_cache_control(_FakeNonStringLlmTypeModel(messages=iter([])))+def test_fallback_preserves_fireworks_cache_settings_for_fireworks() -> None:+ """A Fireworks fallback should keep Fireworks prompt-cache affinity settings."""+ primary_model = _FakeFireworksModel(messages=iter([]))+ fallback_model = _FakeFireworksModel(messages=iter([]))+ middleware = ModelFallbackMiddleware(fallback_model)+ request = _make_request_with_fireworks_cache_settings(primary_model)+ attempts: list[ModelRequest] = []++ def mock_handler(req: ModelRequest) -> ModelResponse:+ attempts.append(req)+ if len(attempts) == 1:+ msg = "Primary model failed"+ raise ValueError(msg)++ assert req.model is fallback_model+ assert req.model_settings == request.model_settings+ return ModelResponse(result=[AIMessage(content="fallback response")])++ middleware.wrap_model_call(request, mock_handler)++ assert len(attempts) == 2++[email protected]("use_async", [False, True])+ ("azure", "extra_headers"),+ [+ pytest.param(False, None, id="openai-without-headers"),+ pytest.param(False, {"X-Session-Affinity": "thread-123"}, id="openai-affinity-only"),+ pytest.param(+ True,+ {"X-Session-Affinity": "thread-123", "X-Request-ID": "request-123"},+ id="azure-preserves-other-headers",+ ),+ ],+)+async def test_openai_fallback_preserves_cache_key(+ *, use_async: bool, azure: bool, extra_headers: dict[str, str] | None+) -> None:+ """OpenAI accepts cache keys independently of Fireworks affinity headers."""+ primary = ChatOpenAI.model_construct(model_name="test-model")+ fallback = (+ AzureChatOpenAI.model_construct(model_name="test-model")+ if azure+ else ChatOpenAI.model_construct(model_name="test-model")+ )+ settings: dict[str, Any] = {"prompt_cache_key": "thread-123", "temperature": 0.3}+ if extra_headers is not None:+ settings["extra_headers"] = extra_headers+ request = _make_request().override(model=primary, model_settings=settings)+ middleware = ModelFallbackMiddleware(fallback)+ attempts: list[ModelRequest] = []++ def handler(req: ModelRequest) -> ModelResponse:+ attempts.append(req)+ if req.model is primary:+ msg = "primary failed"+ raise ValueError(msg)+ assert req.model is fallback+ expected: dict[str, Any] = {"prompt_cache_key": "thread-123", "temperature": 0.3}+ if extra_headers and "X-Request-ID" in extra_headers:+ expected["extra_headers"] = {"X-Request-ID": "request-123"}+ assert req.model_settings == expected+ return ModelResponse(result=[AIMessage(content="ok")])++ async def ahandler(req: ModelRequest) -> ModelResponse:+ return handler(req)++ if use_async:+ await middleware.awrap_model_call(request, ahandler)+ else:+ middleware.wrap_model_call(request, handler)++ assert len(attempts) == 2+ assert request.model_settings == settings+ if extra_headers is not None:+ assert request.model_settings["extra_headers"]["X-Session-Affinity"] == "thread-123"++def test_fallback_preserves_cache_markers_for_anthropic_sync() -> None:"""Anthropic fallback keeps cache markers; non-Anthropic fallback strips them."""primary_model = _FakeAnthropicModel(messages=iter([AIMessage(content="primary response")]))@@ -1053,7 +1214,7 @@ def test_fallback_sanitizer_error_is_not_masked_sync(Anthropic fallback succeeds."""- def _boom(_request: ModelRequest) -> ModelRequest:+ def _boom(_request: ModelRequest, _fallback_model: BaseChatModel | None = None) -> ModelRequest:msg = "sanitizer boom"raise RuntimeError(msg)@@ -1086,7 +1247,7 @@ async def test_fallback_sanitizer_error_is_not_masked_async() -> None:"""Async: a sanitizer bug must surface, not be masked by a later success."""- def _boom(_request: ModelRequest) -> ModelRequest:+ def _boom(_request: ModelRequest, _fallback_model: BaseChatModel | None = None) -> ModelRequest:msg = "sanitizer boom"raise RuntimeError(msg)