Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix: apply prompt caching in agent loop _call_llm

nandanadileep6299f773c203
diff --git a/deeptutor/agents/loop/agent_loop.py b/deeptutor/agents/loop/agent_loop.py
index 854b10a75..68cc81ea5 100644
--- a/deeptutor/agents/loop/agent_loop.py
+++ b/deeptutor/agents/loop/agent_loop.py
@@ -1105,6 +1105,53 @@ class AgentLoop:
# ---- LLM call ----------------------------------------------------------
+ def _apply_prompt_caching(self, kwargs: dict[str, Any]) -> None:
+ """Attach Anthropic-style ``cache_control`` breakpoints to the request.
+
+ The loop builds its own ``chat.completions.create`` kwargs and calls the
+ SDK client directly, so it never passes through
+ ``OpenAICompatProvider._build_kwargs`` — the only other place cache
+ breakpoints are added. Without this, an ``anthropic/`` model on a
+ prompt-caching gateway (OpenRouter) sends no breakpoint and Anthropic
+ caches nothing. Reuse the provider's helper so both paths mark requests
+ identically, and add the end-of-stable-history breakpoint the loop
+ needs: its transient per-turn context message sits just before the
+ learner's message and is never persisted, so the next turn's prefix
+ only matches up to the last durable message.
+ """
+ from deeptutor.services.llm.provider_core.openai_compat_provider import (
+ OpenAICompatProvider,
+ )
+
+ model = str(kwargs.get("model") or "")
+ if not OpenAICompatProvider.supports_prompt_caching_for(model):
+ return
+ if not self._binding_supports_prompt_caching():
+ return
+ messages, tools = OpenAICompatProvider._apply_cache_control(
+ kwargs["messages"],
+ kwargs.get("tools"),
+ stable_history_end=True,
+ )
+ kwargs["messages"] = messages
+ if tools is not None:
+ kwargs["tools"] = tools
+
+ def _binding_supports_prompt_caching(self) -> bool:
+ """Whether this binding reaches the model through OpenAICompatProvider.
+
+ Native Anthropic-backed bindings apply their own ``cache_control``
+ inside their provider, so the loop must only add markers for the
+ OpenAI-compatible path (gateways such as OpenRouter) that otherwise
+ never sees them.
+ """
+ from deeptutor.services.provider_registry import find_by_name
+
+ spec = find_by_name(self.pipeline.binding)
+ if spec is None or not spec.supports_prompt_caching:
+ return False
+ return spec.backend == "openai_compat"
+
async def _call_llm(
self,
*,
@@ -1223,6 +1270,8 @@ class AgentLoop:
messages=list(kwargs["messages"]), tool_schemas=carried
)
+ self._apply_prompt_caching(kwargs)
+
chunk_meta = merge_trace_metadata(trace_meta, {"trace_kind": "llm_chunk"})
for attempt in range(len(_PROVIDER_RETRY_DELAYS) + 1):
diff --git a/deeptutor/services/llm/provider_core/openai_compat_provider.py b/deeptutor/services/llm/provider_core/openai_compat_provider.py
index 5d6e88533..60c1a1bc1 100644
--- a/deeptutor/services/llm/provider_core/openai_compat_provider.py
+++ b/deeptutor/services/llm/provider_core/openai_compat_provider.py
@@ -269,30 +269,38 @@ class OpenAICompatProvider(LLMProvider):
# Prompt caching
# ------------------------------------------------------------------
+ @staticmethod
+ def _mark_cache_control(msg: dict[str, Any]) -> dict[str, Any]:
+ """Attach an ephemeral cache breakpoint to a message's last block."""
+ cache_marker = {"type": "ephemeral"}
+ content = msg.get("content")
+ if isinstance(content, str):
+ return {
+ **msg,
+ "content": [
+ {"type": "text", "text": content, "cache_control": cache_marker},
+ ],
+ }
+ if isinstance(content, list) and content:
+ new_content = list(content)
+ new_content[-1] = {**new_content[-1], "cache_control": cache_marker}
+ return {**msg, "content": new_content}
+ return msg
+
@classmethod
def _apply_cache_control(
cls,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]] | None,
+ *,
+ stable_history_end: bool = False,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]] | None]:
cache_marker = {"type": "ephemeral"}
new_messages = list(messages)
budget = 4
def _mark(msg: dict[str, Any]) -> dict[str, Any]:
- content = msg.get("content")
- if isinstance(content, str):
- return {
- **msg,
- "content": [
- {"type": "text", "text": content, "cache_control": cache_marker},
- ],
- }
- if isinstance(content, list) and content:
- nc = list(content)
- nc[-1] = {**nc[-1], "cache_control": cache_marker}
- return {**msg, "content": nc}
- return msg
+ return cls._mark_cache_control(msg)
if new_messages and new_messages[0].get("role") == "system":
new_messages[0] = _mark(new_messages[0])
@@ -301,6 +309,18 @@ class OpenAICompatProvider(LLMProvider):
new_messages[-1] = _mark(new_messages[-1])
budget -= 1
+ if stable_history_end and budget > 0:
+ # The loop appends a transient per-turn context message just before
+ # the learner's message and never persists it, so a breakpoint on
+ # the last message alone leaves the next turn's prefix diverging at
+ # that transient block. Mark the end of the durable history — the
+ # last non-empty message before the trailing user turn — so the
+ # previous turn's prefix still matches and can be reused.
+ index = cls._stable_history_end_index(new_messages)
+ if index is not None:
+ new_messages[index] = _mark(new_messages[index])
+ budget -= 1
+
new_tools = tools
if tools:
new_tools = list(tools)
@@ -308,6 +328,39 @@ class OpenAICompatProvider(LLMProvider):
new_tools[idx] = {**new_tools[idx], "cache_control": cache_marker}
return new_messages, new_tools
+ @staticmethod
+ def _stable_history_end_index(messages: list[dict[str, Any]]) -> int | None:
+ """Index of the last persisted message before the trailing user turn.
+
+ Walks back over the final user block (the transient context message and
+ the learner's message share the ``user`` role) and skips empty text
+ blocks, which cannot carry ``cache_control``. Returns ``None`` when the
+ marker would fall outside the request.
+ """
+ start = len(messages) - 1
+ while start > 0 and messages[start].get("role") != "user":
+ start -= 1
+ while start > 1 and messages[start - 1].get("role") == "user":
+ start -= 1
+ index = start - 1
+ while index > 0 and not messages[index].get("content"):
+ index -= 1
+ if 0 < index < len(messages) - 1:
+ return index
+ return None
+
+ @classmethod
+ def supports_prompt_caching_for(cls, model: str | None) -> bool:
+ """Whether *model* uses explicit ``cache_control`` breakpoints.
+
+ Anthropic models reached through an OpenAI-compatible gateway cache
+ only when the request carries breakpoints, and only those families
+ understand the marker. Kept here so the agent loop and the provider
+ agree on the same gate.
+ """
+ name = str(model or "").lower()
+ return name.startswith(("anthropic/", "claude"))
+
# ------------------------------------------------------------------
# Message sanitization
# ------------------------------------------------------------------
diff --git a/tests/agents/chat/test_agent_loop.py b/tests/agents/chat/test_agent_loop.py
index 248895570..c53d860c0 100644
--- a/tests/agents/chat/test_agent_loop.py
+++ b/tests/agents/chat/test_agent_loop.py
@@ -1035,6 +1035,140 @@ async def test_finish_first_round_no_tools(monkeypatch: pytest.MonkeyPatch) -> N
assert result.metadata["tool_steps"] == 0
+async def test_anthropic_gateway_request_carries_cache_control_breakpoints(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The loop must apply the provider's prompt-caching treatment (#2078).
+
+ The loop builds its own request kwargs, so an ``anthropic/`` model on a
+ prompt-caching gateway used to send no ``cache_control`` breakpoint at all.
+ It now marks system, the last message, and the end of the durable history.
+ """
+ monkeypatch.setattr(
+ "deeptutor.agents.loop.pipeline.get_llm_config",
+ lambda: SimpleNamespace(
+ binding="openrouter",
+ model="anthropic/claude-haiku-5.5",
+ api_key="k",
+ base_url="https://openrouter.ai/api/v1",
+ api_version=None,
+ extra_headers={},
+ reasoning_effort=None,
+ ),
+ )
+ client = _ScriptedChatClient([[_llm_chunk(content="Answer", finish_reason="stop")]])
+ pipeline = AgenticChatPipeline(language="en")
+ pipeline.registry = _Registry()
+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: [])
+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)
+
+ durable = [
+ {"role": "user", "content": "Question one"},
+ {"role": "assistant", "content": "Answer one"},
+ ]
+ await _run(
+ pipeline,
+ UnifiedContext(
+ session_id="s1",
+ user_message="Question two",
+ runtime=TurnRuntimeContext(model_history=list(durable)),
+ ),
+ )
+
+ sent = client.calls[0]["messages"]
+ marked = [
+ index
+ for index, message in enumerate(sent)
+ if isinstance(message.get("content"), list)
+ and any(
+ isinstance(block, dict) and "cache_control" in block
+ for block in message["content"]
+ )
+ ]
+ # system + end-of-stable-history (the durable assistant turn) + last message.
+ assert len(marked) == 3
+ assert sent[0]["role"] == "system"
+ # The middle breakpoint sits on the last durable assistant turn, before the
+ # trailing (possibly multi-message) user block of this turn.
+ assert sent[marked[1]]["role"] == "assistant"
+ assert sent[marked[1]]["content"][-1]["text"] == "Answer one"
+ assert marked[-1] == len(sent) - 1
+ assert sent[-1]["role"] == "user"
+ assert all(
+ block.get("cache_control") == {"type": "ephemeral"}
+ for index in marked
+ for block in sent[index]["content"]
+ if isinstance(block, dict) and "cache_control" in block
+ )
+
+
+async def test_non_anthropic_gateway_request_gets_no_cache_control(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A non-Anthropic model on the same gateway must not receive markers."""
+ monkeypatch.setattr(
+ "deeptutor.agents.loop.pipeline.get_llm_config",
+ lambda: SimpleNamespace(
+ binding="openrouter",
+ model="google/gemini-2.0-flash",
+ api_key="k",
+ base_url="https://openrouter.ai/api/v1",
+ api_version=None,
+ extra_headers={},
+ reasoning_effort=None,
+ ),
+ )
+ client = _ScriptedChatClient([[_llm_chunk(content="Answer", finish_reason="stop")]])
+ pipeline = AgenticChatPipeline(language="en")
+ pipeline.registry = _Registry()
+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: [])
+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)
+
+ await _run(pipeline, UnifiedContext(session_id="s1", user_message="Hi"))
+
+ assert all(
+ not isinstance(message.get("content"), list)
+ for message in client.calls[0]["messages"]
+ )
+
+
+async def test_native_anthropic_binding_leaves_markers_to_the_provider(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The native Anthropic adapter applies its own cache_control.
+
+ The loop must not pre-mark that path, or the adapter's own breakpoints
+ would be added on top of the loop's.
+ """
+ monkeypatch.setattr(
+ "deeptutor.agents.loop.pipeline.get_llm_config",
+ lambda: SimpleNamespace(
+ binding="anthropic",
+ model="claude-sonnet-4-5",
+ api_key="k",
+ base_url="https://api.anthropic.com",
+ api_version=None,
+ extra_headers={},
+ reasoning_effort=None,
+ ),
+ )
+ client = _ScriptedChatClient([[_llm_chunk(content="Answer", finish_reason="stop")]])
+ pipeline = AgenticChatPipeline(language="en")
+ pipeline.registry = _Registry()
+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: [])
+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)
+
+ await _run(pipeline, UnifiedContext(session_id="s1", user_message="Hi"))
+
+ assert all(
+ not isinstance(message.get("content"), list)
+ for message in client.calls[0]["messages"]
+ )
+
+
@pytest.mark.asyncio
async def test_finish_round_persists_provider_response_state(
monkeypatch: pytest.MonkeyPatch,