DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix: apply prompt caching in agent loop _call_llm
nandanadileep6299f773c203
diff --git a/deeptutor/agents/loop/agent_loop.py b/deeptutor/agents/loop/agent_loop.pyindex 854b10a75..68cc81ea5 100644--- a/deeptutor/agents/loop/agent_loop.py+++ b/deeptutor/agents/loop/agent_loop.py@@ -1105,6 +1105,53 @@ class AgentLoop:# ---- LLM call ----------------------------------------------------------+ def _apply_prompt_caching(self, kwargs: dict[str, Any]) -> None:+ """Attach Anthropic-style ``cache_control`` breakpoints to the request.++ The loop builds its own ``chat.completions.create`` kwargs and calls the+ SDK client directly, so it never passes through+ ``OpenAICompatProvider._build_kwargs`` — the only other place cache+ breakpoints are added. Without this, an ``anthropic/`` model on a+ prompt-caching gateway (OpenRouter) sends no breakpoint and Anthropic+ caches nothing. Reuse the provider's helper so both paths mark requests+ identically, and add the end-of-stable-history breakpoint the loop+ needs: its transient per-turn context message sits just before the+ learner's message and is never persisted, so the next turn's prefix+ only matches up to the last durable message.+ """+ from deeptutor.services.llm.provider_core.openai_compat_provider import (+ OpenAICompatProvider,+ )++ model = str(kwargs.get("model") or "")+ if not OpenAICompatProvider.supports_prompt_caching_for(model):+ return+ if not self._binding_supports_prompt_caching():+ return+ messages, tools = OpenAICompatProvider._apply_cache_control(+ kwargs["messages"],+ kwargs.get("tools"),+ stable_history_end=True,+ )+ kwargs["messages"] = messages+ if tools is not None:+ kwargs["tools"] = tools++ def _binding_supports_prompt_caching(self) -> bool:+ """Whether this binding reaches the model through OpenAICompatProvider.++ Native Anthropic-backed bindings apply their own ``cache_control``+ inside their provider, so the loop must only add markers for the+ OpenAI-compatible path (gateways such as OpenRouter) that otherwise+ never sees them.+ """+ from deeptutor.services.provider_registry import find_by_name++ spec = find_by_name(self.pipeline.binding)+ if spec is None or not spec.supports_prompt_caching:+ return False+ return spec.backend == "openai_compat"+async def _call_llm(self,*,@@ -1223,6 +1270,8 @@ class AgentLoop:messages=list(kwargs["messages"]), tool_schemas=carried)+ self._apply_prompt_caching(kwargs)+chunk_meta = merge_trace_metadata(trace_meta, {"trace_kind": "llm_chunk"})for attempt in range(len(_PROVIDER_RETRY_DELAYS) + 1):diff --git a/deeptutor/services/llm/provider_core/openai_compat_provider.py b/deeptutor/services/llm/provider_core/openai_compat_provider.pyindex 5d6e88533..60c1a1bc1 100644--- a/deeptutor/services/llm/provider_core/openai_compat_provider.py+++ b/deeptutor/services/llm/provider_core/openai_compat_provider.py@@ -269,30 +269,38 @@ class OpenAICompatProvider(LLMProvider):# Prompt caching# ------------------------------------------------------------------+ @staticmethod+ def _mark_cache_control(msg: dict[str, Any]) -> dict[str, Any]:+ """Attach an ephemeral cache breakpoint to a message's last block."""+ cache_marker = {"type": "ephemeral"}+ content = msg.get("content")+ if isinstance(content, str):+ return {+ **msg,+ "content": [+ {"type": "text", "text": content, "cache_control": cache_marker},+ ],+ }+ if isinstance(content, list) and content:+ new_content = list(content)+ new_content[-1] = {**new_content[-1], "cache_control": cache_marker}+ return {**msg, "content": new_content}+ return msg+@classmethoddef _apply_cache_control(cls,messages: list[dict[str, Any]],tools: list[dict[str, Any]] | None,+ *,+ stable_history_end: bool = False,) -> tuple[list[dict[str, Any]], list[dict[str, Any]] | None]:cache_marker = {"type": "ephemeral"}new_messages = list(messages)budget = 4def _mark(msg: dict[str, Any]) -> dict[str, Any]:- content = msg.get("content")- if isinstance(content, str):- return {- **msg,- "content": [- {"type": "text", "text": content, "cache_control": cache_marker},- ],- }- if isinstance(content, list) and content:- nc = list(content)- nc[-1] = {**nc[-1], "cache_control": cache_marker}- return {**msg, "content": nc}- return msg+ return cls._mark_cache_control(msg)if new_messages and new_messages[0].get("role") == "system":new_messages[0] = _mark(new_messages[0])@@ -301,6 +309,18 @@ class OpenAICompatProvider(LLMProvider):new_messages[-1] = _mark(new_messages[-1])budget -= 1+ if stable_history_end and budget > 0:+ # The loop appends a transient per-turn context message just before+ # the learner's message and never persists it, so a breakpoint on+ # the last message alone leaves the next turn's prefix diverging at+ # that transient block. Mark the end of the durable history — the+ # last non-empty message before the trailing user turn — so the+ # previous turn's prefix still matches and can be reused.+ index = cls._stable_history_end_index(new_messages)+ if index is not None:+ new_messages[index] = _mark(new_messages[index])+ budget -= 1+new_tools = toolsif tools:new_tools = list(tools)@@ -308,6 +328,39 @@ class OpenAICompatProvider(LLMProvider):new_tools[idx] = {**new_tools[idx], "cache_control": cache_marker}return new_messages, new_tools+ @staticmethod+ def _stable_history_end_index(messages: list[dict[str, Any]]) -> int | None:+ """Index of the last persisted message before the trailing user turn.++ Walks back over the final user block (the transient context message and+ the learner's message share the ``user`` role) and skips empty text+ blocks, which cannot carry ``cache_control``. Returns ``None`` when the+ marker would fall outside the request.+ """+ start = len(messages) - 1+ while start > 0 and messages[start].get("role") != "user":+ start -= 1+ while start > 1 and messages[start - 1].get("role") == "user":+ start -= 1+ index = start - 1+ while index > 0 and not messages[index].get("content"):+ index -= 1+ if 0 < index < len(messages) - 1:+ return index+ return None++ @classmethod+ def supports_prompt_caching_for(cls, model: str | None) -> bool:+ """Whether *model* uses explicit ``cache_control`` breakpoints.++ Anthropic models reached through an OpenAI-compatible gateway cache+ only when the request carries breakpoints, and only those families+ understand the marker. Kept here so the agent loop and the provider+ agree on the same gate.+ """+ name = str(model or "").lower()+ return name.startswith(("anthropic/", "claude"))+# ------------------------------------------------------------------# Message sanitization# ------------------------------------------------------------------diff --git a/tests/agents/chat/test_agent_loop.py b/tests/agents/chat/test_agent_loop.pyindex 248895570..c53d860c0 100644--- a/tests/agents/chat/test_agent_loop.py+++ b/tests/agents/chat/test_agent_loop.py@@ -1035,6 +1035,140 @@ async def test_finish_first_round_no_tools(monkeypatch: pytest.MonkeyPatch) -> Nassert result.metadata["tool_steps"] == 0+async def test_anthropic_gateway_request_carries_cache_control_breakpoints(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The loop must apply the provider's prompt-caching treatment (#2078).++ The loop builds its own request kwargs, so an ``anthropic/`` model on a+ prompt-caching gateway used to send no ``cache_control`` breakpoint at all.+ It now marks system, the last message, and the end of the durable history.+ """+ monkeypatch.setattr(+ "deeptutor.agents.loop.pipeline.get_llm_config",+ lambda: SimpleNamespace(+ binding="openrouter",+ model="anthropic/claude-haiku-5.5",+ api_key="k",+ base_url="https://openrouter.ai/api/v1",+ api_version=None,+ extra_headers={},+ reasoning_effort=None,+ ),+ )+ client = _ScriptedChatClient([[_llm_chunk(content="Answer", finish_reason="stop")]])+ pipeline = AgenticChatPipeline(language="en")+ pipeline.registry = _Registry()+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: [])+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)++ durable = [+ {"role": "user", "content": "Question one"},+ {"role": "assistant", "content": "Answer one"},+ ]+ await _run(+ pipeline,+ UnifiedContext(+ session_id="s1",+ user_message="Question two",+ runtime=TurnRuntimeContext(model_history=list(durable)),+ ),+ )++ sent = client.calls[0]["messages"]+ marked = [+ index+ for index, message in enumerate(sent)+ if isinstance(message.get("content"), list)+ and any(+ isinstance(block, dict) and "cache_control" in block+ for block in message["content"]+ )+ ]+ # system + end-of-stable-history (the durable assistant turn) + last message.+ assert len(marked) == 3+ assert sent[0]["role"] == "system"+ # The middle breakpoint sits on the last durable assistant turn, before the+ # trailing (possibly multi-message) user block of this turn.+ assert sent[marked[1]]["role"] == "assistant"+ assert sent[marked[1]]["content"][-1]["text"] == "Answer one"+ assert marked[-1] == len(sent) - 1+ assert sent[-1]["role"] == "user"+ assert all(+ block.get("cache_control") == {"type": "ephemeral"}+ for index in marked+ for block in sent[index]["content"]+ if isinstance(block, dict) and "cache_control" in block+ )+++async def test_non_anthropic_gateway_request_gets_no_cache_control(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A non-Anthropic model on the same gateway must not receive markers."""+ monkeypatch.setattr(+ "deeptutor.agents.loop.pipeline.get_llm_config",+ lambda: SimpleNamespace(+ binding="openrouter",+ model="google/gemini-2.0-flash",+ api_key="k",+ base_url="https://openrouter.ai/api/v1",+ api_version=None,+ extra_headers={},+ reasoning_effort=None,+ ),+ )+ client = _ScriptedChatClient([[_llm_chunk(content="Answer", finish_reason="stop")]])+ pipeline = AgenticChatPipeline(language="en")+ pipeline.registry = _Registry()+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: [])+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)++ await _run(pipeline, UnifiedContext(session_id="s1", user_message="Hi"))++ assert all(+ not isinstance(message.get("content"), list)+ for message in client.calls[0]["messages"]+ )+++async def test_native_anthropic_binding_leaves_markers_to_the_provider(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The native Anthropic adapter applies its own cache_control.++ The loop must not pre-mark that path, or the adapter's own breakpoints+ would be added on top of the loop's.+ """+ monkeypatch.setattr(+ "deeptutor.agents.loop.pipeline.get_llm_config",+ lambda: SimpleNamespace(+ binding="anthropic",+ model="claude-sonnet-4-5",+ api_key="k",+ base_url="https://api.anthropic.com",+ api_version=None,+ extra_headers={},+ reasoning_effort=None,+ ),+ )+ client = _ScriptedChatClient([[_llm_chunk(content="Answer", finish_reason="stop")]])+ pipeline = AgenticChatPipeline(language="en")+ pipeline.registry = _Registry()+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: [])+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)++ await _run(pipeline, UnifiedContext(session_id="s1", user_message="Hi"))++ assert all(+ not isinstance(message.get("content"), list)+ for message in client.calls[0]["messages"]+ )++@pytest.mark.asyncioasync def test_finish_round_persists_provider_response_state(monkeypatch: pytest.MonkeyPatch,