DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(mastery): deliver retrieved source images through the active agent loop (#1611)
Bingxi Zhao (Frank)2ccd8a75fb3c
diff --git a/deeptutor/agents/loop/agent_loop.py b/deeptutor/agents/loop/agent_loop.pyindex 456c02883..9752f105f 100644--- a/deeptutor/agents/loop/agent_loop.py+++ b/deeptutor/agents/loop/agent_loop.py@@ -46,7 +46,11 @@ from deeptutor.agents.loop.context_budget import LLMRequestSnapshotfrom deeptutor.agents.loop.dsml_tool_calls import DSMLStreamFilter, extract_dsml_tool_callsfrom deeptutor.core.context import UnifiedContextfrom deeptutor.core.trace import build_trace_metadata, merge_trace_metadata, new_call_id-from deeptutor.runtime.agentic.messages import assistant_message_with_tool_calls+from deeptutor.runtime.agentic.messages import (+ assistant_message_with_tool_calls,+ extend_transient_model_messages,+ with_transient_model_messages,+)from deeptutor.runtime.agentic.think_stream import InlineThinkFilterfrom deeptutor.runtime.agentic.tool_call_stream import ToolCallAccumulatorfrom deeptutor.runtime.agentic.tool_dispatch import DispatchOutcome@@ -278,6 +282,9 @@ class AgentLoop:self._tool_schema_catalog = tool_schemasself._last_request: LLMRequestSnapshot | None = Noneself._request_tools: list[dict[str, Any]] | None = tool_schemas+ # #1611: retrieved pixels belong to this turn's requests, never to+ # durable history. Both loop engines share the same ordering/budget.+ self._transient_model_messages: list[dict[str, Any]] = []self._request_fingerprint = (context.runtime.previous_model_turn or {}).get("request_fingerprint")@@ -704,6 +711,7 @@ class AgentLoop:state.tool_steps += 1state.sources.extend(dispatch.sources)messages.extend(dispatch.tool_messages)+ extend_transient_model_messages(self._transient_model_messages, dispatch.model_messages)if dispatch.pause:resumed = await self.pipeline._await_user_reply_and_resolve(@@ -1023,7 +1031,8 @@ class AgentLoop:defer_visible_output: bool = False,tool_choice: str | None = None,) -> LLMCallResult:- await self.pipeline._guard_context_window(messages, self.stream)+ request_messages = with_transient_model_messages(messages, self._transient_model_messages)+ await self.pipeline._guard_context_window(request_messages, self.stream)stage = self.stagecall_id = new_call_id(f"{self.source}-{stage}")trace_meta = build_trace_metadata(@@ -1050,7 +1059,7 @@ class AgentLoop:"messages": deduplicate_user_images([{key: value for key, value in message.items() if key != "_context_snapshot"}- for message in messages+ for message in request_messages]),"stream": True,@@ -1246,8 +1255,11 @@ class AgentLoop:# replaced only on the wire. Otherwise the next round would# retry images the provider has already rejected.strip_image_parts_inplace(messages)+ strip_image_parts_inplace(self._transient_model_messages)else:- for original, accepted, sent in zip(messages, kwargs["messages"], sent_content):+ for original, accepted, sent in zip(+ request_messages, kwargs["messages"], sent_content+ ):if "content" in accepted and accepted["content"] is not sent:original["content"] = accepted["content"]async for chunk in response_stream:diff --git a/deeptutor/runtime/agentic/loop.py b/deeptutor/runtime/agentic/loop.pyindex d79bcdec3..22e761670 100644--- a/deeptutor/runtime/agentic/loop.py+++ b/deeptutor/runtime/agentic/loop.py@@ -33,6 +33,8 @@ from deeptutor.runtime.agentic.labels import LABEL_UNKNOWN, find_inline_labelsfrom deeptutor.runtime.agentic.messages import (assistant_message,assistant_message_with_tool_calls,+ extend_transient_model_messages,+ with_transient_model_messages,)from deeptutor.runtime.agentic.tool_dispatch import DispatchOutcomefrom deeptutor.runtime.agentic.usage import UsageTracker@@ -173,39 +175,6 @@ class LoopHost(Protocol):return None-def _with_transient_model_messages(- messages: list[dict[str, Any]], transient: list[dict[str, Any]]-) -> list[dict[str, Any]]:- """Place model-only images after their tool result without mutating history."""- if not transient:- return messages- anchored: dict[str, list[dict[str, Any]]] = {}- for item in transient:- tool_call_id = item.get("_after_tool_call_id")- if not isinstance(tool_call_id, str) or not tool_call_id:- continue- anchored.setdefault(tool_call_id, []).append({"role": "user", "content": item["content"]})- request_messages: list[dict[str, Any]] = []- pending: list[dict[str, Any]] = []- for index, message in enumerate(messages):- request_messages.append(message)- if message.get("role") == "tool":- pending.extend(anchored.pop(str(message.get("tool_call_id") or ""), []))- # Providers require all replies to one assistant tool-call batch- # before another user message. Inject images after the batch.- if index + 1 == len(messages) or messages[index + 1].get("role") != "tool":- request_messages.extend(pending)- pending.clear()- return request_messages---def _transient_image_count(message: dict[str, Any]) -> int:- content = message.get("content")- if not isinstance(content, list):- return 0- return sum(1 for part in content if isinstance(part, dict) and part.get("type") == "image_url")--async def run_agentic_loop(*,initial_messages: list[dict[str, Any]],@@ -249,7 +218,6 @@ async def run_agentic_loop(# durable conversation (where a synthetic user message would be persisted# and displayed as if the person authored it).transient_model_messages: list[dict[str, Any]] = []- max_transient_images = 2aggregated_sources: list[dict[str, Any]] = []final_text = ""final_label_seen = ""@@ -259,7 +227,7 @@ async def run_agentic_loop(for iteration in range(max_iter):await host.guard_context_window(- _with_transient_model_messages(messages, transient_model_messages)+ with_transient_model_messages(messages, transient_model_messages))before_iteration = getattr(host, "before_iteration", None)if before_iteration is not None:@@ -269,7 +237,7 @@ async def run_agentic_loop(max_iterations=max_iter,)iter_meta, final_meta = host.build_iteration_trace_meta(iteration)- request_messages = _with_transient_model_messages(messages, transient_model_messages)+ request_messages = with_transient_model_messages(messages, transient_model_messages)step = await run_labeled_step(client=client,@@ -357,12 +325,7 @@ async def run_agentic_loop()aggregated_sources.extend(outcome.sources)messages.extend(outcome.tool_messages)- transient_model_messages.extend(outcome.model_messages)- while (- sum(_transient_image_count(item) for item in transient_model_messages)- > max_transient_images- ):- transient_model_messages.pop(0)+ extend_transient_model_messages(transient_model_messages, outcome.model_messages)if outcome.pause:resumed = await host.resolve_pause(outcome)if not resumed:diff --git a/deeptutor/runtime/agentic/messages.py b/deeptutor/runtime/agentic/messages.pyindex 010d5357f..649a38f6b 100644--- a/deeptutor/runtime/agentic/messages.py+++ b/deeptutor/runtime/agentic/messages.py@@ -76,4 +76,51 @@ def assistant_message(return message-__all__ = ["assistant_message", "assistant_message_with_tool_calls"]+def with_transient_model_messages(+ messages: list[dict[str, Any]], transient: list[dict[str, Any]]+) -> list[dict[str, Any]]:+ """Place model-only images after their tool result without mutating history."""+ if not transient:+ return messages+ anchored: dict[str, list[dict[str, Any]]] = {}+ for item in transient:+ tool_call_id = item.get("_after_tool_call_id")+ if not isinstance(tool_call_id, str) or not tool_call_id:+ continue+ anchored.setdefault(tool_call_id, []).append({"role": "user", "content": item["content"]})+ request_messages: list[dict[str, Any]] = []+ pending: list[dict[str, Any]] = []+ for index, message in enumerate(messages):+ request_messages.append(message)+ if message.get("role") == "tool":+ pending.extend(anchored.pop(str(message.get("tool_call_id") or ""), []))+ # Providers require all replies to one assistant tool-call batch+ # before another user message. Inject images after the batch.+ if index + 1 == len(messages) or messages[index + 1].get("role") != "tool":+ request_messages.extend(pending)+ pending.clear()+ return request_messages+++def _transient_image_count(message: dict[str, Any]) -> int:+ content = message.get("content")+ if not isinstance(content, list):+ return 0+ return sum(1 for part in content if isinstance(part, dict) and part.get("type") == "image_url")+++def extend_transient_model_messages(+ transient: list[dict[str, Any]], incoming: list[dict[str, Any]], *, max_images: int = 2+) -> None:+ """Keep the latest bounded source evidence across tool rounds."""+ transient.extend(incoming)+ while sum(_transient_image_count(item) for item in transient) > max_images:+ transient.pop(0)+++__all__ = [+ "assistant_message",+ "assistant_message_with_tool_calls",+ "extend_transient_model_messages",+ "with_transient_model_messages",+]diff --git a/tests/agents/chat/test_agent_loop.py b/tests/agents/chat/test_agent_loop.pyindex 6b1fbf1a7..97b154e12 100644--- a/tests/agents/chat/test_agent_loop.py+++ b/tests/agents/chat/test_agent_loop.py@@ -205,6 +205,201 @@ async def _run(pipeline: AgenticChatPipeline, context: UnifiedContext):return events[email protected]("evidence", ["available", "text_only", "missing", "inaccessible"])+async def test_active_mastery_retrieves_verified_pixels_without_persisting_them(+ monkeypatch, tmp_path, evidence+):+ """#1611: exercise the active Mastery loop, real RAG tool and verified store."""+ import base64++ from deeptutor.multi_user import knowledge_access+ from deeptutor.services.rag.visual_assets import VisualAssetStore, collect_visual_assets+ from deeptutor.tools import rag_tool+ from deeptutor.tools.builtin import RAGTool+ from tests.services.rag.test_visual_assets import _fixture++ kb_dir, source, image, parsed = _fixture(tmp_path)+ candidate = collect_visual_assets(parsed, source, kb_dir)[0]+ store = VisualAssetStore(kb_dir)+ store.publish([candidate])+ if evidence == "missing":+ (store.root / (candidate.record["asset_id"] + ".png")).unlink()+ monkeypatch.setattr(+ knowledge_access,+ "resolve_for_rag",+ lambda _name: (+ None if evidence == "inaccessible" else SimpleNamespace(name="kb", base_dir=tmp_path)+ ),+ )++ async def search(**_kwargs):+ return {+ "content": "The source contains a learning curve.",+ "sources": [{"visual_asset_id": candidate.record["asset_id"]}],+ }++ monkeypatch.setattr(rag_tool, "rag_search", search)++ class Registry(_Registry):+ def build_openai_schemas(self, _enabled):+ schemas = super().build_openai_schemas(_enabled)+ return [+ *schemas,+ {+ "type": "function",+ "function": {+ "name": "rag",+ "description": "Search selected sources",+ "parameters": {+ "type": "object",+ "properties": {+ "query": {"type": "string"},+ "kb_name": {"type": "string"},+ },+ "required": ["query", "kb_name"],+ },+ },+ },+ ]++ async def execute(self, name, **kwargs):+ if name == "rag":+ return await RAGTool().execute(**kwargs)+ return await super().execute(name, **kwargs)++ client = _ScriptedChatClient(+ [+ [+ _llm_chunk(+ tool_calls=[+ {+ "id": "figure",+ "name": "rag",+ "arguments": '{"query":"curve","kb_name":"kb"}',+ },+ {"id": "other", "name": "web_search", "arguments": '{"query":"context"}'},+ ],+ finish_reason="tool_calls",+ )+ ],+ [_llm_chunk(content="Source-backed explanation.", finish_reason="stop")],+ ]+ )+ pipeline = MasteryLoopPipeline(language="en")+ pipeline.model = "gpt-3.5-turbo" if evidence == "text_only" else "gpt-4o"+ pipeline.registry = Registry()+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _: ["rag", "web_search"])+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)+ monkeypatch.setattr(pipeline, "_capability_finish_instruction", lambda *_: None)+ context = UnifiedContext(session_id="s", user_message="Explain the original figure.")+ events = await _run(pipeline, context)++ request = client.calls[1]["messages"]+ images = [+ part+ for msg in request+ if isinstance(msg.get("content"), list)+ for part in msg["content"]+ if part.get("type") == "image_url"+ ]+ if evidence == "available":+ assert len(images) == 1+ assert (+ base64.b64decode(images[0]["image_url"]["url"].split(",", 1)[1]) == image.read_bytes()+ )+ tool_positions = [i for i, msg in enumerate(request) if msg.get("role") == "tool"]+ image_position = next(+ i+ for i, msg in enumerate(request)+ if msg.get("content") and isinstance(msg["content"], list)+ )+ assert image_position > max(tool_positions)+ assert candidate.record["asset_id"] in str(request[image_position])+ else:+ assert images == []+ tool_text = next(msg["content"] for msg in request if msg.get("tool_call_id") == "figure")+ assert (+ "cannot inspect source image pixels"+ if evidence == "text_only"+ else "unavailable for pixel inspection"+ ) in tool_text+ assert "base64" not in json.dumps(context.runtime.model_turn)+ assert "data:image" not in str(events)+++async def test_active_loop_bounds_source_images_across_rounds_and_retains_fallback(monkeypatch):+ """Private source evidence is bounded and rejected pixels are not retried each round."""+ from copy import deepcopy++ from deeptutor.services.llm.multimodal import has_image_parts++ class Registry(_Registry):+ async def execute(self, name, **kwargs):+ result = await super().execute(name, **kwargs)+ count = len(self.executed)+ result.model_message = {+ "role": "user",+ "content": [+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{count}{i}"}}+ for i in range(2)+ ],+ }+ return result++ client = _ScriptedChatClient(+ [+ [+ _llm_chunk(+ tool_calls=[+ {"id": "one", "name": "web_search", "arguments": '{"query":"one"}'}+ ],+ finish_reason="tool_calls",+ )+ ],+ [+ _llm_chunk(+ tool_calls=[+ {"id": "two", "name": "web_search", "arguments": '{"query":"two"}'}+ ],+ finish_reason="tool_calls",+ )+ ],+ [_llm_chunk(content="Continue.", finish_reason="length")],+ [_llm_chunk(content="Done.", finish_reason="stop")],+ ]+ )+ create = client.chat.completions.create+ attempts = []++ async def reject_latest_images(**kwargs):+ attempts.append(deepcopy(kwargs["messages"]))+ if client.call_count == 2 and has_image_parts(kwargs["messages"]):+ raise RuntimeError("image input is not supported")+ return await create(**kwargs)++ client.chat.completions.create = reject_latest_images+ pipeline = AgenticChatPipeline(language="en")+ pipeline.model = "gpt-3.5-turbo"+ pipeline.registry = Registry()+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _: ["web_search"])+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)+ context = UnifiedContext(session_id="s", user_message="Explain source figures.")+ await _run(pipeline, context)+ images = [+ part["image_url"]["url"]+ for msg in attempts[2]+ if isinstance(msg.get("content"), list)+ for part in msg["content"]+ if part.get("type") == "image_url"+ ]+ assert images == ["data:image/png;base64,20", "data:image/png;base64,21"]+ assert not has_image_parts(attempts[3])+ assert not has_image_parts(attempts[4])+ assert "base64" not in json.dumps(context.runtime.model_turn)++@pytest.mark.asyncioasync def test_rejected_forced_choice_keeps_schemas_for_all_later_rounds(monkeypatch):client = _ScriptedChatClient(diff --git a/tests/services/rag/test_visual_assets.py b/tests/services/rag/test_visual_assets.pyindex 7e19084f1..cc8808c2a 100644--- a/tests/services/rag/test_visual_assets.py+++ b/tests/services/rag/test_visual_assets.py@@ -651,7 +651,7 @@ def test_visual_message_reaches_next_model_request_without_stream_leak(tmp_path:def test_visual_messages_follow_complete_tool_reply_batch():- from deeptutor.runtime.agentic.loop import _with_transient_model_messages+ from deeptutor.runtime.agentic.messages import with_transient_model_messagesmessages = [{"role": "assistant", "tool_calls": [{"id": "rag"}, {"id": "other"}]},@@ -665,7 +665,7 @@ def test_visual_messages_follow_complete_tool_reply_batch():"_after_tool_call_id": "rag",}]- request = _with_transient_model_messages(messages, transient)+ request = with_transient_model_messages(messages, transient)assert [item["role"] for item in request] == ["assistant", "tool", "tool", "user"]assert request[-1]["content"] == transient[0]["content"]assert messages[-1]["role"] == "tool" # canonical history was not changed