Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(mastery): deliver retrieved source images through the active agent loop (#1611)

Bingxi Zhao (Frank)2ccd8a75fb3c
diff --git a/deeptutor/agents/loop/agent_loop.py b/deeptutor/agents/loop/agent_loop.py
index 456c02883..9752f105f 100644
--- a/deeptutor/agents/loop/agent_loop.py
+++ b/deeptutor/agents/loop/agent_loop.py
@@ -46,7 +46,11 @@ from deeptutor.agents.loop.context_budget import LLMRequestSnapshot
from deeptutor.agents.loop.dsml_tool_calls import DSMLStreamFilter, extract_dsml_tool_calls
from deeptutor.core.context import UnifiedContext
from deeptutor.core.trace import build_trace_metadata, merge_trace_metadata, new_call_id
-from deeptutor.runtime.agentic.messages import assistant_message_with_tool_calls
+from deeptutor.runtime.agentic.messages import (
+ assistant_message_with_tool_calls,
+ extend_transient_model_messages,
+ with_transient_model_messages,
+)
from deeptutor.runtime.agentic.think_stream import InlineThinkFilter
from deeptutor.runtime.agentic.tool_call_stream import ToolCallAccumulator
from deeptutor.runtime.agentic.tool_dispatch import DispatchOutcome
@@ -278,6 +282,9 @@ class AgentLoop:
self._tool_schema_catalog = tool_schemas
self._last_request: LLMRequestSnapshot | None = None
self._request_tools: list[dict[str, Any]] | None = tool_schemas
+ # #1611: retrieved pixels belong to this turn's requests, never to
+ # durable history. Both loop engines share the same ordering/budget.
+ self._transient_model_messages: list[dict[str, Any]] = []
self._request_fingerprint = (context.runtime.previous_model_turn or {}).get(
"request_fingerprint"
)
@@ -704,6 +711,7 @@ class AgentLoop:
state.tool_steps += 1
state.sources.extend(dispatch.sources)
messages.extend(dispatch.tool_messages)
+ extend_transient_model_messages(self._transient_model_messages, dispatch.model_messages)
if dispatch.pause:
resumed = await self.pipeline._await_user_reply_and_resolve(
@@ -1023,7 +1031,8 @@ class AgentLoop:
defer_visible_output: bool = False,
tool_choice: str | None = None,
) -> LLMCallResult:
- await self.pipeline._guard_context_window(messages, self.stream)
+ request_messages = with_transient_model_messages(messages, self._transient_model_messages)
+ await self.pipeline._guard_context_window(request_messages, self.stream)
stage = self.stage
call_id = new_call_id(f"{self.source}-{stage}")
trace_meta = build_trace_metadata(
@@ -1050,7 +1059,7 @@ class AgentLoop:
"messages": deduplicate_user_images(
[
{key: value for key, value in message.items() if key != "_context_snapshot"}
- for message in messages
+ for message in request_messages
]
),
"stream": True,
@@ -1246,8 +1255,11 @@ class AgentLoop:
# replaced only on the wire. Otherwise the next round would
# retry images the provider has already rejected.
strip_image_parts_inplace(messages)
+ strip_image_parts_inplace(self._transient_model_messages)
else:
- for original, accepted, sent in zip(messages, kwargs["messages"], sent_content):
+ for original, accepted, sent in zip(
+ request_messages, kwargs["messages"], sent_content
+ ):
if "content" in accepted and accepted["content"] is not sent:
original["content"] = accepted["content"]
async for chunk in response_stream:
diff --git a/deeptutor/runtime/agentic/loop.py b/deeptutor/runtime/agentic/loop.py
index d79bcdec3..22e761670 100644
--- a/deeptutor/runtime/agentic/loop.py
+++ b/deeptutor/runtime/agentic/loop.py
@@ -33,6 +33,8 @@ from deeptutor.runtime.agentic.labels import LABEL_UNKNOWN, find_inline_labels
from deeptutor.runtime.agentic.messages import (
assistant_message,
assistant_message_with_tool_calls,
+ extend_transient_model_messages,
+ with_transient_model_messages,
)
from deeptutor.runtime.agentic.tool_dispatch import DispatchOutcome
from deeptutor.runtime.agentic.usage import UsageTracker
@@ -173,39 +175,6 @@ class LoopHost(Protocol):
return None
-def _with_transient_model_messages(
- messages: list[dict[str, Any]], transient: list[dict[str, Any]]
-) -> list[dict[str, Any]]:
- """Place model-only images after their tool result without mutating history."""
- if not transient:
- return messages
- anchored: dict[str, list[dict[str, Any]]] = {}
- for item in transient:
- tool_call_id = item.get("_after_tool_call_id")
- if not isinstance(tool_call_id, str) or not tool_call_id:
- continue
- anchored.setdefault(tool_call_id, []).append({"role": "user", "content": item["content"]})
- request_messages: list[dict[str, Any]] = []
- pending: list[dict[str, Any]] = []
- for index, message in enumerate(messages):
- request_messages.append(message)
- if message.get("role") == "tool":
- pending.extend(anchored.pop(str(message.get("tool_call_id") or ""), []))
- # Providers require all replies to one assistant tool-call batch
- # before another user message. Inject images after the batch.
- if index + 1 == len(messages) or messages[index + 1].get("role") != "tool":
- request_messages.extend(pending)
- pending.clear()
- return request_messages
-
-
-def _transient_image_count(message: dict[str, Any]) -> int:
- content = message.get("content")
- if not isinstance(content, list):
- return 0
- return sum(1 for part in content if isinstance(part, dict) and part.get("type") == "image_url")
-
-
async def run_agentic_loop(
*,
initial_messages: list[dict[str, Any]],
@@ -249,7 +218,6 @@ async def run_agentic_loop(
# durable conversation (where a synthetic user message would be persisted
# and displayed as if the person authored it).
transient_model_messages: list[dict[str, Any]] = []
- max_transient_images = 2
aggregated_sources: list[dict[str, Any]] = []
final_text = ""
final_label_seen = ""
@@ -259,7 +227,7 @@ async def run_agentic_loop(
for iteration in range(max_iter):
await host.guard_context_window(
- _with_transient_model_messages(messages, transient_model_messages)
+ with_transient_model_messages(messages, transient_model_messages)
)
before_iteration = getattr(host, "before_iteration", None)
if before_iteration is not None:
@@ -269,7 +237,7 @@ async def run_agentic_loop(
max_iterations=max_iter,
)
iter_meta, final_meta = host.build_iteration_trace_meta(iteration)
- request_messages = _with_transient_model_messages(messages, transient_model_messages)
+ request_messages = with_transient_model_messages(messages, transient_model_messages)
step = await run_labeled_step(
client=client,
@@ -357,12 +325,7 @@ async def run_agentic_loop(
)
aggregated_sources.extend(outcome.sources)
messages.extend(outcome.tool_messages)
- transient_model_messages.extend(outcome.model_messages)
- while (
- sum(_transient_image_count(item) for item in transient_model_messages)
- > max_transient_images
- ):
- transient_model_messages.pop(0)
+ extend_transient_model_messages(transient_model_messages, outcome.model_messages)
if outcome.pause:
resumed = await host.resolve_pause(outcome)
if not resumed:
diff --git a/deeptutor/runtime/agentic/messages.py b/deeptutor/runtime/agentic/messages.py
index 010d5357f..649a38f6b 100644
--- a/deeptutor/runtime/agentic/messages.py
+++ b/deeptutor/runtime/agentic/messages.py
@@ -76,4 +76,51 @@ def assistant_message(
return message
-__all__ = ["assistant_message", "assistant_message_with_tool_calls"]
+def with_transient_model_messages(
+ messages: list[dict[str, Any]], transient: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+ """Place model-only images after their tool result without mutating history."""
+ if not transient:
+ return messages
+ anchored: dict[str, list[dict[str, Any]]] = {}
+ for item in transient:
+ tool_call_id = item.get("_after_tool_call_id")
+ if not isinstance(tool_call_id, str) or not tool_call_id:
+ continue
+ anchored.setdefault(tool_call_id, []).append({"role": "user", "content": item["content"]})
+ request_messages: list[dict[str, Any]] = []
+ pending: list[dict[str, Any]] = []
+ for index, message in enumerate(messages):
+ request_messages.append(message)
+ if message.get("role") == "tool":
+ pending.extend(anchored.pop(str(message.get("tool_call_id") or ""), []))
+ # Providers require all replies to one assistant tool-call batch
+ # before another user message. Inject images after the batch.
+ if index + 1 == len(messages) or messages[index + 1].get("role") != "tool":
+ request_messages.extend(pending)
+ pending.clear()
+ return request_messages
+
+
+def _transient_image_count(message: dict[str, Any]) -> int:
+ content = message.get("content")
+ if not isinstance(content, list):
+ return 0
+ return sum(1 for part in content if isinstance(part, dict) and part.get("type") == "image_url")
+
+
+def extend_transient_model_messages(
+ transient: list[dict[str, Any]], incoming: list[dict[str, Any]], *, max_images: int = 2
+) -> None:
+ """Keep the latest bounded source evidence across tool rounds."""
+ transient.extend(incoming)
+ while sum(_transient_image_count(item) for item in transient) > max_images:
+ transient.pop(0)
+
+
+__all__ = [
+ "assistant_message",
+ "assistant_message_with_tool_calls",
+ "extend_transient_model_messages",
+ "with_transient_model_messages",
+]
diff --git a/tests/agents/chat/test_agent_loop.py b/tests/agents/chat/test_agent_loop.py
index 6b1fbf1a7..97b154e12 100644
--- a/tests/agents/chat/test_agent_loop.py
+++ b/tests/agents/chat/test_agent_loop.py
@@ -205,6 +205,201 @@ async def _run(pipeline: AgenticChatPipeline, context: UnifiedContext):
return events
[email protected]("evidence", ["available", "text_only", "missing", "inaccessible"])
+async def test_active_mastery_retrieves_verified_pixels_without_persisting_them(
+ monkeypatch, tmp_path, evidence
+):
+ """#1611: exercise the active Mastery loop, real RAG tool and verified store."""
+ import base64
+
+ from deeptutor.multi_user import knowledge_access
+ from deeptutor.services.rag.visual_assets import VisualAssetStore, collect_visual_assets
+ from deeptutor.tools import rag_tool
+ from deeptutor.tools.builtin import RAGTool
+ from tests.services.rag.test_visual_assets import _fixture
+
+ kb_dir, source, image, parsed = _fixture(tmp_path)
+ candidate = collect_visual_assets(parsed, source, kb_dir)[0]
+ store = VisualAssetStore(kb_dir)
+ store.publish([candidate])
+ if evidence == "missing":
+ (store.root / (candidate.record["asset_id"] + ".png")).unlink()
+ monkeypatch.setattr(
+ knowledge_access,
+ "resolve_for_rag",
+ lambda _name: (
+ None if evidence == "inaccessible" else SimpleNamespace(name="kb", base_dir=tmp_path)
+ ),
+ )
+
+ async def search(**_kwargs):
+ return {
+ "content": "The source contains a learning curve.",
+ "sources": [{"visual_asset_id": candidate.record["asset_id"]}],
+ }
+
+ monkeypatch.setattr(rag_tool, "rag_search", search)
+
+ class Registry(_Registry):
+ def build_openai_schemas(self, _enabled):
+ schemas = super().build_openai_schemas(_enabled)
+ return [
+ *schemas,
+ {
+ "type": "function",
+ "function": {
+ "name": "rag",
+ "description": "Search selected sources",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "query": {"type": "string"},
+ "kb_name": {"type": "string"},
+ },
+ "required": ["query", "kb_name"],
+ },
+ },
+ },
+ ]
+
+ async def execute(self, name, **kwargs):
+ if name == "rag":
+ return await RAGTool().execute(**kwargs)
+ return await super().execute(name, **kwargs)
+
+ client = _ScriptedChatClient(
+ [
+ [
+ _llm_chunk(
+ tool_calls=[
+ {
+ "id": "figure",
+ "name": "rag",
+ "arguments": '{"query":"curve","kb_name":"kb"}',
+ },
+ {"id": "other", "name": "web_search", "arguments": '{"query":"context"}'},
+ ],
+ finish_reason="tool_calls",
+ )
+ ],
+ [_llm_chunk(content="Source-backed explanation.", finish_reason="stop")],
+ ]
+ )
+ pipeline = MasteryLoopPipeline(language="en")
+ pipeline.model = "gpt-3.5-turbo" if evidence == "text_only" else "gpt-4o"
+ pipeline.registry = Registry()
+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _: ["rag", "web_search"])
+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)
+ monkeypatch.setattr(pipeline, "_capability_finish_instruction", lambda *_: None)
+ context = UnifiedContext(session_id="s", user_message="Explain the original figure.")
+ events = await _run(pipeline, context)
+
+ request = client.calls[1]["messages"]
+ images = [
+ part
+ for msg in request
+ if isinstance(msg.get("content"), list)
+ for part in msg["content"]
+ if part.get("type") == "image_url"
+ ]
+ if evidence == "available":
+ assert len(images) == 1
+ assert (
+ base64.b64decode(images[0]["image_url"]["url"].split(",", 1)[1]) == image.read_bytes()
+ )
+ tool_positions = [i for i, msg in enumerate(request) if msg.get("role") == "tool"]
+ image_position = next(
+ i
+ for i, msg in enumerate(request)
+ if msg.get("content") and isinstance(msg["content"], list)
+ )
+ assert image_position > max(tool_positions)
+ assert candidate.record["asset_id"] in str(request[image_position])
+ else:
+ assert images == []
+ tool_text = next(msg["content"] for msg in request if msg.get("tool_call_id") == "figure")
+ assert (
+ "cannot inspect source image pixels"
+ if evidence == "text_only"
+ else "unavailable for pixel inspection"
+ ) in tool_text
+ assert "base64" not in json.dumps(context.runtime.model_turn)
+ assert "data:image" not in str(events)
+
+
+async def test_active_loop_bounds_source_images_across_rounds_and_retains_fallback(monkeypatch):
+ """Private source evidence is bounded and rejected pixels are not retried each round."""
+ from copy import deepcopy
+
+ from deeptutor.services.llm.multimodal import has_image_parts
+
+ class Registry(_Registry):
+ async def execute(self, name, **kwargs):
+ result = await super().execute(name, **kwargs)
+ count = len(self.executed)
+ result.model_message = {
+ "role": "user",
+ "content": [
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{count}{i}"}}
+ for i in range(2)
+ ],
+ }
+ return result
+
+ client = _ScriptedChatClient(
+ [
+ [
+ _llm_chunk(
+ tool_calls=[
+ {"id": "one", "name": "web_search", "arguments": '{"query":"one"}'}
+ ],
+ finish_reason="tool_calls",
+ )
+ ],
+ [
+ _llm_chunk(
+ tool_calls=[
+ {"id": "two", "name": "web_search", "arguments": '{"query":"two"}'}
+ ],
+ finish_reason="tool_calls",
+ )
+ ],
+ [_llm_chunk(content="Continue.", finish_reason="length")],
+ [_llm_chunk(content="Done.", finish_reason="stop")],
+ ]
+ )
+ create = client.chat.completions.create
+ attempts = []
+
+ async def reject_latest_images(**kwargs):
+ attempts.append(deepcopy(kwargs["messages"]))
+ if client.call_count == 2 and has_image_parts(kwargs["messages"]):
+ raise RuntimeError("image input is not supported")
+ return await create(**kwargs)
+
+ client.chat.completions.create = reject_latest_images
+ pipeline = AgenticChatPipeline(language="en")
+ pipeline.model = "gpt-3.5-turbo"
+ pipeline.registry = Registry()
+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _: ["web_search"])
+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)
+ context = UnifiedContext(session_id="s", user_message="Explain source figures.")
+ await _run(pipeline, context)
+ images = [
+ part["image_url"]["url"]
+ for msg in attempts[2]
+ if isinstance(msg.get("content"), list)
+ for part in msg["content"]
+ if part.get("type") == "image_url"
+ ]
+ assert images == ["data:image/png;base64,20", "data:image/png;base64,21"]
+ assert not has_image_parts(attempts[3])
+ assert not has_image_parts(attempts[4])
+ assert "base64" not in json.dumps(context.runtime.model_turn)
+
+
@pytest.mark.asyncio
async def test_rejected_forced_choice_keeps_schemas_for_all_later_rounds(monkeypatch):
client = _ScriptedChatClient(
diff --git a/tests/services/rag/test_visual_assets.py b/tests/services/rag/test_visual_assets.py
index 7e19084f1..cc8808c2a 100644
--- a/tests/services/rag/test_visual_assets.py
+++ b/tests/services/rag/test_visual_assets.py
@@ -651,7 +651,7 @@ def test_visual_message_reaches_next_model_request_without_stream_leak(tmp_path:
def test_visual_messages_follow_complete_tool_reply_batch():
- from deeptutor.runtime.agentic.loop import _with_transient_model_messages
+ from deeptutor.runtime.agentic.messages import with_transient_model_messages
messages = [
{"role": "assistant", "tool_calls": [{"id": "rag"}, {"id": "other"}]},
@@ -665,7 +665,7 @@ def test_visual_messages_follow_complete_tool_reply_batch():
"_after_tool_call_id": "rag",
}
]
- request = _with_transient_model_messages(messages, transient)
+ request = with_transient_model_messages(messages, transient)
assert [item["role"] for item in request] == ["assistant", "tool", "tool", "user"]
assert request[-1]["content"] == transient[0]["content"]
assert messages[-1]["role"] == "tool" # canonical history was not changed