DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(rag): keep visual tool replies valid and pin asset scope
Bingxi Zhao (Frank)0bb753868ea9
diff --git a/deeptutor/runtime/agentic/loop.py b/deeptutor/runtime/agentic/loop.pyindex 49d2a77f8..9255f84e6 100644--- a/deeptutor/runtime/agentic/loop.py+++ b/deeptutor/runtime/agentic/loop.py@@ -186,10 +186,16 @@ def _with_transient_model_messages(continueanchored.setdefault(tool_call_id, []).append({"role": "user", "content": item["content"]})request_messages: list[dict[str, Any]] = []- for message in messages:+ pending: list[dict[str, Any]] = []+ for index, message in enumerate(messages):request_messages.append(message)if message.get("role") == "tool":- request_messages.extend(anchored.pop(str(message.get("tool_call_id") or ""), []))+ pending.extend(anchored.pop(str(message.get("tool_call_id") or ""), []))+ # Providers require all replies to one assistant tool-call batch+ # before another user message. Inject images after the batch.+ if index + 1 == len(messages) or messages[index + 1].get("role") != "tool":+ request_messages.extend(pending)+ pending.clear()return request_messagesdiff --git a/deeptutor/tools/builtin/__init__.py b/deeptutor/tools/builtin/__init__.pyindex 242fd52d7..e36995aa3 100644--- a/deeptutor/tools/builtin/__init__.py+++ b/deeptutor/tools/builtin/__init__.py@@ -88,12 +88,14 @@ def _rag_sources(result: dict[str, Any], *, query: str, kb_name: str) -> list[direturn [{"type": "rag", "query": query, "kb_name": kb_name}]from urllib.parse import quote+ from deeptutor.services.workspace.context import workspace_url+sources = []for item in retrieved:source = {"type": "rag", "kb_name": kb_name, **item}asset_id = source.get("visual_asset_id")if asset_id:- source["visual_asset_url"] = (+ source["visual_asset_url"] = workspace_url(f"/api/knowledge-bases/{quote(kb_name, safe='')}/visual-assets/{asset_id}")sources.append(source)diff --git a/tests/services/rag/test_visual_assets.py b/tests/services/rag/test_visual_assets.pyindex e810f18c4..7da9f1cb5 100644--- a/tests/services/rag/test_visual_assets.py+++ b/tests/services/rag/test_visual_assets.py@@ -248,6 +248,7 @@ def test_visual_preview_enforces_kb_scope_and_verified_bytes(tmp_path: Path, mondef test_rag_tool_sends_exact_retrieved_pixels_to_vision_model(tmp_path: Path, monkeypatch):from deeptutor.multi_user import knowledge_access+ from deeptutor.services.workspace import context as workspace_contextfrom deeptutor.tools import rag_toolfrom deeptutor.tools.builtin import RAGTool@@ -259,6 +260,7 @@ def test_rag_tool_sends_exact_retrieved_pixels_to_vision_model(tmp_path: Path, m"resolve_for_rag",lambda kb_name: SimpleNamespace(name="kb", base_dir=tmp_path) if kb_name == "kb" else None,)+ monkeypatch.setattr(workspace_context, "current_workspace_id", lambda: "study")async def search(**_kwargs):return {@@ -286,7 +288,9 @@ def test_rag_tool_sends_exact_retrieved_pixels_to_vision_model(tmp_path: Path, mdata_uri = parts[1]["image_url"]["url"]assert data_uri.startswith("data:image/png;base64,")assert base64.b64decode(data_uri.split(",", 1)[1]) == image.read_bytes()- assert result.sources[0]["visual_asset_url"].endswith(record.record["asset_id"])+ assert result.sources[0]["visual_asset_url"].endswith(+ f"{record.record['asset_id']}?dt_workspace=study"+ )assert "base64" not in str(result.metadata)text_only = asyncio.run(@@ -482,3 +486,24 @@ def test_visual_message_reaches_next_model_request_without_stream_leak(tmp_path:assert anthropic_messages[-1]["role"] == "user"assert anthropic_messages[-1]["content"][0]["type"] == "tool_result"assert anthropic_messages[-1]["content"][-1]["type"] == "image"+++def test_visual_messages_follow_complete_tool_reply_batch():+ from deeptutor.runtime.agentic.loop import _with_transient_model_messages++ messages = [+ {"role": "assistant", "tool_calls": [{"id": "rag"}, {"id": "other"}]},+ {"role": "tool", "tool_call_id": "rag", "content": "Figure found"},+ {"role": "tool", "tool_call_id": "other", "content": "Other result"},+ ]+ transient = [+ {+ "role": "user",+ "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}],+ "_after_tool_call_id": "rag",+ }+ ]+ request = _with_transient_model_messages(messages, transient)+ assert [item["role"] for item in request] == ["assistant", "tool", "tool", "user"]+ assert request[-1]["content"] == transient[0]["content"]+ assert messages[-1]["role"] == "tool" # canonical history was not changed