Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(rag): keep visual tool replies valid and pin asset scope

Bingxi Zhao (Frank)0bb753868ea9
diff --git a/deeptutor/runtime/agentic/loop.py b/deeptutor/runtime/agentic/loop.py
index 49d2a77f8..9255f84e6 100644
--- a/deeptutor/runtime/agentic/loop.py
+++ b/deeptutor/runtime/agentic/loop.py
@@ -186,10 +186,16 @@ def _with_transient_model_messages(
continue
anchored.setdefault(tool_call_id, []).append({"role": "user", "content": item["content"]})
request_messages: list[dict[str, Any]] = []
- for message in messages:
+ pending: list[dict[str, Any]] = []
+ for index, message in enumerate(messages):
request_messages.append(message)
if message.get("role") == "tool":
- request_messages.extend(anchored.pop(str(message.get("tool_call_id") or ""), []))
+ pending.extend(anchored.pop(str(message.get("tool_call_id") or ""), []))
+ # Providers require all replies to one assistant tool-call batch
+ # before another user message. Inject images after the batch.
+ if index + 1 == len(messages) or messages[index + 1].get("role") != "tool":
+ request_messages.extend(pending)
+ pending.clear()
return request_messages
diff --git a/deeptutor/tools/builtin/__init__.py b/deeptutor/tools/builtin/__init__.py
index 242fd52d7..e36995aa3 100644
--- a/deeptutor/tools/builtin/__init__.py
+++ b/deeptutor/tools/builtin/__init__.py
@@ -88,12 +88,14 @@ def _rag_sources(result: dict[str, Any], *, query: str, kb_name: str) -> list[di
return [{"type": "rag", "query": query, "kb_name": kb_name}]
from urllib.parse import quote
+ from deeptutor.services.workspace.context import workspace_url
+
sources = []
for item in retrieved:
source = {"type": "rag", "kb_name": kb_name, **item}
asset_id = source.get("visual_asset_id")
if asset_id:
- source["visual_asset_url"] = (
+ source["visual_asset_url"] = workspace_url(
f"/api/knowledge-bases/{quote(kb_name, safe='')}/visual-assets/{asset_id}"
)
sources.append(source)
diff --git a/tests/services/rag/test_visual_assets.py b/tests/services/rag/test_visual_assets.py
index e810f18c4..7da9f1cb5 100644
--- a/tests/services/rag/test_visual_assets.py
+++ b/tests/services/rag/test_visual_assets.py
@@ -248,6 +248,7 @@ def test_visual_preview_enforces_kb_scope_and_verified_bytes(tmp_path: Path, mon
def test_rag_tool_sends_exact_retrieved_pixels_to_vision_model(tmp_path: Path, monkeypatch):
from deeptutor.multi_user import knowledge_access
+ from deeptutor.services.workspace import context as workspace_context
from deeptutor.tools import rag_tool
from deeptutor.tools.builtin import RAGTool
@@ -259,6 +260,7 @@ def test_rag_tool_sends_exact_retrieved_pixels_to_vision_model(tmp_path: Path, m
"resolve_for_rag",
lambda kb_name: SimpleNamespace(name="kb", base_dir=tmp_path) if kb_name == "kb" else None,
)
+ monkeypatch.setattr(workspace_context, "current_workspace_id", lambda: "study")
async def search(**_kwargs):
return {
@@ -286,7 +288,9 @@ def test_rag_tool_sends_exact_retrieved_pixels_to_vision_model(tmp_path: Path, m
data_uri = parts[1]["image_url"]["url"]
assert data_uri.startswith("data:image/png;base64,")
assert base64.b64decode(data_uri.split(",", 1)[1]) == image.read_bytes()
- assert result.sources[0]["visual_asset_url"].endswith(record.record["asset_id"])
+ assert result.sources[0]["visual_asset_url"].endswith(
+ f"{record.record['asset_id']}?dt_workspace=study"
+ )
assert "base64" not in str(result.metadata)
text_only = asyncio.run(
@@ -482,3 +486,24 @@ def test_visual_message_reaches_next_model_request_without_stream_leak(tmp_path:
assert anthropic_messages[-1]["role"] == "user"
assert anthropic_messages[-1]["content"][0]["type"] == "tool_result"
assert anthropic_messages[-1]["content"][-1]["type"] == "image"
+
+
+def test_visual_messages_follow_complete_tool_reply_batch():
+ from deeptutor.runtime.agentic.loop import _with_transient_model_messages
+
+ messages = [
+ {"role": "assistant", "tool_calls": [{"id": "rag"}, {"id": "other"}]},
+ {"role": "tool", "tool_call_id": "rag", "content": "Figure found"},
+ {"role": "tool", "tool_call_id": "other", "content": "Other result"},
+ ]
+ transient = [
+ {
+ "role": "user",
+ "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}],
+ "_after_tool_call_id": "rag",
+ }
+ ]
+ request = _with_transient_model_messages(messages, transient)
+ assert [item["role"] for item in request] == ["assistant", "tool", "tool", "user"]
+ assert request[-1]["content"] == transient[0]["content"]
+ assert messages[-1]["role"] == "tool" # canonical history was not changed