DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(rag): bound visual manifests and per-turn image context
Bingxi Zhao (Frank)bb39f0d91155
diff --git a/deeptutor/agents/loop/pipeline.py b/deeptutor/agents/loop/pipeline.pyindex fdcee04a6..f6b0087de 100644--- a/deeptutor/agents/loop/pipeline.py+++ b/deeptutor/agents/loop/pipeline.py@@ -105,6 +105,9 @@ KB_SEED_CHARS_PER_KB = 4000# tool calls ends the loop early — that is the normal exit.DEFAULT_MAX_ROUNDS = 8CONTEXT_WINDOW_GUARD_RATIO = 0.9+# Provider image token accounting varies by model and resolution. Reserve a+# conservative amount for each image instead of treating source pixels as free.+IMAGE_TOKEN_GUARD_RESERVE = 4096_DispatchOutcome = DispatchOutcome@@ -1662,6 +1665,8 @@ class AgenticLoopPipeline:for part in content:if isinstance(part, dict) and part.get("type") == "text":total += count_tokens(str(part.get("text") or ""))+ elif isinstance(part, dict) and part.get("type") == "image_url":+ total += IMAGE_TOKEN_GUARD_RESERVEreturn total# ---- LLM client ------------------------------------------------------diff --git a/deeptutor/runtime/agentic/loop.py b/deeptutor/runtime/agentic/loop.pyindex 9255f84e6..d79bcdec3 100644--- a/deeptutor/runtime/agentic/loop.py+++ b/deeptutor/runtime/agentic/loop.py@@ -199,6 +199,13 @@ def _with_transient_model_messages(return request_messages+def _transient_image_count(message: dict[str, Any]) -> int:+ content = message.get("content")+ if not isinstance(content, list):+ return 0+ return sum(1 for part in content if isinstance(part, dict) and part.get("type") == "image_url")++async def run_agentic_loop(*,initial_messages: list[dict[str, Any]],@@ -242,6 +249,7 @@ async def run_agentic_loop(# durable conversation (where a synthetic user message would be persisted# and displayed as if the person authored it).transient_model_messages: list[dict[str, Any]] = []+ max_transient_images = 2aggregated_sources: list[dict[str, Any]] = []final_text = ""final_label_seen = ""@@ -250,7 +258,9 @@ async def run_agentic_loop(max_iter = max(1, max_iterations)for iteration in range(max_iter):- await host.guard_context_window(messages)+ await host.guard_context_window(+ _with_transient_model_messages(messages, transient_model_messages)+ )before_iteration = getattr(host, "before_iteration", None)if before_iteration is not None:await before_iteration(@@ -259,11 +269,12 @@ async def run_agentic_loop(max_iterations=max_iter,)iter_meta, final_meta = host.build_iteration_trace_meta(iteration)+ request_messages = _with_transient_model_messages(messages, transient_model_messages)step = await run_labeled_step(client=client,model=model,- messages=_with_transient_model_messages(messages, transient_model_messages),+ messages=request_messages,completion_kwargs=completion_kwargs,tool_schemas=tool_schemas,allowed_labels=protocol.allowed,@@ -347,6 +358,11 @@ async def run_agentic_loop(aggregated_sources.extend(outcome.sources)messages.extend(outcome.tool_messages)transient_model_messages.extend(outcome.model_messages)+ while (+ sum(_transient_image_count(item) for item in transient_model_messages)+ > max_transient_images+ ):+ transient_model_messages.pop(0)if outcome.pause:resumed = await host.resolve_pause(outcome)if not resumed:diff --git a/deeptutor/services/rag/visual_assets.py b/deeptutor/services/rag/visual_assets.pyindex 0541ad0f3..52e084bcb 100644--- a/deeptutor/services/rag/visual_assets.py+++ b/deeptutor/services/rag/visual_assets.py@@ -274,6 +274,12 @@ class VisualAssetStore:if not value.get("managed_source")or self._source_exists(value.get("source_path"))}+ updated = {**remaining, **incoming}+ manifest = json.dumps(+ {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True+ ).encode()+ if len(manifest) > MAX_MANIFEST_BYTES:+ raise OSError("Visual asset manifest exceeds its size limit")for candidate in candidates:record = candidate.recordloaded = _image_bytes(candidate.path)@@ -287,13 +293,7 @@ class VisualAssetStore:existing = _image_bytes(target)if existing is None or sha256(existing[0]).hexdigest() != record["image_sha256"]:self._atomic_write(target, loaded[0])- updated = {**remaining, **incoming}- self._atomic_write(- self.manifest_path,- json.dumps(- {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True- ).encode(),- )+ self._atomic_write(self.manifest_path, manifest)for old_id, old_record in prior.items():if old_id not in updated:self._path(old_id, str(old_record.get("mime_type"))).unlink(missing_ok=True)diff --git a/tests/services/rag/test_visual_assets.py b/tests/services/rag/test_visual_assets.pyindex 7da9f1cb5..fe6ab1eba 100644--- a/tests/services/rag/test_visual_assets.py+++ b/tests/services/rag/test_visual_assets.py@@ -104,6 +104,26 @@ def test_verified_source_visual_contract_and_lifecycle(tmp_path: Path, suffix: sassert not persisted.exists()+def test_manifest_limit_does_not_replace_existing_assets(tmp_path: Path, monkeypatch):+ import deeptutor.services.rag.visual_assets as assets_module++ kb_dir, source, image, parsed = _fixture(tmp_path)+ original = collect_visual_assets(parsed, source, kb_dir)[0]+ store = VisualAssetStore(kb_dir)+ store.publish([original])+ manifest_before = store.manifest_path.read_bytes()+ monkeypatch.setattr(assets_module, "MAX_MANIFEST_BYTES", len(manifest_before) + 10)+ second = assets_module.VisualAssetCandidate(+ path=image,+ record={**original.record, "asset_id": "a" * 64},+ )+ with pytest.raises(OSError, match="manifest exceeds"):+ store.publish([second])+ assert store.manifest_path.read_bytes() == manifest_before+ assert store.read(original.record["asset_id"]) is not None+ assert store.read(second.record["asset_id"]) is None++def test_size_count_and_rebuild_cleanup(tmp_path: Path, monkeypatch):import deeptutor.services.rag.visual_assets as assets_module@@ -507,3 +527,88 @@ def test_visual_messages_follow_complete_tool_reply_batch():assert [item["role"] for item in request] == ["assistant", "tool", "tool", "user"]assert request[-1]["content"] == transient[0]["content"]assert messages[-1]["role"] == "tool" # canonical history was not changed+++def test_visual_images_are_bounded_across_tool_rounds(monkeypatch):+ from deeptutor.agents.loop.pipeline import IMAGE_TOKEN_GUARD_RESERVE, AgenticLoopPipeline+ from deeptutor.runtime.agentic import loop as agent_loop+ from deeptutor.runtime.agentic.labeled_step import LabeledStepResult+ from deeptutor.runtime.agentic.tool_dispatch import DispatchOutcome++ requests = []+ guard_estimates = []++ async def fake_step(**kwargs):+ requests.append(kwargs["messages"])+ if len(requests) <= 2:+ tool_id = f"tool-{len(requests)}"+ return LabeledStepResult(+ label="CALL_TOOLS",+ text="",+ tool_calls=[{"id": tool_id, "name": "rag", "arguments": "{}"}],+ )+ return LabeledStepResult(label="FINISH", text="Done")++ monkeypatch.setattr(agent_loop, "run_labeled_step", fake_step)++ class Host:+ async def guard_context_window(self, messages):+ guard_estimates.append(AgenticLoopPipeline._estimate_messages_tokens(messages))++ def build_iteration_trace_meta(self, _iteration):+ return {}, {}++ async def dispatch_tools(self, *, iteration, **_kwargs):+ tool_id = f"tool-{iteration + 1}"+ return DispatchOutcome(+ tool_messages=[+ {"role": "tool", "tool_call_id": tool_id, "content": "Figure found"}+ ],+ model_messages=[+ {+ "role": "user",+ "_after_tool_call_id": tool_id,+ "content": [+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{iteration}{image}"},+ }+ for image in range(2)+ ],+ }+ ],+ )++ protocol = agent_loop.LabelProtocol(+ allowed=("CALL_TOOLS", "FINISH"),+ terminal=frozenset({"FINISH"}),+ intermediate=frozenset(),+ final=frozenset(),+ tool_label="CALL_TOOLS",+ )+ result = asyncio.run(+ agent_loop.run_agentic_loop(+ initial_messages=[{"role": "user", "content": "Explain the figures"}],+ protocol=protocol,+ client=object(),+ model="vision-model",+ completion_kwargs={},+ binding="openai",+ tool_schemas=[],+ stream=object(),+ source="chat",+ stage="exploring",+ max_iterations=3,+ host=Host(),+ )+ )+ assert result.completed+ images = [+ part["image_url"]["url"]+ for message in requests[2]+ for part in (message.get("content") or [])+ if isinstance(part, dict) and part.get("type") == "image_url"+ ]+ assert images == ["data:image/png;base64,10", "data:image/png;base64,11"]+ assert guard_estimates[2] >= 2 * IMAGE_TOKEN_GUARD_RESERVE+ assert "base64" not in str(result.messages)