Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(rag): bound visual manifests and per-turn image context

Bingxi Zhao (Frank)bb39f0d91155
diff --git a/deeptutor/agents/loop/pipeline.py b/deeptutor/agents/loop/pipeline.py
index fdcee04a6..f6b0087de 100644
--- a/deeptutor/agents/loop/pipeline.py
+++ b/deeptutor/agents/loop/pipeline.py
@@ -105,6 +105,9 @@ KB_SEED_CHARS_PER_KB = 4000
# tool calls ends the loop early — that is the normal exit.
DEFAULT_MAX_ROUNDS = 8
CONTEXT_WINDOW_GUARD_RATIO = 0.9
+# Provider image token accounting varies by model and resolution. Reserve a
+# conservative amount for each image instead of treating source pixels as free.
+IMAGE_TOKEN_GUARD_RESERVE = 4096
_DispatchOutcome = DispatchOutcome
@@ -1662,6 +1665,8 @@ class AgenticLoopPipeline:
for part in content:
if isinstance(part, dict) and part.get("type") == "text":
total += count_tokens(str(part.get("text") or ""))
+ elif isinstance(part, dict) and part.get("type") == "image_url":
+ total += IMAGE_TOKEN_GUARD_RESERVE
return total
# ---- LLM client ------------------------------------------------------
diff --git a/deeptutor/runtime/agentic/loop.py b/deeptutor/runtime/agentic/loop.py
index 9255f84e6..d79bcdec3 100644
--- a/deeptutor/runtime/agentic/loop.py
+++ b/deeptutor/runtime/agentic/loop.py
@@ -199,6 +199,13 @@ def _with_transient_model_messages(
return request_messages
+def _transient_image_count(message: dict[str, Any]) -> int:
+ content = message.get("content")
+ if not isinstance(content, list):
+ return 0
+ return sum(1 for part in content if isinstance(part, dict) and part.get("type") == "image_url")
+
+
async def run_agentic_loop(
*,
initial_messages: list[dict[str, Any]],
@@ -242,6 +249,7 @@ async def run_agentic_loop(
# durable conversation (where a synthetic user message would be persisted
# and displayed as if the person authored it).
transient_model_messages: list[dict[str, Any]] = []
+ max_transient_images = 2
aggregated_sources: list[dict[str, Any]] = []
final_text = ""
final_label_seen = ""
@@ -250,7 +258,9 @@ async def run_agentic_loop(
max_iter = max(1, max_iterations)
for iteration in range(max_iter):
- await host.guard_context_window(messages)
+ await host.guard_context_window(
+ _with_transient_model_messages(messages, transient_model_messages)
+ )
before_iteration = getattr(host, "before_iteration", None)
if before_iteration is not None:
await before_iteration(
@@ -259,11 +269,12 @@ async def run_agentic_loop(
max_iterations=max_iter,
)
iter_meta, final_meta = host.build_iteration_trace_meta(iteration)
+ request_messages = _with_transient_model_messages(messages, transient_model_messages)
step = await run_labeled_step(
client=client,
model=model,
- messages=_with_transient_model_messages(messages, transient_model_messages),
+ messages=request_messages,
completion_kwargs=completion_kwargs,
tool_schemas=tool_schemas,
allowed_labels=protocol.allowed,
@@ -347,6 +358,11 @@ async def run_agentic_loop(
aggregated_sources.extend(outcome.sources)
messages.extend(outcome.tool_messages)
transient_model_messages.extend(outcome.model_messages)
+ while (
+ sum(_transient_image_count(item) for item in transient_model_messages)
+ > max_transient_images
+ ):
+ transient_model_messages.pop(0)
if outcome.pause:
resumed = await host.resolve_pause(outcome)
if not resumed:
diff --git a/deeptutor/services/rag/visual_assets.py b/deeptutor/services/rag/visual_assets.py
index 0541ad0f3..52e084bcb 100644
--- a/deeptutor/services/rag/visual_assets.py
+++ b/deeptutor/services/rag/visual_assets.py
@@ -274,6 +274,12 @@ class VisualAssetStore:
if not value.get("managed_source")
or self._source_exists(value.get("source_path"))
}
+ updated = {**remaining, **incoming}
+ manifest = json.dumps(
+ {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True
+ ).encode()
+ if len(manifest) > MAX_MANIFEST_BYTES:
+ raise OSError("Visual asset manifest exceeds its size limit")
for candidate in candidates:
record = candidate.record
loaded = _image_bytes(candidate.path)
@@ -287,13 +293,7 @@ class VisualAssetStore:
existing = _image_bytes(target)
if existing is None or sha256(existing[0]).hexdigest() != record["image_sha256"]:
self._atomic_write(target, loaded[0])
- updated = {**remaining, **incoming}
- self._atomic_write(
- self.manifest_path,
- json.dumps(
- {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True
- ).encode(),
- )
+ self._atomic_write(self.manifest_path, manifest)
for old_id, old_record in prior.items():
if old_id not in updated:
self._path(old_id, str(old_record.get("mime_type"))).unlink(missing_ok=True)
diff --git a/tests/services/rag/test_visual_assets.py b/tests/services/rag/test_visual_assets.py
index 7da9f1cb5..fe6ab1eba 100644
--- a/tests/services/rag/test_visual_assets.py
+++ b/tests/services/rag/test_visual_assets.py
@@ -104,6 +104,26 @@ def test_verified_source_visual_contract_and_lifecycle(tmp_path: Path, suffix: s
assert not persisted.exists()
+def test_manifest_limit_does_not_replace_existing_assets(tmp_path: Path, monkeypatch):
+ import deeptutor.services.rag.visual_assets as assets_module
+
+ kb_dir, source, image, parsed = _fixture(tmp_path)
+ original = collect_visual_assets(parsed, source, kb_dir)[0]
+ store = VisualAssetStore(kb_dir)
+ store.publish([original])
+ manifest_before = store.manifest_path.read_bytes()
+ monkeypatch.setattr(assets_module, "MAX_MANIFEST_BYTES", len(manifest_before) + 10)
+ second = assets_module.VisualAssetCandidate(
+ path=image,
+ record={**original.record, "asset_id": "a" * 64},
+ )
+ with pytest.raises(OSError, match="manifest exceeds"):
+ store.publish([second])
+ assert store.manifest_path.read_bytes() == manifest_before
+ assert store.read(original.record["asset_id"]) is not None
+ assert store.read(second.record["asset_id"]) is None
+
+
def test_size_count_and_rebuild_cleanup(tmp_path: Path, monkeypatch):
import deeptutor.services.rag.visual_assets as assets_module
@@ -507,3 +527,88 @@ def test_visual_messages_follow_complete_tool_reply_batch():
assert [item["role"] for item in request] == ["assistant", "tool", "tool", "user"]
assert request[-1]["content"] == transient[0]["content"]
assert messages[-1]["role"] == "tool" # canonical history was not changed
+
+
+def test_visual_images_are_bounded_across_tool_rounds(monkeypatch):
+ from deeptutor.agents.loop.pipeline import IMAGE_TOKEN_GUARD_RESERVE, AgenticLoopPipeline
+ from deeptutor.runtime.agentic import loop as agent_loop
+ from deeptutor.runtime.agentic.labeled_step import LabeledStepResult
+ from deeptutor.runtime.agentic.tool_dispatch import DispatchOutcome
+
+ requests = []
+ guard_estimates = []
+
+ async def fake_step(**kwargs):
+ requests.append(kwargs["messages"])
+ if len(requests) <= 2:
+ tool_id = f"tool-{len(requests)}"
+ return LabeledStepResult(
+ label="CALL_TOOLS",
+ text="",
+ tool_calls=[{"id": tool_id, "name": "rag", "arguments": "{}"}],
+ )
+ return LabeledStepResult(label="FINISH", text="Done")
+
+ monkeypatch.setattr(agent_loop, "run_labeled_step", fake_step)
+
+ class Host:
+ async def guard_context_window(self, messages):
+ guard_estimates.append(AgenticLoopPipeline._estimate_messages_tokens(messages))
+
+ def build_iteration_trace_meta(self, _iteration):
+ return {}, {}
+
+ async def dispatch_tools(self, *, iteration, **_kwargs):
+ tool_id = f"tool-{iteration + 1}"
+ return DispatchOutcome(
+ tool_messages=[
+ {"role": "tool", "tool_call_id": tool_id, "content": "Figure found"}
+ ],
+ model_messages=[
+ {
+ "role": "user",
+ "_after_tool_call_id": tool_id,
+ "content": [
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{iteration}{image}"},
+ }
+ for image in range(2)
+ ],
+ }
+ ],
+ )
+
+ protocol = agent_loop.LabelProtocol(
+ allowed=("CALL_TOOLS", "FINISH"),
+ terminal=frozenset({"FINISH"}),
+ intermediate=frozenset(),
+ final=frozenset(),
+ tool_label="CALL_TOOLS",
+ )
+ result = asyncio.run(
+ agent_loop.run_agentic_loop(
+ initial_messages=[{"role": "user", "content": "Explain the figures"}],
+ protocol=protocol,
+ client=object(),
+ model="vision-model",
+ completion_kwargs={},
+ binding="openai",
+ tool_schemas=[],
+ stream=object(),
+ source="chat",
+ stage="exploring",
+ max_iterations=3,
+ host=Host(),
+ )
+ )
+ assert result.completed
+ images = [
+ part["image_url"]["url"]
+ for message in requests[2]
+ for part in (message.get("content") or [])
+ if isinstance(part, dict) and part.get("type") == "image_url"
+ ]
+ assert images == ["data:image/png;base64,10", "data:image/png;base64,11"]
+ assert guard_estimates[2] >= 2 * IMAGE_TOKEN_GUARD_RESERVE
+ assert "base64" not in str(result.messages)