DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
perf(llm): deduplicate repeated inline user images per request (#1714)
ZyPulsef3b840ce46a2
diff --git a/README.md b/README.mdindex 65dd79024..449d46fcd 100644--- a/README.md+++ b/README.md@@ -1160,3 +1160,14 @@ Licensed under the [Apache License 2.0](LICENSE).</p></div>++### Repeated image attachments++The agent loop sends identical inline user images once per request and replaces later+copies with references to the retained image. Stable request-only labels identify+the retained inline images across provider translations and appended turns. Unique+image blocks, remote URLs, differing+image options, assistant messages, and tool results remain complete. Saved history+keeps every original attachment; references are rebuilt after history is trimmed.+This reduces repeated image payloads, but provider-reported usage remains the+source of truth for billing and cache hits.diff --git a/deeptutor/agents/loop/agent_loop.py b/deeptutor/agents/loop/agent_loop.pyindex 3a8e2cc30..456c02883 100644--- a/deeptutor/agents/loop/agent_loop.py+++ b/deeptutor/agents/loop/agent_loop.py@@ -60,7 +60,12 @@ from deeptutor.services.llm import ()from deeptutor.services.llm import finish_was_truncated as _finish_was_truncatedfrom deeptutor.services.llm.capabilities import threads_session_id-from deeptutor.services.llm.multimodal import should_degrade_to_text, strip_image_parts_inplace+from deeptutor.services.llm.image_replay import deduplicate_user_images+from deeptutor.services.llm.multimodal import (+ has_image_parts,+ should_degrade_to_text,+ strip_image_parts_inplace,+)from deeptutor.services.llm.request_compat import (is_forced_tool_choice_unsupported,is_image_input_unsupported,@@ -1042,10 +1047,12 @@ class AgentLoop:kwargs: dict[str, Any] = {"model": self.pipeline.model,- "messages": [- {key: value for key, value in message.items() if key != "_context_snapshot"}- for message in messages- ],+ "messages": deduplicate_user_images(+ [+ {key: value for key, value in message.items() if key != "_context_snapshot"}+ for message in messages+ ]+ ),"stream": True,**self.pipeline._completion_kwargs(max_tokens=max_tokens),}@@ -1098,7 +1105,9 @@ class AgentLoop:carried = list(tool_schemas or [])if not carried and self._last_request is not None:carried = self._last_request.tool_schemas- self._last_request = LLMRequestSnapshot(messages=list(messages), tool_schemas=carried)+ self._last_request = LLMRequestSnapshot(+ messages=list(kwargs["messages"]), tool_schemas=carried+ )chunk_meta = merge_trace_metadata(trace_meta, {"trace_kind": "llm_chunk"})@@ -1227,12 +1236,20 @@ class AgentLoop:response_stream = Nonetry:+ sent_content = [message.get("content") for message in kwargs["messages"]]+ sent_had_images = has_image_parts(kwargs["messages"])response_stream = await self._create_response_stream(kwargs, trace_meta, stage)- # A provider's image fallback can replace content in the wire- # copy. Retain that accepted representation for later rounds.- for original, accepted in zip(messages, kwargs["messages"]):- if "content" in accepted:- original["content"] = accepted["content"]+ # Retain an actual provider fallback for later rounds. The+ # request-only image deduplication must never overwrite history.+ if sent_had_images and not has_image_parts(kwargs["messages"]):+ # Degrade all canonical copies too, including duplicates+ # replaced only on the wire. Otherwise the next round would+ # retry images the provider has already rejected.+ strip_image_parts_inplace(messages)+ else:+ for original, accepted, sent in zip(messages, kwargs["messages"], sent_content):+ if "content" in accepted and accepted["content"] is not sent:+ original["content"] = accepted["content"]async for chunk in response_stream:usage = getattr(chunk, "usage", None)if usage is not None:diff --git a/deeptutor/services/llm/image_replay.py b/deeptutor/services/llm/image_replay.pynew file mode 100644index 000000000..90b6179e5--- /dev/null+++ b/deeptutor/services/llm/image_replay.py@@ -0,0 +1,89 @@+"""Deduplicate immutable user images in a request-only projection.++Durable model turns remain complete. Keep the first identical inline image+at its original position, preserving the request prefix and all unique images.+Remote URLs can change underneath a stable address and are never deduplicated.+"""++from __future__ import annotations++import hashlib+import json+from typing import Any+++def _inline_image_key(block: Any) -> str | None:+ if not isinstance(block, dict):+ return None+ kind = block.get("type")+ if kind in {"image_url", "input_image"}:+ image = block.get("image_url")+ url = image.get("url") if isinstance(image, dict) else image+ if not isinstance(url, str) or not url.startswith("data:image/"):+ return None+ payload_hash = hashlib.sha256(url.encode()).hexdigest()+ compact = {+ **block,+ "image_url": {**image, "url": payload_hash}+ if isinstance(image, dict)+ else payload_hash,+ }+ elif kind == "image":+ source = block.get("source")+ if not (+ isinstance(source, dict)+ and source.get("type") == "base64"+ and isinstance(source.get("data"), str)+ and source["data"]+ ):+ return None+ compact = {+ **block,+ "source": {**source, "data": hashlib.sha256(source["data"].encode()).hexdigest()},+ }+ else:+ return None+ # Include detail, MIME and other options: only exactly equal image blocks+ # are interchangeable. No bytes are decoded and no URLs are fetched.+ encoded = json.dumps(compact, sort_keys=True, ensure_ascii=False, separators=(",", ":"))+ return hashlib.sha256(encoded.encode()).hexdigest()+++def deduplicate_user_images(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:+ """Project repeated inline user images to stable references in this request.++ Label each retained image even when unique so appending a repeat never+ changes earlier request prefixes. Labels also survive provider translation,+ which may move system messages or combine tool results with user messages.+ Recompute from full history after a cut: every reference still has real+ bytes in the current request. Original image blocks and history are intact.+ """+ seen: set[str] = set()+ projected = []+ for message in messages:+ content = message.get("content")+ if message.get("role") != "user" or not isinstance(content, list):+ projected.append(dict(message))+ continue+ parts = []+ for block in content:+ key = _inline_image_key(block)+ if key is None:+ parts.append(block)+ continue+ label = key[:32]+ if key not in seen:+ seen.add(key)+ marker = {"type": "text", "text": f"[Image {label}]"}+ if not parts or parts[-1] != marker:+ parts.append(marker)+ parts.append(block)+ continue+ parts.append(+ {+ "type": "text",+ "text": f"[Repeated image {label}; see the identical image included earlier in this request.]",+ }+ )+ projected.append({**message, "content": parts})+ return projecteddiff --git a/tests/agents/chat/test_agent_loop.py b/tests/agents/chat/test_agent_loop.pyindex 99449c146..215410480 100644--- a/tests/agents/chat/test_agent_loop.py+++ b/tests/agents/chat/test_agent_loop.py@@ -3492,3 +3492,80 @@ async def test_truncated_tool_call_says_the_output_hit_the_limit(# The notice is a report, not a control-flow change: the round is handled# exactly as before and the turn still finishes.assert _answer_text(events) == "答案"+++async def test_wire_deduplicates_user_images_without_mutating_model_history(monkeypatch):+ from copy import deepcopy++ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}+ messages = [+ {"role": "user", "content": [image]},+ {"role": "assistant", "content": "answer"},+ {"role": "user", "content": [deepcopy(image)]},+ ]+ original = deepcopy(messages)+ client = _ScriptedChatClient([[_llm_chunk(content="answer")]])+ pipeline = AgenticChatPipeline(language="en")+ pipeline.registry = _Registry()+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: [])+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)+ pipeline._model_turn_start = 0+ monkeypatch.setattr(pipeline, "_build_loop_messages", lambda *_args, **_kwargs: messages)+ await _run(pipeline, UnifiedContext(session_id="s", user_message="again"))+ wire = client.calls[0]["messages"]+ assert wire[0]["content"][-1] == original[0]["content"][0]+ assert "Repeated image" in wire[2]["content"][0]["text"]+ assert [message["content"] for message in messages[:3]] == [+ message["content"] for message in original+ ]+++async def test_image_fallback_strips_all_canonical_copies_after_wire_deduplication(monkeypatch):+ from copy import deepcopy++ from deeptutor.services.llm.multimodal import has_image_parts++ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}+ messages = [+ {"role": "user", "content": [image]},+ {"role": "user", "content": [deepcopy(image)]},+ ]+ client = _ScriptedChatClient(+ [+ [+ _llm_chunk(+ tool_calls=[+ {"id": "search-1", "name": "web_search", "arguments": '{"query":"text"}'}+ ],+ finish_reason="tool_calls",+ )+ ],+ [_llm_chunk(content="Text answer.")],+ ]+ )+ create = client.chat.completions.create+ attempted = []++ async def reject_images(**kwargs):+ attempted.append(deepcopy(kwargs["messages"]))+ if has_image_parts(kwargs["messages"]):+ raise RuntimeError("image input is not supported")+ return await create(**kwargs)++ client.chat.completions.create = reject_images+ pipeline = AgenticChatPipeline(language="en")+ pipeline.model = "gpt-3.5-turbo"+ pipeline.registry = _Registry()+ pipeline._model_turn_start = 0+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: ["web_search"])+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)+ monkeypatch.setattr(pipeline, "_build_loop_messages", lambda *_args, **_kwargs: messages)+ context = UnifiedContext(session_id="s", user_message="again")+ await _run(pipeline, context)+ assert len(attempted) == 3+ assert not has_image_parts(attempted[1])+ assert not has_image_parts(attempted[2])+ assert not has_image_parts(messages)+ assert "Repeated image" not in str(messages)diff --git a/tests/services/llm/test_image_replay.py b/tests/services/llm/test_image_replay.pynew file mode 100644index 000000000..12022eee5--- /dev/null+++ b/tests/services/llm/test_image_replay.py@@ -0,0 +1,88 @@+from copy import deepcopy++import pytest++from deeptutor.services.llm.image_replay import deduplicate_user_images+++ "image",+ [+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}},+ {"type": "input_image", "image_url": "data:image/png;base64,YWJj"},+ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "YWJj"}},+ ],+)+def test_only_duplicate_user_images_are_projected_and_history_is_complete(image):+ messages = [+ {"role": "user", "content": [{"type": "text", "text": "first"}, image]},+ {"role": "assistant", "content": "answer", "reasoning_content": "unchanged"},+ {"role": "user", "content": [{"type": "text", "text": "again"}, deepcopy(image)]},+ {"role": "tool", "tool_call_id": "t", "content": [deepcopy(image)]},+ ]+ before = deepcopy(messages)+ wire = deduplicate_user_images(messages)+ assert messages == before+ assert wire[0]["content"][-1] == image+ assert wire[0]["content"][0] == messages[0]["content"][0]+ assert "Repeated image" in wire[2]["content"][-1]["text"]+ assert wire[1] == messages[1]+ assert wire[3] == messages[3]+ # Projecting again is harmless; a later history cut still has real bytes.+ assert deduplicate_user_images(wire) == wire+ assert deduplicate_user_images(messages[2:])[0]["content"][-1] == image+++def test_remote_images_unique_images_and_detail_options_are_preserved():+ blocks = [+ {"type": "image_url", "image_url": {"url": "https://example.com/changing.png"}},+ {"type": "image_url", "image_url": {"url": "https://example.com/changing.png"}},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj", "detail": "low"}},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj", "detail": "high"}},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,ZGVm"}},+ ]+ messages = [{"role": "user", "content": blocks}]+ assert [+ part for part in deduplicate_user_images(messages)[0]["content"] if part["type"] != "text"+ ] == blocks+++def test_projection_keeps_previous_wire_prefix_when_a_turn_is_appended():+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}+ messages = [{"role": "user", "content": [image]}]+ first = deduplicate_user_images(messages)+ later = deduplicate_user_images([*messages, {"role": "user", "content": [image]}])+ assert later[: len(first)] == first+++def test_duplicate_inside_one_message_references_retained_image_and_preserves_count():+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}+ wire = deduplicate_user_images([{"role": "user", "content": [image, image, image]}])+ assert wire[0]["content"][1] == image+ assert len(wire[0]["content"]) == 4+ label = wire[0]["content"][0]["text"].removeprefix("[Image ").removesuffix("]")+ assert all(label in part["text"] for part in wire[0]["content"][2:])+++def test_changed_mime_and_provider_options_are_not_duplicates():+ image = {+ "type": "image",+ "source": {"type": "base64", "media_type": "image/png", "data": "YWJj"},+ }+ other = deepcopy(image)+ other["source"]["media_type"] = "image/jpeg"+ with_cache = {**image, "cache_control": {"type": "ephemeral"}}+ messages = [{"role": "user", "content": [image, other, with_cache]}]+ assert [+ part for part in deduplicate_user_images(messages)[0]["content"] if part["type"] != "text"+ ] == [image, other, with_cache]+++def test_stable_label_references_inline_image_among_remote_images():+ remote = {"type": "image_url", "image_url": {"url": "https://example.org/image.png"}}+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}+ wire = deduplicate_user_images([{"role": "user", "content": [remote, image, image]}])+ assert wire[0]["content"][0] == remote+ assert wire[0]["content"][2] == image+ label = wire[0]["content"][1]["text"].removeprefix("[Image ").removesuffix("]")+ assert label in wire[0]["content"][3]["text"]