Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

perf(llm): deduplicate repeated inline user images per request (#1714)

ZyPulsef3b840ce46a2
diff --git a/README.md b/README.md
index 65dd79024..449d46fcd 100644
--- a/README.md
+++ b/README.md
@@ -1160,3 +1160,14 @@ Licensed under the [Apache License 2.0](LICENSE).
</p>
</div>
+
+### Repeated image attachments
+
+The agent loop sends identical inline user images once per request and replaces later
+copies with references to the retained image. Stable request-only labels identify
+the retained inline images across provider translations and appended turns. Unique
+image blocks, remote URLs, differing
+image options, assistant messages, and tool results remain complete. Saved history
+keeps every original attachment; references are rebuilt after history is trimmed.
+This reduces repeated image payloads, but provider-reported usage remains the
+source of truth for billing and cache hits.
diff --git a/deeptutor/agents/loop/agent_loop.py b/deeptutor/agents/loop/agent_loop.py
index 3a8e2cc30..456c02883 100644
--- a/deeptutor/agents/loop/agent_loop.py
+++ b/deeptutor/agents/loop/agent_loop.py
@@ -60,7 +60,12 @@ from deeptutor.services.llm import (
)
from deeptutor.services.llm import finish_was_truncated as _finish_was_truncated
from deeptutor.services.llm.capabilities import threads_session_id
-from deeptutor.services.llm.multimodal import should_degrade_to_text, strip_image_parts_inplace
+from deeptutor.services.llm.image_replay import deduplicate_user_images
+from deeptutor.services.llm.multimodal import (
+ has_image_parts,
+ should_degrade_to_text,
+ strip_image_parts_inplace,
+)
from deeptutor.services.llm.request_compat import (
is_forced_tool_choice_unsupported,
is_image_input_unsupported,
@@ -1042,10 +1047,12 @@ class AgentLoop:
kwargs: dict[str, Any] = {
"model": self.pipeline.model,
- "messages": [
- {key: value for key, value in message.items() if key != "_context_snapshot"}
- for message in messages
- ],
+ "messages": deduplicate_user_images(
+ [
+ {key: value for key, value in message.items() if key != "_context_snapshot"}
+ for message in messages
+ ]
+ ),
"stream": True,
**self.pipeline._completion_kwargs(max_tokens=max_tokens),
}
@@ -1098,7 +1105,9 @@ class AgentLoop:
carried = list(tool_schemas or [])
if not carried and self._last_request is not None:
carried = self._last_request.tool_schemas
- self._last_request = LLMRequestSnapshot(messages=list(messages), tool_schemas=carried)
+ self._last_request = LLMRequestSnapshot(
+ messages=list(kwargs["messages"]), tool_schemas=carried
+ )
chunk_meta = merge_trace_metadata(trace_meta, {"trace_kind": "llm_chunk"})
@@ -1227,12 +1236,20 @@ class AgentLoop:
response_stream = None
try:
+ sent_content = [message.get("content") for message in kwargs["messages"]]
+ sent_had_images = has_image_parts(kwargs["messages"])
response_stream = await self._create_response_stream(kwargs, trace_meta, stage)
- # A provider's image fallback can replace content in the wire
- # copy. Retain that accepted representation for later rounds.
- for original, accepted in zip(messages, kwargs["messages"]):
- if "content" in accepted:
- original["content"] = accepted["content"]
+ # Retain an actual provider fallback for later rounds. The
+ # request-only image deduplication must never overwrite history.
+ if sent_had_images and not has_image_parts(kwargs["messages"]):
+ # Degrade all canonical copies too, including duplicates
+ # replaced only on the wire. Otherwise the next round would
+ # retry images the provider has already rejected.
+ strip_image_parts_inplace(messages)
+ else:
+ for original, accepted, sent in zip(messages, kwargs["messages"], sent_content):
+ if "content" in accepted and accepted["content"] is not sent:
+ original["content"] = accepted["content"]
async for chunk in response_stream:
usage = getattr(chunk, "usage", None)
if usage is not None:
diff --git a/deeptutor/services/llm/image_replay.py b/deeptutor/services/llm/image_replay.py
new file mode 100644
index 000000000..90b6179e5
--- /dev/null
+++ b/deeptutor/services/llm/image_replay.py
@@ -0,0 +1,89 @@
+"""Deduplicate immutable user images in a request-only projection.
+
+Durable model turns remain complete. Keep the first identical inline image
+at its original position, preserving the request prefix and all unique images.
+Remote URLs can change underneath a stable address and are never deduplicated.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+
+def _inline_image_key(block: Any) -> str | None:
+ if not isinstance(block, dict):
+ return None
+ kind = block.get("type")
+ if kind in {"image_url", "input_image"}:
+ image = block.get("image_url")
+ url = image.get("url") if isinstance(image, dict) else image
+ if not isinstance(url, str) or not url.startswith("data:image/"):
+ return None
+ payload_hash = hashlib.sha256(url.encode()).hexdigest()
+ compact = {
+ **block,
+ "image_url": {**image, "url": payload_hash}
+ if isinstance(image, dict)
+ else payload_hash,
+ }
+ elif kind == "image":
+ source = block.get("source")
+ if not (
+ isinstance(source, dict)
+ and source.get("type") == "base64"
+ and isinstance(source.get("data"), str)
+ and source["data"]
+ ):
+ return None
+ compact = {
+ **block,
+ "source": {**source, "data": hashlib.sha256(source["data"].encode()).hexdigest()},
+ }
+ else:
+ return None
+ # Include detail, MIME and other options: only exactly equal image blocks
+ # are interchangeable. No bytes are decoded and no URLs are fetched.
+ encoded = json.dumps(compact, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+ return hashlib.sha256(encoded.encode()).hexdigest()
+
+
+def deduplicate_user_images(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Project repeated inline user images to stable references in this request.
+
+ Label each retained image even when unique so appending a repeat never
+ changes earlier request prefixes. Labels also survive provider translation,
+ which may move system messages or combine tool results with user messages.
+ Recompute from full history after a cut: every reference still has real
+ bytes in the current request. Original image blocks and history are intact.
+ """
+ seen: set[str] = set()
+ projected = []
+ for message in messages:
+ content = message.get("content")
+ if message.get("role") != "user" or not isinstance(content, list):
+ projected.append(dict(message))
+ continue
+ parts = []
+ for block in content:
+ key = _inline_image_key(block)
+ if key is None:
+ parts.append(block)
+ continue
+ label = key[:32]
+ if key not in seen:
+ seen.add(key)
+ marker = {"type": "text", "text": f"[Image {label}]"}
+ if not parts or parts[-1] != marker:
+ parts.append(marker)
+ parts.append(block)
+ continue
+ parts.append(
+ {
+ "type": "text",
+ "text": f"[Repeated image {label}; see the identical image included earlier in this request.]",
+ }
+ )
+ projected.append({**message, "content": parts})
+ return projected
diff --git a/tests/agents/chat/test_agent_loop.py b/tests/agents/chat/test_agent_loop.py
index 99449c146..215410480 100644
--- a/tests/agents/chat/test_agent_loop.py
+++ b/tests/agents/chat/test_agent_loop.py
@@ -3492,3 +3492,80 @@ async def test_truncated_tool_call_says_the_output_hit_the_limit(
# The notice is a report, not a control-flow change: the round is handled
# exactly as before and the turn still finishes.
assert _answer_text(events) == "答案"
+
+
+async def test_wire_deduplicates_user_images_without_mutating_model_history(monkeypatch):
+ from copy import deepcopy
+
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}
+ messages = [
+ {"role": "user", "content": [image]},
+ {"role": "assistant", "content": "answer"},
+ {"role": "user", "content": [deepcopy(image)]},
+ ]
+ original = deepcopy(messages)
+ client = _ScriptedChatClient([[_llm_chunk(content="answer")]])
+ pipeline = AgenticChatPipeline(language="en")
+ pipeline.registry = _Registry()
+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: [])
+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)
+ pipeline._model_turn_start = 0
+ monkeypatch.setattr(pipeline, "_build_loop_messages", lambda *_args, **_kwargs: messages)
+ await _run(pipeline, UnifiedContext(session_id="s", user_message="again"))
+ wire = client.calls[0]["messages"]
+ assert wire[0]["content"][-1] == original[0]["content"][0]
+ assert "Repeated image" in wire[2]["content"][0]["text"]
+ assert [message["content"] for message in messages[:3]] == [
+ message["content"] for message in original
+ ]
+
+
+async def test_image_fallback_strips_all_canonical_copies_after_wire_deduplication(monkeypatch):
+ from copy import deepcopy
+
+ from deeptutor.services.llm.multimodal import has_image_parts
+
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}
+ messages = [
+ {"role": "user", "content": [image]},
+ {"role": "user", "content": [deepcopy(image)]},
+ ]
+ client = _ScriptedChatClient(
+ [
+ [
+ _llm_chunk(
+ tool_calls=[
+ {"id": "search-1", "name": "web_search", "arguments": '{"query":"text"}'}
+ ],
+ finish_reason="tool_calls",
+ )
+ ],
+ [_llm_chunk(content="Text answer.")],
+ ]
+ )
+ create = client.chat.completions.create
+ attempted = []
+
+ async def reject_images(**kwargs):
+ attempted.append(deepcopy(kwargs["messages"]))
+ if has_image_parts(kwargs["messages"]):
+ raise RuntimeError("image input is not supported")
+ return await create(**kwargs)
+
+ client.chat.completions.create = reject_images
+ pipeline = AgenticChatPipeline(language="en")
+ pipeline.model = "gpt-3.5-turbo"
+ pipeline.registry = _Registry()
+ pipeline._model_turn_start = 0
+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: ["web_search"])
+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)
+ monkeypatch.setattr(pipeline, "_build_loop_messages", lambda *_args, **_kwargs: messages)
+ context = UnifiedContext(session_id="s", user_message="again")
+ await _run(pipeline, context)
+ assert len(attempted) == 3
+ assert not has_image_parts(attempted[1])
+ assert not has_image_parts(attempted[2])
+ assert not has_image_parts(messages)
+ assert "Repeated image" not in str(messages)
diff --git a/tests/services/llm/test_image_replay.py b/tests/services/llm/test_image_replay.py
new file mode 100644
index 000000000..12022eee5
--- /dev/null
+++ b/tests/services/llm/test_image_replay.py
@@ -0,0 +1,88 @@
+from copy import deepcopy
+
+import pytest
+
+from deeptutor.services.llm.image_replay import deduplicate_user_images
+
+
+ "image",
+ [
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}},
+ {"type": "input_image", "image_url": "data:image/png;base64,YWJj"},
+ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "YWJj"}},
+ ],
+)
+def test_only_duplicate_user_images_are_projected_and_history_is_complete(image):
+ messages = [
+ {"role": "user", "content": [{"type": "text", "text": "first"}, image]},
+ {"role": "assistant", "content": "answer", "reasoning_content": "unchanged"},
+ {"role": "user", "content": [{"type": "text", "text": "again"}, deepcopy(image)]},
+ {"role": "tool", "tool_call_id": "t", "content": [deepcopy(image)]},
+ ]
+ before = deepcopy(messages)
+ wire = deduplicate_user_images(messages)
+ assert messages == before
+ assert wire[0]["content"][-1] == image
+ assert wire[0]["content"][0] == messages[0]["content"][0]
+ assert "Repeated image" in wire[2]["content"][-1]["text"]
+ assert wire[1] == messages[1]
+ assert wire[3] == messages[3]
+ # Projecting again is harmless; a later history cut still has real bytes.
+ assert deduplicate_user_images(wire) == wire
+ assert deduplicate_user_images(messages[2:])[0]["content"][-1] == image
+
+
+def test_remote_images_unique_images_and_detail_options_are_preserved():
+ blocks = [
+ {"type": "image_url", "image_url": {"url": "https://example.com/changing.png"}},
+ {"type": "image_url", "image_url": {"url": "https://example.com/changing.png"}},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj", "detail": "low"}},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj", "detail": "high"}},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,ZGVm"}},
+ ]
+ messages = [{"role": "user", "content": blocks}]
+ assert [
+ part for part in deduplicate_user_images(messages)[0]["content"] if part["type"] != "text"
+ ] == blocks
+
+
+def test_projection_keeps_previous_wire_prefix_when_a_turn_is_appended():
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}
+ messages = [{"role": "user", "content": [image]}]
+ first = deduplicate_user_images(messages)
+ later = deduplicate_user_images([*messages, {"role": "user", "content": [image]}])
+ assert later[: len(first)] == first
+
+
+def test_duplicate_inside_one_message_references_retained_image_and_preserves_count():
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}
+ wire = deduplicate_user_images([{"role": "user", "content": [image, image, image]}])
+ assert wire[0]["content"][1] == image
+ assert len(wire[0]["content"]) == 4
+ label = wire[0]["content"][0]["text"].removeprefix("[Image ").removesuffix("]")
+ assert all(label in part["text"] for part in wire[0]["content"][2:])
+
+
+def test_changed_mime_and_provider_options_are_not_duplicates():
+ image = {
+ "type": "image",
+ "source": {"type": "base64", "media_type": "image/png", "data": "YWJj"},
+ }
+ other = deepcopy(image)
+ other["source"]["media_type"] = "image/jpeg"
+ with_cache = {**image, "cache_control": {"type": "ephemeral"}}
+ messages = [{"role": "user", "content": [image, other, with_cache]}]
+ assert [
+ part for part in deduplicate_user_images(messages)[0]["content"] if part["type"] != "text"
+ ] == [image, other, with_cache]
+
+
+def test_stable_label_references_inline_image_among_remote_images():
+ remote = {"type": "image_url", "image_url": {"url": "https://example.org/image.png"}}
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}
+ wire = deduplicate_user_images([{"role": "user", "content": [remote, image, image]}])
+ assert wire[0]["content"][0] == remote
+ assert wire[0]["content"][2] == image
+ label = wire[0]["content"][1]["text"].removeprefix("[Image ").removesuffix("]")
+ assert label in wire[0]["content"][3]["text"]