Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(session): budget projected image replay before compaction

YizukiAmee747f28705c4
diff --git a/deeptutor/agents/loop/agent_loop.py b/deeptutor/agents/loop/agent_loop.py
index 7f1a76821..1a3b9f7da 100644
--- a/deeptutor/agents/loop/agent_loop.py
+++ b/deeptutor/agents/loop/agent_loop.py
@@ -1047,7 +1047,15 @@ class AgentLoop:
tool_choice: str | None = None,
) -> LLMCallResult:
request_messages = with_transient_model_messages(messages, self._transient_model_messages)
- await self.pipeline._guard_context_window(request_messages, self.stream)
+ # Guard the same request representation we send. Raw repeated images
+ # must not cause tool evidence to be snipped before image projection.
+ projected_messages = deduplicate_user_images(
+ [
+ {key: value for key, value in message.items() if key != "_context_snapshot"}
+ for message in request_messages
+ ]
+ )
+ await self.pipeline._guard_context_window(projected_messages, self.stream)
stage = self.stage
call_id = new_call_id(f"{self.source}-{stage}")
trace_meta = build_trace_metadata(
@@ -1071,12 +1079,7 @@ class AgentLoop:
kwargs: dict[str, Any] = {
"model": self.pipeline.model,
- "messages": deduplicate_user_images(
- [
- {key: value for key, value in message.items() if key != "_context_snapshot"}
- for message in request_messages
- ]
- ),
+ "messages": projected_messages,
"stream": True,
**self.pipeline._completion_kwargs(max_tokens=max_tokens),
}
diff --git a/deeptutor/services/session/context_builder.py b/deeptutor/services/session/context_builder.py
index 74479b1f0..624040944 100644
--- a/deeptutor/services/session/context_builder.py
+++ b/deeptutor/services/session/context_builder.py
@@ -4,6 +4,7 @@ from __future__ import annotations
from dataclasses import dataclass
import json
+import logging
from typing import Any, Awaitable, Callable
from deeptutor.agents.base_agent import BaseAgent
@@ -14,6 +15,7 @@ from deeptutor.services.llm.context_window import (
coerce_positive_int,
resolve_effective_context_window,
)
+from deeptutor.services.llm.image_replay import deduplicate_user_images
from deeptutor.services.prompt.language import language_label
from .ask_user_trace import (
@@ -24,6 +26,8 @@ from .model_history import history_groups, model_turn, replay_history
from .protocol import SessionStoreProtocol
from .provider_response_state import normalize_provider_response_state
+logger = logging.getLogger(__name__)
+
#: When the summarizer's output lands within this fraction of its hard token
#: cap, assume the provider cut it mid-sentence and trim the partial tail.
TRUNCATION_GUARD_RATIO = 0.95
@@ -42,7 +46,8 @@ MAX_RAW_REBUILD_TOKENS = 131_072
# Planning allowance per image, not provider-reported usage. Counting encoded
# image bytes as text can exhaust the entire history budget on one screenshot.
# Reserve nonzero headroom for vision while keeping it independent of PNG/JPEG
-# compression and URL length. Repeated images each consume this allowance.
+# compression and URL length. Apply request projection before counting so only
+# images retained in the request consume this allowance; references remain text.
IMAGE_CONTEXT_TOKEN_ESTIMATE = 4096
@@ -90,6 +95,17 @@ def _count_model_context_tokens(value: Any) -> int:
return count_tokens(serialized) + image_count * IMAGE_CONTEXT_TOKEN_ESTIMATE
+def _count_projected_model_tokens(replay: list[dict[str, Any]]) -> int:
+ projected_tokens = _count_model_context_tokens(deduplicate_user_images(replay))
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug(
+ "History budget: raw_replay_tokens=%d projected_replay_tokens=%d",
+ _count_model_context_tokens(replay),
+ projected_tokens,
+ )
+ return projected_tokens
+
+
def trim_incomplete_tail(text: str) -> str:
"""Drop the trailing partial line from output that hit a hard token cap.
@@ -284,21 +300,35 @@ class ContextBuilder:
self,
messages: list[dict[str, Any]],
recent_budget: int,
+ route: dict[str, str] | None = None,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+ groups = history_groups(messages)
+ if any(model_turn(row) is not None for row in messages):
+ # Rebuild each group once, then find the longest fitting suffix.
+ # Images are shared across turns: adding per-group costs overcounts
+ # repeats, while replaying every growing tail is quadratic work.
+ replay_groups = [replay_history(group, route=route) for group in groups]
+ start, end = 0, len(groups) - 1
+ while start < end:
+ middle = (start + end) // 2
+ candidate = [message for group in replay_groups[middle:] for message in group]
+ if _count_projected_model_tokens(candidate) <= recent_budget:
+ end = middle
+ else:
+ start = middle + 1
+ # Always retain the newest whole turn, even if it alone overflows.
+ cutoff = sum(len(group) for group in groups[:start])
+ return messages[:cutoff], messages[cutoff:]
+
selected: list[dict[str, Any]] = []
total = 0
- for group in reversed(history_groups(messages)):
- if any(model_turn(row) is not None for row in group):
- tokens = self._model_tokens(group)
- else:
- tokens = 0
- for item in group:
- content = str(item.get("content", "") or "")
- clarification = extract_ask_user_clarifications(item)
- tokens += count_tokens(
- f"{content}\n{clarification}" if clarification else content
- )
- tokens += _provider_response_state_tokens(item)
+ for group in reversed(groups):
+ tokens = 0
+ for item in group:
+ content = str(item.get("content", "") or "")
+ clarification = extract_ask_user_clarifications(item)
+ tokens += count_tokens(f"{content}\n{clarification}" if clarification else content)
+ tokens += _provider_response_state_tokens(item)
if selected and total + tokens > recent_budget:
break
selected[0:0] = group
@@ -306,9 +336,14 @@ class ContextBuilder:
cutoff = len(messages) - len(selected)
return messages[:cutoff], selected
- def _model_tokens(self, messages: list[dict[str, Any]], summary: str = "") -> int:
+ def _model_tokens(
+ self,
+ messages: list[dict[str, Any]],
+ summary: str = "",
+ route: dict[str, str] | None = None,
+ ) -> int:
if any(model_turn(row) is not None for row in messages):
- return _count_model_context_tokens(replay_history(messages, summary))
+ return _count_projected_model_tokens(replay_history(messages, summary, route))
return count_tokens(build_history_text(self._build_history(summary, messages))) + sum(
_provider_response_state_tokens(row) for row in messages
)
@@ -473,10 +508,12 @@ class ContextBuilder:
f"{target_tokens} tokens. Output only the summary; do not call tools."
)
replay_kwargs = {
- "messages": [
- *replay_request["messages"],
- {"role": "user", "content": instruction},
- ],
+ "messages": deduplicate_user_images(
+ [
+ *replay_request["messages"],
+ {"role": "user", "content": instruction},
+ ]
+ ),
"tools": replay_request.get("tools"),
}
async for _c in agent.stream_llm(
@@ -542,16 +579,16 @@ class ContextBuilder:
item for item in messages if int(item.get("id", 0) or 0) > summary_up_to_msg_id
]
+ binding = getattr(llm_config, "binding", None)
+ route = (
+ {"provider": binding, "model": llm_config.model} if isinstance(binding, str) else None
+ )
current_history = self._build_history(stored_summary, unsummarized)
- current_tokens = self._model_tokens(unsummarized, stored_summary)
+ current_tokens = self._model_tokens(unsummarized, stored_summary, route)
previous_model_turn = next(
(record for row in reversed(messages) if (record := model_turn(row)) is not None),
None,
)
- binding = getattr(llm_config, "binding", None)
- route = (
- {"provider": binding, "model": llm_config.model} if isinstance(binding, str) else None
- )
if current_tokens <= budget:
return ContextBuildResult(
conversation_history=current_history,
@@ -565,7 +602,7 @@ class ContextBuilder:
)
older_unsummarized, recent_messages = self._select_recent_messages(
- unsummarized, recent_budget
+ unsummarized, recent_budget, route=route
)
# Everything not retained verbatim: previously summarized messages
# plus the older unsummarized turns.
@@ -640,7 +677,8 @@ class ContextBuilder:
retained_rows = unsummarized
retained_groups = history_groups(retained_rows)
while (
- len(retained_groups) > 1 and self._model_tokens(retained_rows, stored_summary) > budget
+ len(retained_groups) > 1
+ and self._model_tokens(retained_rows, stored_summary, route) > budget
):
retained_groups.pop(0)
retained_rows = [row for group in retained_groups for row in group]
@@ -652,7 +690,7 @@ class ContextBuilder:
conversation_summary=stored_summary,
context_text=final_text,
events=events,
- token_count=self._model_tokens(retained_rows, stored_summary),
+ token_count=self._model_tokens(retained_rows, stored_summary, route),
budget=budget,
model_history=replay_history(retained_rows, stored_summary, route),
previous_model_turn=previous_model_turn,
diff --git a/tests/agents/chat/test_agent_loop.py b/tests/agents/chat/test_agent_loop.py
index bdd1840e4..f301e4265 100644
--- a/tests/agents/chat/test_agent_loop.py
+++ b/tests/agents/chat/test_agent_loop.py
@@ -3781,6 +3781,66 @@ async def test_wire_deduplicates_user_images_without_mutating_model_history(monk
]
[email protected]("distinct", [False, True])
+async def test_request_guard_budgets_projected_images_and_preserves_tool_history(
+ monkeypatch, distinct
+):
+ from copy import deepcopy
+
+ messages = []
+ for turn in range(4):
+ images = [
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": f"data:image/png;base64,image-{4 * turn + i if distinct else i}"
+ },
+ }
+ for i in range(4)
+ ]
+ messages.extend(
+ [
+ {"role": "user", "content": images},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": f"call-{turn}",
+ "type": "function",
+ "function": {"name": "web_search", "arguments": '{"query":"source"}'},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": f"call-{turn}", "content": f"evidence {turn}"},
+ {"role": "assistant", "content": "answer"},
+ ]
+ )
+ original = deepcopy(messages)
+ client = _ScriptedChatClient([[_llm_chunk(content="answer")]])
+ pipeline = AgenticChatPipeline(language="en")
+ pipeline.llm_config = SimpleNamespace(context_window=60000, max_tokens=4096)
+ pipeline.registry = _Registry()
+ pipeline._model_turn_start = 0
+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: [])
+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)
+ monkeypatch.setattr(pipeline, "_build_loop_messages", lambda *_args, **_kwargs: messages)
+
+ await _run(pipeline, UnifiedContext(session_id="s", user_message="again"))
+
+ wire_tools = [
+ message["content"] for message in client.calls[0]["messages"] if message["role"] == "tool"
+ ]
+ expected = (
+ [pipeline._tool_result_snip_marker()] * 4
+ if distinct
+ else [f"evidence {turn}" for turn in range(4)]
+ )
+ assert wire_tools == expected
+ assert messages[: len(original)] == original
+
+
@pytest.mark.asyncio
async def test_image_fallback_strips_all_canonical_copies_after_wire_deduplication(monkeypatch):
from copy import deepcopy
diff --git a/tests/services/session/test_image_context_budget.py b/tests/services/session/test_image_context_budget.py
index f470c4e22..2700d22b9 100644
--- a/tests/services/session/test_image_context_budget.py
+++ b/tests/services/session/test_image_context_budget.py
@@ -8,6 +8,7 @@ from types import SimpleNamespace
import unittest
from unittest.mock import AsyncMock
+from deeptutor.services.llm.image_replay import deduplicate_user_images
from deeptutor.services.session.context_builder import ContextBuilder, count_tokens
from deeptutor.services.session.model_history import replay_history
@@ -53,6 +54,10 @@ def image(url=RANDOM_IMAGE):
return {"type": "image_url", "image_url": {"url": url, "detail": "auto"}}
+def distinct_image(index):
+ return image("data:image/png;base64," + base64.b64encode(bytes([index])).decode())
+
+
class ImageTokenTests(unittest.TestCase):
def setUp(self):
self.builder = ContextBuilder(None)
@@ -60,12 +65,15 @@ class ImageTokenTests(unittest.TestCase):
def test_image_encoding_size_does_not_change_budget(self):
small = self.builder._model_tokens(rows_with_parts([image("data:image/png;base64,AA==")]))
large = self.builder._model_tokens(rows_with_parts([image()]))
- self.assertEqual(small, large)
- self.assertGreaterEqual(large, IMAGE_ALLOWANCE)
+ # Stable image labels also consume a small amount of text; their
+ # token counts need not match for two different image identities.
+ for tokens in (small, large):
+ self.assertGreaterEqual(tokens, IMAGE_ALLOWANCE)
+ self.assertLess(tokens, IMAGE_ALLOWANCE + 200)
def test_reader_three_screenshots_fit_history_budget(self):
rows = rows_with_parts(
- [{"type": "text", "text": "继续解释贝叶斯"}, *[image() for _ in range(3)]]
+ [{"type": "text", "text": "继续解释贝叶斯"}, *[distinct_image(i) for i in range(3)]]
)
tokens = self.builder._model_tokens(rows)
self.assertGreaterEqual(tokens, 3 * IMAGE_ALLOWANCE)
@@ -76,7 +84,9 @@ class ImageTokenTests(unittest.TestCase):
remote = self.builder._model_tokens(
rows_with_parts([image("https://example.test/image.png")])
)
- self.assertEqual(local, remote)
+ for tokens in (local, remote):
+ self.assertGreaterEqual(tokens, IMAGE_ALLOWANCE)
+ self.assertLess(tokens, IMAGE_ALLOWANCE + 200)
def test_responses_and_anthropic_images_have_nonzero_allowance(self):
parts = [
@@ -115,10 +125,65 @@ class ImageTokenTests(unittest.TestCase):
self.assertEqual(rows, original)
self.assertEqual(replay_history(rows), replay_before)
- def test_each_image_occurrence_is_counted(self):
+ def test_identical_images_share_allowance_but_references_still_count(self):
one = self.builder._model_tokens(rows_with_parts([image()]))
two = self.builder._model_tokens(rows_with_parts([image(), image()]))
- self.assertGreaterEqual(two - one, IMAGE_ALLOWANCE)
+ self.assertGreater(two, one)
+ self.assertLess(two - one, 200)
+
+ def test_budget_diagnostics_distinguish_raw_and_projected_replay(self):
+ rows = rows_with_parts([distinct_image(1), distinct_image(1)])
+ with self.assertLogs("deeptutor.services.session.context_builder", level="DEBUG") as logs:
+ effective = self.builder._model_tokens(rows)
+
+ raw_tokens = logs.records[0].args[0]
+ projected_tokens = logs.records[0].args[1]
+ self.assertGreater(raw_tokens, 2 * IMAGE_ALLOWANCE)
+ self.assertEqual(projected_tokens, effective)
+ self.assertLess(effective, IMAGE_ALLOWANCE + 200)
+
+ def test_distinct_options_and_mutable_urls_each_keep_an_image_allowance(self):
+ low = distinct_image(1)
+ low["image_url"]["detail"] = "low"
+ high = distinct_image(1)
+ high["image_url"]["detail"] = "high"
+ jpeg = image("data:image/jpeg;base64,AQ==")
+ cropped = {**low, "crop": [0, 0, 10, 10]}
+ remote = image("https://example.test/changing.png")
+ parts = [low, high, jpeg, cropped, remote, remote]
+ tokens = self.builder._model_tokens(rows_with_parts(parts))
+ self.assertGreaterEqual(tokens, 6 * IMAGE_ALLOWANCE)
+ self.assertLess(tokens, 6 * IMAGE_ALLOWANCE + 500)
+
+ def test_recent_selection_keeps_the_longest_whole_turn_suffix(self):
+ rows = []
+ for i in range(4):
+ rows += rows_with_parts([distinct_image(i)], start=2 * i + 1)
+ for budget, retained_turns in [(0, 1), (5000, 1), (9000, 2), (13000, 3), (18000, 4)]:
+ with self.subTest(budget=budget):
+ older, recent = self.builder._select_recent_messages(rows, budget)
+ self.assertEqual(recent, rows[-2 * retained_turns :])
+ self.assertEqual(older + recent, rows)
+
+ def test_mixed_history_selection_uses_replay_for_every_candidate(self):
+ rows = rows_with_parts([distinct_image(1)])
+ rows += [
+ {"id": 3, "role": "user", "content": "earlier question"},
+ {
+ "id": 4,
+ "role": "assistant",
+ "content": "",
+ "metadata": {"provider_response_state": {"reasoning_content": "private " * 7000}},
+ },
+ {"id": 5, "role": "user", "content": "recent question"},
+ ]
+ # Empty legacy display rows are omitted from replay. Switching to
+ # transcript accounting for a legacy-only candidate would count its
+ # unused state and incorrectly discard the earlier model turn.
+ self.assertGreater(self.builder._model_tokens(rows[2:]), 5000)
+ older, recent = self.builder._select_recent_messages(rows, 5000)
+ self.assertEqual(older, [])
+ self.assertEqual(recent, rows)
class BuilderTests(unittest.IsolatedAsyncioTestCase):
@@ -155,12 +220,64 @@ class BuilderTests(unittest.IsolatedAsyncioTestCase):
async def test_many_images_still_consume_finite_budget(self):
rows = []
for i in range(25):
- rows += rows_with_parts([image("data:image/png;base64,AA==")], start=i * 2 + 1)
+ rows += rows_with_parts([distinct_image(i)], start=i * 2 + 1)
builder, store = self.builder(rows)
result = await builder.build(session_id="test", llm_config=CONFIG)
builder._summarize.assert_awaited_once()
self.assertLess(result.token_count, result.budget)
+ async def test_repeated_images_fit_without_summarizing_or_changing_durable_history(self):
+ rows = []
+ for i in range(4):
+ rows += rows_with_parts([distinct_image(j) for j in range(4)], start=2 * i + 1)
+ before = copy.deepcopy(rows)
+ builder, store = self.builder(rows)
+ config = SimpleNamespace(model="test", context_window=60000, max_tokens=4096)
+
+ result = await builder.build(session_id="test", llm_config=config)
+
+ builder._summarize.assert_not_awaited()
+ store.update_summary.assert_not_awaited()
+ self.assertGreaterEqual(result.token_count, 4 * IMAGE_ALLOWANCE)
+ self.assertLess(result.token_count, result.budget)
+ self.assertEqual(result.model_history, replay_history(rows))
+ self.assertEqual(rows, before)
+
+ async def test_recent_selection_budgets_repeated_images_together(self):
+ old = rows_with_parts("old evidence " * 60000)
+ recent = []
+ for i in range(3):
+ recent += rows_with_parts([distinct_image(1)], start=3 + 2 * i)
+ builder, store = self.builder(old + recent)
+ config = SimpleNamespace(model="test", context_window=24000, max_tokens=1024)
+
+ result = await builder.build(session_id="test", llm_config=config)
+
+ # The three retained turns share one image, fitting the 5040-token
+ # recent budget. Counting them separately wrongly summarizes two.
+ store.update_summary.assert_awaited_once_with("test", "compact summary", 2)
+ self.assertEqual(result.model_history[1:], replay_history(recent))
+
+ async def test_failed_summary_reprojects_images_after_removing_canonical_turn(self):
+ rows = rows_with_parts([distinct_image(1), {"type": "text", "text": "old " * 20000}])
+ recent = rows_with_parts([distinct_image(1)], start=3, tools=True)
+ recent += rows_with_parts([distinct_image(1)], start=5)
+ rows += recent
+ before = copy.deepcopy(rows)
+ builder, store = self.builder(rows)
+ builder._summarize.side_effect = RuntimeError("offline simulated failure")
+ config = SimpleNamespace(model="test", context_window=16000, max_tokens=1024)
+
+ result = await builder.build(session_id="test", llm_config=config)
+
+ store.update_summary.assert_not_awaited()
+ self.assertEqual(result.model_history, replay_history(recent))
+ projected = deduplicate_user_images(result.model_history)
+ self.assertEqual(projected[0]["content"][-1], distinct_image(1))
+ self.assertIn("Repeated image", projected[-2]["content"][-1]["text"])
+ self.assertTrue(any(message["role"] == "tool" for message in projected))
+ self.assertEqual(rows, before)
+
async def test_summary_failure_keeps_existing_watermark(self):
rows = rows_with_parts("long text " * 60000)
rows += rows_with_parts([image()], start=3)
diff --git a/tests/services/session/test_model_history.py b/tests/services/session/test_model_history.py
index 8448091f8..d9d37ecf7 100644
--- a/tests/services/session/test_model_history.py
+++ b/tests/services/session/test_model_history.py
@@ -175,6 +175,84 @@ async def test_summary_instruction_is_appended_after_original_messages_and_tools
assert request == original
+async def test_summary_replay_projects_duplicate_images_without_losing_provider_state(monkeypatch):
+ captured = {}
+
+ async def stream(_self, **kwargs):
+ captured.update(kwargs)
+ yield "Summary"
+
+ monkeypatch.setattr(_ContextSummaryAgent, "stream_llm", stream)
+ record = turn_record()
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}
+ record["messages"][0]["content"] = [image]
+ record["messages"][-1]["thinking_blocks"] = [
+ {"type": "thinking", "thinking": "private thought", "signature": "signed"}
+ ]
+ request = {
+ "messages": [
+ {"role": "system", "content": record["system"]},
+ *record["messages"],
+ {"role": "user", "content": [deepcopy(image)]},
+ ],
+ "tools": record["tools"],
+ }
+ original = deepcopy(request)
+
+ await ContextBuilder(MagicMock())._summarize(
+ session_id="s",
+ language="en",
+ source_text="fallback transcript",
+ summary_budget=256,
+ replay_request=request,
+ )
+
+ messages = captured["messages"]
+ assert messages[1]["content"][-1] == image
+ assert all(part["type"] == "text" for part in messages[-2]["content"])
+ assert "Repeated image" in messages[-2]["content"][0]["text"]
+ assert messages[2:5] == record["messages"][1:]
+ assert captured["tools"] == record["tools"]
+ assert messages[-1]["role"] == "user"
+ assert request == original
+
+
+async def test_budget_uses_the_replayed_route_and_preserves_stored_signed_state():
+ record = turn_record()
+ record["messages"][-1]["thinking_blocks"] = [
+ {"type": "thinking", "thinking": "private thought " * 4000, "signature": "signed"}
+ ]
+ records = rows(record)
+ before = deepcopy(records)
+ store = MagicMock()
+ store.get_session = AsyncMock(
+ return_value={"compressed_summary": "", "summary_up_to_msg_id": 0}
+ )
+ store.get_messages_for_context = AsyncMock(return_value=records)
+ store.update_summary = AsyncMock()
+ builder = ContextBuilder(store)
+ builder._summarize = AsyncMock(return_value=("Summary", []))
+
+ switched = await builder.build(
+ session_id="s",
+ llm_config=SimpleNamespace(binding="openai", model="other-model", context_window=16000),
+ )
+
+ builder._summarize.assert_not_awaited()
+ assert switched.token_count < switched.budget
+ assert "thinking_blocks" not in switched.model_history[-1]
+ assert switched.model_history[2]["content"] == "tool evidence"
+
+ await builder.build(
+ session_id="s",
+ llm_config=SimpleNamespace(binding="openai", model="gpt-test", context_window=16000),
+ )
+ builder._summarize.assert_awaited_once()
+ assert records == before
+
+
@pytest.mark.asyncio
async def test_branch_replay_and_public_export_are_isolated(tmp_path):
from deeptutor.services.session.sqlite_store import SQLiteSessionStore