DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(session): budget projected image replay before compaction
YizukiAmee747f28705c4
diff --git a/deeptutor/agents/loop/agent_loop.py b/deeptutor/agents/loop/agent_loop.pyindex 7f1a76821..1a3b9f7da 100644--- a/deeptutor/agents/loop/agent_loop.py+++ b/deeptutor/agents/loop/agent_loop.py@@ -1047,7 +1047,15 @@ class AgentLoop:tool_choice: str | None = None,) -> LLMCallResult:request_messages = with_transient_model_messages(messages, self._transient_model_messages)- await self.pipeline._guard_context_window(request_messages, self.stream)+ # Guard the same request representation we send. Raw repeated images+ # must not cause tool evidence to be snipped before image projection.+ projected_messages = deduplicate_user_images(+ [+ {key: value for key, value in message.items() if key != "_context_snapshot"}+ for message in request_messages+ ]+ )+ await self.pipeline._guard_context_window(projected_messages, self.stream)stage = self.stagecall_id = new_call_id(f"{self.source}-{stage}")trace_meta = build_trace_metadata(@@ -1071,12 +1079,7 @@ class AgentLoop:kwargs: dict[str, Any] = {"model": self.pipeline.model,- "messages": deduplicate_user_images(- [- {key: value for key, value in message.items() if key != "_context_snapshot"}- for message in request_messages- ]- ),+ "messages": projected_messages,"stream": True,**self.pipeline._completion_kwargs(max_tokens=max_tokens),}diff --git a/deeptutor/services/session/context_builder.py b/deeptutor/services/session/context_builder.pyindex 74479b1f0..624040944 100644--- a/deeptutor/services/session/context_builder.py+++ b/deeptutor/services/session/context_builder.py@@ -4,6 +4,7 @@ from __future__ import annotationsfrom dataclasses import dataclassimport json+import loggingfrom typing import Any, Awaitable, Callablefrom deeptutor.agents.base_agent import BaseAgent@@ -14,6 +15,7 @@ from deeptutor.services.llm.context_window import (coerce_positive_int,resolve_effective_context_window,)+from deeptutor.services.llm.image_replay import deduplicate_user_imagesfrom deeptutor.services.prompt.language import language_labelfrom .ask_user_trace import (@@ -24,6 +26,8 @@ from .model_history import history_groups, model_turn, replay_historyfrom .protocol import SessionStoreProtocolfrom .provider_response_state import normalize_provider_response_state+logger = logging.getLogger(__name__)+#: When the summarizer's output lands within this fraction of its hard token#: cap, assume the provider cut it mid-sentence and trim the partial tail.TRUNCATION_GUARD_RATIO = 0.95@@ -42,7 +46,8 @@ MAX_RAW_REBUILD_TOKENS = 131_072# Planning allowance per image, not provider-reported usage. Counting encoded# image bytes as text can exhaust the entire history budget on one screenshot.# Reserve nonzero headroom for vision while keeping it independent of PNG/JPEG-# compression and URL length. Repeated images each consume this allowance.+# compression and URL length. Apply request projection before counting so only+# images retained in the request consume this allowance; references remain text.IMAGE_CONTEXT_TOKEN_ESTIMATE = 4096@@ -90,6 +95,17 @@ def _count_model_context_tokens(value: Any) -> int:return count_tokens(serialized) + image_count * IMAGE_CONTEXT_TOKEN_ESTIMATE+def _count_projected_model_tokens(replay: list[dict[str, Any]]) -> int:+ projected_tokens = _count_model_context_tokens(deduplicate_user_images(replay))+ if logger.isEnabledFor(logging.DEBUG):+ logger.debug(+ "History budget: raw_replay_tokens=%d projected_replay_tokens=%d",+ _count_model_context_tokens(replay),+ projected_tokens,+ )+ return projected_tokens++def trim_incomplete_tail(text: str) -> str:"""Drop the trailing partial line from output that hit a hard token cap.@@ -284,21 +300,35 @@ class ContextBuilder:self,messages: list[dict[str, Any]],recent_budget: int,+ route: dict[str, str] | None = None,) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:+ groups = history_groups(messages)+ if any(model_turn(row) is not None for row in messages):+ # Rebuild each group once, then find the longest fitting suffix.+ # Images are shared across turns: adding per-group costs overcounts+ # repeats, while replaying every growing tail is quadratic work.+ replay_groups = [replay_history(group, route=route) for group in groups]+ start, end = 0, len(groups) - 1+ while start < end:+ middle = (start + end) // 2+ candidate = [message for group in replay_groups[middle:] for message in group]+ if _count_projected_model_tokens(candidate) <= recent_budget:+ end = middle+ else:+ start = middle + 1+ # Always retain the newest whole turn, even if it alone overflows.+ cutoff = sum(len(group) for group in groups[:start])+ return messages[:cutoff], messages[cutoff:]+selected: list[dict[str, Any]] = []total = 0- for group in reversed(history_groups(messages)):- if any(model_turn(row) is not None for row in group):- tokens = self._model_tokens(group)- else:- tokens = 0- for item in group:- content = str(item.get("content", "") or "")- clarification = extract_ask_user_clarifications(item)- tokens += count_tokens(- f"{content}\n{clarification}" if clarification else content- )- tokens += _provider_response_state_tokens(item)+ for group in reversed(groups):+ tokens = 0+ for item in group:+ content = str(item.get("content", "") or "")+ clarification = extract_ask_user_clarifications(item)+ tokens += count_tokens(f"{content}\n{clarification}" if clarification else content)+ tokens += _provider_response_state_tokens(item)if selected and total + tokens > recent_budget:breakselected[0:0] = group@@ -306,9 +336,14 @@ class ContextBuilder:cutoff = len(messages) - len(selected)return messages[:cutoff], selected- def _model_tokens(self, messages: list[dict[str, Any]], summary: str = "") -> int:+ def _model_tokens(+ self,+ messages: list[dict[str, Any]],+ summary: str = "",+ route: dict[str, str] | None = None,+ ) -> int:if any(model_turn(row) is not None for row in messages):- return _count_model_context_tokens(replay_history(messages, summary))+ return _count_projected_model_tokens(replay_history(messages, summary, route))return count_tokens(build_history_text(self._build_history(summary, messages))) + sum(_provider_response_state_tokens(row) for row in messages)@@ -473,10 +508,12 @@ class ContextBuilder:f"{target_tokens} tokens. Output only the summary; do not call tools.")replay_kwargs = {- "messages": [- *replay_request["messages"],- {"role": "user", "content": instruction},- ],+ "messages": deduplicate_user_images(+ [+ *replay_request["messages"],+ {"role": "user", "content": instruction},+ ]+ ),"tools": replay_request.get("tools"),}async for _c in agent.stream_llm(@@ -542,16 +579,16 @@ class ContextBuilder:item for item in messages if int(item.get("id", 0) or 0) > summary_up_to_msg_id]+ binding = getattr(llm_config, "binding", None)+ route = (+ {"provider": binding, "model": llm_config.model} if isinstance(binding, str) else None+ )current_history = self._build_history(stored_summary, unsummarized)- current_tokens = self._model_tokens(unsummarized, stored_summary)+ current_tokens = self._model_tokens(unsummarized, stored_summary, route)previous_model_turn = next((record for row in reversed(messages) if (record := model_turn(row)) is not None),None,)- binding = getattr(llm_config, "binding", None)- route = (- {"provider": binding, "model": llm_config.model} if isinstance(binding, str) else None- )if current_tokens <= budget:return ContextBuildResult(conversation_history=current_history,@@ -565,7 +602,7 @@ class ContextBuilder:)older_unsummarized, recent_messages = self._select_recent_messages(- unsummarized, recent_budget+ unsummarized, recent_budget, route=route)# Everything not retained verbatim: previously summarized messages# plus the older unsummarized turns.@@ -640,7 +677,8 @@ class ContextBuilder:retained_rows = unsummarizedretained_groups = history_groups(retained_rows)while (- len(retained_groups) > 1 and self._model_tokens(retained_rows, stored_summary) > budget+ len(retained_groups) > 1+ and self._model_tokens(retained_rows, stored_summary, route) > budget):retained_groups.pop(0)retained_rows = [row for group in retained_groups for row in group]@@ -652,7 +690,7 @@ class ContextBuilder:conversation_summary=stored_summary,context_text=final_text,events=events,- token_count=self._model_tokens(retained_rows, stored_summary),+ token_count=self._model_tokens(retained_rows, stored_summary, route),budget=budget,model_history=replay_history(retained_rows, stored_summary, route),previous_model_turn=previous_model_turn,diff --git a/tests/agents/chat/test_agent_loop.py b/tests/agents/chat/test_agent_loop.pyindex bdd1840e4..f301e4265 100644--- a/tests/agents/chat/test_agent_loop.py+++ b/tests/agents/chat/test_agent_loop.py@@ -3781,6 +3781,66 @@ async def test_wire_deduplicates_user_images_without_mutating_model_history(monk][email protected]("distinct", [False, True])+async def test_request_guard_budgets_projected_images_and_preserves_tool_history(+ monkeypatch, distinct+):+ from copy import deepcopy++ messages = []+ for turn in range(4):+ images = [+ {+ "type": "image_url",+ "image_url": {+ "url": f"data:image/png;base64,image-{4 * turn + i if distinct else i}"+ },+ }+ for i in range(4)+ ]+ messages.extend(+ [+ {"role": "user", "content": images},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": f"call-{turn}",+ "type": "function",+ "function": {"name": "web_search", "arguments": '{"query":"source"}'},+ }+ ],+ },+ {"role": "tool", "tool_call_id": f"call-{turn}", "content": f"evidence {turn}"},+ {"role": "assistant", "content": "answer"},+ ]+ )+ original = deepcopy(messages)+ client = _ScriptedChatClient([[_llm_chunk(content="answer")]])+ pipeline = AgenticChatPipeline(language="en")+ pipeline.llm_config = SimpleNamespace(context_window=60000, max_tokens=4096)+ pipeline.registry = _Registry()+ pipeline._model_turn_start = 0+ monkeypatch.setattr(pipeline, "_compose_enabled_tools", lambda _context: [])+ monkeypatch.setattr(pipeline, "_build_openai_client", lambda: client)+ monkeypatch.setattr(pipeline, "_build_loop_messages", lambda *_args, **_kwargs: messages)++ await _run(pipeline, UnifiedContext(session_id="s", user_message="again"))++ wire_tools = [+ message["content"] for message in client.calls[0]["messages"] if message["role"] == "tool"+ ]+ expected = (+ [pipeline._tool_result_snip_marker()] * 4+ if distinct+ else [f"evidence {turn}" for turn in range(4)]+ )+ assert wire_tools == expected+ assert messages[: len(original)] == original++@pytest.mark.asyncioasync def test_image_fallback_strips_all_canonical_copies_after_wire_deduplication(monkeypatch):from copy import deepcopydiff --git a/tests/services/session/test_image_context_budget.py b/tests/services/session/test_image_context_budget.pyindex f470c4e22..2700d22b9 100644--- a/tests/services/session/test_image_context_budget.py+++ b/tests/services/session/test_image_context_budget.py@@ -8,6 +8,7 @@ from types import SimpleNamespaceimport unittestfrom unittest.mock import AsyncMock+from deeptutor.services.llm.image_replay import deduplicate_user_imagesfrom deeptutor.services.session.context_builder import ContextBuilder, count_tokensfrom deeptutor.services.session.model_history import replay_history@@ -53,6 +54,10 @@ def image(url=RANDOM_IMAGE):return {"type": "image_url", "image_url": {"url": url, "detail": "auto"}}+def distinct_image(index):+ return image("data:image/png;base64," + base64.b64encode(bytes([index])).decode())++class ImageTokenTests(unittest.TestCase):def setUp(self):self.builder = ContextBuilder(None)@@ -60,12 +65,15 @@ class ImageTokenTests(unittest.TestCase):def test_image_encoding_size_does_not_change_budget(self):small = self.builder._model_tokens(rows_with_parts([image("data:image/png;base64,AA==")]))large = self.builder._model_tokens(rows_with_parts([image()]))- self.assertEqual(small, large)- self.assertGreaterEqual(large, IMAGE_ALLOWANCE)+ # Stable image labels also consume a small amount of text; their+ # token counts need not match for two different image identities.+ for tokens in (small, large):+ self.assertGreaterEqual(tokens, IMAGE_ALLOWANCE)+ self.assertLess(tokens, IMAGE_ALLOWANCE + 200)def test_reader_three_screenshots_fit_history_budget(self):rows = rows_with_parts(- [{"type": "text", "text": "继续解释贝叶斯"}, *[image() for _ in range(3)]]+ [{"type": "text", "text": "继续解释贝叶斯"}, *[distinct_image(i) for i in range(3)]])tokens = self.builder._model_tokens(rows)self.assertGreaterEqual(tokens, 3 * IMAGE_ALLOWANCE)@@ -76,7 +84,9 @@ class ImageTokenTests(unittest.TestCase):remote = self.builder._model_tokens(rows_with_parts([image("https://example.test/image.png")]))- self.assertEqual(local, remote)+ for tokens in (local, remote):+ self.assertGreaterEqual(tokens, IMAGE_ALLOWANCE)+ self.assertLess(tokens, IMAGE_ALLOWANCE + 200)def test_responses_and_anthropic_images_have_nonzero_allowance(self):parts = [@@ -115,10 +125,65 @@ class ImageTokenTests(unittest.TestCase):self.assertEqual(rows, original)self.assertEqual(replay_history(rows), replay_before)- def test_each_image_occurrence_is_counted(self):+ def test_identical_images_share_allowance_but_references_still_count(self):one = self.builder._model_tokens(rows_with_parts([image()]))two = self.builder._model_tokens(rows_with_parts([image(), image()]))- self.assertGreaterEqual(two - one, IMAGE_ALLOWANCE)+ self.assertGreater(two, one)+ self.assertLess(two - one, 200)++ def test_budget_diagnostics_distinguish_raw_and_projected_replay(self):+ rows = rows_with_parts([distinct_image(1), distinct_image(1)])+ with self.assertLogs("deeptutor.services.session.context_builder", level="DEBUG") as logs:+ effective = self.builder._model_tokens(rows)++ raw_tokens = logs.records[0].args[0]+ projected_tokens = logs.records[0].args[1]+ self.assertGreater(raw_tokens, 2 * IMAGE_ALLOWANCE)+ self.assertEqual(projected_tokens, effective)+ self.assertLess(effective, IMAGE_ALLOWANCE + 200)++ def test_distinct_options_and_mutable_urls_each_keep_an_image_allowance(self):+ low = distinct_image(1)+ low["image_url"]["detail"] = "low"+ high = distinct_image(1)+ high["image_url"]["detail"] = "high"+ jpeg = image("data:image/jpeg;base64,AQ==")+ cropped = {**low, "crop": [0, 0, 10, 10]}+ remote = image("https://example.test/changing.png")+ parts = [low, high, jpeg, cropped, remote, remote]+ tokens = self.builder._model_tokens(rows_with_parts(parts))+ self.assertGreaterEqual(tokens, 6 * IMAGE_ALLOWANCE)+ self.assertLess(tokens, 6 * IMAGE_ALLOWANCE + 500)++ def test_recent_selection_keeps_the_longest_whole_turn_suffix(self):+ rows = []+ for i in range(4):+ rows += rows_with_parts([distinct_image(i)], start=2 * i + 1)+ for budget, retained_turns in [(0, 1), (5000, 1), (9000, 2), (13000, 3), (18000, 4)]:+ with self.subTest(budget=budget):+ older, recent = self.builder._select_recent_messages(rows, budget)+ self.assertEqual(recent, rows[-2 * retained_turns :])+ self.assertEqual(older + recent, rows)++ def test_mixed_history_selection_uses_replay_for_every_candidate(self):+ rows = rows_with_parts([distinct_image(1)])+ rows += [+ {"id": 3, "role": "user", "content": "earlier question"},+ {+ "id": 4,+ "role": "assistant",+ "content": "",+ "metadata": {"provider_response_state": {"reasoning_content": "private " * 7000}},+ },+ {"id": 5, "role": "user", "content": "recent question"},+ ]+ # Empty legacy display rows are omitted from replay. Switching to+ # transcript accounting for a legacy-only candidate would count its+ # unused state and incorrectly discard the earlier model turn.+ self.assertGreater(self.builder._model_tokens(rows[2:]), 5000)+ older, recent = self.builder._select_recent_messages(rows, 5000)+ self.assertEqual(older, [])+ self.assertEqual(recent, rows)class BuilderTests(unittest.IsolatedAsyncioTestCase):@@ -155,12 +220,64 @@ class BuilderTests(unittest.IsolatedAsyncioTestCase):async def test_many_images_still_consume_finite_budget(self):rows = []for i in range(25):- rows += rows_with_parts([image("data:image/png;base64,AA==")], start=i * 2 + 1)+ rows += rows_with_parts([distinct_image(i)], start=i * 2 + 1)builder, store = self.builder(rows)result = await builder.build(session_id="test", llm_config=CONFIG)builder._summarize.assert_awaited_once()self.assertLess(result.token_count, result.budget)+ async def test_repeated_images_fit_without_summarizing_or_changing_durable_history(self):+ rows = []+ for i in range(4):+ rows += rows_with_parts([distinct_image(j) for j in range(4)], start=2 * i + 1)+ before = copy.deepcopy(rows)+ builder, store = self.builder(rows)+ config = SimpleNamespace(model="test", context_window=60000, max_tokens=4096)++ result = await builder.build(session_id="test", llm_config=config)++ builder._summarize.assert_not_awaited()+ store.update_summary.assert_not_awaited()+ self.assertGreaterEqual(result.token_count, 4 * IMAGE_ALLOWANCE)+ self.assertLess(result.token_count, result.budget)+ self.assertEqual(result.model_history, replay_history(rows))+ self.assertEqual(rows, before)++ async def test_recent_selection_budgets_repeated_images_together(self):+ old = rows_with_parts("old evidence " * 60000)+ recent = []+ for i in range(3):+ recent += rows_with_parts([distinct_image(1)], start=3 + 2 * i)+ builder, store = self.builder(old + recent)+ config = SimpleNamespace(model="test", context_window=24000, max_tokens=1024)++ result = await builder.build(session_id="test", llm_config=config)++ # The three retained turns share one image, fitting the 5040-token+ # recent budget. Counting them separately wrongly summarizes two.+ store.update_summary.assert_awaited_once_with("test", "compact summary", 2)+ self.assertEqual(result.model_history[1:], replay_history(recent))++ async def test_failed_summary_reprojects_images_after_removing_canonical_turn(self):+ rows = rows_with_parts([distinct_image(1), {"type": "text", "text": "old " * 20000}])+ recent = rows_with_parts([distinct_image(1)], start=3, tools=True)+ recent += rows_with_parts([distinct_image(1)], start=5)+ rows += recent+ before = copy.deepcopy(rows)+ builder, store = self.builder(rows)+ builder._summarize.side_effect = RuntimeError("offline simulated failure")+ config = SimpleNamespace(model="test", context_window=16000, max_tokens=1024)++ result = await builder.build(session_id="test", llm_config=config)++ store.update_summary.assert_not_awaited()+ self.assertEqual(result.model_history, replay_history(recent))+ projected = deduplicate_user_images(result.model_history)+ self.assertEqual(projected[0]["content"][-1], distinct_image(1))+ self.assertIn("Repeated image", projected[-2]["content"][-1]["text"])+ self.assertTrue(any(message["role"] == "tool" for message in projected))+ self.assertEqual(rows, before)+async def test_summary_failure_keeps_existing_watermark(self):rows = rows_with_parts("long text " * 60000)rows += rows_with_parts([image()], start=3)diff --git a/tests/services/session/test_model_history.py b/tests/services/session/test_model_history.pyindex 8448091f8..d9d37ecf7 100644--- a/tests/services/session/test_model_history.py+++ b/tests/services/session/test_model_history.py@@ -175,6 +175,84 @@ async def test_summary_instruction_is_appended_after_original_messages_and_toolsassert request == original+async def test_summary_replay_projects_duplicate_images_without_losing_provider_state(monkeypatch):+ captured = {}++ async def stream(_self, **kwargs):+ captured.update(kwargs)+ yield "Summary"++ monkeypatch.setattr(_ContextSummaryAgent, "stream_llm", stream)+ record = turn_record()+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}+ record["messages"][0]["content"] = [image]+ record["messages"][-1]["thinking_blocks"] = [+ {"type": "thinking", "thinking": "private thought", "signature": "signed"}+ ]+ request = {+ "messages": [+ {"role": "system", "content": record["system"]},+ *record["messages"],+ {"role": "user", "content": [deepcopy(image)]},+ ],+ "tools": record["tools"],+ }+ original = deepcopy(request)++ await ContextBuilder(MagicMock())._summarize(+ session_id="s",+ language="en",+ source_text="fallback transcript",+ summary_budget=256,+ replay_request=request,+ )++ messages = captured["messages"]+ assert messages[1]["content"][-1] == image+ assert all(part["type"] == "text" for part in messages[-2]["content"])+ assert "Repeated image" in messages[-2]["content"][0]["text"]+ assert messages[2:5] == record["messages"][1:]+ assert captured["tools"] == record["tools"]+ assert messages[-1]["role"] == "user"+ assert request == original+++async def test_budget_uses_the_replayed_route_and_preserves_stored_signed_state():+ record = turn_record()+ record["messages"][-1]["thinking_blocks"] = [+ {"type": "thinking", "thinking": "private thought " * 4000, "signature": "signed"}+ ]+ records = rows(record)+ before = deepcopy(records)+ store = MagicMock()+ store.get_session = AsyncMock(+ return_value={"compressed_summary": "", "summary_up_to_msg_id": 0}+ )+ store.get_messages_for_context = AsyncMock(return_value=records)+ store.update_summary = AsyncMock()+ builder = ContextBuilder(store)+ builder._summarize = AsyncMock(return_value=("Summary", []))++ switched = await builder.build(+ session_id="s",+ llm_config=SimpleNamespace(binding="openai", model="other-model", context_window=16000),+ )++ builder._summarize.assert_not_awaited()+ assert switched.token_count < switched.budget+ assert "thinking_blocks" not in switched.model_history[-1]+ assert switched.model_history[2]["content"] == "tool evidence"++ await builder.build(+ session_id="s",+ llm_config=SimpleNamespace(binding="openai", model="gpt-test", context_window=16000),+ )+ builder._summarize.assert_awaited_once()+ assert records == before++@pytest.mark.asyncioasync def test_branch_replay_and_public_export_are_isolated(tmp_path):from deeptutor.services.session.sqlite_store import SQLiteSessionStore