Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(session): budget image blocks without tokenizing encoded payloads

ZyPulse05aabcbdf8f6
diff --git a/deeptutor/services/session/context_builder.py b/deeptutor/services/session/context_builder.py
index e1530af4d..74479b1f0 100644
--- a/deeptutor/services/session/context_builder.py
+++ b/deeptutor/services/session/context_builder.py
@@ -39,6 +39,13 @@ MAX_SUMMARY_OUTPUT_TOKENS = 16_384
MAX_RAW_REBUILD_TOKENS = 131_072
+# Planning allowance per image, not provider-reported usage. Counting encoded
+# image bytes as text can exhaust the entire history budget on one screenshot.
+# Reserve nonzero headroom for vision while keeping it independent of PNG/JPEG
+# compression and URL length. Repeated images each consume this allowance.
+IMAGE_CONTEXT_TOKEN_ESTIMATE = 4096
+
+
def count_tokens(text: str) -> int:
"""Estimate token count with tiktoken when available."""
if not text:
@@ -52,6 +59,37 @@ def count_tokens(text: str) -> int:
return max(1, len(text) // 4)
+def _count_model_context_tokens(value: Any) -> int:
+ """Measure a temporary accounting view; never alter the replay payload.
+
+ Keep all text, tool arguments/results and provider replay state in the
+ existing serialized-text estimate. Only recognized multimodal image blocks
+ use a separate allowance; their encoded bytes/URLs are not language tokens.
+ This is a context-planning heuristic, not an exact vision billing counter.
+ """
+ image_count = 0
+
+ def accounting_view(item: Any) -> Any:
+ nonlocal image_count
+ if isinstance(item, dict):
+ kind = item.get("type")
+ is_image = (
+ (kind == "image_url" and "image_url" in item)
+ or (kind == "input_image" and ("image_url" in item or "file_id" in item))
+ or (kind == "image" and "source" in item)
+ )
+ if is_image:
+ image_count += 1
+ return {"type": "image"}
+ return {key: accounting_view(child) for key, child in item.items()}
+ if isinstance(item, list):
+ return [accounting_view(child) for child in item]
+ return item
+
+ serialized = json.dumps(accounting_view(value), ensure_ascii=False)
+ return count_tokens(serialized) + image_count * IMAGE_CONTEXT_TOKEN_ESTIMATE
+
+
def trim_incomplete_tail(text: str) -> str:
"""Drop the trailing partial line from output that hit a hard token cap.
@@ -270,7 +308,7 @@ class ContextBuilder:
def _model_tokens(self, messages: list[dict[str, Any]], summary: str = "") -> int:
if any(model_turn(row) is not None for row in messages):
- return count_tokens(json.dumps(replay_history(messages, summary), ensure_ascii=False))
+ return _count_model_context_tokens(replay_history(messages, summary))
return count_tokens(build_history_text(self._build_history(summary, messages))) + sum(
_provider_response_state_tokens(row) for row in messages
)
diff --git a/tests/services/session/test_image_context_budget.py b/tests/services/session/test_image_context_budget.py
new file mode 100644
index 000000000..f470c4e22
--- /dev/null
+++ b/tests/services/session/test_image_context_budget.py
@@ -0,0 +1,176 @@
+"""Offline regressions: vision budgeting must not rewrite or drop history."""
+
+import base64
+import copy
+import json
+import random
+from types import SimpleNamespace
+import unittest
+from unittest.mock import AsyncMock
+
+from deeptutor.services.session.context_builder import ContextBuilder, count_tokens
+from deeptutor.services.session.model_history import replay_history
+
+CONFIG = SimpleNamespace(model="test", context_window=272000, max_tokens=4096, binding="openai")
+IMAGE_ALLOWANCE = 4096
+RANDOM_IMAGE = (
+ "data:image/png;base64," + base64.b64encode(random.Random(17).randbytes(340000)).decode()
+)
+
+
+def rows_with_parts(parts, *, start=1, answer="answer", tools=False):
+ messages = [{"role": "user", "content": parts}]
+ if tools:
+ messages.extend(
+ [
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call-1",
+ "type": "function",
+ "function": {"name": "read_material", "arguments": '{"page":147}'},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call-1", "content": "source evidence"},
+ ]
+ )
+ messages.append({"role": "assistant", "content": answer})
+ return [
+ {"id": start, "role": "user", "content": "continue"},
+ {
+ "id": start + 1,
+ "role": "assistant",
+ "content": answer,
+ "metadata": {"model_turn": {"version": 1, "messages": messages}},
+ },
+ ]
+
+
+def image(url=RANDOM_IMAGE):
+ return {"type": "image_url", "image_url": {"url": url, "detail": "auto"}}
+
+
+class ImageTokenTests(unittest.TestCase):
+ def setUp(self):
+ self.builder = ContextBuilder(None)
+
+ def test_image_encoding_size_does_not_change_budget(self):
+ small = self.builder._model_tokens(rows_with_parts([image("data:image/png;base64,AA==")]))
+ large = self.builder._model_tokens(rows_with_parts([image()]))
+ self.assertEqual(small, large)
+ self.assertGreaterEqual(large, IMAGE_ALLOWANCE)
+
+ def test_reader_three_screenshots_fit_history_budget(self):
+ rows = rows_with_parts(
+ [{"type": "text", "text": "继续解释贝叶斯"}, *[image() for _ in range(3)]]
+ )
+ tokens = self.builder._model_tokens(rows)
+ self.assertGreaterEqual(tokens, 3 * IMAGE_ALLOWANCE)
+ self.assertLess(tokens, self.builder._history_budget(CONFIG))
+
+ def test_remote_image_url_is_not_language_text(self):
+ local = self.builder._model_tokens(rows_with_parts([image()]))
+ remote = self.builder._model_tokens(
+ rows_with_parts([image("https://example.test/image.png")])
+ )
+ self.assertEqual(local, remote)
+
+ def test_responses_and_anthropic_images_have_nonzero_allowance(self):
+ parts = [
+ {"type": "input_image", "image_url": RANDOM_IMAGE},
+ {"type": "input_image", "file_id": "file-test"},
+ {
+ "type": "image",
+ "source": {"type": "base64", "media_type": "image/png", "data": RANDOM_IMAGE},
+ },
+ ]
+ n = self.builder._model_tokens(rows_with_parts(parts))
+ self.assertGreaterEqual(n, 3 * IMAGE_ALLOWANCE)
+ self.assertLess(n, 3 * IMAGE_ALLOWANCE + 200)
+
+ def test_text_tool_and_reasoning_accounting_unchanged(self):
+ rows = rows_with_parts([{"type": "text", "text": "some text"}], tools=True)
+ record = rows[-1]["metadata"]["model_turn"]
+ record["messages"][-1]["_provider_response_state"] = {
+ "reasoning_content": "retained reasoning"
+ }
+ record["messages"][-1]["thinking_blocks"] = [
+ {"type": "thinking", "thinking": "reasoning", "signature": "sig"}
+ ]
+ expected = count_tokens(json.dumps(replay_history(rows, "summary"), ensure_ascii=False))
+ self.assertEqual(self.builder._model_tokens(rows, "summary"), expected)
+
+ def test_encoding_in_user_text_is_still_text(self):
+ rows = rows_with_parts([{"type": "text", "text": RANDOM_IMAGE}])
+ self.assertGreater(self.builder._model_tokens(rows), self.builder._history_budget(CONFIG))
+
+ def test_never_mutates_images_tools_or_saved_rows(self):
+ rows = rows_with_parts([image(), {"type": "text", "text": "keep"}], tools=True)
+ original = copy.deepcopy(rows)
+ replay_before = replay_history(rows)
+ self.builder._model_tokens(rows)
+ self.assertEqual(rows, original)
+ self.assertEqual(replay_history(rows), replay_before)
+
+ def test_each_image_occurrence_is_counted(self):
+ one = self.builder._model_tokens(rows_with_parts([image()]))
+ two = self.builder._model_tokens(rows_with_parts([image(), image()]))
+ self.assertGreaterEqual(two - one, IMAGE_ALLOWANCE)
+
+
+class BuilderTests(unittest.IsolatedAsyncioTestCase):
+ def builder(self, rows):
+ store = SimpleNamespace(
+ get_session=AsyncMock(
+ return_value={"compressed_summary": "", "summary_up_to_msg_id": 0}
+ ),
+ get_messages_for_context=AsyncMock(return_value=rows),
+ update_summary=AsyncMock(),
+ )
+ builder = ContextBuilder(store)
+ builder._summarize = AsyncMock(return_value=("compact summary", []))
+ return builder, store
+
+ async def test_screenshots_do_not_trigger_summary_or_write(self):
+ rows = rows_with_parts([image(), image(), image()], tools=True)
+ builder, store = self.builder(rows)
+ result = await builder.build(session_id="test", llm_config=CONFIG)
+ builder._summarize.assert_not_awaited()
+ store.update_summary.assert_not_awaited()
+ self.assertEqual(result.model_history, replay_history(rows))
+
+ async def test_real_text_overflow_still_summarizes_and_preserves_latest(self):
+ rows = rows_with_parts("long text " * 60000)
+ latest = rows_with_parts([image(), {"type": "text", "text": "latest"}], start=3, tools=True)
+ rows += latest
+ builder, store = self.builder(rows)
+ result = await builder.build(session_id="test", llm_config=CONFIG)
+ builder._summarize.assert_awaited_once()
+ store.update_summary.assert_awaited_once_with("test", "compact summary", 2)
+ self.assertEqual(result.model_history[1:], replay_history(latest))
+
+ async def test_many_images_still_consume_finite_budget(self):
+ rows = []
+ for i in range(25):
+ rows += rows_with_parts([image("data:image/png;base64,AA==")], start=i * 2 + 1)
+ builder, store = self.builder(rows)
+ result = await builder.build(session_id="test", llm_config=CONFIG)
+ builder._summarize.assert_awaited_once()
+ self.assertLess(result.token_count, result.budget)
+
+ async def test_summary_failure_keeps_existing_watermark(self):
+ rows = rows_with_parts("long text " * 60000)
+ rows += rows_with_parts([image()], start=3)
+ builder, store = self.builder(rows)
+ builder._summarize.side_effect = RuntimeError("offline simulated failure")
+ result = await builder.build(session_id="test", llm_config=CONFIG)
+ builder._summarize.assert_awaited_once()
+ store.update_summary.assert_not_awaited()
+ self.assertEqual(result.model_history, replay_history(rows[-2:]))
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)