DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(usage): exclude encoded image payloads from token estimates
ZyPulse31c8f0acc079
diff --git a/deeptutor/services/llm/metrics.py b/deeptutor/services/llm/metrics.pyindex 952f6c13a..b5753257f 100644--- a/deeptutor/services/llm/metrics.py+++ b/deeptutor/services/llm/metrics.py@@ -12,6 +12,7 @@ import timefrom typing import Anyimport uuid+from .usage_estimation import estimate_prompt_tokensfrom .usage_frame import usage_breakdown@@ -90,7 +91,7 @@ class CallMeasurement:self.last: float | None = Noneself.output_chars = 0# Estimate only when the API does not report usage; never persist message text.- self.input_chars = len(str(messages or "")) if self.enabled else 0+ self.estimated_prompt_tokens = estimate_prompt_tokens(messages) if self.enabled else 0self.usage: Any = Noneself.finished = False@@ -133,7 +134,7 @@ class CallMeasurement:response_text += str(getattr(message, "reasoning_content", "") or "")output = self.output_chars or len(response_text)counts = {- "prompt_tokens": int(self.input_chars / 3.5),+ "prompt_tokens": self.estimated_prompt_tokens,"completion_tokens": int(output / 3.5),}counts["total_tokens"] = sum(counts.values())diff --git a/deeptutor/services/llm/usage_estimation.py b/deeptutor/services/llm/usage_estimation.pynew file mode 100644index 000000000..eca19d9b9--- /dev/null+++ b/deeptutor/services/llm/usage_estimation.py@@ -0,0 +1,45 @@+"""Coarse fallback accounting when a provider omits usage.++Image transport bytes are not language tokens. Keep the historical chars/3.5+text heuristic, with a fixed allowance per recognized image content block.+This is not a model-specific vision tokenizer or a billing calculation.+"""++from __future__ import annotations++from typing import Any++ESTIMATED_IMAGE_TOKENS = 1024+++def estimate_prompt_tokens(messages: Any) -> int:+ """Estimate without serializing image bytes/URLs or mutating the request.++ Only structured content blocks are images. Image-looking text, tool-call+ arguments, tool results and other message fields still count as text.+ No image is decoded, fetched, or retained by this estimator.+ """+ images = 0++ def without_image_payloads(value: Any, *, content: bool = False) -> Any:+ nonlocal images+ if isinstance(value, dict):+ kind = value.get("type")+ if content and (+ (kind == "image_url" and "image_url" in value)+ or (kind == "input_image" and ("image_url" in value or "file_id" in value))+ or (kind == "image" and "source" in value)+ ):+ images += 1+ return {"type": kind}+ return {+ key: without_image_payloads(item, content=key == "content")+ for key, item in value.items()+ }+ if isinstance(value, (list, tuple)):+ items = [without_image_payloads(item, content=content) for item in value]+ return tuple(items) if isinstance(value, tuple) else items+ return value++ text_chars = len(str(without_image_payloads(messages or "")))+ return int(text_chars / 3.5) + images * ESTIMATED_IMAGE_TOKENSdiff --git a/docs-for-user/SOURCE_VISUALS.md b/docs-for-user/SOURCE_VISUALS.mdindex 674b8d1fc..7d23418cb 100644--- a/docs-for-user/SOURCE_VISUALS.md+++ b/docs-for-user/SOURCE_VISUALS.md@@ -26,3 +26,18 @@ source image assets. Images that are only vector drawing commands, or pagesthat require OCR when no usable OCR engine is configured, may be absent.Other RAG providers do not yet index this visual manifest. Interactiveexercises, visual annotations, and mastery updates are separate future work.+++## Token estimates for image requests++When the provider returns usage, DeepTutor uses those reported counters. When+usage is absent, the conversation statistics are marked as estimates. The+fallback counts serialized text at roughly 3.5 characters per token and adds+1,024 tokens per structured image block. It excludes image URLs and Base64+payloads from the text estimate, so the encoded file size does not inflate the+counter into millions of tokens.++The image allowance is a rough placeholder, independent of resolution, detail,+model and provider; it is not a billing calculation. Each model call in a turn+still counts its own input, including any replayed images. Existing stored+estimates are not rewritten by this change, and conversation content is kept.diff --git a/tests/services/llm/test_usage_estimation.py b/tests/services/llm/test_usage_estimation.pynew file mode 100644index 000000000..da0318253--- /dev/null+++ b/tests/services/llm/test_usage_estimation.py@@ -0,0 +1,151 @@+"""Regression: transport encodings must not inflate fallback usage."""++from copy import deepcopy+import json+from types import SimpleNamespace as NS++import pytest++from deeptutor.services.llm import metrics+from deeptutor.services.llm.usage_estimation import (+ ESTIMATED_IMAGE_TOKENS,+ estimate_prompt_tokens,+)+++def image_part(payload, dialect):+ if dialect == "chat":+ return {+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{payload}", "detail": "high"},+ }+ if dialect == "responses":+ return {"type": "input_image", "image_url": f"data:image/png;base64,{payload}"}+ return {+ "type": "image",+ "source": {"type": "base64", "media_type": "image/png", "data": payload},+ }++[email protected]("dialect", ["chat", "responses", "anthropic"])+def test_encoded_size_does_not_change_estimate_or_request(dialect):+ small = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "Describe this image"},+ image_part("AAAA", dialect),+ ],+ }+ ]+ large = deepcopy(small)+ large[0]["content"][1] = image_part("A" * 8_212_712, dialect)+ before = deepcopy(large)+ assert estimate_prompt_tokens(large) == estimate_prompt_tokens(small)+ assert ESTIMATED_IMAGE_TOKENS <= estimate_prompt_tokens(large) < ESTIMATED_IMAGE_TOKENS + 100+ assert large == before+++ "messages",+ [+ None,+ "",+ "plain text",+ [+ {"role": "system", "content": "You are a tutor"},+ {"role": "user", "content": [{"type": "text", "text": "Explain 光学"}]},+ {+ "role": "assistant",+ "content": None,+ "reasoning_content": "reasoning",+ "tool_calls": [{"function": {"name": "lookup", "arguments": '{"q":"optics"}'}}],+ },+ {"role": "tool", "tool_call_id": "1", "content": "Long source evidence " * 300},+ ],+ ],+)+def test_text_only_estimates_keep_existing_behavior(messages):+ assert estimate_prompt_tokens(messages) == int(len(str(messages or "")) / 3.5)+++def test_remote_urls_file_ids_and_repeated_images_are_bounded():+ images = [+ {"type": "image_url", "image_url": {"url": "https://example.invalid/" + "a" * 5000}},+ {"type": "input_image", "file_id": "file-123"},+ {"type": "image", "source": {"type": "url", "url": "https://example.invalid/a.png"}},+ ]+ messages = [{"role": "user", "content": images + images}]+ estimate = estimate_prompt_tokens(messages)+ assert 6 * ESTIMATED_IMAGE_TOKENS <= estimate < 6 * ESTIMATED_IMAGE_TOKENS + 100+++def test_image_looking_text_and_tool_arguments_are_not_removed():+ text = json.dumps(image_part("A" * 4000, "chat"))+ messages = [+ {"role": "user", "content": text},+ {"role": "assistant", "tool_calls": [{"function": {"arguments": text}}]},+ ]+ assert estimate_prompt_tokens(messages) == int(len(str(messages)) / 3.5)++[email protected]("reported", [False, True])+def test_meter_uses_bounded_fallback_but_provider_usage_wins(tmp_path, monkeypatch, reported):+ from deeptutor.services.llm import usage_ledger++ monkeypatch.setattr(usage_ledger, "ledger_path", lambda: tmp_path / "usage.sqlite3")+ collector = metrics.TurnUsage(session_id="synthetic", turn_id="turn")+ token = metrics.current_usage.set(collector)+ messages = [{"role": "user", "content": [image_part("A" * 8_212_712, "chat")]}]+ try:+ meter = metrics.CallMeasurement(messages=messages)+ meter.delta("an image")+ meter.finish(+ NS(usage={"prompt_tokens": 1700, "completion_tokens": 9} if reported else None)+ )+ meter.finish() # caller cleanup must not duplicate the call+ finally:+ metrics.current_usage.reset(token)+ summary = collector.summary()+ assert summary["total_calls"] == 1+ assert summary["prompt_tokens"] == (1700 if reported else estimate_prompt_tokens(messages))+ assert summary["estimated_calls"] == (0 if reported else 1)+ assert summary["total_tokens"] == summary["prompt_tokens"] + summary["completion_tokens"]+ # The durable ledger must carry the same corrected measurement, never image data.+ import sqlite3++ with sqlite3.connect(tmp_path / "usage.sqlite3") as conn:+ raw = conn.execute("SELECT usage_json FROM llm_calls").fetchone()[0]+ assert json.loads(raw)["prompt_tokens"] == summary["prompt_tokens"]+ assert "data:image" not in raw+ assert len(raw) < 2000+++async def test_stream_without_usage_aggregates_calls_once(tmp_path, monkeypatch):+ from deeptutor.services.llm import usage_ledger++ monkeypatch.setattr(usage_ledger, "ledger_path", lambda: tmp_path / "usage.sqlite3")+ messages = [{"role": "user", "content": [image_part("A" * 5000, "chat")]}]++ async def chunks():+ yield NS(choices=[NS(delta=NS(content="answer"))], usage=None)++ async def create(**kwargs):+ return chunks()++ collector = metrics.TurnUsage()+ token = metrics.current_usage.set(collector)+ client = NS(chat=NS(completions=NS(create=create)))+ metrics.instrument_client(client)+ try:+ for _ in range(2):+ stream = await client.chat.completions.create(messages=messages, stream=True)+ async for _ in stream:+ pass+ await stream.close()+ finally:+ metrics.current_usage.reset(token)+ summary = collector.summary()+ assert summary["total_calls"] == summary["estimated_calls"] == 2+ assert summary["prompt_tokens"] == 2 * estimate_prompt_tokens(messages)