Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

feat(reading): let the reader look at any stored figure

flower-moone0e3e02663a1
diff --git a/deeptutor/capabilities/reading/_tool_base.py b/deeptutor/capabilities/reading/_tool_base.py
index 27bf7d7b7..c6eab60b5 100644
--- a/deeptutor/capabilities/reading/_tool_base.py
+++ b/deeptutor/capabilities/reading/_tool_base.py
@@ -83,4 +83,4 @@ def _guard(func):
logger.warning("reading tool %s failed", getattr(self, "name", "?"), exc_info=True)
return self._failure("The reader could not complete that request.")
- return wrapper
\ No newline at end of file
+ return wrapper
diff --git a/deeptutor/capabilities/reading/figure_view.py b/deeptutor/capabilities/reading/figure_view.py
new file mode 100644
index 000000000..b965045aa
--- /dev/null
+++ b/deeptutor/capabilities/reading/figure_view.py
@@ -0,0 +1,167 @@
+"""The ``view_figure`` tool: read one stored figure through a vision model.
+
+Kept in its own module so :mod:`deeptutor.capabilities.reading.tools` stays under
+the repository's file-size limit. Behaviour is unchanged from when it lived
+beside the other reading tools; ``tools`` re-exports the class.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from deeptutor.capabilities.reading._tool_base import _ReadingToolBase, _guard
+from deeptutor.core.tool_protocol import ToolDefinition, ToolParameter, ToolResult
+
+# A missing-name failure lists the figures the model could have asked for, capped
+# so a media-heavy material does not answer one typo with a wall of file names.
+_FIGURE_NAME_LIST_LIMIT = 20
+
+_FIGURE_SYSTEM_PROMPT = (
+ "You describe figures for a reader who cannot see them. You state only "
+ "what is visibly present and you never guess at intent."
+)
+
+_DEFAULT_FIGURE_QUESTION = (
+ "Describe this image completely: the text in it, any formulas, and the "
+ "structure and labelling of any figure."
+)
+
+
+class ViewFigureTool(_ReadingToolBase):
+ """Look at one stored figure through a vision model, as text.
+
+ The chat turn only carries the images on the page the user has open, so any
+ other figure is invisible to a text model. This tool closes that gap by
+ resolving the named image on disk, asking a vision model about it, and
+ returning the *text* — the only channel a tool result has. The pixels never
+ travel; the answer does.
+ """
+
+ name = "view_figure"
+
+ def get_definition(self) -> ToolDefinition:
+ return ToolDefinition(
+ name=self.name,
+ description=(
+ "Look at the pixels of a single figure in the document the user "
+ "is reading and answer a question about it. Use it when the body "
+ "text and the image list's caption are not enough to tell what "
+ "the figure actually shows; if the caption already answers, do "
+ "not call it."
+ ),
+ parameters=[
+ ToolParameter(
+ name="name",
+ type="string",
+ description=(
+ "The figure's file name, e.g. 'image-05.png', exactly as "
+ "it appears in the image list returned by read_material."
+ ),
+ ),
+ ToolParameter(
+ name="question",
+ type="string",
+ description=(
+ "What you want to know about the figure. If omitted, a "
+ "full description is produced."
+ ),
+ required=False,
+ ),
+ ],
+ )
+
+ @_guard
+ async def execute(self, **kwargs: Any) -> ToolResult:
+ import base64
+ import mimetypes
+
+ from deeptutor.services.llm.client import get_llm_client
+ from deeptutor.services.rag.pipelines.llamaindex.config import image_description_limits
+ from deeptutor.utils.document_images import MAX_IMAGE_BYTES
+
+ material_id = self._material_id(kwargs)
+ name = str(kwargs.get("name") or "").strip()
+ if not name:
+ return self._failure("view_figure needs a figure name, e.g. 'image-05.png'.")
+
+ store = self._store()
+ rows = await asyncio.to_thread(store.media_items, material_id)
+ row = next((item for item in rows if str(item.get("name") or "") == name), None)
+ if row is None:
+ available = [str(item.get("name") or "") for item in rows]
+ listed = ", ".join(available[:_FIGURE_NAME_LIST_LIMIT])
+ if len(available) > _FIGURE_NAME_LIST_LIMIT:
+ listed += f" … ({len(available)} images in total)"
+ return self._failure(
+ f"No figure named “{name}” in this document. Available figures: {listed or 'none'}."
+ )
+
+ caption = str(row.get("caption") or "").strip()
+ locator = row.get("locator")
+ path = await asyncio.to_thread(store.media_path, material_id, name)
+ if path is None:
+ return self._failure(f"The image “{name}” is missing on disk and cannot be viewed.")
+ size = await asyncio.to_thread(lambda: path.stat().st_size)
+ if size > MAX_IMAGE_BYTES:
+ return self._failure(
+ f"The image “{name}” is {size} bytes, over the {MAX_IMAGE_BYTES}-byte "
+ "limit for a single vision call, so it cannot be viewed."
+ )
+
+ question = str(kwargs.get("question") or "").strip() or _DEFAULT_FIGURE_QUESTION
+
+ try:
+ client = get_llm_client()
+ except Exception: # noqa: BLE001 - no client means no vision, not a crash
+ client = None
+ if client is None or not client.supports_multimodal_images():
+ # Never answer with silence: hand back the caption if there is one,
+ # and say plainly that the pixels were not read.
+ content = (
+ f"Vision parsing is unavailable for this model, so “{name}” could "
+ "not be read."
+ )
+ if caption:
+ content += f"\nCaption on record: {caption}"
+ return ToolResult(
+ content=content,
+ metadata={
+ "material_id": material_id,
+ "image": name,
+ "locator": locator,
+ "caption": caption,
+ },
+ )
+
+ mime = str(row.get("mime") or "") or (
+ mimetypes.guess_type(name)[0] or "application/octet-stream"
+ )
+ _, timeout_seconds = image_description_limits()
+ data = await asyncio.to_thread(path.read_bytes)
+ encoded = base64.b64encode(data).decode("ascii")
+ text = await asyncio.wait_for(
+ client.complete(
+ question,
+ system_prompt=_FIGURE_SYSTEM_PROMPT,
+ image_data=encoded,
+ image_mime_type=mime,
+ image_filename=name,
+ ),
+ timeout=timeout_seconds,
+ )
+ answer = str(text or "").strip()
+ if not answer:
+ return self._failure(
+ f"The vision model returned nothing for “{name}”. The caption on "
+ f"record is: {caption or '(none)'}"
+ )
+ return ToolResult(
+ content=answer,
+ metadata={
+ "material_id": material_id,
+ "image": name,
+ "locator": locator,
+ "caption": caption,
+ },
+ )
\ No newline at end of file
diff --git a/deeptutor/capabilities/reading/prompts/en/reading.yaml b/deeptutor/capabilities/reading/prompts/en/reading.yaml
index d99d321fb..13622208a 100644
--- a/deeptutor/capabilities/reading/prompts/en/reading.yaml
+++ b/deeptutor/capabilities/reading/prompts/en/reading.yaml
@@ -40,7 +40,12 @@ playbook: |
Write the sentence however reads best; you do not have to avoid naming the
location in words. If you write "…on page 3 … [p.3]", the reader merges the
two into one link on the phrase itself, so it never comes out doubled.
- 5. `reader_annotate` leaves a durable mark the user can export with their own
+ 5. `view_figure` looks at a single figure's pixels through a vision model.
+ Call it with the figure's name from the image list when the caption there
+ is not enough to tell what the figure shows — a non-current page, another
+ tab, or any figure the turn does not carry. If the caption already answers,
+ do not call it.
+ 6. `reader_annotate` leaves a durable mark the user can export with their own
annotations. Use it when they ask you to mark something up — not as a
substitute for explaining.
diff --git a/deeptutor/capabilities/reading/prompts/zh/reading.yaml b/deeptutor/capabilities/reading/prompts/zh/reading.yaml
index 7991d97af..e8183bd14 100644
--- a/deeptutor/capabilities/reading/prompts/zh/reading.yaml
+++ b/deeptutor/capabilities/reading/prompts/zh/reading.yaml
@@ -30,7 +30,10 @@ playbook: |
句子怎么读得顺就怎么写,不必回避用文字说出位置。如果你写「……位于第 3 页
……[p.3]」,阅读器会把两者合并成短语上的一个链接,不会重复出现。
- 5. `reader_annotate` 会留下持久的标注,用户导出标注文件时会带上。用户要求你
+ 5. `view_figure` 通过视觉模型看单张图的像素内容。当图片清单里的 caption 不足以
+ 判断图里画了什么时(图不在当前页、在别的标签页、或本轮消息没有携带这张图),
+ 用清单里的图名调用它。caption 够用就不要调。
+ 6. `reader_annotate` 会留下持久的标注,用户导出标注文件时会带上。用户要求你
划重点/做批注时才用,它不能替代讲解。
如果文档回答不了这个问题,就直接说明,并说清文档实际覆盖了什么。不要在不标注
diff --git a/deeptutor/capabilities/reading/tools.py b/deeptutor/capabilities/reading/tools.py
index e7eb45177..f58f289fd 100644
--- a/deeptutor/capabilities/reading/tools.py
+++ b/deeptutor/capabilities/reading/tools.py
@@ -44,6 +44,7 @@ from deeptutor.capabilities.reading._tool_base import (
_NoMaterial,
_ReadingToolBase,
)
+from deeptutor.capabilities.reading.figure_view import ViewFigureTool
from deeptutor.capabilities.reading.media_notes import render_media_note as _media_note
from deeptutor.core.tool_protocol import BaseTool, ToolDefinition, ToolParameter, ToolResult
@@ -55,6 +56,7 @@ READING_TOOL_NAMES: tuple[str, ...] = (
"material_outline",
"search_material",
"read_material",
+ "view_figure",
"reader_goto",
"reader_annotate",
)
@@ -656,6 +658,7 @@ READING_TOOL_TYPES: tuple[type[BaseTool], ...] = (
MaterialOutlineTool,
SearchMaterialTool,
ReadMaterialTool,
+ ViewFigureTool,
ReaderGotoTool,
ReaderAnnotateTool,
)
@@ -670,6 +673,7 @@ __all__ = [
"ReadingListTabsTool",
"ReadingSwitchTabTool",
"ReadMaterialTool",
+ "ViewFigureTool",
"ReaderAnnotateTool",
"ReaderGotoTool",
"SearchMaterialTool",
diff --git a/deeptutor/tools/builtin_specs.py b/deeptutor/tools/builtin_specs.py
index 07facfed8..5242843a0 100644
--- a/deeptutor/tools/builtin_specs.py
+++ b/deeptutor/tools/builtin_specs.py
@@ -167,6 +167,7 @@ BUILTIN_TOOL_SPECS: tuple[BuiltinToolSpec, ...] = (
("material_outline", "MaterialOutlineTool"),
("search_material", "SearchMaterialTool"),
("read_material", "ReadMaterialTool"),
+ ("view_figure", "ViewFigureTool"),
("reader_goto", "ReaderGotoTool"),
("reader_annotate", "ReaderAnnotateTool"),
),
diff --git a/tests/reading/test_capability.py b/tests/reading/test_capability.py
index 03fa9f193..dd5fec466 100644
--- a/tests/reading/test_capability.py
+++ b/tests/reading/test_capability.py
@@ -125,7 +125,7 @@ def test_reading_turns_do_not_activate_the_explore_context_pre_pass() -> None:
def test_capability_owns_reading_and_workspace_navigation_tools() -> None:
assert ReadingCapability().owned_tools == READING_TOOL_NAMES
- assert len(READING_TOOL_NAMES) == 7
+ assert len(READING_TOOL_NAMES) == 8
assert READING_TOOL_NAMES[:2] == ("reading_list_tabs", "reading_switch_tab")
# Additive, not exclusive: chat keeps its own surface on a reading turn.
assert getattr(ReadingCapability(), "exclusive_tools", False) is False