DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
feat(reading): let the reader look at any stored figure
flower-moone0e3e02663a1
diff --git a/deeptutor/capabilities/reading/_tool_base.py b/deeptutor/capabilities/reading/_tool_base.pyindex 27bf7d7b7..c6eab60b5 100644--- a/deeptutor/capabilities/reading/_tool_base.py+++ b/deeptutor/capabilities/reading/_tool_base.py@@ -83,4 +83,4 @@ def _guard(func):logger.warning("reading tool %s failed", getattr(self, "name", "?"), exc_info=True)return self._failure("The reader could not complete that request.")- return wrapper\ No newline at end of file+ return wrapperdiff --git a/deeptutor/capabilities/reading/figure_view.py b/deeptutor/capabilities/reading/figure_view.pynew file mode 100644index 000000000..b965045aa--- /dev/null+++ b/deeptutor/capabilities/reading/figure_view.py@@ -0,0 +1,167 @@+"""The ``view_figure`` tool: read one stored figure through a vision model.++Kept in its own module so :mod:`deeptutor.capabilities.reading.tools` stays under+the repository's file-size limit. Behaviour is unchanged from when it lived+beside the other reading tools; ``tools`` re-exports the class.+"""++from __future__ import annotations++import asyncio+from typing import Any++from deeptutor.capabilities.reading._tool_base import _ReadingToolBase, _guard+from deeptutor.core.tool_protocol import ToolDefinition, ToolParameter, ToolResult++# A missing-name failure lists the figures the model could have asked for, capped+# so a media-heavy material does not answer one typo with a wall of file names.+_FIGURE_NAME_LIST_LIMIT = 20++_FIGURE_SYSTEM_PROMPT = (+ "You describe figures for a reader who cannot see them. You state only "+ "what is visibly present and you never guess at intent."+)++_DEFAULT_FIGURE_QUESTION = (+ "Describe this image completely: the text in it, any formulas, and the "+ "structure and labelling of any figure."+)+++class ViewFigureTool(_ReadingToolBase):+ """Look at one stored figure through a vision model, as text.++ The chat turn only carries the images on the page the user has open, so any+ other figure is invisible to a text model. This tool closes that gap by+ resolving the named image on disk, asking a vision model about it, and+ returning the *text* — the only channel a tool result has. The pixels never+ travel; the answer does.+ """++ name = "view_figure"++ def get_definition(self) -> ToolDefinition:+ return ToolDefinition(+ name=self.name,+ description=(+ "Look at the pixels of a single figure in the document the user "+ "is reading and answer a question about it. Use it when the body "+ "text and the image list's caption are not enough to tell what "+ "the figure actually shows; if the caption already answers, do "+ "not call it."+ ),+ parameters=[+ ToolParameter(+ name="name",+ type="string",+ description=(+ "The figure's file name, e.g. 'image-05.png', exactly as "+ "it appears in the image list returned by read_material."+ ),+ ),+ ToolParameter(+ name="question",+ type="string",+ description=(+ "What you want to know about the figure. If omitted, a "+ "full description is produced."+ ),+ required=False,+ ),+ ],+ )++ @_guard+ async def execute(self, **kwargs: Any) -> ToolResult:+ import base64+ import mimetypes++ from deeptutor.services.llm.client import get_llm_client+ from deeptutor.services.rag.pipelines.llamaindex.config import image_description_limits+ from deeptutor.utils.document_images import MAX_IMAGE_BYTES++ material_id = self._material_id(kwargs)+ name = str(kwargs.get("name") or "").strip()+ if not name:+ return self._failure("view_figure needs a figure name, e.g. 'image-05.png'.")++ store = self._store()+ rows = await asyncio.to_thread(store.media_items, material_id)+ row = next((item for item in rows if str(item.get("name") or "") == name), None)+ if row is None:+ available = [str(item.get("name") or "") for item in rows]+ listed = ", ".join(available[:_FIGURE_NAME_LIST_LIMIT])+ if len(available) > _FIGURE_NAME_LIST_LIMIT:+ listed += f" … ({len(available)} images in total)"+ return self._failure(+ f"No figure named “{name}” in this document. Available figures: {listed or 'none'}."+ )++ caption = str(row.get("caption") or "").strip()+ locator = row.get("locator")+ path = await asyncio.to_thread(store.media_path, material_id, name)+ if path is None:+ return self._failure(f"The image “{name}” is missing on disk and cannot be viewed.")+ size = await asyncio.to_thread(lambda: path.stat().st_size)+ if size > MAX_IMAGE_BYTES:+ return self._failure(+ f"The image “{name}” is {size} bytes, over the {MAX_IMAGE_BYTES}-byte "+ "limit for a single vision call, so it cannot be viewed."+ )++ question = str(kwargs.get("question") or "").strip() or _DEFAULT_FIGURE_QUESTION++ try:+ client = get_llm_client()+ except Exception: # noqa: BLE001 - no client means no vision, not a crash+ client = None+ if client is None or not client.supports_multimodal_images():+ # Never answer with silence: hand back the caption if there is one,+ # and say plainly that the pixels were not read.+ content = (+ f"Vision parsing is unavailable for this model, so “{name}” could "+ "not be read."+ )+ if caption:+ content += f"\nCaption on record: {caption}"+ return ToolResult(+ content=content,+ metadata={+ "material_id": material_id,+ "image": name,+ "locator": locator,+ "caption": caption,+ },+ )++ mime = str(row.get("mime") or "") or (+ mimetypes.guess_type(name)[0] or "application/octet-stream"+ )+ _, timeout_seconds = image_description_limits()+ data = await asyncio.to_thread(path.read_bytes)+ encoded = base64.b64encode(data).decode("ascii")+ text = await asyncio.wait_for(+ client.complete(+ question,+ system_prompt=_FIGURE_SYSTEM_PROMPT,+ image_data=encoded,+ image_mime_type=mime,+ image_filename=name,+ ),+ timeout=timeout_seconds,+ )+ answer = str(text or "").strip()+ if not answer:+ return self._failure(+ f"The vision model returned nothing for “{name}”. The caption on "+ f"record is: {caption or '(none)'}"+ )+ return ToolResult(+ content=answer,+ metadata={+ "material_id": material_id,+ "image": name,+ "locator": locator,+ "caption": caption,+ },+ )\ No newline at end of filediff --git a/deeptutor/capabilities/reading/prompts/en/reading.yaml b/deeptutor/capabilities/reading/prompts/en/reading.yamlindex d99d321fb..13622208a 100644--- a/deeptutor/capabilities/reading/prompts/en/reading.yaml+++ b/deeptutor/capabilities/reading/prompts/en/reading.yaml@@ -40,7 +40,12 @@ playbook: |Write the sentence however reads best; you do not have to avoid naming thelocation in words. If you write "…on page 3 … [p.3]", the reader merges thetwo into one link on the phrase itself, so it never comes out doubled.- 5. `reader_annotate` leaves a durable mark the user can export with their own+ 5. `view_figure` looks at a single figure's pixels through a vision model.+ Call it with the figure's name from the image list when the caption there+ is not enough to tell what the figure shows — a non-current page, another+ tab, or any figure the turn does not carry. If the caption already answers,+ do not call it.+ 6. `reader_annotate` leaves a durable mark the user can export with their ownannotations. Use it when they ask you to mark something up — not as asubstitute for explaining.diff --git a/deeptutor/capabilities/reading/prompts/zh/reading.yaml b/deeptutor/capabilities/reading/prompts/zh/reading.yamlindex 7991d97af..e8183bd14 100644--- a/deeptutor/capabilities/reading/prompts/zh/reading.yaml+++ b/deeptutor/capabilities/reading/prompts/zh/reading.yaml@@ -30,7 +30,10 @@ playbook: |句子怎么读得顺就怎么写,不必回避用文字说出位置。如果你写「……位于第 3 页……[p.3]」,阅读器会把两者合并成短语上的一个链接,不会重复出现。- 5. `reader_annotate` 会留下持久的标注,用户导出标注文件时会带上。用户要求你+ 5. `view_figure` 通过视觉模型看单张图的像素内容。当图片清单里的 caption 不足以+ 判断图里画了什么时(图不在当前页、在别的标签页、或本轮消息没有携带这张图),+ 用清单里的图名调用它。caption 够用就不要调。+ 6. `reader_annotate` 会留下持久的标注,用户导出标注文件时会带上。用户要求你划重点/做批注时才用,它不能替代讲解。如果文档回答不了这个问题,就直接说明,并说清文档实际覆盖了什么。不要在不标注diff --git a/deeptutor/capabilities/reading/tools.py b/deeptutor/capabilities/reading/tools.pyindex e7eb45177..f58f289fd 100644--- a/deeptutor/capabilities/reading/tools.py+++ b/deeptutor/capabilities/reading/tools.py@@ -44,6 +44,7 @@ from deeptutor.capabilities.reading._tool_base import (_NoMaterial,_ReadingToolBase,)+from deeptutor.capabilities.reading.figure_view import ViewFigureToolfrom deeptutor.capabilities.reading.media_notes import render_media_note as _media_notefrom deeptutor.core.tool_protocol import BaseTool, ToolDefinition, ToolParameter, ToolResult@@ -55,6 +56,7 @@ READING_TOOL_NAMES: tuple[str, ...] = ("material_outline","search_material","read_material",+ "view_figure","reader_goto","reader_annotate",)@@ -656,6 +658,7 @@ READING_TOOL_TYPES: tuple[type[BaseTool], ...] = (MaterialOutlineTool,SearchMaterialTool,ReadMaterialTool,+ ViewFigureTool,ReaderGotoTool,ReaderAnnotateTool,)@@ -670,6 +673,7 @@ __all__ = ["ReadingListTabsTool","ReadingSwitchTabTool","ReadMaterialTool",+ "ViewFigureTool","ReaderAnnotateTool","ReaderGotoTool","SearchMaterialTool",diff --git a/deeptutor/tools/builtin_specs.py b/deeptutor/tools/builtin_specs.pyindex 07facfed8..5242843a0 100644--- a/deeptutor/tools/builtin_specs.py+++ b/deeptutor/tools/builtin_specs.py@@ -167,6 +167,7 @@ BUILTIN_TOOL_SPECS: tuple[BuiltinToolSpec, ...] = (("material_outline", "MaterialOutlineTool"),("search_material", "SearchMaterialTool"),("read_material", "ReadMaterialTool"),+ ("view_figure", "ViewFigureTool"),("reader_goto", "ReaderGotoTool"),("reader_annotate", "ReaderAnnotateTool"),),diff --git a/tests/reading/test_capability.py b/tests/reading/test_capability.pyindex 03fa9f193..dd5fec466 100644--- a/tests/reading/test_capability.py+++ b/tests/reading/test_capability.py@@ -125,7 +125,7 @@ def test_reading_turns_do_not_activate_the_explore_context_pre_pass() -> None:def test_capability_owns_reading_and_workspace_navigation_tools() -> None:assert ReadingCapability().owned_tools == READING_TOOL_NAMES- assert len(READING_TOOL_NAMES) == 7+ assert len(READING_TOOL_NAMES) == 8assert READING_TOOL_NAMES[:2] == ("reading_list_tabs", "reading_switch_tab")# Additive, not exclusive: chat keeps its own surface on a reading turn.assert getattr(ReadingCapability(), "exclusive_tools", False) is False