DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(reading): cite figure locations and cover vision fallback
Bingxi Zhao (Frank)842ed970e8b0
diff --git a/deeptutor/capabilities/reading/figure_view.py b/deeptutor/capabilities/reading/figure_view.pyindex b965045aa..affd2185d 100644--- a/deeptutor/capabilities/reading/figure_view.py+++ b/deeptutor/capabilities/reading/figure_view.py@@ -10,7 +10,7 @@ from __future__ import annotationsimport asynciofrom typing import Any-from deeptutor.capabilities.reading._tool_base import _ReadingToolBase, _guard+from deeptutor.capabilities.reading._tool_base import _guard, _ReadingToolBasefrom deeptutor.core.tool_protocol import ToolDefinition, ToolParameter, ToolResult# A missing-name failure lists the figures the model could have asked for, capped@@ -99,9 +99,29 @@ class ViewFigureTool(_ReadingToolBase):caption = str(row.get("caption") or "").strip()locator = row.get("locator")+ manifest = await asyncio.to_thread(store.manifest, material_id)+ if type(locator) is not int or not 1 <= locator <= manifest.unit_count:+ return self._failure(f"The image “{name}” has no valid document locator.")+ source = {+ "type": "reading",+ "material_id": material_id,+ "material_revision": manifest.revision,+ "title": manifest.filename,+ "page": locator,+ }+ metadata = {+ "material_id": material_id,+ "material_revision": manifest.revision,+ "image": name,+ "locator": locator,+ "caption": caption,+ }path = await asyncio.to_thread(store.media_path, material_id, name)if path is None:- return self._failure(f"The image “{name}” is missing on disk and cannot be viewed.")+ return self._failure(+ f"The image “{name}” on {manifest.unit} {locator} is missing on disk "+ "and cannot be viewed."+ )size = await asyncio.to_thread(lambda: path.stat().st_size)if size > MAX_IMAGE_BYTES:return self._failure(@@ -119,19 +139,16 @@ class ViewFigureTool(_ReadingToolBase):# Never answer with silence: hand back the caption if there is one,# and say plainly that the pixels were not read.content = (- f"Vision parsing is unavailable for this model, so “{name}” could "+ f"{manifest.unit} {locator}: Vision parsing is unavailable for this model, "+ f"so “{name}” could ""not be read.")if caption:content += f"\nCaption on record: {caption}"return ToolResult(content=content,- metadata={- "material_id": material_id,- "image": name,- "locator": locator,- "caption": caption,- },+ sources=[source],+ metadata=metadata,)mime = str(row.get("mime") or "") or (@@ -157,11 +174,7 @@ class ViewFigureTool(_ReadingToolBase):f"record is: {caption or '(none)'}")return ToolResult(- content=answer,- metadata={- "material_id": material_id,- "image": name,- "locator": locator,- "caption": caption,- },- )\ No newline at end of file+ content=f"{manifest.unit} {locator} ({name}): {answer}",+ sources=[source],+ metadata=metadata,+ )diff --git a/deeptutor/capabilities/reading/tools.py b/deeptutor/capabilities/reading/tools.pyindex f58f289fd..68df91f51 100644--- a/deeptutor/capabilities/reading/tools.py+++ b/deeptutor/capabilities/reading/tools.py@@ -41,7 +41,6 @@ from deeptutor.capabilities.reading._tool_base import (MATERIAL_KWARG,WORKSPACE_KWARG,_guard,- _NoMaterial,_ReadingToolBase,)from deeptutor.capabilities.reading.figure_view import ViewFigureTooldiff --git a/tests/reading/test_view_figure.py b/tests/reading/test_view_figure.pynew file mode 100644index 000000000..d64cb71eb--- /dev/null+++ b/tests/reading/test_view_figure.py@@ -0,0 +1,126 @@+"""Figure viewing returns a grounded locator, including when vision is unavailable."""++from __future__ import annotations++import asyncio+import base64+from pathlib import Path+from types import SimpleNamespace++import pytest++from deeptutor.capabilities.reading.figure_view import ViewFigureTool+++class _FigureStore:+ def __init__(self, image: Path | None) -> None:+ self.image = image++ def media_items(self, material_id: str) -> list[dict]:+ assert material_id == "material-1"+ return [+ {+ "name": "image-05.png",+ "locator": 2,+ "mime": "image/png",+ "caption": "A chart with two axes.",+ }+ ]++ def media_path(self, material_id: str, name: str) -> Path | None:+ assert (material_id, name) == ("material-1", "image-05.png")+ return self.image++ def manifest(self, material_id: str) -> SimpleNamespace:+ assert material_id == "material-1"+ return SimpleNamespace(unit="page", unit_count=3, revision=4, filename="paper.pdf")+++def configure_tool(monkeypatch):+ monkeypatch.setattr(ViewFigureTool, "_material_id", staticmethod(lambda _kwargs: "material-1"))+ monkeypatch.setattr(+ "deeptutor.services.rag.pipelines.llamaindex.config.image_description_limits",+ lambda: (0, 2.0),+ )++ def configure(image: Path | None, client: object) -> ViewFigureTool:+ monkeypatch.setattr(ViewFigureTool, "_store", staticmethod(lambda: _FigureStore(image)))+ monkeypatch.setattr("deeptutor.services.llm.client.get_llm_client", lambda: client)+ return ViewFigureTool()++ return configure+++def test_view_figure_sends_the_stored_pixels_and_returns_page_source(+ configure_tool, tmp_path: Path+) -> None:+ image = tmp_path / "image-05.png"+ image.write_bytes(b"\x89PNG\r\nfigure")++ class _VisionClient:+ call: tuple[str, dict] | None = None++ def supports_multimodal_images(self) -> bool:+ return True++ async def complete(self, question: str, **kwargs):+ self.call = (question, kwargs)+ return "The vertical axis is probability."++ client = _VisionClient()+ tool = configure_tool(image, client)+ result = asyncio.run(tool.execute(name="image-05.png", question="What is the vertical axis?"))++ assert result.success is True+ assert result.content == "page 2 (image-05.png): The vertical axis is probability."+ assert result.sources == [+ {+ "type": "reading",+ "material_id": "material-1",+ "material_revision": 4,+ "title": "paper.pdf",+ "page": 2,+ }+ ]+ assert client.call is not None+ assert client.call[0] == "What is the vertical axis?"+ assert client.call[1]["image_data"] == base64.b64encode(image.read_bytes()).decode("ascii")+ assert client.call[1]["image_mime_type"] == "image/png"+ assert client.call[1]["image_filename"] == "image-05.png"+++def test_view_figure_without_vision_identifies_the_unread_page_and_caption(+ configure_tool, tmp_path: Path+) -> None:+ image = tmp_path / "image-05.png"+ image.write_bytes(b"image")++ class _TextOnlyClient:+ def supports_multimodal_images(self) -> bool:+ return False++ async def complete(self, *_args, **_kwargs):+ raise AssertionError("The tool must not send pixels to a text-only client")++ result = asyncio.run(configure_tool(image, _TextOnlyClient()).execute(name="image-05.png"))++ assert result.success is True+ assert "page 2" in result.content+ assert "could not be read" in result.content+ assert "A chart with two axes." in result.content+ assert result.sources[0]["page"] == 2+++def test_view_figure_reports_a_missing_stored_image_without_calling_vision(+ configure_tool,+) -> None:+ class _UnexpectedClient:+ def supports_multimodal_images(self) -> bool:+ raise AssertionError("Missing image must not reach the vision client")++ result = asyncio.run(configure_tool(None, _UnexpectedClient()).execute(name="image-05.png"))++ assert result.success is False+ assert "page 2" in result.content+ assert "missing on disk" in result.content