Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(reading): cite figure locations and cover vision fallback

Bingxi Zhao (Frank)842ed970e8b0
diff --git a/deeptutor/capabilities/reading/figure_view.py b/deeptutor/capabilities/reading/figure_view.py
index b965045aa..affd2185d 100644
--- a/deeptutor/capabilities/reading/figure_view.py
+++ b/deeptutor/capabilities/reading/figure_view.py
@@ -10,7 +10,7 @@ from __future__ import annotations
import asyncio
from typing import Any
-from deeptutor.capabilities.reading._tool_base import _ReadingToolBase, _guard
+from deeptutor.capabilities.reading._tool_base import _guard, _ReadingToolBase
from deeptutor.core.tool_protocol import ToolDefinition, ToolParameter, ToolResult
# A missing-name failure lists the figures the model could have asked for, capped
@@ -99,9 +99,29 @@ class ViewFigureTool(_ReadingToolBase):
caption = str(row.get("caption") or "").strip()
locator = row.get("locator")
+ manifest = await asyncio.to_thread(store.manifest, material_id)
+ if type(locator) is not int or not 1 <= locator <= manifest.unit_count:
+ return self._failure(f"The image “{name}” has no valid document locator.")
+ source = {
+ "type": "reading",
+ "material_id": material_id,
+ "material_revision": manifest.revision,
+ "title": manifest.filename,
+ "page": locator,
+ }
+ metadata = {
+ "material_id": material_id,
+ "material_revision": manifest.revision,
+ "image": name,
+ "locator": locator,
+ "caption": caption,
+ }
path = await asyncio.to_thread(store.media_path, material_id, name)
if path is None:
- return self._failure(f"The image “{name}” is missing on disk and cannot be viewed.")
+ return self._failure(
+ f"The image “{name}” on {manifest.unit} {locator} is missing on disk "
+ "and cannot be viewed."
+ )
size = await asyncio.to_thread(lambda: path.stat().st_size)
if size > MAX_IMAGE_BYTES:
return self._failure(
@@ -119,19 +139,16 @@ class ViewFigureTool(_ReadingToolBase):
# Never answer with silence: hand back the caption if there is one,
# and say plainly that the pixels were not read.
content = (
- f"Vision parsing is unavailable for this model, so “{name}” could "
+ f"{manifest.unit} {locator}: Vision parsing is unavailable for this model, "
+ f"so “{name}” could "
"not be read."
)
if caption:
content += f"\nCaption on record: {caption}"
return ToolResult(
content=content,
- metadata={
- "material_id": material_id,
- "image": name,
- "locator": locator,
- "caption": caption,
- },
+ sources=[source],
+ metadata=metadata,
)
mime = str(row.get("mime") or "") or (
@@ -157,11 +174,7 @@ class ViewFigureTool(_ReadingToolBase):
f"record is: {caption or '(none)'}"
)
return ToolResult(
- content=answer,
- metadata={
- "material_id": material_id,
- "image": name,
- "locator": locator,
- "caption": caption,
- },
- )
\ No newline at end of file
+ content=f"{manifest.unit} {locator} ({name}): {answer}",
+ sources=[source],
+ metadata=metadata,
+ )
diff --git a/deeptutor/capabilities/reading/tools.py b/deeptutor/capabilities/reading/tools.py
index f58f289fd..68df91f51 100644
--- a/deeptutor/capabilities/reading/tools.py
+++ b/deeptutor/capabilities/reading/tools.py
@@ -41,7 +41,6 @@ from deeptutor.capabilities.reading._tool_base import (
MATERIAL_KWARG,
WORKSPACE_KWARG,
_guard,
- _NoMaterial,
_ReadingToolBase,
)
from deeptutor.capabilities.reading.figure_view import ViewFigureTool
diff --git a/tests/reading/test_view_figure.py b/tests/reading/test_view_figure.py
new file mode 100644
index 000000000..d64cb71eb
--- /dev/null
+++ b/tests/reading/test_view_figure.py
@@ -0,0 +1,126 @@
+"""Figure viewing returns a grounded locator, including when vision is unavailable."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from deeptutor.capabilities.reading.figure_view import ViewFigureTool
+
+
+class _FigureStore:
+ def __init__(self, image: Path | None) -> None:
+ self.image = image
+
+ def media_items(self, material_id: str) -> list[dict]:
+ assert material_id == "material-1"
+ return [
+ {
+ "name": "image-05.png",
+ "locator": 2,
+ "mime": "image/png",
+ "caption": "A chart with two axes.",
+ }
+ ]
+
+ def media_path(self, material_id: str, name: str) -> Path | None:
+ assert (material_id, name) == ("material-1", "image-05.png")
+ return self.image
+
+ def manifest(self, material_id: str) -> SimpleNamespace:
+ assert material_id == "material-1"
+ return SimpleNamespace(unit="page", unit_count=3, revision=4, filename="paper.pdf")
+
+
+def configure_tool(monkeypatch):
+ monkeypatch.setattr(ViewFigureTool, "_material_id", staticmethod(lambda _kwargs: "material-1"))
+ monkeypatch.setattr(
+ "deeptutor.services.rag.pipelines.llamaindex.config.image_description_limits",
+ lambda: (0, 2.0),
+ )
+
+ def configure(image: Path | None, client: object) -> ViewFigureTool:
+ monkeypatch.setattr(ViewFigureTool, "_store", staticmethod(lambda: _FigureStore(image)))
+ monkeypatch.setattr("deeptutor.services.llm.client.get_llm_client", lambda: client)
+ return ViewFigureTool()
+
+ return configure
+
+
+def test_view_figure_sends_the_stored_pixels_and_returns_page_source(
+ configure_tool, tmp_path: Path
+) -> None:
+ image = tmp_path / "image-05.png"
+ image.write_bytes(b"\x89PNG\r\nfigure")
+
+ class _VisionClient:
+ call: tuple[str, dict] | None = None
+
+ def supports_multimodal_images(self) -> bool:
+ return True
+
+ async def complete(self, question: str, **kwargs):
+ self.call = (question, kwargs)
+ return "The vertical axis is probability."
+
+ client = _VisionClient()
+ tool = configure_tool(image, client)
+ result = asyncio.run(tool.execute(name="image-05.png", question="What is the vertical axis?"))
+
+ assert result.success is True
+ assert result.content == "page 2 (image-05.png): The vertical axis is probability."
+ assert result.sources == [
+ {
+ "type": "reading",
+ "material_id": "material-1",
+ "material_revision": 4,
+ "title": "paper.pdf",
+ "page": 2,
+ }
+ ]
+ assert client.call is not None
+ assert client.call[0] == "What is the vertical axis?"
+ assert client.call[1]["image_data"] == base64.b64encode(image.read_bytes()).decode("ascii")
+ assert client.call[1]["image_mime_type"] == "image/png"
+ assert client.call[1]["image_filename"] == "image-05.png"
+
+
+def test_view_figure_without_vision_identifies_the_unread_page_and_caption(
+ configure_tool, tmp_path: Path
+) -> None:
+ image = tmp_path / "image-05.png"
+ image.write_bytes(b"image")
+
+ class _TextOnlyClient:
+ def supports_multimodal_images(self) -> bool:
+ return False
+
+ async def complete(self, *_args, **_kwargs):
+ raise AssertionError("The tool must not send pixels to a text-only client")
+
+ result = asyncio.run(configure_tool(image, _TextOnlyClient()).execute(name="image-05.png"))
+
+ assert result.success is True
+ assert "page 2" in result.content
+ assert "could not be read" in result.content
+ assert "A chart with two axes." in result.content
+ assert result.sources[0]["page"] == 2
+
+
+def test_view_figure_reports_a_missing_stored_image_without_calling_vision(
+ configure_tool,
+) -> None:
+ class _UnexpectedClient:
+ def supports_multimodal_images(self) -> bool:
+ raise AssertionError("Missing image must not reach the vision client")
+
+ result = asyncio.run(configure_tool(None, _UnexpectedClient()).execute(name="image-05.png"))
+
+ assert result.success is False
+ assert "page 2" in result.content
+ assert "missing on disk" in result.content