Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

Preserve retrieved source visuals for vision answers

Bingxi Zhao (Frank)5e7be659a224
diff --git a/deeptutor/agents/loop/pipeline.py b/deeptutor/agents/loop/pipeline.py
index 7e347e5c4..fdcee04a6 100644
--- a/deeptutor/agents/loop/pipeline.py
+++ b/deeptutor/agents/loop/pipeline.py
@@ -1325,6 +1325,9 @@ class AgenticLoopPipeline:
task_dir = Path(runtime_workspace.output_dir) if runtime_workspace is not None else task_dir
if tool_name == "rag":
kwargs.setdefault("mode", "hybrid")
+ from deeptutor.services.llm.capabilities import supports_vision
+
+ kwargs["_vision_supported"] = supports_vision(self.binding, self.model)
elif tool_name == "kb_files":
# The report is read by the user as much as by the model, so it is
# written in the turn's language. Injected server-side; the tool
diff --git a/deeptutor/api/routers/auth.py b/deeptutor/api/routers/auth.py
index 4248d4440..5764c3819 100644
--- a/deeptutor/api/routers/auth.py
+++ b/deeptutor/api/routers/auth.py
@@ -601,6 +601,7 @@ _LEARNER_KB_READ_ROUTES = frozenset(
"/api/knowledge-bases/{kb_name}/files",
"/api/knowledge-bases/{kb_name}/files/{filename:path}",
"/api/knowledge-bases/{kb_name}/file-preview-text/{filename:path}",
+ "/api/knowledge-bases/{kb_name}/visual-assets/{asset_id}",
"/api/knowledge-bases/{kb_name}/progress",
}
)
diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.py
index 6915f01da..c92014a0c 100644
--- a/deeptutor/api/routers/knowledge.py
+++ b/deeptutor/api/routers/knowledge.py
@@ -33,7 +33,7 @@ from fastapi import (
WebSocket,
WebSocketDisconnect,
)
-from fastapi.responses import FileResponse, PlainTextResponse, StreamingResponse
+from fastapi.responses import FileResponse, PlainTextResponse, Response, StreamingResponse
from pydantic import BaseModel, Field
from deeptutor.api.routers.auth import require_admin
@@ -96,6 +96,7 @@ from deeptutor.services.rag.pipelines.ima.config import (
ImaCredentials,
get_account_credentials,
)
+from deeptutor.services.rag.visual_assets import VisualAssetStore
from deeptutor.services.web_source.scheduler import get_web_source_sync_scheduler
from deeptutor.utils.document_extractor import (
MAX_EXTRACTED_CHARS_PER_DOC,
@@ -2978,6 +2979,9 @@ async def move_kb_file(kb_name: str, payload: MoveFilePayload):
dest_dir.mkdir(parents=True, exist_ok=True)
shutil.move(str(src), str(dest))
+ VisualAssetStore(manager.get_knowledge_base_path(kb_name)).move_source(
+ source_rel, dest.relative_to(raw_dir.resolve()).as_posix()
+ )
return {"status": "ok", "path": dest.relative_to(raw_dir.resolve()).as_posix()}
@@ -3020,6 +3024,22 @@ async def serve_kb_raw_file(kb_name: str, filename: str):
)
[email protected]("/knowledge-bases/{kb_name}/visual-assets/{asset_id}")
+async def serve_kb_visual_asset(kb_name: str, asset_id: str):
+ """Serve a verified source image from an access checked local KB."""
+ raw_dir = _resolve_kb_raw_dir(kb_name)
+ assert raw_dir is not None
+ loaded = VisualAssetStore(raw_dir.parent).read(asset_id)
+ if loaded is None:
+ raise HTTPException(status_code=404, detail="Visual asset not found")
+ record, data = loaded
+ return Response(
+ content=data,
+ media_type=record["mime_type"],
+ headers={"Cache-Control": "private, no-store", "X-Content-Type-Options": "nosniff"},
+ )
+
+
@router.delete("/knowledge-bases/{kb_name}/files/{filename:path}")
async def delete_kb_file(kb_name: str, filename: str):
"""Remove a single raw document from a knowledge base.
@@ -3045,6 +3065,7 @@ async def delete_kb_file(kb_name: str, filename: str):
kb_name,
target.name,
)
+ VisualAssetStore(kb_dir).remove_source(target.relative_to(kb_dir / "raw").as_posix())
removal = remove_raw_document(Path(kb_dir), target)
return {
"status": "ok",
diff --git a/deeptutor/core/tool_protocol.py b/deeptutor/core/tool_protocol.py
index 0ce950ade..edbfd8dc1 100644
--- a/deeptutor/core/tool_protocol.py
+++ b/deeptutor/core/tool_protocol.py
@@ -151,6 +151,9 @@ class ToolResult:
``ask_user`` to keep the turn alive across the user's
answer instead of ending and starting a new turn.
Shape mirrors ``AskUserPayload.to_dict()``.
+ model_message: Request-local multimodal context attached after this
+ tool result. The loop must never add it to durable messages or
+ user-facing stream metadata.
"""
content: str = ""
@@ -159,6 +162,9 @@ class ToolResult:
success: bool = True
terminate_turn: bool = False
pause_for_user: dict[str, Any] | None = None
+ # Private model-only follow-up (for example verified source image parts).
+ # Never copy this into stream events, citation rows, or persisted metadata.
+ model_message: dict[str, Any] | None = field(default=None, repr=False)
def __str__(self) -> str:
return self.content
diff --git a/deeptutor/runtime/agentic/loop.py b/deeptutor/runtime/agentic/loop.py
index e54298c1e..49d2a77f8 100644
--- a/deeptutor/runtime/agentic/loop.py
+++ b/deeptutor/runtime/agentic/loop.py
@@ -173,6 +173,26 @@ class LoopHost(Protocol):
return None
+def _with_transient_model_messages(
+ messages: list[dict[str, Any]], transient: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+ """Place model-only images after their tool result without mutating history."""
+ if not transient:
+ return messages
+ anchored: dict[str, list[dict[str, Any]]] = {}
+ for item in transient:
+ tool_call_id = item.get("_after_tool_call_id")
+ if not isinstance(tool_call_id, str) or not tool_call_id:
+ continue
+ anchored.setdefault(tool_call_id, []).append({"role": "user", "content": item["content"]})
+ request_messages: list[dict[str, Any]] = []
+ for message in messages:
+ request_messages.append(message)
+ if message.get("role") == "tool":
+ request_messages.extend(anchored.pop(str(message.get("tool_call_id") or ""), []))
+ return request_messages
+
+
async def run_agentic_loop(
*,
initial_messages: list[dict[str, Any]],
@@ -212,6 +232,10 @@ async def run_agentic_loop(
iterations don't spawn empty "Reasoning…" cards.
"""
messages = initial_messages
+ # Retrieved image bytes are request-local. They must never enter the
+ # durable conversation (where a synthetic user message would be persisted
+ # and displayed as if the person authored it).
+ transient_model_messages: list[dict[str, Any]] = []
aggregated_sources: list[dict[str, Any]] = []
final_text = ""
final_label_seen = ""
@@ -233,7 +257,7 @@ async def run_agentic_loop(
step = await run_labeled_step(
client=client,
model=model,
- messages=messages,
+ messages=_with_transient_model_messages(messages, transient_model_messages),
completion_kwargs=completion_kwargs,
tool_schemas=tool_schemas,
allowed_labels=protocol.allowed,
@@ -316,6 +340,7 @@ async def run_agentic_loop(
)
aggregated_sources.extend(outcome.sources)
messages.extend(outcome.tool_messages)
+ transient_model_messages.extend(outcome.model_messages)
if outcome.pause:
resumed = await host.resolve_pause(outcome)
if not resumed:
diff --git a/deeptutor/runtime/agentic/tool_dispatch.py b/deeptutor/runtime/agentic/tool_dispatch.py
index c924ac467..6376ec0a4 100644
--- a/deeptutor/runtime/agentic/tool_dispatch.py
+++ b/deeptutor/runtime/agentic/tool_dispatch.py
@@ -110,6 +110,7 @@ class DispatchOutcome:
sources: list[dict[str, Any]] = field(default_factory=list)
tool_messages: list[dict[str, Any]] = field(default_factory=list)
+ model_messages: list[dict[str, Any]] = field(default_factory=list, repr=False)
tool_metadata_by_id: dict[str, dict[str, Any]] = field(default_factory=dict)
terminate: bool = False
terminate_payload: dict[str, Any] | None = None
@@ -777,6 +778,7 @@ async def execute_tool_call(
"metadata": result.metadata,
"terminate_turn": getattr(result, "terminate_turn", False),
"pause_for_user": getattr(result, "pause_for_user", None),
+ "model_message": getattr(result, "model_message", None),
}
except Exception as exc:
# Unknown tool names arrive here too (the registry raises KeyError), so
@@ -843,6 +845,8 @@ async def _collect_outcome(
"""
aggregated_sources: list[dict[str, Any]] = []
tool_messages: list[dict[str, Any]] = []
+ model_messages: list[dict[str, Any]] = []
+ model_image_count = 0
tool_metadata_by_id: dict[str, dict[str, Any]] = {}
terminate = False
terminate_payload: dict[str, Any] | None = None
@@ -879,6 +883,18 @@ async def _collect_outcome(
"content": result_text,
}
)
+ private_message = result.get("model_message")
+ if isinstance(private_message, dict) and private_message.get("role") == "user":
+ content = private_message.get("content")
+ if isinstance(content, list):
+ image_count = sum(
+ 1
+ for part in content
+ if isinstance(part, dict) and part.get("type") == "image_url"
+ )
+ if image_count and model_image_count + image_count <= 2:
+ model_messages.append({**private_message, "_after_tool_call_id": tool_call_id})
+ model_image_count += image_count
if isinstance(tool_extra_meta, dict) and tool_extra_meta:
tool_metadata_by_id[tool_call_id] = dict(tool_extra_meta)
if result.get("terminate_turn") and not terminate:
@@ -900,6 +916,7 @@ async def _collect_outcome(
return DispatchOutcome(
sources=aggregated_sources,
tool_messages=tool_messages,
+ model_messages=model_messages,
tool_metadata_by_id=tool_metadata_by_id,
terminate=terminate,
terminate_payload=terminate_payload,
diff --git a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
index f5fded8ce..9a8cef74e 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py
@@ -26,6 +26,7 @@ from deeptutor.services.config.runtime_settings import DOCUMENT_PARSING_ENGINE_L
from deeptutor.services.embedding import get_embedding_client
from deeptutor.services.llm.client import get_llm_client
from deeptutor.services.rag.file_routing import FileTypeRouter
+from deeptutor.services.rag.visual_assets import VisualAssetCandidate, collect_visual_assets
from deeptutor.utils.document_validator import DocumentValidator
from .config import image_description_limits
@@ -59,6 +60,7 @@ class _ImageSource:
path: Path
origin: Path
+ visual: VisualAssetCandidate | None = None
class LlamaIndexDocumentLoader:
@@ -72,6 +74,9 @@ class LlamaIndexDocumentLoader:
self,
file_paths: Iterable[str],
image_progress_callback: Callable[[int, int], None] | None = None,
+ *,
+ kb_dir: Path | None = None,
+ visual_candidates: list[VisualAssetCandidate] | None = None,
) -> list[Any]:
documents: list[Any] = []
image_sources: list[_ImageSource] = []
@@ -85,7 +90,7 @@ class LlamaIndexDocumentLoader:
# the event loop stalls every other request for the whole PDF
# (same class of bug as upstream #761/#777). Hand it to a thread.
text, extracted_images, parse_engine = await asyncio.to_thread(
- self._parse_document, file_path
+ self._parse_document, file_path, kb_dir=kb_dir
)
scanned_pdf_needs_ocr = (
file_path.suffix.lower() == ".pdf"
@@ -98,6 +103,7 @@ class LlamaIndexDocumentLoader:
file_path,
None,
_PDF_OCR_FALLBACK_ENGINE,
+ kb_dir=kb_dir,
)
if fallback[0].strip() or fallback[1]:
text, extracted_images, parse_engine = fallback
@@ -117,6 +123,7 @@ class LlamaIndexDocumentLoader:
extracted_image_count=len(extracted_images),
)
image_sources.extend(extracted_images)
+ self._append_visual_documents(documents, extracted_images, visual_candidates)
for file_path_str in classification.text_files:
file_path = Path(file_path_str)
@@ -133,7 +140,7 @@ class LlamaIndexDocumentLoader:
if supports(path):
self.logger.info(f"Parsing image with active document parser: {path.name}")
text, extracted_images, parse_engine = await asyncio.to_thread(
- self._parse_document, path, parse_service
+ self._parse_document, path, parse_service, kb_dir=kb_dir
)
if text.strip() or extracted_images:
self._append_if_nonempty(
@@ -144,6 +151,7 @@ class LlamaIndexDocumentLoader:
extracted_image_count=len(extracted_images),
)
image_sources.extend(extracted_images)
+ self._append_visual_documents(documents, extracted_images, visual_candidates)
else:
# Preserve the pre-parser behavior when an image-capable
# engine fails or yields no usable IR.
@@ -151,10 +159,11 @@ class LlamaIndexDocumentLoader:
else:
image_sources.append(_ImageSource(path=path, origin=path))
- if image_sources:
+ legacy_image_sources = [source for source in image_sources if source.visual is None]
+ if legacy_image_sources:
documents.extend(
await self._load_image_nodes(
- image_sources, image_progress_callback=image_progress_callback
+ legacy_image_sources, image_progress_callback=image_progress_callback
)
)
@@ -168,6 +177,8 @@ class LlamaIndexDocumentLoader:
file_path: Path,
parse_service=None, # noqa: ANN001
engine: str | None = None,
+ *,
+ kb_dir: Path | None = None,
) -> tuple[str, list[_ImageSource], str]:
"""Parse a document through the shared, engine-pluggable parse layer.
@@ -197,8 +208,54 @@ class LlamaIndexDocumentLoader:
text = parsed.markdown.strip() or self._text_from_blocks(parsed.blocks)
images = self._collect_asset_images(parsed.asset_dir, origin=file_path)
+ if kb_dir is not None and images:
+ by_path = {
+ candidate.path.resolve(): candidate
+ for candidate in collect_visual_assets(parsed, file_path, kb_dir)
+ }
+ images = [
+ _ImageSource(
+ path=image.path, origin=image.origin, visual=by_path[image.path.resolve()]
+ )
+ for image in images
+ if image.path.resolve() in by_path
+ ]
return text, images, str(parsed.engine or "")
+ @staticmethod
+ def _append_visual_documents(
+ documents: list[Any],
+ images: list[_ImageSource],
+ candidates: list[VisualAssetCandidate] | None,
+ ) -> None:
+ for image in images:
+ if image.visual is None:
+ continue
+ record = image.visual.record
+ if candidates is not None:
+ candidates.append(image.visual)
+ caption = record["caption"] or "Source figure"
+ context = record["context"]
+ text = f"[Source visual] {image.origin.name}: {caption}"
+ if context:
+ text += f"\nContext: {context}"
+ documents.append(
+ Document(
+ text=text,
+ metadata={
+ "file_name": image.origin.name,
+ "file_path": str(image.origin),
+ "content_type": "source_visual",
+ "visual_asset_id": record["asset_id"],
+ "source_document_id": record["source_document_id"],
+ "page": record["page_number"] or "",
+ "bbox": record["bbox"],
+ "caption": record["caption"],
+ "source_locator": record["source_locator"],
+ },
+ )
+ )
+
def _log_scanned_pdf_without_ocr(
self,
file_path: Path,
diff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
index 3c9453752..c6a589b6a 100644
--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py
@@ -22,6 +22,7 @@ from deeptutor.services.rag.index_versioning import (
write_version_meta,
)
from deeptutor.services.rag.kb_paths import resolve_kb_dir
+from deeptutor.services.rag.visual_assets import VisualAssetStore
from . import storage
from .config import default_top_k, should_show_progress
@@ -212,8 +213,12 @@ class LlamaIndexPipeline:
try:
await self._verify_embedding_connectivity()
+ visual_candidates = []
documents = await self.document_loader.load(
- file_paths, image_progress_callback=image_progress_callback
+ file_paths,
+ image_progress_callback=image_progress_callback,
+ kb_dir=kb_dir,
+ visual_candidates=visual_candidates,
)
if not documents:
self.logger.error("No valid documents found")
@@ -233,6 +238,7 @@ class LlamaIndexPipeline:
)
self.logger.info(f"Index persisted to {storage_dir}")
+ VisualAssetStore(kb_dir).publish(visual_candidates, prune_missing=True)
if signature is not None:
write_version_meta(kb_dir, signature, storage_dir=storage_dir)
@@ -351,6 +357,17 @@ class LlamaIndexPipeline:
"page": meta.get("page_label", meta.get("page", "")),
"chunk_id": node.node.node_id or str(i),
"score": round(node.score, 4) if node.score is not None else "",
+ **(
+ {
+ "visual_asset_id": meta["visual_asset_id"],
+ "source_document_id": meta.get("source_document_id", ""),
+ "bbox": meta.get("bbox"),
+ "caption": meta.get("caption", ""),
+ "source_locator": meta.get("source_locator", ""),
+ }
+ if meta.get("visual_asset_id")
+ else {}
+ ),
}
)
@@ -378,8 +395,12 @@ class LlamaIndexPipeline:
try:
await self._verify_embedding_connectivity()
+ visual_candidates = []
documents = await self.document_loader.load(
- file_paths, image_progress_callback=image_progress_callback
+ file_paths,
+ image_progress_callback=image_progress_callback,
+ kb_dir=kb_dir,
+ visual_candidates=visual_candidates,
)
if not documents:
self.logger.warning("No valid documents to add")
@@ -395,6 +416,7 @@ class LlamaIndexPipeline:
worker_key=str(kb_dir.resolve()),
)
self.logger.info(f"Added {num_added} documents to existing index")
+ VisualAssetStore(kb_dir).publish(visual_candidates)
if signature is not None and plan.storage_dir != plan.existing_storage:
write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)
else:
@@ -408,6 +430,7 @@ class LlamaIndexPipeline:
worker_key=str(kb_dir.resolve()),
)
self.logger.info(f"Created new index with {num_added} documents")
+ VisualAssetStore(kb_dir).publish(visual_candidates)
if signature is not None:
write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)
diff --git a/deeptutor/services/rag/visual_assets.py b/deeptutor/services/rag/visual_assets.py
new file mode 100644
index 000000000..0541ad0f3
--- /dev/null
+++ b/deeptutor/services/rag/visual_assets.py
@@ -0,0 +1,389 @@
+"""Verified, parser-independent source images owned by a knowledge base.
+
+The manifest is deliberately outside any vector index. Retrieval engines may
+index ``asset_id`` as metadata, but only this store resolves it to image bytes.
+The caller must first resolve the KB through the normal access guard.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from hashlib import sha256
+from io import BytesIO
+import json
+import os
+from pathlib import Path
+import re
+import tempfile
+from typing import Any
+import warnings
+
+from deeptutor.services.parsing.types import ParsedDocument
+
+MAX_IMAGE_BYTES = 5 * 1024 * 1024
+MAX_ASSETS_PER_DOCUMENT = 64
+MAX_MODEL_IMAGES = 2
+MAX_IMAGE_PIXELS = 30_000_000
+MAX_MANIFEST_BYTES = 16 * 1024 * 1024
+_ID_RE = re.compile(r"^[0-9a-f]{64}$")
+_MARKDOWN_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
+_MIME_EXT = {
+ "image/png": ".png",
+ "image/jpeg": ".jpg",
+ "image/gif": ".gif",
+ "image/webp": ".webp",
+}
+
+
+def _image_mime(data: bytes) -> str | None:
+ if data.startswith(b"\x89PNG\r\n\x1a\n"):
+ mime = "image/png"
+ elif data.startswith(b"\xff\xd8\xff"):
+ mime = "image/jpeg"
+ elif data.startswith((b"GIF87a", b"GIF89a")):
+ mime = "image/gif"
+ elif data.startswith(b"RIFF") and data[8:12] == b"WEBP":
+ mime = "image/webp"
+ else:
+ return None
+ # Check that the bytes actually decode as an image, rather than trusting
+ # an extension or a short magic prefix supplied by a parser.
+ try:
+ from PIL import Image
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", Image.DecompressionBombWarning)
+ with Image.open(BytesIO(data)) as image:
+ if image.width * image.height > MAX_IMAGE_PIXELS:
+ return None
+ image.verify()
+ except Exception: # malformed or oversized images must not abort document indexing
+ return None
+ return mime
+
+
+def _image_bytes(path: Path) -> tuple[bytes, str] | None:
+ if path.is_symlink() or not path.is_file():
+ return None
+ size = path.stat().st_size
+ if not 0 < size <= MAX_IMAGE_BYTES:
+ return None
+ data = path.read_bytes()
+ if len(data) > MAX_IMAGE_BYTES:
+ return None
+ mime = _image_mime(data)
+ return (data, mime) if mime is not None else None
+
+
+def source_key_for(kb_dir: Path, source: Path) -> str:
+ try:
+ return source.resolve().relative_to((kb_dir / "raw").resolve()).as_posix()
+ except ValueError:
+ # Direct SDK callers can index paths outside raw/. Never persist the
+ # host path in an asset manifest or expose it through citations.
+ return source.name
+
+
+def _sha256_file(path: Path) -> str:
+ digest = sha256()
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1 << 20), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def _managed_source(kb_dir: Path, source: Path) -> bool:
+ return source.resolve().is_relative_to((kb_dir / "raw").resolve())
+
+
+def _text(value: Any) -> str:
+ if isinstance(value, str):
+ return value.strip()
+ if isinstance(value, list):
+ return " ".join(filter(None, (_text(item) for item in value))).strip()
+ return ""
+
+
+def _block_details(
+ blocks: list[dict[str, Any]] | None, asset: Path
+) -> tuple[str, str, int | None, int | None, list[float] | None, str]:
+ if not blocks:
+ return "", "", None, None, None, ""
+ for index, block in enumerate(blocks):
+ if not isinstance(block, dict):
+ continue
+ raw = str(block.get("img_path") or block.get("path") or "")
+ if not raw or Path(raw).resolve() != asset.resolve():
+ continue
+ caption = _text(
+ block.get("caption")
+ or block.get("image_caption")
+ or block.get("chart_caption")
+ or block.get("captions")
+ )
+ context = _text(block.get("text") or block.get("content"))
+ if not context:
+ neighbors = blocks[max(0, index - 1) : index] + blocks[index + 1 : index + 2]
+ context = " ".join(
+ _text(item.get("text") or item.get("content"))
+ for item in neighbors
+ if isinstance(item, dict)
+ ).strip()
+ has_page_index = block.get("page_idx") is not None
+ page_raw = block.get("page_idx") if has_page_index else block.get("page")
+ try:
+ page_value = int(page_raw) if page_raw is not None else None
+ except (ValueError, TypeError):
+ page_value = None
+ page_index = page_value if has_page_index else None
+ page_number = (
+ (page_value + 1 if has_page_index else page_value) if page_value is not None else None
+ )
+ raw_bbox = block.get("bbox")
+ try:
+ bbox = (
+ [float(value) for value in raw_bbox[:4]]
+ if isinstance(raw_bbox, (list, tuple)) and len(raw_bbox) >= 4
+ else None
+ )
+ except (ValueError, TypeError):
+ bbox = None
+ return caption[:500], context[:500], page_index, page_number, bbox, f"blocks.json#/{index}"
+ return "", "", None, None, None, ""
+
+
+def _markdown_details(markdown: str, asset: Path) -> tuple[str, str, str]:
+ for match in _MARKDOWN_IMAGE_RE.finditer(markdown):
+ if Path(match.group(2).split("?", 1)[0]).name != asset.name:
+ continue
+ excerpt = markdown[max(0, match.start() - 250) : min(len(markdown), match.end() + 250)]
+ excerpt = _MARKDOWN_IMAGE_RE.sub(" ", excerpt)
+ return (
+ match.group(1).strip()[:500],
+ " ".join(excerpt.split())[:500],
+ f"markdown:{match.start()}",
+ )
+ return "", "", ""
+
+
+@dataclass(frozen=True)
+class VisualAssetCandidate:
+ path: Path
+ record: dict[str, Any]
+
+
+def collect_visual_assets(
+ parsed: ParsedDocument, source: Path, kb_dir: Path
+) -> list[VisualAssetCandidate]:
+ """Normalize extracted PDF/EPUB images with provenance and byte identity."""
+ asset_dir = parsed.asset_dir
+ if asset_dir is None or asset_dir.is_symlink() or not asset_dir.is_dir():
+ return []
+ source_key = source_key_for(kb_dir, source)
+ source_hash = _sha256_file(source)
+ candidates: list[VisualAssetCandidate] = []
+ for path in sorted(asset_dir.iterdir()):
+ if len(candidates) >= MAX_ASSETS_PER_DOCUMENT:
+ break
+ loaded = _image_bytes(path)
+ if loaded is None:
+ continue
+ image, mime = loaded
+ image_hash = sha256(image).hexdigest()
+ caption, context, page_index, page_number, bbox, locator = _block_details(
+ parsed.blocks, path
+ )
+ md_caption, md_context, md_locator = _markdown_details(parsed.markdown, path)
+ caption = caption or md_caption
+ context = context or md_context
+ locator = locator or md_locator or f"asset:{path.name}"
+ asset_id = sha256(
+ f"{source_key}\0{source_hash}\0{parsed.parser_signature}\0{locator}\0{image_hash}".encode()
+ ).hexdigest()
+ record = {
+ "asset_id": asset_id,
+ "source_document_id": source_hash,
+ "source_path": source_key,
+ "managed_source": _managed_source(kb_dir, source),
+ "parser_engine": parsed.engine,
+ "parser_signature": parsed.parser_signature,
+ "source_hash": parsed.source_hash,
+ "source_locator": locator,
+ "page_index": page_index,
+ "page_number": page_number,
+ "bbox": bbox,
+ "caption": caption,
+ "context": context,
+ "image_sha256": image_hash,
+ "mime_type": mime,
+ "size": len(image),
+ }
+ candidates.append(VisualAssetCandidate(path=path, record=record))
+ return candidates
+
+
+class VisualAssetStore:
+ """KB-scoped immutable image files plus an atomically replaced manifest."""
+
+ def __init__(self, kb_dir: Path):
+ self.kb_dir = Path(kb_dir)
+ self.root = self.kb_dir / "visual_assets"
+ self.manifest_path = self.root / "manifest.json"
+
+ def records(self) -> dict[str, dict[str, Any]]:
+ if self.root.is_symlink() or not self.root.resolve().is_relative_to(self.kb_dir.resolve()):
+ return {}
+ try:
+ if self.manifest_path.stat().st_size > MAX_MANIFEST_BYTES:
+ return {}
+ payload = json.loads(self.manifest_path.read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ return {}
+ records = payload.get("assets") if isinstance(payload, dict) else None
+ return records if isinstance(records, dict) else {}
+
+ def publish(
+ self,
+ candidates: list[VisualAssetCandidate],
+ *,
+ replace: bool = False,
+ source_paths: set[str] | None = None,
+ prune_missing: bool = False,
+ ) -> None:
+ if self.root.is_symlink() or not self.root.resolve().is_relative_to(self.kb_dir.resolve()):
+ raise OSError("Visual asset directory escapes knowledge base")
+ self.root.mkdir(parents=True, exist_ok=True)
+ prior = self.records()
+ incoming = {item.record["asset_id"]: item.record for item in candidates}
+ if replace:
+ remaining: dict[str, dict[str, Any]] = {}
+ else:
+ # Keep older index versions' asset IDs reachable when the same
+ # source is re-parsed with a new parser or embedding signature.
+ # Explicit raw-file deletion removes all IDs for that source.
+ replaced_sources = source_paths or set()
+ remaining = {
+ key: value
+ for key, value in prior.items()
+ if value.get("source_path") not in replaced_sources
+ }
+ if prune_missing:
+ remaining = {
+ key: value
+ for key, value in remaining.items()
+ if not value.get("managed_source")
+ or self._source_exists(value.get("source_path"))
+ }
+ for candidate in candidates:
+ record = candidate.record
+ loaded = _image_bytes(candidate.path)
+ if (
+ loaded is None
+ or sha256(loaded[0]).hexdigest() != record["image_sha256"]
+ or loaded[1] != record["mime_type"]
+ ):
+ raise OSError(f"Source image changed during indexing: {candidate.path.name}")
+ target = self._path(record["asset_id"], record["mime_type"])
+ existing = _image_bytes(target)
+ if existing is None or sha256(existing[0]).hexdigest() != record["image_sha256"]:
+ self._atomic_write(target, loaded[0])
+ updated = {**remaining, **incoming}
+ self._atomic_write(
+ self.manifest_path,
+ json.dumps(
+ {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True
+ ).encode(),
+ )
+ for old_id, old_record in prior.items():
+ if old_id not in updated:
+ self._path(old_id, str(old_record.get("mime_type"))).unlink(missing_ok=True)
+
+ def read(self, asset_id: str) -> tuple[dict[str, Any], bytes] | None:
+ if not _ID_RE.fullmatch(asset_id):
+ return None
+ record = self.records().get(asset_id)
+ if not isinstance(record, dict) or record.get("asset_id") != asset_id:
+ return None
+ mime = record.get("mime_type")
+ if mime not in _MIME_EXT:
+ return None
+ path = self._path(asset_id, mime)
+ loaded = _image_bytes(path)
+ if loaded is None or loaded[1] != mime:
+ return None
+ data = loaded[0]
+ if len(data) != record.get("size") or sha256(data).hexdigest() != record.get(
+ "image_sha256"
+ ):
+ return None
+ return record, data
+
+ def remove_source(self, source_relative_path: str) -> None:
+ prior = self.records()
+ updated = {
+ key: value
+ for key, value in prior.items()
+ if value.get("source_path") != source_relative_path
+ }
+ if len(updated) == len(prior):
+ return
+ self._atomic_write(
+ self.manifest_path,
+ json.dumps(
+ {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True
+ ).encode(),
+ )
+ for old_id, old_record in prior.items():
+ if old_id not in updated:
+ self._path(old_id, str(old_record.get("mime_type"))).unlink(missing_ok=True)
+
+ def move_source(self, old_relative_path: str, new_relative_path: str) -> None:
+ prior = self.records()
+ updated = {}
+ changed = False
+ for key, value in prior.items():
+ source = value.get("source_path")
+ if source == old_relative_path or (
+ isinstance(source, str) and source.startswith(old_relative_path + "/")
+ ):
+ value = {
+ **value,
+ "source_path": new_relative_path + source[len(old_relative_path) :],
+ }
+ changed = True
+ updated[key] = value
+ if changed:
+ self._atomic_write(
+ self.manifest_path,
+ json.dumps(
+ {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True
+ ).encode(),
+ )
+
+ def _path(self, asset_id: str, mime: str) -> Path:
+ if not _ID_RE.fullmatch(asset_id) or mime not in _MIME_EXT:
+ raise ValueError("Invalid visual asset record")
+ return self.root / (asset_id + _MIME_EXT[mime])
+
+ def _source_exists(self, relative_path: Any) -> bool:
+ if not isinstance(relative_path, str):
+ return False
+ relative = Path(relative_path)
+ if relative.is_absolute() or ".." in relative.parts:
+ return False
+ raw = (self.kb_dir / "raw").resolve()
+ source = (raw / relative).resolve()
+ return source.is_relative_to(raw) and source.is_file()
+
+ @staticmethod
+ def _atomic_write(path: Path, data: bytes) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ fd, temp_name = tempfile.mkstemp(prefix=".visual-", dir=path.parent)
+ try:
+ with os.fdopen(fd, "wb") as handle:
+ handle.write(data)
+ handle.flush()
+ os.fsync(handle.fileno())
+ os.replace(temp_name, path)
+ finally:
+ Path(temp_name).unlink(missing_ok=True)
diff --git a/deeptutor/tools/builtin/__init__.py b/deeptutor/tools/builtin/__init__.py
index bd139026b..33b294c16 100644
--- a/deeptutor/tools/builtin/__init__.py
+++ b/deeptutor/tools/builtin/__init__.py
@@ -86,7 +86,18 @@ def _rag_sources(result: dict[str, Any], *, query: str, kb_name: str) -> list[di
return []
if not retrieved:
return [{"type": "rag", "query": query, "kb_name": kb_name}]
- return [{"type": "rag", "kb_name": kb_name, **item} for item in retrieved]
+ from urllib.parse import quote
+
+ sources = []
+ for item in retrieved:
+ source = {"type": "rag", "kb_name": kb_name, **item}
+ asset_id = source.get("visual_asset_id")
+ if asset_id:
+ source["visual_asset_url"] = (
+ f"/api/knowledge-bases/{quote(kb_name, safe='')}/visual-assets/{asset_id}"
+ )
+ sources.append(source)
+ return sources
class RAGTool(_PromptHintsMixin, BaseTool):
@@ -118,10 +129,11 @@ class RAGTool(_PromptHintsMixin, BaseTool):
if not kb_name:
raise ValueError("RAG requires an explicit kb_name.")
event_sink = kwargs.get("event_sink")
+ vision_supported = bool(kwargs.get("_vision_supported", False))
extra_kwargs = {
key: value
for key, value in kwargs.items()
- if key not in {"query", "kb_name", "event_sink"}
+ if key not in {"query", "kb_name", "event_sink", "_vision_supported"}
}
result = await rag_search(
@@ -141,12 +153,71 @@ class RAGTool(_PromptHintsMixin, BaseTool):
f"No matching content was found in knowledge base '{kb_name}'. "
"The search completed successfully."
)
+ visual_sources = [
+ source
+ for source in (result.get("sources") or [])
+ if isinstance(source, dict) and source.get("visual_asset_id")
+ ]
+ model_message = None
+ if visual_sources and vision_supported:
+ model_message = _rag_visual_model_message(kb_name, visual_sources)
+ if model_message is None:
+ content += "\nRetrieved source visuals were unavailable for pixel inspection."
+ elif visual_sources:
+ content += (
+ "\nThe selected model cannot inspect source image pixels. "
+ "Use the retrieved caption and text context; do not claim to have seen the image."
+ )
return ToolResult(
content=content,
sources=_rag_sources(result, query=query, kb_name=kb_name),
metadata=result,
success=not failed,
+ model_message=model_message,
+ )
+
+
+def _rag_visual_model_message(kb_name: str, sources: list[dict[str, Any]]) -> dict[str, Any] | None:
+ """Hydrate retrieved source pixels only after resolving this user's KB scope."""
+ import base64
+
+ from deeptutor.multi_user.knowledge_access import resolve_for_rag
+ from deeptutor.services.rag.kb_paths import resolve_kb_dir
+ from deeptutor.services.rag.visual_assets import MAX_MODEL_IMAGES, VisualAssetStore
+
+ resource = resolve_for_rag(kb_name)
+ if resource is None:
+ return None
+ store = VisualAssetStore(resolve_kb_dir(resource.base_dir, resource.name))
+ parts: list[dict[str, Any]] = []
+ seen: set[str] = set()
+ for source in sources:
+ asset_id = str(source.get("visual_asset_id") or "")
+ if asset_id in seen:
+ continue
+ seen.add(asset_id)
+ loaded = store.read(asset_id)
+ if loaded is None:
+ continue
+ record, data = loaded
+ label = record.get("caption") or record.get("source_locator") or "source visual"
+ parts.append(
+ {
+ "type": "text",
+ "text": f"Retrieved source visual {asset_id}: {label}. Inspect its pixels when answering.",
+ }
+ )
+ parts.append(
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": f"data:{record['mime_type']};base64,{base64.b64encode(data).decode('ascii')}"
+ },
+ }
)
+ if len(parts) // 2 >= MAX_MODEL_IMAGES:
+ break
+ return {"role": "user", "content": parts} if parts else None
class KbFilesTool(_PromptHintsMixin, BaseTool):
diff --git a/docs-for-user/SOURCE_VISUALS.md b/docs-for-user/SOURCE_VISUALS.md
new file mode 100644
index 000000000..674b8d1fc
--- /dev/null
+++ b/docs-for-user/SOURCE_VISUALS.md
@@ -0,0 +1,28 @@
+# Source visuals in knowledge bases
+
+DeepTutor can retain extracted source images when a PDF or EPUB parser emits
+image files. The KB stores verified parser-extracted PNG, JPEG, GIF, or WebP
+bytes with a deterministic asset ID, document hash, parser identity, source
+locator, caption and nearby text. Structured parser blocks also preserve a page
+index and bounding box
+when available. The visual record lives under the KB's `visual_assets/`
+directory, separate from the vector index. The access checked API route
+`/api/knowledge-bases/{kb_name}/visual-assets/{asset_id}` serves those exact
+bytes, and deleting a raw source file removes its visual assets.
+
+The LlamaIndex KB pipeline indexes a text record for each source visual, so a
+text embedding model can retrieve it by its caption and context. When `rag`
+retrieves that record in the chat loop, a vision capable answer model receives
+the verified image pixels in its next request. A text only model receives the
+caption and context with an explicit warning that it has not seen the pixels.
+At most 64 images per document are retained, each image is limited to 5 MiB,
+and at most two retrieved images are sent in one model continuation.
+
+Current extraction coverage depends on the selected parser. MinerU can emit
+structured PDF figures. PyMuPDF4LLM can emit PDF and EPUB images when image
+extraction is enabled, but its Markdown output does not supply page boxes.
+The default text only parser and the current Docling adapter do not emit
+source image assets. Images that are only vector drawing commands, or pages
+that require OCR when no usable OCR engine is configured, may be absent.
+Other RAG providers do not yet index this visual manifest. Interactive
+exercises, visual annotations, and mastery updates are separate future work.
diff --git a/packaging/deeptutor-cli/pyproject.toml b/packaging/deeptutor-cli/pyproject.toml
index e0c091076..43d7b824a 100644
--- a/packaging/deeptutor-cli/pyproject.toml
+++ b/packaging/deeptutor-cli/pyproject.toml
@@ -54,6 +54,7 @@ dependencies = [
"faiss-cpu>=1.8.0,<2.0.0; python_version < '3.14'",
"faiss-cpu>=1.12.0,<2.0.0; python_version >= '3.14'",
"PyMuPDF>=1.26.0",
+ "Pillow>=10.0.0",
"numpy>=1.24.0,<3.0.0",
"arxiv>=2.0.0",
"python-docx>=1.1.0",
diff --git a/pyproject.toml b/pyproject.toml
index 4ef9b69c5..345ab6968 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -64,6 +64,7 @@ dependencies = [
"faiss-cpu>=1.8.0,<2.0.0; python_version < '3.14'",
"faiss-cpu>=1.12.0,<2.0.0; python_version >= '3.14'",
"PyMuPDF>=1.26.0",
+ "Pillow>=10.0.0",
"numpy>=1.24.0,<3.0.0",
"arxiv>=2.0.0",
"python-docx>=1.1.0",
@@ -123,6 +124,7 @@ cli = [
"faiss-cpu>=1.8.0,<2.0.0; python_version < '3.14'",
"faiss-cpu>=1.12.0,<2.0.0; python_version >= '3.14'",
"PyMuPDF>=1.26.0",
+ "Pillow>=10.0.0",
"numpy>=1.24.0,<3.0.0",
"arxiv>=2.0.0",
# Document text extraction (chat attachments) + office-skill authoring libs
diff --git a/requirements/cli.txt b/requirements/cli.txt
index ed38869a2..f93536dee 100644
--- a/requirements/cli.txt
+++ b/requirements/cli.txt
@@ -48,6 +48,7 @@ llama-index-vector-stores-faiss>=0.4.0,<1.0.0
faiss-cpu>=1.8.0,<2.0.0; python_version < "3.14"
faiss-cpu>=1.12.0,<2.0.0; python_version >= "3.14"
PyMuPDF>=1.26.0
+Pillow>=10.0.0
numpy>=1.24.0,<3.0.0
arxiv>=2.0.0
diff --git a/tests/api/test_knowledge_router.py b/tests/api/test_knowledge_router.py
index 2ddf5e776..237e2b46c 100644
--- a/tests/api/test_knowledge_router.py
+++ b/tests/api/test_knowledge_router.py
@@ -99,6 +99,11 @@ def _materialized_routes(app: "FastAPI"):
"/api/knowledge-bases/{kb_name}/file-preview-text/{filename:path}",
"reading",
),
+ (
+ "/api/knowledge-bases/demo/visual-assets/abc123",
+ "/api/knowledge-bases/{kb_name}/visual-assets/{asset_id}",
+ "reading",
+ ),
(
"/api/knowledge-bases/demo/progress",
"/api/knowledge-bases/{kb_name}/progress",
diff --git a/tests/services/rag/test_visual_assets.py b/tests/services/rag/test_visual_assets.py
new file mode 100644
index 000000000..1e85e4831
--- /dev/null
+++ b/tests/services/rag/test_visual_assets.py
@@ -0,0 +1,483 @@
+"""Source visual provenance, KB scope, and model pixel delivery."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+from pathlib import Path
+from types import SimpleNamespace
+import zipfile
+
+from PIL import Image
+import pytest
+
+from deeptutor.services.parsing.types import ParsedDocument
+from deeptutor.services.rag.visual_assets import (
+ VisualAssetStore,
+ collect_visual_assets,
+)
+
+
+def _fixture(tmp_path: Path, *, suffix: str = ".pdf", engine: str = "mineru"):
+ kb_dir = tmp_path / "kb"
+ raw = kb_dir / "raw"
+ raw.mkdir(parents=True)
+ source = raw / f"lesson{suffix}"
+ source.write_bytes(b"source document with a figure")
+ assets = tmp_path / "parse-cache" / "images"
+ assets.mkdir(parents=True)
+ image = assets / "figure.png"
+ Image.new("RGB", (3, 2), color=(17, 99, 211)).save(image)
+ parsed = ParsedDocument(
+ markdown="The learning curve is shown below.\n\n![Learning curve](images/figure.png)",
+ blocks=(
+ [
+ {
+ "type": "image",
+ "img_path": str(image),
+ "page_idx": 2,
+ "bbox": [10, 20, 100, 200],
+ "image_caption": ["Learning curve"],
+ }
+ ]
+ if engine == "mineru"
+ else None
+ ),
+ asset_dir=assets,
+ source_hash="source-hash",
+ parser_signature=f"{engine}-signature",
+ engine=engine,
+ )
+ return kb_dir, source, image, parsed
+
+
+ ("suffix", "engine"),
+ [(".pdf", "mineru"), (".epub", "pymupdf4llm")],
+)
+def test_verified_source_visual_contract_and_lifecycle(tmp_path: Path, suffix: str, engine: str):
+ kb_dir, source, image, parsed = _fixture(tmp_path, suffix=suffix, engine=engine)
+ candidates = collect_visual_assets(parsed, source, kb_dir)
+ assert len(candidates) == 1
+ record = candidates[0].record
+ assert record["asset_id"] == collect_visual_assets(parsed, source, kb_dir)[0].record["asset_id"]
+ assert record["source_path"] == source.name
+ assert record["source_document_id"]
+ assert record["parser_engine"] == engine
+ assert record["caption"] == "Learning curve"
+ if engine == "mineru":
+ assert record["page_index"] == 2
+ assert record["page_number"] == 3
+ assert record["bbox"] == [10, 20, 100, 200]
+ assert record["source_locator"] == "blocks.json#/0"
+ else:
+ assert record["page_index"] is None
+ assert record["source_locator"].startswith("markdown:")
+
+ store = VisualAssetStore(kb_dir)
+ store.publish(candidates, replace=True)
+ assert store.read(record["asset_id"]) == (record, image.read_bytes())
+ assert store.read("../manifest.json") is None
+ persisted = store.root / (record["asset_id"] + ".png")
+ persisted.write_bytes(b"\x89PNG\r\n\x1a\ncorrupted")
+ assert store.read(record["asset_id"]) is None
+ store.publish(candidates, replace=True)
+ newer = ParsedDocument(
+ markdown=parsed.markdown,
+ blocks=parsed.blocks,
+ asset_dir=parsed.asset_dir,
+ parser_signature="new-parser-signature",
+ engine=parsed.engine,
+ )
+ newer_candidate = collect_visual_assets(newer, source, kb_dir)[0]
+ store.publish([newer_candidate])
+ assert store.read(record["asset_id"]) is not None # older index version remains valid
+ assert store.read(newer_candidate.record["asset_id"]) is not None
+ moved = kb_dir / "raw" / "folder" / source.name
+ moved.parent.mkdir()
+ source.rename(moved)
+ store.move_source(source.name, f"folder/{source.name}")
+ assert store.records()[record["asset_id"]]["source_path"] == f"folder/{source.name}"
+ store.remove_source(f"folder/{source.name}")
+ assert store.read(record["asset_id"]) is None
+ assert store.read(newer_candidate.record["asset_id"]) is None
+ assert not persisted.exists()
+
+
+def test_size_count_and_rebuild_cleanup(tmp_path: Path, monkeypatch):
+ import deeptutor.services.rag.visual_assets as assets_module
+
+ kb_dir, source, image, parsed = _fixture(tmp_path)
+ store = VisualAssetStore(kb_dir)
+ first = collect_visual_assets(parsed, source, kb_dir)[0]
+ store.publish([first])
+ source.unlink()
+ store.publish([], prune_missing=True)
+ assert store.read(first.record["asset_id"]) is None
+
+ source.write_bytes(b"source document with a figure")
+ monkeypatch.setattr(assets_module, "MAX_IMAGE_BYTES", 10)
+ assert collect_visual_assets(parsed, source, kb_dir) == []
+ monkeypatch.setattr(assets_module, "MAX_IMAGE_BYTES", 5 * 1024 * 1024)
+ for index in range(2):
+ Image.new("RGB", (3, 2), color=(index * 60, 0, 0)).save(image.parent / f"extra-{index}.png")
+ monkeypatch.setattr(assets_module, "MAX_ASSETS_PER_DOCUMENT", 2)
+ assert len(collect_visual_assets(parsed, source, kb_dir)) == 2
+
+
[email protected]("suffix", [".pdf", ".epub"])
+def test_real_pymupdf_parser_extracts_pdf_and_epub_pixels(tmp_path: Path, suffix: str):
+ fitz = pytest.importorskip("fitz")
+ pytest.importorskip("pymupdf4llm")
+ from deeptutor.services.parsing.cache import load_ir
+ from deeptutor.services.parsing.engines.pymupdf4llm.config import PyMuPDF4LLMConfig
+ from deeptutor.services.parsing.engines.pymupdf4llm.engine import PyMuPDF4LLMParser
+
+ kb_dir = tmp_path / "kb"
+ raw = kb_dir / "raw"
+ raw.mkdir(parents=True)
+ source = raw / f"lesson{suffix}"
+ image = tmp_path / "figure.png"
+ Image.new("RGB", (100, 80), color="blue").save(image)
+ if suffix == ".pdf":
+ pdf = fitz.open()
+ page = pdf.new_page()
+ page.insert_text((50, 50), "Learning curve")
+ page.insert_image(fitz.Rect(50, 80, 250, 240), filename=str(image))
+ pdf.save(source)
+ pdf.close()
+ else:
+ with zipfile.ZipFile(source, "w") as epub:
+ epub.writestr("mimetype", "application/epub+zip", compress_type=zipfile.ZIP_STORED)
+ epub.writestr(
+ "META-INF/container.xml",
+ '<?xml version="1.0"?><container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container"><rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles></container>',
+ )
+ epub.writestr(
+ "OEBPS/content.opf",
+ '<?xml version="1.0"?><package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="id"><metadata xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:title>Lesson</dc:title><dc:language>en</dc:language><dc:identifier id="id">lesson</dc:identifier></metadata><manifest><item id="page" href="page.xhtml" media-type="application/xhtml+xml"/><item id="fig" href="figure.png" media-type="image/png"/></manifest><spine><itemref idref="page"/></spine></package>',
+ )
+ epub.writestr(
+ "OEBPS/page.xhtml",
+ '<html xmlns="http://www.w3.org/1999/xhtml"><head><title>Lesson</title></head><body><p>Learning curve</p><img src="figure.png" alt="Learning curve"/></body></html>',
+ )
+ epub.write(image, "OEBPS/figure.png")
+
+ workdir = tmp_path / "parse"
+ workdir.mkdir()
+ PyMuPDF4LLMParser().parse(source, workdir, config=PyMuPDF4LLMConfig())
+ markdown, blocks, asset_dir = load_ir(workdir)
+ parsed = ParsedDocument(
+ markdown=markdown,
+ blocks=blocks,
+ asset_dir=asset_dir,
+ parser_signature="pymupdf-test",
+ engine="pymupdf4llm",
+ )
+ candidates = collect_visual_assets(parsed, source, kb_dir)
+ assert len(candidates) == 1
+ assert candidates[0].record["mime_type"] == "image/png"
+ assert candidates[0].record["source_locator"].startswith("markdown:")
+ assert candidates[0].record["context"].startswith("Learning curve")
+
+
+def test_visual_document_is_searchable_with_text_embeddings(tmp_path: Path, monkeypatch):
+ pytest.importorskip("llama_index.core")
+ import deeptutor.services.parsing as parsing
+ from deeptutor.services.rag.pipelines.llamaindex.document_loader import LlamaIndexDocumentLoader
+
+ kb_dir, source, _image, parsed = _fixture(tmp_path, suffix=".epub", engine="pymupdf4llm")
+
+ class Parser:
+ def parse(self, _source, **_kwargs):
+ return parsed
+
+ monkeypatch.setattr(parsing, "get_parse_service", lambda: Parser())
+ candidates = []
+ documents = asyncio.run(
+ LlamaIndexDocumentLoader().load([str(source)], kb_dir=kb_dir, visual_candidates=candidates)
+ )
+ visual = [doc for doc in documents if doc.metadata.get("content_type") == "source_visual"]
+ assert len(visual) == 1
+ assert len(candidates) == 1
+ assert visual[0].metadata["visual_asset_id"] == candidates[0].record["asset_id"]
+ assert "Learning curve" in visual[0].text
+ assert "The learning curve" in visual[0].text
+ from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline
+
+ retrieved = SimpleNamespace(
+ node=SimpleNamespace(
+ text=visual[0].text,
+ metadata=visual[0].metadata,
+ node_id="visual-node",
+ ),
+ score=0.9,
+ )
+ result = LlamaIndexPipeline._nodes_to_result(None, "learning curve", [retrieved])
+ assert result["sources"][0]["visual_asset_id"] == candidates[0].record["asset_id"]
+ assert result["sources"][0]["chunk_id"] == "visual-node"
+
+
+def test_visual_preview_enforces_kb_scope_and_verified_bytes(tmp_path: Path, monkeypatch):
+ import importlib
+
+ from fastapi import HTTPException
+
+ import deeptutor.services.config as config_module
+
+ monkeypatch.setattr(config_module, "load_config_with_main", lambda *_args: {})
+ knowledge = importlib.import_module("deeptutor.api.routers.knowledge")
+
+ kb_dir, source, image, parsed = _fixture(tmp_path)
+ candidate = collect_visual_assets(parsed, source, kb_dir)[0]
+ VisualAssetStore(kb_dir).publish([candidate])
+ monkeypatch.setattr(knowledge, "_resolve_kb_raw_dir", lambda _kb_name: kb_dir / "raw")
+ response = asyncio.run(knowledge.serve_kb_visual_asset("kb", candidate.record["asset_id"]))
+ assert response.body == image.read_bytes()
+ assert response.headers["cache-control"] == "private, no-store"
+ assert response.headers["x-content-type-options"] == "nosniff"
+
+ def denied(_kb_name):
+ raise HTTPException(status_code=403, detail="Knowledge base is not assigned to you")
+
+ monkeypatch.setattr(knowledge, "_resolve_kb_raw_dir", denied)
+ with pytest.raises(HTTPException) as exc:
+ asyncio.run(knowledge.serve_kb_visual_asset("kb", candidate.record["asset_id"]))
+ assert exc.value.status_code == 403
+
+
+def test_rag_tool_sends_exact_retrieved_pixels_to_vision_model(tmp_path: Path, monkeypatch):
+ from deeptutor.multi_user import knowledge_access
+ from deeptutor.tools import rag_tool
+ from deeptutor.tools.builtin import RAGTool
+
+ kb_dir, source, image, parsed = _fixture(tmp_path)
+ record = collect_visual_assets(parsed, source, kb_dir)[0]
+ VisualAssetStore(kb_dir).publish([record], replace=True)
+ monkeypatch.setattr(
+ knowledge_access,
+ "resolve_for_rag",
+ lambda kb_name: SimpleNamespace(name="kb", base_dir=tmp_path) if kb_name == "kb" else None,
+ )
+
+ async def search(**_kwargs):
+ return {
+ "answer": "A learning curve is cited.",
+ "sources": [
+ {
+ "title": "lesson.pdf",
+ "visual_asset_id": record.record["asset_id"],
+ "caption": "Learning curve",
+ "source": "/etc/passwd",
+ }
+ ],
+ }
+
+ monkeypatch.setattr(rag_tool, "rag_search", search)
+ result = asyncio.run(
+ RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=True)
+ )
+ assert result.model_message is not None
+ parts = result.model_message["content"]
+ assert parts[0]["type"] == "text"
+ assert record.record["asset_id"] in parts[0]["text"]
+ assert parts[1]["type"] == "image_url"
+ data_uri = parts[1]["image_url"]["url"]
+ assert data_uri.startswith("data:image/png;base64,")
+ assert base64.b64decode(data_uri.split(",", 1)[1]) == image.read_bytes()
+ assert result.sources[0]["visual_asset_url"].endswith(record.record["asset_id"])
+ assert "base64" not in str(result.metadata)
+
+ text_only = asyncio.run(
+ RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=False)
+ )
+ assert text_only.model_message is None
+ assert "cannot inspect source image pixels" in text_only.content
+
+ monkeypatch.setattr(knowledge_access, "resolve_for_rag", lambda _kb_name: None)
+ inaccessible = asyncio.run(
+ RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=True)
+ )
+ assert inaccessible.model_message is None
+
+ monkeypatch.setattr(
+ knowledge_access,
+ "resolve_for_rag",
+ lambda _kb_name: SimpleNamespace(name="different-kb", base_dir=tmp_path),
+ )
+ cross_kb = asyncio.run(
+ RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=True)
+ )
+ assert cross_kb.model_message is None
+
+
+def test_rag_model_payload_is_bounded_to_two_images(tmp_path: Path, monkeypatch):
+ from deeptutor.multi_user import knowledge_access
+ from deeptutor.tools import rag_tool
+ from deeptutor.tools.builtin import RAGTool
+
+ kb_dir, source, image, parsed = _fixture(tmp_path)
+ for index in range(2):
+ Image.new("RGB", (3, 2), color=(0, index * 60, 10)).save(
+ image.parent / f"extra-{index}.png"
+ )
+ candidates = collect_visual_assets(parsed, source, kb_dir)
+ assert len(candidates) == 3
+ VisualAssetStore(kb_dir).publish(candidates)
+ monkeypatch.setattr(
+ knowledge_access,
+ "resolve_for_rag",
+ lambda _kb_name: SimpleNamespace(name="kb", base_dir=tmp_path),
+ )
+
+ async def search(**_kwargs):
+ return {
+ "answer": "Three figures found.",
+ "sources": [{"visual_asset_id": item.record["asset_id"]} for item in candidates],
+ }
+
+ monkeypatch.setattr(rag_tool, "rag_search", search)
+ result = asyncio.run(RAGTool().execute(query="figures", kb_name="kb", _vision_supported=True))
+ assert sum(part["type"] == "image_url" for part in result.model_message["content"]) == 2
+
+
+def test_visual_message_reaches_next_model_request_without_stream_leak(tmp_path: Path, monkeypatch):
+ from deeptutor.runtime.agentic import loop as agent_loop
+ from deeptutor.runtime.agentic.labeled_step import LabeledStepResult
+ from deeptutor.runtime.agentic.tool_dispatch import DispatchOutcome, _collect_outcome
+
+ kb_dir, source, image, parsed = _fixture(tmp_path)
+ candidate = collect_visual_assets(parsed, source, kb_dir)[0]
+ VisualAssetStore(kb_dir).publish([candidate], replace=True)
+ record, pixels = VisualAssetStore(kb_dir).read(candidate.record["asset_id"])
+ private_message = {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": f"Inspect {record['asset_id']}"},
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64," + base64.b64encode(pixels).decode()},
+ },
+ ],
+ }
+
+ class Stream:
+ def __init__(self):
+ self.events = []
+
+ async def tool_result(self, **kwargs):
+ self.events.append(kwargs)
+
+ stream = Stream()
+ outcome = asyncio.run(
+ _collect_outcome(
+ prepared=[("tool-1", "rag", {})],
+ results=[
+ {
+ "result_text": "Source visual found",
+ "sources": [],
+ "metadata": {},
+ "model_message": private_message,
+ }
+ ],
+ per_tool_trace_meta=[{}],
+ stream=stream,
+ source="chat",
+ stage="exploring",
+ )
+ )
+ assert outcome.model_messages == [{**private_message, "_after_tool_call_id": "tool-1"}]
+ assert "base64" not in str(stream.events)
+ capped = asyncio.run(
+ _collect_outcome(
+ prepared=[(f"tool-{index}", "rag", {}) for index in range(3)],
+ results=[
+ {
+ "result_text": "Source visual found",
+ "sources": [],
+ "metadata": {},
+ "model_message": private_message,
+ }
+ for _ in range(3)
+ ],
+ per_tool_trace_meta=[{} for _ in range(3)],
+ stream=stream,
+ source="chat",
+ stage="exploring",
+ )
+ )
+ assert len(capped.model_messages) == 2
+
+ captured = []
+
+ async def fake_step(**kwargs):
+ captured.append(list(kwargs["messages"]))
+ if len(captured) == 1:
+ return LabeledStepResult(
+ label="CALL_TOOLS",
+ text="",
+ tool_calls=[{"id": "tool-1", "name": "rag", "arguments": "{}"}],
+ )
+ return LabeledStepResult(label="FINISH", text="Done")
+
+ monkeypatch.setattr(agent_loop, "run_labeled_step", fake_step)
+
+ class Host:
+ async def guard_context_window(self, _messages):
+ pass
+
+ def build_iteration_trace_meta(self, _iteration):
+ return {}, {}
+
+ async def dispatch_tools(self, **_kwargs):
+ return outcome
+
+ protocol = agent_loop.LabelProtocol(
+ allowed=("CALL_TOOLS", "FINISH"),
+ terminal=frozenset({"FINISH"}),
+ intermediate=frozenset(),
+ final=frozenset(),
+ tool_label="CALL_TOOLS",
+ )
+ loop_result = asyncio.run(
+ agent_loop.run_agentic_loop(
+ initial_messages=[{"role": "user", "content": "What does the figure show?"}],
+ protocol=protocol,
+ client=object(),
+ model="gpt-4o",
+ completion_kwargs={},
+ binding="openai",
+ tool_schemas=[],
+ stream=stream,
+ source="chat",
+ stage="exploring",
+ max_iterations=2,
+ host=Host(),
+ )
+ )
+ assert loop_result.completed
+ assert "base64" not in str(loop_result.messages)
+ assert all(message.get("role") != "user" for message in loop_result.messages[1:])
+ next_request = captured[1]
+ assert next_request[-2]["role"] == "tool"
+ assert next_request[-1] == private_message
+ assert (
+ base64.b64decode(next_request[-1]["content"][1]["image_url"]["url"].split(",", 1)[1])
+ == image.read_bytes()
+ )
+ from deeptutor.services.llm.provider_core.anthropic_provider import AnthropicProvider
+ from deeptutor.services.llm.provider_core.openai_responses.converters import convert_messages
+
+ _system, responses_input = convert_messages(next_request)
+ response_image = responses_input[-1]["content"][1]
+ assert response_image["type"] == "input_image"
+ assert base64.b64decode(response_image["image_url"].split(",", 1)[1]) == image.read_bytes()
+ anthropic_image = AnthropicProvider._convert_image_block(next_request[-1]["content"][1])
+ assert anthropic_image["type"] == "image"
+ assert base64.b64decode(anthropic_image["source"]["data"]) == image.read_bytes()
+ _system, anthropic_messages = AnthropicProvider.__new__(AnthropicProvider)._convert_messages(
+ next_request
+ )
+ assert anthropic_messages[-1]["role"] == "user"
+ assert anthropic_messages[-1]["content"][0]["type"] == "tool_result"
+ assert anthropic_messages[-1]["content"][-1]["type"] == "image"