DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
Preserve retrieved source visuals for vision answers
Bingxi Zhao (Frank)5e7be659a224
diff --git a/deeptutor/agents/loop/pipeline.py b/deeptutor/agents/loop/pipeline.pyindex 7e347e5c4..fdcee04a6 100644--- a/deeptutor/agents/loop/pipeline.py+++ b/deeptutor/agents/loop/pipeline.py@@ -1325,6 +1325,9 @@ class AgenticLoopPipeline:task_dir = Path(runtime_workspace.output_dir) if runtime_workspace is not None else task_dirif tool_name == "rag":kwargs.setdefault("mode", "hybrid")+ from deeptutor.services.llm.capabilities import supports_vision++ kwargs["_vision_supported"] = supports_vision(self.binding, self.model)elif tool_name == "kb_files":# The report is read by the user as much as by the model, so it is# written in the turn's language. Injected server-side; the tooldiff --git a/deeptutor/api/routers/auth.py b/deeptutor/api/routers/auth.pyindex 4248d4440..5764c3819 100644--- a/deeptutor/api/routers/auth.py+++ b/deeptutor/api/routers/auth.py@@ -601,6 +601,7 @@ _LEARNER_KB_READ_ROUTES = frozenset("/api/knowledge-bases/{kb_name}/files","/api/knowledge-bases/{kb_name}/files/{filename:path}","/api/knowledge-bases/{kb_name}/file-preview-text/{filename:path}",+ "/api/knowledge-bases/{kb_name}/visual-assets/{asset_id}","/api/knowledge-bases/{kb_name}/progress",})diff --git a/deeptutor/api/routers/knowledge.py b/deeptutor/api/routers/knowledge.pyindex 6915f01da..c92014a0c 100644--- a/deeptutor/api/routers/knowledge.py+++ b/deeptutor/api/routers/knowledge.py@@ -33,7 +33,7 @@ from fastapi import (WebSocket,WebSocketDisconnect,)-from fastapi.responses import FileResponse, PlainTextResponse, StreamingResponse+from fastapi.responses import FileResponse, PlainTextResponse, Response, StreamingResponsefrom pydantic import BaseModel, Fieldfrom deeptutor.api.routers.auth import require_admin@@ -96,6 +96,7 @@ from deeptutor.services.rag.pipelines.ima.config import (ImaCredentials,get_account_credentials,)+from deeptutor.services.rag.visual_assets import VisualAssetStorefrom deeptutor.services.web_source.scheduler import get_web_source_sync_schedulerfrom deeptutor.utils.document_extractor import (MAX_EXTRACTED_CHARS_PER_DOC,@@ -2978,6 +2979,9 @@ async def move_kb_file(kb_name: str, payload: MoveFilePayload):dest_dir.mkdir(parents=True, exist_ok=True)shutil.move(str(src), str(dest))+ VisualAssetStore(manager.get_knowledge_base_path(kb_name)).move_source(+ source_rel, dest.relative_to(raw_dir.resolve()).as_posix()+ )return {"status": "ok", "path": dest.relative_to(raw_dir.resolve()).as_posix()}@@ -3020,6 +3024,22 @@ async def serve_kb_raw_file(kb_name: str, filename: str):)[email protected]("/knowledge-bases/{kb_name}/visual-assets/{asset_id}")+async def serve_kb_visual_asset(kb_name: str, asset_id: str):+ """Serve a verified source image from an access checked local KB."""+ raw_dir = _resolve_kb_raw_dir(kb_name)+ assert raw_dir is not None+ loaded = VisualAssetStore(raw_dir.parent).read(asset_id)+ if loaded is None:+ raise HTTPException(status_code=404, detail="Visual asset not found")+ record, data = loaded+ return Response(+ content=data,+ media_type=record["mime_type"],+ headers={"Cache-Control": "private, no-store", "X-Content-Type-Options": "nosniff"},+ )++@router.delete("/knowledge-bases/{kb_name}/files/{filename:path}")async def delete_kb_file(kb_name: str, filename: str):"""Remove a single raw document from a knowledge base.@@ -3045,6 +3065,7 @@ async def delete_kb_file(kb_name: str, filename: str):kb_name,target.name,)+ VisualAssetStore(kb_dir).remove_source(target.relative_to(kb_dir / "raw").as_posix())removal = remove_raw_document(Path(kb_dir), target)return {"status": "ok",diff --git a/deeptutor/core/tool_protocol.py b/deeptutor/core/tool_protocol.pyindex 0ce950ade..edbfd8dc1 100644--- a/deeptutor/core/tool_protocol.py+++ b/deeptutor/core/tool_protocol.py@@ -151,6 +151,9 @@ class ToolResult:``ask_user`` to keep the turn alive across the user'sanswer instead of ending and starting a new turn.Shape mirrors ``AskUserPayload.to_dict()``.+ model_message: Request-local multimodal context attached after this+ tool result. The loop must never add it to durable messages or+ user-facing stream metadata."""content: str = ""@@ -159,6 +162,9 @@ class ToolResult:success: bool = Trueterminate_turn: bool = Falsepause_for_user: dict[str, Any] | None = None+ # Private model-only follow-up (for example verified source image parts).+ # Never copy this into stream events, citation rows, or persisted metadata.+ model_message: dict[str, Any] | None = field(default=None, repr=False)def __str__(self) -> str:return self.contentdiff --git a/deeptutor/runtime/agentic/loop.py b/deeptutor/runtime/agentic/loop.pyindex e54298c1e..49d2a77f8 100644--- a/deeptutor/runtime/agentic/loop.py+++ b/deeptutor/runtime/agentic/loop.py@@ -173,6 +173,26 @@ class LoopHost(Protocol):return None+def _with_transient_model_messages(+ messages: list[dict[str, Any]], transient: list[dict[str, Any]]+) -> list[dict[str, Any]]:+ """Place model-only images after their tool result without mutating history."""+ if not transient:+ return messages+ anchored: dict[str, list[dict[str, Any]]] = {}+ for item in transient:+ tool_call_id = item.get("_after_tool_call_id")+ if not isinstance(tool_call_id, str) or not tool_call_id:+ continue+ anchored.setdefault(tool_call_id, []).append({"role": "user", "content": item["content"]})+ request_messages: list[dict[str, Any]] = []+ for message in messages:+ request_messages.append(message)+ if message.get("role") == "tool":+ request_messages.extend(anchored.pop(str(message.get("tool_call_id") or ""), []))+ return request_messages++async def run_agentic_loop(*,initial_messages: list[dict[str, Any]],@@ -212,6 +232,10 @@ async def run_agentic_loop(iterations don't spawn empty "Reasoning…" cards."""messages = initial_messages+ # Retrieved image bytes are request-local. They must never enter the+ # durable conversation (where a synthetic user message would be persisted+ # and displayed as if the person authored it).+ transient_model_messages: list[dict[str, Any]] = []aggregated_sources: list[dict[str, Any]] = []final_text = ""final_label_seen = ""@@ -233,7 +257,7 @@ async def run_agentic_loop(step = await run_labeled_step(client=client,model=model,- messages=messages,+ messages=_with_transient_model_messages(messages, transient_model_messages),completion_kwargs=completion_kwargs,tool_schemas=tool_schemas,allowed_labels=protocol.allowed,@@ -316,6 +340,7 @@ async def run_agentic_loop()aggregated_sources.extend(outcome.sources)messages.extend(outcome.tool_messages)+ transient_model_messages.extend(outcome.model_messages)if outcome.pause:resumed = await host.resolve_pause(outcome)if not resumed:diff --git a/deeptutor/runtime/agentic/tool_dispatch.py b/deeptutor/runtime/agentic/tool_dispatch.pyindex c924ac467..6376ec0a4 100644--- a/deeptutor/runtime/agentic/tool_dispatch.py+++ b/deeptutor/runtime/agentic/tool_dispatch.py@@ -110,6 +110,7 @@ class DispatchOutcome:sources: list[dict[str, Any]] = field(default_factory=list)tool_messages: list[dict[str, Any]] = field(default_factory=list)+ model_messages: list[dict[str, Any]] = field(default_factory=list, repr=False)tool_metadata_by_id: dict[str, dict[str, Any]] = field(default_factory=dict)terminate: bool = Falseterminate_payload: dict[str, Any] | None = None@@ -777,6 +778,7 @@ async def execute_tool_call("metadata": result.metadata,"terminate_turn": getattr(result, "terminate_turn", False),"pause_for_user": getattr(result, "pause_for_user", None),+ "model_message": getattr(result, "model_message", None),}except Exception as exc:# Unknown tool names arrive here too (the registry raises KeyError), so@@ -843,6 +845,8 @@ async def _collect_outcome("""aggregated_sources: list[dict[str, Any]] = []tool_messages: list[dict[str, Any]] = []+ model_messages: list[dict[str, Any]] = []+ model_image_count = 0tool_metadata_by_id: dict[str, dict[str, Any]] = {}terminate = Falseterminate_payload: dict[str, Any] | None = None@@ -879,6 +883,18 @@ async def _collect_outcome("content": result_text,})+ private_message = result.get("model_message")+ if isinstance(private_message, dict) and private_message.get("role") == "user":+ content = private_message.get("content")+ if isinstance(content, list):+ image_count = sum(+ 1+ for part in content+ if isinstance(part, dict) and part.get("type") == "image_url"+ )+ if image_count and model_image_count + image_count <= 2:+ model_messages.append({**private_message, "_after_tool_call_id": tool_call_id})+ model_image_count += image_countif isinstance(tool_extra_meta, dict) and tool_extra_meta:tool_metadata_by_id[tool_call_id] = dict(tool_extra_meta)if result.get("terminate_turn") and not terminate:@@ -900,6 +916,7 @@ async def _collect_outcome(return DispatchOutcome(sources=aggregated_sources,tool_messages=tool_messages,+ model_messages=model_messages,tool_metadata_by_id=tool_metadata_by_id,terminate=terminate,terminate_payload=terminate_payload,diff --git a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py b/deeptutor/services/rag/pipelines/llamaindex/document_loader.pyindex f5fded8ce..9a8cef74e 100644--- a/deeptutor/services/rag/pipelines/llamaindex/document_loader.py+++ b/deeptutor/services/rag/pipelines/llamaindex/document_loader.py@@ -26,6 +26,7 @@ from deeptutor.services.config.runtime_settings import DOCUMENT_PARSING_ENGINE_Lfrom deeptutor.services.embedding import get_embedding_clientfrom deeptutor.services.llm.client import get_llm_clientfrom deeptutor.services.rag.file_routing import FileTypeRouter+from deeptutor.services.rag.visual_assets import VisualAssetCandidate, collect_visual_assetsfrom deeptutor.utils.document_validator import DocumentValidatorfrom .config import image_description_limits@@ -59,6 +60,7 @@ class _ImageSource:path: Pathorigin: Path+ visual: VisualAssetCandidate | None = Noneclass LlamaIndexDocumentLoader:@@ -72,6 +74,9 @@ class LlamaIndexDocumentLoader:self,file_paths: Iterable[str],image_progress_callback: Callable[[int, int], None] | None = None,+ *,+ kb_dir: Path | None = None,+ visual_candidates: list[VisualAssetCandidate] | None = None,) -> list[Any]:documents: list[Any] = []image_sources: list[_ImageSource] = []@@ -85,7 +90,7 @@ class LlamaIndexDocumentLoader:# the event loop stalls every other request for the whole PDF# (same class of bug as upstream #761/#777). Hand it to a thread.text, extracted_images, parse_engine = await asyncio.to_thread(- self._parse_document, file_path+ self._parse_document, file_path, kb_dir=kb_dir)scanned_pdf_needs_ocr = (file_path.suffix.lower() == ".pdf"@@ -98,6 +103,7 @@ class LlamaIndexDocumentLoader:file_path,None,_PDF_OCR_FALLBACK_ENGINE,+ kb_dir=kb_dir,)if fallback[0].strip() or fallback[1]:text, extracted_images, parse_engine = fallback@@ -117,6 +123,7 @@ class LlamaIndexDocumentLoader:extracted_image_count=len(extracted_images),)image_sources.extend(extracted_images)+ self._append_visual_documents(documents, extracted_images, visual_candidates)for file_path_str in classification.text_files:file_path = Path(file_path_str)@@ -133,7 +140,7 @@ class LlamaIndexDocumentLoader:if supports(path):self.logger.info(f"Parsing image with active document parser: {path.name}")text, extracted_images, parse_engine = await asyncio.to_thread(- self._parse_document, path, parse_service+ self._parse_document, path, parse_service, kb_dir=kb_dir)if text.strip() or extracted_images:self._append_if_nonempty(@@ -144,6 +151,7 @@ class LlamaIndexDocumentLoader:extracted_image_count=len(extracted_images),)image_sources.extend(extracted_images)+ self._append_visual_documents(documents, extracted_images, visual_candidates)else:# Preserve the pre-parser behavior when an image-capable# engine fails or yields no usable IR.@@ -151,10 +159,11 @@ class LlamaIndexDocumentLoader:else:image_sources.append(_ImageSource(path=path, origin=path))- if image_sources:+ legacy_image_sources = [source for source in image_sources if source.visual is None]+ if legacy_image_sources:documents.extend(await self._load_image_nodes(- image_sources, image_progress_callback=image_progress_callback+ legacy_image_sources, image_progress_callback=image_progress_callback))@@ -168,6 +177,8 @@ class LlamaIndexDocumentLoader:file_path: Path,parse_service=None, # noqa: ANN001engine: str | None = None,+ *,+ kb_dir: Path | None = None,) -> tuple[str, list[_ImageSource], str]:"""Parse a document through the shared, engine-pluggable parse layer.@@ -197,8 +208,54 @@ class LlamaIndexDocumentLoader:text = parsed.markdown.strip() or self._text_from_blocks(parsed.blocks)images = self._collect_asset_images(parsed.asset_dir, origin=file_path)+ if kb_dir is not None and images:+ by_path = {+ candidate.path.resolve(): candidate+ for candidate in collect_visual_assets(parsed, file_path, kb_dir)+ }+ images = [+ _ImageSource(+ path=image.path, origin=image.origin, visual=by_path[image.path.resolve()]+ )+ for image in images+ if image.path.resolve() in by_path+ ]return text, images, str(parsed.engine or "")+ @staticmethod+ def _append_visual_documents(+ documents: list[Any],+ images: list[_ImageSource],+ candidates: list[VisualAssetCandidate] | None,+ ) -> None:+ for image in images:+ if image.visual is None:+ continue+ record = image.visual.record+ if candidates is not None:+ candidates.append(image.visual)+ caption = record["caption"] or "Source figure"+ context = record["context"]+ text = f"[Source visual] {image.origin.name}: {caption}"+ if context:+ text += f"\nContext: {context}"+ documents.append(+ Document(+ text=text,+ metadata={+ "file_name": image.origin.name,+ "file_path": str(image.origin),+ "content_type": "source_visual",+ "visual_asset_id": record["asset_id"],+ "source_document_id": record["source_document_id"],+ "page": record["page_number"] or "",+ "bbox": record["bbox"],+ "caption": record["caption"],+ "source_locator": record["source_locator"],+ },+ )+ )+def _log_scanned_pdf_without_ocr(self,file_path: Path,diff --git a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py b/deeptutor/services/rag/pipelines/llamaindex/pipeline.pyindex 3c9453752..c6a589b6a 100644--- a/deeptutor/services/rag/pipelines/llamaindex/pipeline.py+++ b/deeptutor/services/rag/pipelines/llamaindex/pipeline.py@@ -22,6 +22,7 @@ from deeptutor.services.rag.index_versioning import (write_version_meta,)from deeptutor.services.rag.kb_paths import resolve_kb_dir+from deeptutor.services.rag.visual_assets import VisualAssetStorefrom . import storagefrom .config import default_top_k, should_show_progress@@ -212,8 +213,12 @@ class LlamaIndexPipeline:try:await self._verify_embedding_connectivity()+ visual_candidates = []documents = await self.document_loader.load(- file_paths, image_progress_callback=image_progress_callback+ file_paths,+ image_progress_callback=image_progress_callback,+ kb_dir=kb_dir,+ visual_candidates=visual_candidates,)if not documents:self.logger.error("No valid documents found")@@ -233,6 +238,7 @@ class LlamaIndexPipeline:)self.logger.info(f"Index persisted to {storage_dir}")+ VisualAssetStore(kb_dir).publish(visual_candidates, prune_missing=True)if signature is not None:write_version_meta(kb_dir, signature, storage_dir=storage_dir)@@ -351,6 +357,17 @@ class LlamaIndexPipeline:"page": meta.get("page_label", meta.get("page", "")),"chunk_id": node.node.node_id or str(i),"score": round(node.score, 4) if node.score is not None else "",+ **(+ {+ "visual_asset_id": meta["visual_asset_id"],+ "source_document_id": meta.get("source_document_id", ""),+ "bbox": meta.get("bbox"),+ "caption": meta.get("caption", ""),+ "source_locator": meta.get("source_locator", ""),+ }+ if meta.get("visual_asset_id")+ else {}+ ),})@@ -378,8 +395,12 @@ class LlamaIndexPipeline:try:await self._verify_embedding_connectivity()+ visual_candidates = []documents = await self.document_loader.load(- file_paths, image_progress_callback=image_progress_callback+ file_paths,+ image_progress_callback=image_progress_callback,+ kb_dir=kb_dir,+ visual_candidates=visual_candidates,)if not documents:self.logger.warning("No valid documents to add")@@ -395,6 +416,7 @@ class LlamaIndexPipeline:worker_key=str(kb_dir.resolve()),)self.logger.info(f"Added {num_added} documents to existing index")+ VisualAssetStore(kb_dir).publish(visual_candidates)if signature is not None and plan.storage_dir != plan.existing_storage:write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)else:@@ -408,6 +430,7 @@ class LlamaIndexPipeline:worker_key=str(kb_dir.resolve()),)self.logger.info(f"Created new index with {num_added} documents")+ VisualAssetStore(kb_dir).publish(visual_candidates)if signature is not None:write_version_meta(kb_dir, signature, storage_dir=plan.storage_dir)diff --git a/deeptutor/services/rag/visual_assets.py b/deeptutor/services/rag/visual_assets.pynew file mode 100644index 000000000..0541ad0f3--- /dev/null+++ b/deeptutor/services/rag/visual_assets.py@@ -0,0 +1,389 @@+"""Verified, parser-independent source images owned by a knowledge base.++The manifest is deliberately outside any vector index. Retrieval engines may+index ``asset_id`` as metadata, but only this store resolves it to image bytes.+The caller must first resolve the KB through the normal access guard.+"""++from __future__ import annotations++from dataclasses import dataclass+from hashlib import sha256+from io import BytesIO+import json+import os+from pathlib import Path+import re+import tempfile+from typing import Any+import warnings++from deeptutor.services.parsing.types import ParsedDocument++MAX_IMAGE_BYTES = 5 * 1024 * 1024+MAX_ASSETS_PER_DOCUMENT = 64+MAX_MODEL_IMAGES = 2+MAX_IMAGE_PIXELS = 30_000_000+MAX_MANIFEST_BYTES = 16 * 1024 * 1024+_ID_RE = re.compile(r"^[0-9a-f]{64}$")+_MARKDOWN_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")+_MIME_EXT = {+ "image/png": ".png",+ "image/jpeg": ".jpg",+ "image/gif": ".gif",+ "image/webp": ".webp",+}+++def _image_mime(data: bytes) -> str | None:+ if data.startswith(b"\x89PNG\r\n\x1a\n"):+ mime = "image/png"+ elif data.startswith(b"\xff\xd8\xff"):+ mime = "image/jpeg"+ elif data.startswith((b"GIF87a", b"GIF89a")):+ mime = "image/gif"+ elif data.startswith(b"RIFF") and data[8:12] == b"WEBP":+ mime = "image/webp"+ else:+ return None+ # Check that the bytes actually decode as an image, rather than trusting+ # an extension or a short magic prefix supplied by a parser.+ try:+ from PIL import Image++ with warnings.catch_warnings():+ warnings.simplefilter("error", Image.DecompressionBombWarning)+ with Image.open(BytesIO(data)) as image:+ if image.width * image.height > MAX_IMAGE_PIXELS:+ return None+ image.verify()+ except Exception: # malformed or oversized images must not abort document indexing+ return None+ return mime+++def _image_bytes(path: Path) -> tuple[bytes, str] | None:+ if path.is_symlink() or not path.is_file():+ return None+ size = path.stat().st_size+ if not 0 < size <= MAX_IMAGE_BYTES:+ return None+ data = path.read_bytes()+ if len(data) > MAX_IMAGE_BYTES:+ return None+ mime = _image_mime(data)+ return (data, mime) if mime is not None else None+++def source_key_for(kb_dir: Path, source: Path) -> str:+ try:+ return source.resolve().relative_to((kb_dir / "raw").resolve()).as_posix()+ except ValueError:+ # Direct SDK callers can index paths outside raw/. Never persist the+ # host path in an asset manifest or expose it through citations.+ return source.name+++def _sha256_file(path: Path) -> str:+ digest = sha256()+ with path.open("rb") as handle:+ for chunk in iter(lambda: handle.read(1 << 20), b""):+ digest.update(chunk)+ return digest.hexdigest()+++def _managed_source(kb_dir: Path, source: Path) -> bool:+ return source.resolve().is_relative_to((kb_dir / "raw").resolve())+++def _text(value: Any) -> str:+ if isinstance(value, str):+ return value.strip()+ if isinstance(value, list):+ return " ".join(filter(None, (_text(item) for item in value))).strip()+ return ""+++def _block_details(+ blocks: list[dict[str, Any]] | None, asset: Path+) -> tuple[str, str, int | None, int | None, list[float] | None, str]:+ if not blocks:+ return "", "", None, None, None, ""+ for index, block in enumerate(blocks):+ if not isinstance(block, dict):+ continue+ raw = str(block.get("img_path") or block.get("path") or "")+ if not raw or Path(raw).resolve() != asset.resolve():+ continue+ caption = _text(+ block.get("caption")+ or block.get("image_caption")+ or block.get("chart_caption")+ or block.get("captions")+ )+ context = _text(block.get("text") or block.get("content"))+ if not context:+ neighbors = blocks[max(0, index - 1) : index] + blocks[index + 1 : index + 2]+ context = " ".join(+ _text(item.get("text") or item.get("content"))+ for item in neighbors+ if isinstance(item, dict)+ ).strip()+ has_page_index = block.get("page_idx") is not None+ page_raw = block.get("page_idx") if has_page_index else block.get("page")+ try:+ page_value = int(page_raw) if page_raw is not None else None+ except (ValueError, TypeError):+ page_value = None+ page_index = page_value if has_page_index else None+ page_number = (+ (page_value + 1 if has_page_index else page_value) if page_value is not None else None+ )+ raw_bbox = block.get("bbox")+ try:+ bbox = (+ [float(value) for value in raw_bbox[:4]]+ if isinstance(raw_bbox, (list, tuple)) and len(raw_bbox) >= 4+ else None+ )+ except (ValueError, TypeError):+ bbox = None+ return caption[:500], context[:500], page_index, page_number, bbox, f"blocks.json#/{index}"+ return "", "", None, None, None, ""+++def _markdown_details(markdown: str, asset: Path) -> tuple[str, str, str]:+ for match in _MARKDOWN_IMAGE_RE.finditer(markdown):+ if Path(match.group(2).split("?", 1)[0]).name != asset.name:+ continue+ excerpt = markdown[max(0, match.start() - 250) : min(len(markdown), match.end() + 250)]+ excerpt = _MARKDOWN_IMAGE_RE.sub(" ", excerpt)+ return (+ match.group(1).strip()[:500],+ " ".join(excerpt.split())[:500],+ f"markdown:{match.start()}",+ )+ return "", "", ""+++@dataclass(frozen=True)+class VisualAssetCandidate:+ path: Path+ record: dict[str, Any]+++def collect_visual_assets(+ parsed: ParsedDocument, source: Path, kb_dir: Path+) -> list[VisualAssetCandidate]:+ """Normalize extracted PDF/EPUB images with provenance and byte identity."""+ asset_dir = parsed.asset_dir+ if asset_dir is None or asset_dir.is_symlink() or not asset_dir.is_dir():+ return []+ source_key = source_key_for(kb_dir, source)+ source_hash = _sha256_file(source)+ candidates: list[VisualAssetCandidate] = []+ for path in sorted(asset_dir.iterdir()):+ if len(candidates) >= MAX_ASSETS_PER_DOCUMENT:+ break+ loaded = _image_bytes(path)+ if loaded is None:+ continue+ image, mime = loaded+ image_hash = sha256(image).hexdigest()+ caption, context, page_index, page_number, bbox, locator = _block_details(+ parsed.blocks, path+ )+ md_caption, md_context, md_locator = _markdown_details(parsed.markdown, path)+ caption = caption or md_caption+ context = context or md_context+ locator = locator or md_locator or f"asset:{path.name}"+ asset_id = sha256(+ f"{source_key}\0{source_hash}\0{parsed.parser_signature}\0{locator}\0{image_hash}".encode()+ ).hexdigest()+ record = {+ "asset_id": asset_id,+ "source_document_id": source_hash,+ "source_path": source_key,+ "managed_source": _managed_source(kb_dir, source),+ "parser_engine": parsed.engine,+ "parser_signature": parsed.parser_signature,+ "source_hash": parsed.source_hash,+ "source_locator": locator,+ "page_index": page_index,+ "page_number": page_number,+ "bbox": bbox,+ "caption": caption,+ "context": context,+ "image_sha256": image_hash,+ "mime_type": mime,+ "size": len(image),+ }+ candidates.append(VisualAssetCandidate(path=path, record=record))+ return candidates+++class VisualAssetStore:+ """KB-scoped immutable image files plus an atomically replaced manifest."""++ def __init__(self, kb_dir: Path):+ self.kb_dir = Path(kb_dir)+ self.root = self.kb_dir / "visual_assets"+ self.manifest_path = self.root / "manifest.json"++ def records(self) -> dict[str, dict[str, Any]]:+ if self.root.is_symlink() or not self.root.resolve().is_relative_to(self.kb_dir.resolve()):+ return {}+ try:+ if self.manifest_path.stat().st_size > MAX_MANIFEST_BYTES:+ return {}+ payload = json.loads(self.manifest_path.read_text(encoding="utf-8"))+ except (OSError, ValueError):+ return {}+ records = payload.get("assets") if isinstance(payload, dict) else None+ return records if isinstance(records, dict) else {}++ def publish(+ self,+ candidates: list[VisualAssetCandidate],+ *,+ replace: bool = False,+ source_paths: set[str] | None = None,+ prune_missing: bool = False,+ ) -> None:+ if self.root.is_symlink() or not self.root.resolve().is_relative_to(self.kb_dir.resolve()):+ raise OSError("Visual asset directory escapes knowledge base")+ self.root.mkdir(parents=True, exist_ok=True)+ prior = self.records()+ incoming = {item.record["asset_id"]: item.record for item in candidates}+ if replace:+ remaining: dict[str, dict[str, Any]] = {}+ else:+ # Keep older index versions' asset IDs reachable when the same+ # source is re-parsed with a new parser or embedding signature.+ # Explicit raw-file deletion removes all IDs for that source.+ replaced_sources = source_paths or set()+ remaining = {+ key: value+ for key, value in prior.items()+ if value.get("source_path") not in replaced_sources+ }+ if prune_missing:+ remaining = {+ key: value+ for key, value in remaining.items()+ if not value.get("managed_source")+ or self._source_exists(value.get("source_path"))+ }+ for candidate in candidates:+ record = candidate.record+ loaded = _image_bytes(candidate.path)+ if (+ loaded is None+ or sha256(loaded[0]).hexdigest() != record["image_sha256"]+ or loaded[1] != record["mime_type"]+ ):+ raise OSError(f"Source image changed during indexing: {candidate.path.name}")+ target = self._path(record["asset_id"], record["mime_type"])+ existing = _image_bytes(target)+ if existing is None or sha256(existing[0]).hexdigest() != record["image_sha256"]:+ self._atomic_write(target, loaded[0])+ updated = {**remaining, **incoming}+ self._atomic_write(+ self.manifest_path,+ json.dumps(+ {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True+ ).encode(),+ )+ for old_id, old_record in prior.items():+ if old_id not in updated:+ self._path(old_id, str(old_record.get("mime_type"))).unlink(missing_ok=True)++ def read(self, asset_id: str) -> tuple[dict[str, Any], bytes] | None:+ if not _ID_RE.fullmatch(asset_id):+ return None+ record = self.records().get(asset_id)+ if not isinstance(record, dict) or record.get("asset_id") != asset_id:+ return None+ mime = record.get("mime_type")+ if mime not in _MIME_EXT:+ return None+ path = self._path(asset_id, mime)+ loaded = _image_bytes(path)+ if loaded is None or loaded[1] != mime:+ return None+ data = loaded[0]+ if len(data) != record.get("size") or sha256(data).hexdigest() != record.get(+ "image_sha256"+ ):+ return None+ return record, data++ def remove_source(self, source_relative_path: str) -> None:+ prior = self.records()+ updated = {+ key: value+ for key, value in prior.items()+ if value.get("source_path") != source_relative_path+ }+ if len(updated) == len(prior):+ return+ self._atomic_write(+ self.manifest_path,+ json.dumps(+ {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True+ ).encode(),+ )+ for old_id, old_record in prior.items():+ if old_id not in updated:+ self._path(old_id, str(old_record.get("mime_type"))).unlink(missing_ok=True)++ def move_source(self, old_relative_path: str, new_relative_path: str) -> None:+ prior = self.records()+ updated = {}+ changed = False+ for key, value in prior.items():+ source = value.get("source_path")+ if source == old_relative_path or (+ isinstance(source, str) and source.startswith(old_relative_path + "/")+ ):+ value = {+ **value,+ "source_path": new_relative_path + source[len(old_relative_path) :],+ }+ changed = True+ updated[key] = value+ if changed:+ self._atomic_write(+ self.manifest_path,+ json.dumps(+ {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True+ ).encode(),+ )++ def _path(self, asset_id: str, mime: str) -> Path:+ if not _ID_RE.fullmatch(asset_id) or mime not in _MIME_EXT:+ raise ValueError("Invalid visual asset record")+ return self.root / (asset_id + _MIME_EXT[mime])++ def _source_exists(self, relative_path: Any) -> bool:+ if not isinstance(relative_path, str):+ return False+ relative = Path(relative_path)+ if relative.is_absolute() or ".." in relative.parts:+ return False+ raw = (self.kb_dir / "raw").resolve()+ source = (raw / relative).resolve()+ return source.is_relative_to(raw) and source.is_file()++ @staticmethod+ def _atomic_write(path: Path, data: bytes) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ fd, temp_name = tempfile.mkstemp(prefix=".visual-", dir=path.parent)+ try:+ with os.fdopen(fd, "wb") as handle:+ handle.write(data)+ handle.flush()+ os.fsync(handle.fileno())+ os.replace(temp_name, path)+ finally:+ Path(temp_name).unlink(missing_ok=True)diff --git a/deeptutor/tools/builtin/__init__.py b/deeptutor/tools/builtin/__init__.pyindex bd139026b..33b294c16 100644--- a/deeptutor/tools/builtin/__init__.py+++ b/deeptutor/tools/builtin/__init__.py@@ -86,7 +86,18 @@ def _rag_sources(result: dict[str, Any], *, query: str, kb_name: str) -> list[direturn []if not retrieved:return [{"type": "rag", "query": query, "kb_name": kb_name}]- return [{"type": "rag", "kb_name": kb_name, **item} for item in retrieved]+ from urllib.parse import quote++ sources = []+ for item in retrieved:+ source = {"type": "rag", "kb_name": kb_name, **item}+ asset_id = source.get("visual_asset_id")+ if asset_id:+ source["visual_asset_url"] = (+ f"/api/knowledge-bases/{quote(kb_name, safe='')}/visual-assets/{asset_id}"+ )+ sources.append(source)+ return sourcesclass RAGTool(_PromptHintsMixin, BaseTool):@@ -118,10 +129,11 @@ class RAGTool(_PromptHintsMixin, BaseTool):if not kb_name:raise ValueError("RAG requires an explicit kb_name.")event_sink = kwargs.get("event_sink")+ vision_supported = bool(kwargs.get("_vision_supported", False))extra_kwargs = {key: valuefor key, value in kwargs.items()- if key not in {"query", "kb_name", "event_sink"}+ if key not in {"query", "kb_name", "event_sink", "_vision_supported"}}result = await rag_search(@@ -141,12 +153,71 @@ class RAGTool(_PromptHintsMixin, BaseTool):f"No matching content was found in knowledge base '{kb_name}'. ""The search completed successfully.")+ visual_sources = [+ source+ for source in (result.get("sources") or [])+ if isinstance(source, dict) and source.get("visual_asset_id")+ ]+ model_message = None+ if visual_sources and vision_supported:+ model_message = _rag_visual_model_message(kb_name, visual_sources)+ if model_message is None:+ content += "\nRetrieved source visuals were unavailable for pixel inspection."+ elif visual_sources:+ content += (+ "\nThe selected model cannot inspect source image pixels. "+ "Use the retrieved caption and text context; do not claim to have seen the image."+ )return ToolResult(content=content,sources=_rag_sources(result, query=query, kb_name=kb_name),metadata=result,success=not failed,+ model_message=model_message,+ )+++def _rag_visual_model_message(kb_name: str, sources: list[dict[str, Any]]) -> dict[str, Any] | None:+ """Hydrate retrieved source pixels only after resolving this user's KB scope."""+ import base64++ from deeptutor.multi_user.knowledge_access import resolve_for_rag+ from deeptutor.services.rag.kb_paths import resolve_kb_dir+ from deeptutor.services.rag.visual_assets import MAX_MODEL_IMAGES, VisualAssetStore++ resource = resolve_for_rag(kb_name)+ if resource is None:+ return None+ store = VisualAssetStore(resolve_kb_dir(resource.base_dir, resource.name))+ parts: list[dict[str, Any]] = []+ seen: set[str] = set()+ for source in sources:+ asset_id = str(source.get("visual_asset_id") or "")+ if asset_id in seen:+ continue+ seen.add(asset_id)+ loaded = store.read(asset_id)+ if loaded is None:+ continue+ record, data = loaded+ label = record.get("caption") or record.get("source_locator") or "source visual"+ parts.append(+ {+ "type": "text",+ "text": f"Retrieved source visual {asset_id}: {label}. Inspect its pixels when answering.",+ }+ )+ parts.append(+ {+ "type": "image_url",+ "image_url": {+ "url": f"data:{record['mime_type']};base64,{base64.b64encode(data).decode('ascii')}"+ },+ })+ if len(parts) // 2 >= MAX_MODEL_IMAGES:+ break+ return {"role": "user", "content": parts} if parts else Noneclass KbFilesTool(_PromptHintsMixin, BaseTool):diff --git a/docs-for-user/SOURCE_VISUALS.md b/docs-for-user/SOURCE_VISUALS.mdnew file mode 100644index 000000000..674b8d1fc--- /dev/null+++ b/docs-for-user/SOURCE_VISUALS.md@@ -0,0 +1,28 @@+# Source visuals in knowledge bases++DeepTutor can retain extracted source images when a PDF or EPUB parser emits+image files. The KB stores verified parser-extracted PNG, JPEG, GIF, or WebP+bytes with a deterministic asset ID, document hash, parser identity, source+locator, caption and nearby text. Structured parser blocks also preserve a page+index and bounding box+when available. The visual record lives under the KB's `visual_assets/`+directory, separate from the vector index. The access checked API route+`/api/knowledge-bases/{kb_name}/visual-assets/{asset_id}` serves those exact+bytes, and deleting a raw source file removes its visual assets.++The LlamaIndex KB pipeline indexes a text record for each source visual, so a+text embedding model can retrieve it by its caption and context. When `rag`+retrieves that record in the chat loop, a vision capable answer model receives+the verified image pixels in its next request. A text only model receives the+caption and context with an explicit warning that it has not seen the pixels.+At most 64 images per document are retained, each image is limited to 5 MiB,+and at most two retrieved images are sent in one model continuation.++Current extraction coverage depends on the selected parser. MinerU can emit+structured PDF figures. PyMuPDF4LLM can emit PDF and EPUB images when image+extraction is enabled, but its Markdown output does not supply page boxes.+The default text only parser and the current Docling adapter do not emit+source image assets. Images that are only vector drawing commands, or pages+that require OCR when no usable OCR engine is configured, may be absent.+Other RAG providers do not yet index this visual manifest. Interactive+exercises, visual annotations, and mastery updates are separate future work.diff --git a/packaging/deeptutor-cli/pyproject.toml b/packaging/deeptutor-cli/pyproject.tomlindex e0c091076..43d7b824a 100644--- a/packaging/deeptutor-cli/pyproject.toml+++ b/packaging/deeptutor-cli/pyproject.toml@@ -54,6 +54,7 @@ dependencies = ["faiss-cpu>=1.8.0,<2.0.0; python_version < '3.14'","faiss-cpu>=1.12.0,<2.0.0; python_version >= '3.14'","PyMuPDF>=1.26.0",+ "Pillow>=10.0.0","numpy>=1.24.0,<3.0.0","arxiv>=2.0.0","python-docx>=1.1.0",diff --git a/pyproject.toml b/pyproject.tomlindex 4ef9b69c5..345ab6968 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -64,6 +64,7 @@ dependencies = ["faiss-cpu>=1.8.0,<2.0.0; python_version < '3.14'","faiss-cpu>=1.12.0,<2.0.0; python_version >= '3.14'","PyMuPDF>=1.26.0",+ "Pillow>=10.0.0","numpy>=1.24.0,<3.0.0","arxiv>=2.0.0","python-docx>=1.1.0",@@ -123,6 +124,7 @@ cli = ["faiss-cpu>=1.8.0,<2.0.0; python_version < '3.14'","faiss-cpu>=1.12.0,<2.0.0; python_version >= '3.14'","PyMuPDF>=1.26.0",+ "Pillow>=10.0.0","numpy>=1.24.0,<3.0.0","arxiv>=2.0.0",# Document text extraction (chat attachments) + office-skill authoring libsdiff --git a/requirements/cli.txt b/requirements/cli.txtindex ed38869a2..f93536dee 100644--- a/requirements/cli.txt+++ b/requirements/cli.txt@@ -48,6 +48,7 @@ llama-index-vector-stores-faiss>=0.4.0,<1.0.0faiss-cpu>=1.8.0,<2.0.0; python_version < "3.14"faiss-cpu>=1.12.0,<2.0.0; python_version >= "3.14"PyMuPDF>=1.26.0+Pillow>=10.0.0numpy>=1.24.0,<3.0.0arxiv>=2.0.0diff --git a/tests/api/test_knowledge_router.py b/tests/api/test_knowledge_router.pyindex 2ddf5e776..237e2b46c 100644--- a/tests/api/test_knowledge_router.py+++ b/tests/api/test_knowledge_router.py@@ -99,6 +99,11 @@ def _materialized_routes(app: "FastAPI"):"/api/knowledge-bases/{kb_name}/file-preview-text/{filename:path}","reading",),+ (+ "/api/knowledge-bases/demo/visual-assets/abc123",+ "/api/knowledge-bases/{kb_name}/visual-assets/{asset_id}",+ "reading",+ ),("/api/knowledge-bases/demo/progress","/api/knowledge-bases/{kb_name}/progress",diff --git a/tests/services/rag/test_visual_assets.py b/tests/services/rag/test_visual_assets.pynew file mode 100644index 000000000..1e85e4831--- /dev/null+++ b/tests/services/rag/test_visual_assets.py@@ -0,0 +1,483 @@+"""Source visual provenance, KB scope, and model pixel delivery."""++from __future__ import annotations++import asyncio+import base64+from pathlib import Path+from types import SimpleNamespace+import zipfile++from PIL import Image+import pytest++from deeptutor.services.parsing.types import ParsedDocument+from deeptutor.services.rag.visual_assets import (+ VisualAssetStore,+ collect_visual_assets,+)+++def _fixture(tmp_path: Path, *, suffix: str = ".pdf", engine: str = "mineru"):+ kb_dir = tmp_path / "kb"+ raw = kb_dir / "raw"+ raw.mkdir(parents=True)+ source = raw / f"lesson{suffix}"+ source.write_bytes(b"source document with a figure")+ assets = tmp_path / "parse-cache" / "images"+ assets.mkdir(parents=True)+ image = assets / "figure.png"+ Image.new("RGB", (3, 2), color=(17, 99, 211)).save(image)+ parsed = ParsedDocument(+ markdown="The learning curve is shown below.\n\n",+ blocks=(+ [+ {+ "type": "image",+ "img_path": str(image),+ "page_idx": 2,+ "bbox": [10, 20, 100, 200],+ "image_caption": ["Learning curve"],+ }+ ]+ if engine == "mineru"+ else None+ ),+ asset_dir=assets,+ source_hash="source-hash",+ parser_signature=f"{engine}-signature",+ engine=engine,+ )+ return kb_dir, source, image, parsed+++ ("suffix", "engine"),+ [(".pdf", "mineru"), (".epub", "pymupdf4llm")],+)+def test_verified_source_visual_contract_and_lifecycle(tmp_path: Path, suffix: str, engine: str):+ kb_dir, source, image, parsed = _fixture(tmp_path, suffix=suffix, engine=engine)+ candidates = collect_visual_assets(parsed, source, kb_dir)+ assert len(candidates) == 1+ record = candidates[0].record+ assert record["asset_id"] == collect_visual_assets(parsed, source, kb_dir)[0].record["asset_id"]+ assert record["source_path"] == source.name+ assert record["source_document_id"]+ assert record["parser_engine"] == engine+ assert record["caption"] == "Learning curve"+ if engine == "mineru":+ assert record["page_index"] == 2+ assert record["page_number"] == 3+ assert record["bbox"] == [10, 20, 100, 200]+ assert record["source_locator"] == "blocks.json#/0"+ else:+ assert record["page_index"] is None+ assert record["source_locator"].startswith("markdown:")++ store = VisualAssetStore(kb_dir)+ store.publish(candidates, replace=True)+ assert store.read(record["asset_id"]) == (record, image.read_bytes())+ assert store.read("../manifest.json") is None+ persisted = store.root / (record["asset_id"] + ".png")+ persisted.write_bytes(b"\x89PNG\r\n\x1a\ncorrupted")+ assert store.read(record["asset_id"]) is None+ store.publish(candidates, replace=True)+ newer = ParsedDocument(+ markdown=parsed.markdown,+ blocks=parsed.blocks,+ asset_dir=parsed.asset_dir,+ parser_signature="new-parser-signature",+ engine=parsed.engine,+ )+ newer_candidate = collect_visual_assets(newer, source, kb_dir)[0]+ store.publish([newer_candidate])+ assert store.read(record["asset_id"]) is not None # older index version remains valid+ assert store.read(newer_candidate.record["asset_id"]) is not None+ moved = kb_dir / "raw" / "folder" / source.name+ moved.parent.mkdir()+ source.rename(moved)+ store.move_source(source.name, f"folder/{source.name}")+ assert store.records()[record["asset_id"]]["source_path"] == f"folder/{source.name}"+ store.remove_source(f"folder/{source.name}")+ assert store.read(record["asset_id"]) is None+ assert store.read(newer_candidate.record["asset_id"]) is None+ assert not persisted.exists()+++def test_size_count_and_rebuild_cleanup(tmp_path: Path, monkeypatch):+ import deeptutor.services.rag.visual_assets as assets_module++ kb_dir, source, image, parsed = _fixture(tmp_path)+ store = VisualAssetStore(kb_dir)+ first = collect_visual_assets(parsed, source, kb_dir)[0]+ store.publish([first])+ source.unlink()+ store.publish([], prune_missing=True)+ assert store.read(first.record["asset_id"]) is None++ source.write_bytes(b"source document with a figure")+ monkeypatch.setattr(assets_module, "MAX_IMAGE_BYTES", 10)+ assert collect_visual_assets(parsed, source, kb_dir) == []+ monkeypatch.setattr(assets_module, "MAX_IMAGE_BYTES", 5 * 1024 * 1024)+ for index in range(2):+ Image.new("RGB", (3, 2), color=(index * 60, 0, 0)).save(image.parent / f"extra-{index}.png")+ monkeypatch.setattr(assets_module, "MAX_ASSETS_PER_DOCUMENT", 2)+ assert len(collect_visual_assets(parsed, source, kb_dir)) == 2++[email protected]("suffix", [".pdf", ".epub"])+def test_real_pymupdf_parser_extracts_pdf_and_epub_pixels(tmp_path: Path, suffix: str):+ fitz = pytest.importorskip("fitz")+ pytest.importorskip("pymupdf4llm")+ from deeptutor.services.parsing.cache import load_ir+ from deeptutor.services.parsing.engines.pymupdf4llm.config import PyMuPDF4LLMConfig+ from deeptutor.services.parsing.engines.pymupdf4llm.engine import PyMuPDF4LLMParser++ kb_dir = tmp_path / "kb"+ raw = kb_dir / "raw"+ raw.mkdir(parents=True)+ source = raw / f"lesson{suffix}"+ image = tmp_path / "figure.png"+ Image.new("RGB", (100, 80), color="blue").save(image)+ if suffix == ".pdf":+ pdf = fitz.open()+ page = pdf.new_page()+ page.insert_text((50, 50), "Learning curve")+ page.insert_image(fitz.Rect(50, 80, 250, 240), filename=str(image))+ pdf.save(source)+ pdf.close()+ else:+ with zipfile.ZipFile(source, "w") as epub:+ epub.writestr("mimetype", "application/epub+zip", compress_type=zipfile.ZIP_STORED)+ epub.writestr(+ "META-INF/container.xml",+ '<?xml version="1.0"?><container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container"><rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles></container>',+ )+ epub.writestr(+ "OEBPS/content.opf",+ '<?xml version="1.0"?><package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="id"><metadata xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:title>Lesson</dc:title><dc:language>en</dc:language><dc:identifier id="id">lesson</dc:identifier></metadata><manifest><item id="page" href="page.xhtml" media-type="application/xhtml+xml"/><item id="fig" href="figure.png" media-type="image/png"/></manifest><spine><itemref idref="page"/></spine></package>',+ )+ epub.writestr(+ "OEBPS/page.xhtml",+ '<html xmlns="http://www.w3.org/1999/xhtml"><head><title>Lesson</title></head><body><p>Learning curve</p><img src="figure.png" alt="Learning curve"/></body></html>',+ )+ epub.write(image, "OEBPS/figure.png")++ workdir = tmp_path / "parse"+ workdir.mkdir()+ PyMuPDF4LLMParser().parse(source, workdir, config=PyMuPDF4LLMConfig())+ markdown, blocks, asset_dir = load_ir(workdir)+ parsed = ParsedDocument(+ markdown=markdown,+ blocks=blocks,+ asset_dir=asset_dir,+ parser_signature="pymupdf-test",+ engine="pymupdf4llm",+ )+ candidates = collect_visual_assets(parsed, source, kb_dir)+ assert len(candidates) == 1+ assert candidates[0].record["mime_type"] == "image/png"+ assert candidates[0].record["source_locator"].startswith("markdown:")+ assert candidates[0].record["context"].startswith("Learning curve")+++def test_visual_document_is_searchable_with_text_embeddings(tmp_path: Path, monkeypatch):+ pytest.importorskip("llama_index.core")+ import deeptutor.services.parsing as parsing+ from deeptutor.services.rag.pipelines.llamaindex.document_loader import LlamaIndexDocumentLoader++ kb_dir, source, _image, parsed = _fixture(tmp_path, suffix=".epub", engine="pymupdf4llm")++ class Parser:+ def parse(self, _source, **_kwargs):+ return parsed++ monkeypatch.setattr(parsing, "get_parse_service", lambda: Parser())+ candidates = []+ documents = asyncio.run(+ LlamaIndexDocumentLoader().load([str(source)], kb_dir=kb_dir, visual_candidates=candidates)+ )+ visual = [doc for doc in documents if doc.metadata.get("content_type") == "source_visual"]+ assert len(visual) == 1+ assert len(candidates) == 1+ assert visual[0].metadata["visual_asset_id"] == candidates[0].record["asset_id"]+ assert "Learning curve" in visual[0].text+ assert "The learning curve" in visual[0].text+ from deeptutor.services.rag.pipelines.llamaindex.pipeline import LlamaIndexPipeline++ retrieved = SimpleNamespace(+ node=SimpleNamespace(+ text=visual[0].text,+ metadata=visual[0].metadata,+ node_id="visual-node",+ ),+ score=0.9,+ )+ result = LlamaIndexPipeline._nodes_to_result(None, "learning curve", [retrieved])+ assert result["sources"][0]["visual_asset_id"] == candidates[0].record["asset_id"]+ assert result["sources"][0]["chunk_id"] == "visual-node"+++def test_visual_preview_enforces_kb_scope_and_verified_bytes(tmp_path: Path, monkeypatch):+ import importlib++ from fastapi import HTTPException++ import deeptutor.services.config as config_module++ monkeypatch.setattr(config_module, "load_config_with_main", lambda *_args: {})+ knowledge = importlib.import_module("deeptutor.api.routers.knowledge")++ kb_dir, source, image, parsed = _fixture(tmp_path)+ candidate = collect_visual_assets(parsed, source, kb_dir)[0]+ VisualAssetStore(kb_dir).publish([candidate])+ monkeypatch.setattr(knowledge, "_resolve_kb_raw_dir", lambda _kb_name: kb_dir / "raw")+ response = asyncio.run(knowledge.serve_kb_visual_asset("kb", candidate.record["asset_id"]))+ assert response.body == image.read_bytes()+ assert response.headers["cache-control"] == "private, no-store"+ assert response.headers["x-content-type-options"] == "nosniff"++ def denied(_kb_name):+ raise HTTPException(status_code=403, detail="Knowledge base is not assigned to you")++ monkeypatch.setattr(knowledge, "_resolve_kb_raw_dir", denied)+ with pytest.raises(HTTPException) as exc:+ asyncio.run(knowledge.serve_kb_visual_asset("kb", candidate.record["asset_id"]))+ assert exc.value.status_code == 403+++def test_rag_tool_sends_exact_retrieved_pixels_to_vision_model(tmp_path: Path, monkeypatch):+ from deeptutor.multi_user import knowledge_access+ from deeptutor.tools import rag_tool+ from deeptutor.tools.builtin import RAGTool++ kb_dir, source, image, parsed = _fixture(tmp_path)+ record = collect_visual_assets(parsed, source, kb_dir)[0]+ VisualAssetStore(kb_dir).publish([record], replace=True)+ monkeypatch.setattr(+ knowledge_access,+ "resolve_for_rag",+ lambda kb_name: SimpleNamespace(name="kb", base_dir=tmp_path) if kb_name == "kb" else None,+ )++ async def search(**_kwargs):+ return {+ "answer": "A learning curve is cited.",+ "sources": [+ {+ "title": "lesson.pdf",+ "visual_asset_id": record.record["asset_id"],+ "caption": "Learning curve",+ "source": "/etc/passwd",+ }+ ],+ }++ monkeypatch.setattr(rag_tool, "rag_search", search)+ result = asyncio.run(+ RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=True)+ )+ assert result.model_message is not None+ parts = result.model_message["content"]+ assert parts[0]["type"] == "text"+ assert record.record["asset_id"] in parts[0]["text"]+ assert parts[1]["type"] == "image_url"+ data_uri = parts[1]["image_url"]["url"]+ assert data_uri.startswith("data:image/png;base64,")+ assert base64.b64decode(data_uri.split(",", 1)[1]) == image.read_bytes()+ assert result.sources[0]["visual_asset_url"].endswith(record.record["asset_id"])+ assert "base64" not in str(result.metadata)++ text_only = asyncio.run(+ RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=False)+ )+ assert text_only.model_message is None+ assert "cannot inspect source image pixels" in text_only.content++ monkeypatch.setattr(knowledge_access, "resolve_for_rag", lambda _kb_name: None)+ inaccessible = asyncio.run(+ RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=True)+ )+ assert inaccessible.model_message is None++ monkeypatch.setattr(+ knowledge_access,+ "resolve_for_rag",+ lambda _kb_name: SimpleNamespace(name="different-kb", base_dir=tmp_path),+ )+ cross_kb = asyncio.run(+ RAGTool().execute(query="learning curve", kb_name="kb", _vision_supported=True)+ )+ assert cross_kb.model_message is None+++def test_rag_model_payload_is_bounded_to_two_images(tmp_path: Path, monkeypatch):+ from deeptutor.multi_user import knowledge_access+ from deeptutor.tools import rag_tool+ from deeptutor.tools.builtin import RAGTool++ kb_dir, source, image, parsed = _fixture(tmp_path)+ for index in range(2):+ Image.new("RGB", (3, 2), color=(0, index * 60, 10)).save(+ image.parent / f"extra-{index}.png"+ )+ candidates = collect_visual_assets(parsed, source, kb_dir)+ assert len(candidates) == 3+ VisualAssetStore(kb_dir).publish(candidates)+ monkeypatch.setattr(+ knowledge_access,+ "resolve_for_rag",+ lambda _kb_name: SimpleNamespace(name="kb", base_dir=tmp_path),+ )++ async def search(**_kwargs):+ return {+ "answer": "Three figures found.",+ "sources": [{"visual_asset_id": item.record["asset_id"]} for item in candidates],+ }++ monkeypatch.setattr(rag_tool, "rag_search", search)+ result = asyncio.run(RAGTool().execute(query="figures", kb_name="kb", _vision_supported=True))+ assert sum(part["type"] == "image_url" for part in result.model_message["content"]) == 2+++def test_visual_message_reaches_next_model_request_without_stream_leak(tmp_path: Path, monkeypatch):+ from deeptutor.runtime.agentic import loop as agent_loop+ from deeptutor.runtime.agentic.labeled_step import LabeledStepResult+ from deeptutor.runtime.agentic.tool_dispatch import DispatchOutcome, _collect_outcome++ kb_dir, source, image, parsed = _fixture(tmp_path)+ candidate = collect_visual_assets(parsed, source, kb_dir)[0]+ VisualAssetStore(kb_dir).publish([candidate], replace=True)+ record, pixels = VisualAssetStore(kb_dir).read(candidate.record["asset_id"])+ private_message = {+ "role": "user",+ "content": [+ {"type": "text", "text": f"Inspect {record['asset_id']}"},+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64," + base64.b64encode(pixels).decode()},+ },+ ],+ }++ class Stream:+ def __init__(self):+ self.events = []++ async def tool_result(self, **kwargs):+ self.events.append(kwargs)++ stream = Stream()+ outcome = asyncio.run(+ _collect_outcome(+ prepared=[("tool-1", "rag", {})],+ results=[+ {+ "result_text": "Source visual found",+ "sources": [],+ "metadata": {},+ "model_message": private_message,+ }+ ],+ per_tool_trace_meta=[{}],+ stream=stream,+ source="chat",+ stage="exploring",+ )+ )+ assert outcome.model_messages == [{**private_message, "_after_tool_call_id": "tool-1"}]+ assert "base64" not in str(stream.events)+ capped = asyncio.run(+ _collect_outcome(+ prepared=[(f"tool-{index}", "rag", {}) for index in range(3)],+ results=[+ {+ "result_text": "Source visual found",+ "sources": [],+ "metadata": {},+ "model_message": private_message,+ }+ for _ in range(3)+ ],+ per_tool_trace_meta=[{} for _ in range(3)],+ stream=stream,+ source="chat",+ stage="exploring",+ )+ )+ assert len(capped.model_messages) == 2++ captured = []++ async def fake_step(**kwargs):+ captured.append(list(kwargs["messages"]))+ if len(captured) == 1:+ return LabeledStepResult(+ label="CALL_TOOLS",+ text="",+ tool_calls=[{"id": "tool-1", "name": "rag", "arguments": "{}"}],+ )+ return LabeledStepResult(label="FINISH", text="Done")++ monkeypatch.setattr(agent_loop, "run_labeled_step", fake_step)++ class Host:+ async def guard_context_window(self, _messages):+ pass++ def build_iteration_trace_meta(self, _iteration):+ return {}, {}++ async def dispatch_tools(self, **_kwargs):+ return outcome++ protocol = agent_loop.LabelProtocol(+ allowed=("CALL_TOOLS", "FINISH"),+ terminal=frozenset({"FINISH"}),+ intermediate=frozenset(),+ final=frozenset(),+ tool_label="CALL_TOOLS",+ )+ loop_result = asyncio.run(+ agent_loop.run_agentic_loop(+ initial_messages=[{"role": "user", "content": "What does the figure show?"}],+ protocol=protocol,+ client=object(),+ model="gpt-4o",+ completion_kwargs={},+ binding="openai",+ tool_schemas=[],+ stream=stream,+ source="chat",+ stage="exploring",+ max_iterations=2,+ host=Host(),+ )+ )+ assert loop_result.completed+ assert "base64" not in str(loop_result.messages)+ assert all(message.get("role") != "user" for message in loop_result.messages[1:])+ next_request = captured[1]+ assert next_request[-2]["role"] == "tool"+ assert next_request[-1] == private_message+ assert (+ base64.b64decode(next_request[-1]["content"][1]["image_url"]["url"].split(",", 1)[1])+ == image.read_bytes()+ )+ from deeptutor.services.llm.provider_core.anthropic_provider import AnthropicProvider+ from deeptutor.services.llm.provider_core.openai_responses.converters import convert_messages++ _system, responses_input = convert_messages(next_request)+ response_image = responses_input[-1]["content"][1]+ assert response_image["type"] == "input_image"+ assert base64.b64decode(response_image["image_url"].split(",", 1)[1]) == image.read_bytes()+ anthropic_image = AnthropicProvider._convert_image_block(next_request[-1]["content"][1])+ assert anthropic_image["type"] == "image"+ assert base64.b64decode(anthropic_image["source"]["data"]) == image.read_bytes()+ _system, anthropic_messages = AnthropicProvider.__new__(AnthropicProvider)._convert_messages(+ next_request+ )+ assert anthropic_messages[-1]["role"] == "user"+ assert anthropic_messages[-1]["content"][0]["type"] == "tool_result"+ assert anthropic_messages[-1]["content"][-1]["type"] == "image"