Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

16.4 KB
Raw Blame
"""Verified, parser-independent source images owned by a knowledge base.

The manifest is deliberately outside any vector index.  Retrieval engines may
index ``asset_id`` as metadata, but only this store resolves it to image bytes.
The caller must first resolve the KB through the normal access guard.
"""

from __future__ import annotations

from dataclasses import dataclass
from hashlib import sha256
from io import BytesIO
import json
import logging
import os
from pathlib import Path
import re
import tempfile
from typing import Any
import warnings

from deeptutor.services.parsing.types import ParsedDocument

MAX_IMAGE_BYTES = 5 * 1024 * 1024
MAX_MODEL_IMAGES = 2
MAX_IMAGE_PIXELS = 30_000_000
logger = logging.getLogger(__name__)
_ID_RE = re.compile(r"^[0-9a-f]{64}$")
_MARKDOWN_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
_MIME_EXT = {
    "image/png": ".png",
    "image/jpeg": ".jpg",
    "image/gif": ".gif",
    "image/webp": ".webp",
}


def _image_mime(data: bytes) -> str | None:
    if data.startswith(b"\x89PNG\r\n\x1a\n"):
        mime = "image/png"
    elif data.startswith(b"\xff\xd8\xff"):
        mime = "image/jpeg"
    elif data.startswith((b"GIF87a", b"GIF89a")):
        mime = "image/gif"
    elif data.startswith(b"RIFF") and data[8:12] == b"WEBP":
        mime = "image/webp"
    else:
        return None
    # Check that the bytes actually decode as an image, rather than trusting
    # an extension or a short magic prefix supplied by a parser.
    try:
        from PIL import Image

        with warnings.catch_warnings():
            warnings.simplefilter("error", Image.DecompressionBombWarning)
            with Image.open(BytesIO(data)) as image:
                if image.width * image.height > MAX_IMAGE_PIXELS:
                    return None
                image.verify()
    except Exception:  # malformed or oversized images must not abort document indexing
        return None
    return mime


def _image_bytes(path: Path) -> tuple[bytes, str] | None:
    if path.is_symlink() or not path.is_file():
        return None
    size = path.stat().st_size
    if not 0 < size <= MAX_IMAGE_BYTES:
        return None
    data = path.read_bytes()
    if len(data) > MAX_IMAGE_BYTES:
        return None
    mime = _image_mime(data)
    return (data, mime) if mime is not None else None


def source_key_for(kb_dir: Path, source: Path) -> str:
    try:
        return source.resolve().relative_to((kb_dir / "raw").resolve()).as_posix()
    except ValueError:
        # Direct SDK callers can index paths outside raw/. Never persist the
        # host path in an asset manifest or expose it through citations.
        return source.name


def _sha256_file(path: Path) -> str:
    digest = sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1 << 20), b""):
            digest.update(chunk)
    return digest.hexdigest()


def _managed_source(kb_dir: Path, source: Path) -> bool:
    return source.resolve().is_relative_to((kb_dir / "raw").resolve())


def _text(value: Any) -> str:
    if isinstance(value, str):
        return value.strip()
    if isinstance(value, list):
        return " ".join(filter(None, (_text(item) for item in value))).strip()
    return ""


def _block_details(
    blocks: list[dict[str, Any]] | None, asset: Path
) -> tuple[str, str, int | None, int | None, list[float] | None, str]:
    if not blocks:
        return "", "", None, None, None, ""
    for index, block in enumerate(blocks):
        if not isinstance(block, dict):
            continue
        raw = str(block.get("img_path") or block.get("path") or "")
        if not raw or Path(raw).resolve() != asset.resolve():
            continue
        caption = _text(
            block.get("caption")
            or block.get("image_caption")
            or block.get("chart_caption")
            or block.get("table_caption")
            or block.get("captions")
        )
        context = _text(block.get("text") or block.get("content"))
        if not context:
            neighbors = blocks[max(0, index - 1) : index] + blocks[index + 1 : index + 2]
            context = " ".join(
                _text(item.get("text") or item.get("content"))
                for item in neighbors
                if isinstance(item, dict)
            ).strip()
        has_page_index = block.get("page_idx") is not None
        page_raw = block.get("page_idx") if has_page_index else block.get("page")
        try:
            page_value = int(page_raw) if page_raw is not None else None
        except (ValueError, TypeError):
            page_value = None
        page_index = page_value if has_page_index else None
        page_number = (
            (page_value + 1 if has_page_index else page_value) if page_value is not None else None
        )
        raw_bbox = block.get("bbox")
        try:
            bbox = (
                [float(value) for value in raw_bbox[:4]]
                if isinstance(raw_bbox, (list, tuple)) and len(raw_bbox) >= 4
                else None
            )
        except (ValueError, TypeError):
            bbox = None
        return caption[:500], context[:500], page_index, page_number, bbox, f"blocks.json#/{index}"
    return "", "", None, None, None, ""


def _markdown_details(markdown: str, asset: Path) -> tuple[str, str, str]:
    for match in _MARKDOWN_IMAGE_RE.finditer(markdown):
        if Path(match.group(2).split("?", 1)[0]).name != asset.name:
            continue
        excerpt = markdown[max(0, match.start() - 250) : min(len(markdown), match.end() + 250)]
        excerpt = _MARKDOWN_IMAGE_RE.sub(" ", excerpt)
        return (
            match.group(1).strip()[:500],
            " ".join(excerpt.split())[:500],
            f"markdown:{match.start()}",
        )
    return "", "", ""


@dataclass(frozen=True)
class VisualAssetCandidate:
    path: Path
    record: dict[str, Any]


def collect_visual_assets(
    parsed: ParsedDocument, source: Path, kb_dir: Path
) -> list[VisualAssetCandidate]:
    """Normalize extracted PDF/EPUB images with provenance and byte identity."""
    asset_dir = parsed.asset_dir
    if asset_dir is None or asset_dir.is_symlink() or not asset_dir.is_dir():
        return []
    source_key = source_key_for(kb_dir, source)
    source_hash = _sha256_file(source)
    candidates: list[VisualAssetCandidate] = []
    for path in sorted(asset_dir.iterdir()):
        try:
            loaded = _image_bytes(path)
        except OSError:
            logger.warning(
                "Unable to read parser asset %s; coverage report will record it", path.name
            )
            continue
        if loaded is None:
            continue
        image, mime = loaded
        image_hash = sha256(image).hexdigest()
        caption, context, page_index, page_number, bbox, locator = _block_details(
            parsed.blocks, path
        )
        md_caption, md_context, md_locator = _markdown_details(parsed.markdown, path)
        caption = caption or md_caption
        context = context or md_context
        locator = locator or md_locator or f"asset:{path.name}"
        asset_id = sha256(
            f"{source_key}\0{source_hash}\0{parsed.parser_signature}\0{locator}\0{image_hash}".encode()
        ).hexdigest()
        record = {
            "asset_id": asset_id,
            "source_document_id": source_hash,
            "source_path": source_key,
            "managed_source": _managed_source(kb_dir, source),
            "parser_engine": parsed.engine,
            "parser_signature": parsed.parser_signature,
            "source_hash": parsed.source_hash,
            "source_locator": locator,
            "page_index": page_index,
            "page_number": page_number,
            "bbox": bbox,
            "caption": caption,
            "context": context,
            "image_sha256": image_hash,
            "mime_type": mime,
            "size": len(image),
        }
        from deeptutor.services.rag.source_visuals import figure_labels

        record["figure_labels"] = figure_labels(caption)
        matching = [
            block
            for block in (parsed.blocks or [])
            if isinstance(block, dict)
            and str(block.get("img_path") or block.get("path") or "")
            and Path(str(block.get("img_path") or block.get("path"))).resolve() == path.resolve()
        ]
        if matching:
            block = matching[0]
            record.update(
                {
                    "kind": str(block.get("type") or "image"),
                    "section": _text(block.get("section") or block.get("section_title")),
                    "group_id": str(block.get("group_id") or block.get("figure_id") or ""),
                    "table_html": _text(block.get("table_body") or block.get("table_html")),
                    "notes": _text(block.get("notes") or block.get("table_footnote")),
                }
            )
        candidates.append(VisualAssetCandidate(path=path, record=record))
    return candidates


class VisualAssetStore:
    """KB-scoped immutable image files plus an atomically replaced manifest."""

    def __init__(self, kb_dir: Path):
        self.kb_dir = Path(kb_dir)
        self.root = self.kb_dir / "visual_assets"
        self.manifest_path = self.root / "manifest.json"

    def records(self) -> dict[str, dict[str, Any]]:
        if self.root.is_symlink() or not self.root.resolve().is_relative_to(self.kb_dir.resolve()):
            return {}
        try:
            return self._read_manifest()
        except OSError as exc:
            logger.warning("Cannot read source figures for KB '%s': %s", self.kb_dir.name, exc)
            return {}

    def _read_manifest(self) -> dict[str, dict[str, Any]]:
        # #1802: the corpus grows independently of the per-image request budget.
        # Never interpret a large or corrupt manifest as an empty store on a write.
        try:
            if self.root.is_symlink() or not self.root.resolve().is_relative_to(
                self.kb_dir.resolve()
            ):
                raise OSError("Visual asset directory escapes knowledge base")
            if self.manifest_path.is_symlink():
                raise OSError("Visual asset manifest must not be a symbolic link")
            payload = json.loads(self.manifest_path.read_text(encoding="utf-8"))
        except FileNotFoundError:
            return {}
        except (OSError, ValueError) as exc:
            raise OSError(f"Cannot read visual asset manifest {self.manifest_path}: {exc}") from exc
        records = payload.get("assets") if isinstance(payload, dict) else None
        if not isinstance(records, dict) or any(not isinstance(v, dict) for v in records.values()):
            raise OSError(f"Invalid visual asset manifest {self.manifest_path}")
        return records

    def publish(
        self,
        candidates: list[VisualAssetCandidate],
        *,
        replace: bool = False,
        source_paths: set[str] | None = None,
        prune_missing: bool = False,
    ) -> None:
        if self.root.is_symlink() or not self.root.resolve().is_relative_to(self.kb_dir.resolve()):
            raise OSError("Visual asset directory escapes knowledge base")
        self.root.mkdir(parents=True, exist_ok=True)
        prior = self._read_manifest()
        incoming = {item.record["asset_id"]: item.record for item in candidates}
        if replace:
            remaining: dict[str, dict[str, Any]] = {}
        else:
            # Keep older index versions' asset IDs reachable when the same
            # source is re-parsed with a new parser or embedding signature.
            # Explicit raw-file deletion removes all IDs for that source.
            replaced_sources = source_paths or set()
            remaining = {
                key: value
                for key, value in prior.items()
                if value.get("source_path") not in replaced_sources
            }
            if prune_missing:
                remaining = {
                    key: value
                    for key, value in remaining.items()
                    if not value.get("managed_source")
                    or self._source_exists(value.get("source_path"))
                }
        updated = {**remaining, **incoming}
        manifest = json.dumps(
            {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True
        ).encode()
        for candidate in candidates:
            record = candidate.record
            loaded = _image_bytes(candidate.path)
            if (
                loaded is None
                or sha256(loaded[0]).hexdigest() != record["image_sha256"]
                or loaded[1] != record["mime_type"]
            ):
                raise OSError(f"Source image changed during indexing: {candidate.path.name}")
            target = self._path(record["asset_id"], record["mime_type"])
            existing = _image_bytes(target)
            if existing is None or sha256(existing[0]).hexdigest() != record["image_sha256"]:
                self._atomic_write(target, loaded[0])
        self._atomic_write(self.manifest_path, manifest)
        for old_id, old_record in prior.items():
            if old_id not in updated:
                self._path(old_id, str(old_record.get("mime_type"))).unlink(missing_ok=True)

    def read(self, asset_id: str) -> tuple[dict[str, Any], bytes] | None:
        if not _ID_RE.fullmatch(asset_id):
            return None
        record = self.records().get(asset_id)
        if not isinstance(record, dict) or record.get("asset_id") != asset_id:
            return None
        mime = record.get("mime_type")
        if mime not in _MIME_EXT:
            return None
        path = self._path(asset_id, mime)
        loaded = _image_bytes(path)
        if loaded is None or loaded[1] != mime:
            return None
        data = loaded[0]
        if len(data) != record.get("size") or sha256(data).hexdigest() != record.get(
            "image_sha256"
        ):
            return None
        return record, data

    def remove_source(self, source_relative_path: str) -> None:
        prior = self._read_manifest()
        updated = {
            key: value
            for key, value in prior.items()
            if value.get("source_path") != source_relative_path
        }
        if len(updated) == len(prior):
            return
        self._atomic_write(
            self.manifest_path,
            json.dumps(
                {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True
            ).encode(),
        )
        for old_id, old_record in prior.items():
            if old_id not in updated:
                self._path(old_id, str(old_record.get("mime_type"))).unlink(missing_ok=True)

    def move_source(self, old_relative_path: str, new_relative_path: str) -> None:
        prior = self._read_manifest()
        updated = {}
        changed = False
        for key, value in prior.items():
            source = value.get("source_path")
            if source == old_relative_path or (
                isinstance(source, str) and source.startswith(old_relative_path + "/")
            ):
                value = {
                    **value,
                    "source_path": new_relative_path + source[len(old_relative_path) :],
                }
                changed = True
            updated[key] = value
        if changed:
            self._atomic_write(
                self.manifest_path,
                json.dumps(
                    {"version": 1, "assets": updated}, ensure_ascii=False, sort_keys=True
                ).encode(),
            )

    def _path(self, asset_id: str, mime: str) -> Path:
        if not _ID_RE.fullmatch(asset_id) or mime not in _MIME_EXT:
            raise ValueError("Invalid visual asset record")
        return self.root / (asset_id + _MIME_EXT[mime])

    def _source_exists(self, relative_path: Any) -> bool:
        if not isinstance(relative_path, str):
            return False
        relative = Path(relative_path)
        if relative.is_absolute() or ".." in relative.parts:
            return False
        raw = (self.kb_dir / "raw").resolve()
        source = (raw / relative).resolve()
        return source.is_relative_to(raw) and source.is_file()

    @staticmethod
    def _atomic_write(path: Path, data: bytes) -> None:
        path.parent.mkdir(parents=True, exist_ok=True)
        fd, temp_name = tempfile.mkstemp(prefix=".visual-", dir=path.parent)
        try:
            with os.fdopen(fd, "wb") as handle:
                handle.write(data)
                handle.flush()
                os.fsync(handle.fileno())
            os.replace(temp_name, path)
        finally:
            Path(temp_name).unlink(missing_ok=True)