Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

26.4 KB
Raw Blame
"""Embedded-image extraction for Office documents and PDFs.

Text extraction everywhere in DeepTutor is text-only by design, which silently
drops the pictures inside DOCX/PPTX/PDF attachments: the reader never sees
them and neither does the model. This module is the shared fix — bytes in,
embedded raster images out, plus the *marker strings* that let the plain-text
extraction record where each image belonged in the reading order.

Two consumers:
  * ``utils.document_extractor`` — chat attachments (and the KB/reading text
    paths that share its parsers) append the markers to the extracted text and
    emit the images as image-type attachment records so the existing
    multimodal injection pipeline forwards them to vision-capable models.
  * ``reading.extract`` — maps markers to section locators so the reader pane
    can display each image next to the section that contains it.

Markers use a fixed, greppable format (``[图片 3: image-03.png]``) so both
consumers and the model itself can tie an image back to its position without
any side-channel. Numbering is 1-based and document-wide, matching the
``name`` given to each :class:`EmbeddedImage`.

Vector formats (EMF/WMF) and other formats vision models cannot consume are
counted in the returned :class:`ImageCollection` and never emitted, but the
text keeps an unnumbered note so their existence is not silently erased.
"""

from __future__ import annotations

from dataclasses import dataclass, field
import io
import logging
import posixpath
import re
from typing import Any
import zipfile

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Budgets. Defaults serve the chat-attachment path; the reading-area ingest
# passes looser per-document caps because images are stored on disk there and
# never inflated into a single LLM request.
# ---------------------------------------------------------------------------
MAX_IMAGES_PER_DOC = 12
MAX_TOTAL_IMAGE_BYTES_PER_DOC = 12 * 1024 * 1024
MAX_IMAGE_BYTES = 4 * 1024 * 1024
MIN_IMAGE_BYTES = 1_200  # below this it is a bullet point or a divider, not a figure

#: Long-edge bound for an image that will be base64-inlined into one provider
#: request. Byte budgets alone do not bound what a *page-sized* raster costs:
#: a scanned A4 page arrives at ~2900x4032 (~600 KB), so seven of them are
#: ~4.2 MB of body and ~5.6 MB encoded, while a vision model is served a raster
#: scaled toward ~1.5k on the long edge anyway. Bounding the long edge keeps the
#: detail a model can actually read and drops the rest. The reading-area ingest
#: is exempt (``max_inline_edge=0``) because those bytes are written to disk for
#: the reader pane and never inflated into a request.
#:
#: The number is deliberately looser than ``PAGE_RENDER_DPI`` (110 dpi, ~1286 px
#: on A4, ~132 KB/page), which the reading viewport already uses for its
#: whole-page fallback raster: a harvested scan is the only channel carrying a
#: page's content, so it keeps more resolution than a supplementary render.
MAX_INLINE_IMAGE_EDGE = 1600

#: Matches ``PAGE_RENDER_DPI``'s jpg_quality so a fitted raster is encoded the
#: same way the reading render already is.
INLINE_IMAGE_JPEG_QUALITY = 82

#: Fitting decodes pixels, so refuse images past this many pixels before Pillow
#: allocates for them. Mirrors ``MAX_IMAGE_PIXELS`` in
#: ``services/rag/visual_assets`` (kept as a literal so this utils module stays
#: free of a services import). Oversized candidates are left untouched and fall
#: through to the existing byte budgets.
MAX_INLINE_FIT_PIXELS = 30_000_000


def reading_image_budget() -> ImageBudget:
    """Looser per-document caps for the reading-area ingest path.

    The reading store keeps the image bytes on disk, and a single chat turn
    inflates only the *current page's* images into the request, so the caps
    split in two directions: the document-wide caps can be far looser than the
    chat-attachment defaults, while the per-page cap is tightened to what one
    request can carry. ``max_images_per_page=4`` mirrors
    ``READING_VIEWPORT_MAX_IMAGES`` in ``services/session/_turn_runtime_shared``
    (kept as a literal so this utils module stays free of a services import).

    ``max_inline_edge=0`` keeps source resolution: these bytes are stored for
    the reader pane to display, not inlined into a request.
    """
    return ImageBudget(
        max_images=120,
        max_total_bytes=60 * 1024 * 1024,
        max_images_per_page=4,
        max_inline_edge=0,
    )


# Raster formats vision models and the browser can both consume. Everything
# else (EMF/WMF vector, TIFF, SVG, …) is skipped with a note.
SUPPORTED_IMAGE_EXTENSIONS: frozenset[str] = frozenset(
    {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"}
)

_MIME_BY_EXT = {
    ".png": "image/png",
    ".jpg": "image/jpeg",
    ".jpeg": "image/jpeg",
    ".gif": "image/gif",
    ".webp": "image/webp",
    ".bmp": "image/bmp",
}


@dataclass(frozen=True, slots=True)
class EmbeddedImage:
    """One extracted raster image. ``name`` is unique within a document."""

    name: str
    mime_type: str
    data: bytes


@dataclass
class ImageBudget:
    """Running caps and skip counters while collecting one document's images.

    Mutable by design: the walk functions thread one instance through the
    whole document and read the counters afterwards.
    """

    max_images: int = MAX_IMAGES_PER_DOC
    max_total_bytes: int = MAX_TOTAL_IMAGE_BYTES_PER_DOC
    max_image_bytes: int = MAX_IMAGE_BYTES
    min_image_bytes: int = MIN_IMAGE_BYTES
    # 0 = unlimited. Only the per-page walkers (PDF) enforce this; it keeps one
    # page from monopolising the document budget when a single request can only
    # carry a handful of images.
    max_images_per_page: int = 0
    # 0 = keep the source resolution. Non-zero fits every admitted raster within
    # this long-edge bound, for budgets whose images are inlined into one
    # request. See ``MAX_INLINE_IMAGE_EDGE``.
    max_inline_edge: int = MAX_INLINE_IMAGE_EDGE
    count: int = 0
    total_bytes: int = 0
    skipped_vector: int = 0
    skipped_other: int = 0
    skipped_budget: int = 0
    skipped_page_budget: int = 0

    def register_vector_skip(self) -> None:
        self.skipped_vector += 1

    def register_other_skip(self) -> None:
        self.skipped_other += 1

    def try_admit(self, ext: str, data: bytes) -> EmbeddedImage | None:
        """Admit one candidate image, applying every budget. ``None`` = skip.

        The candidate is fitted to ``max_inline_edge`` first, so the byte
        budgets below weigh what will actually ship, and a page-sized raster no
        provider would read at source size still qualifies.
        """
        ext, data = fit_inline_image(ext, data, max_edge=self.max_inline_edge)
        if len(data) < self.min_image_bytes or len(data) > self.max_image_bytes:
            self.skipped_budget += 1
            return None
        if self.count >= self.max_images or self.total_bytes + len(data) > self.max_total_bytes:
            self.skipped_budget += 1
            return None
        self.count += 1
        self.total_bytes += len(data)
        return EmbeddedImage(
            name=f"image-{self.count:02d}{ext}",
            mime_type=_MIME_BY_EXT.get(ext, "application/octet-stream"),
            data=data,
        )


@dataclass(frozen=True, slots=True)
class ImageCollection:
    """Images from one document plus how many were left behind."""

    images: tuple[EmbeddedImage, ...] = ()
    skipped_vector: int = 0
    skipped_other: int = 0
    skipped_budget: int = 0
    skipped_page_budget: int = 0

    def summary_note(self) -> str:
        """One-line note for the extracted text describing what was dropped."""
        parts: list[str] = []
        if self.skipped_vector:
            parts.append(f"{self.skipped_vector} 张矢量图(EMF/WMF)未提取")
        if self.skipped_other:
            parts.append(f"{self.skipped_other} 张不支持的图片格式未提取")
        if self.skipped_budget:
            parts.append(f"{self.skipped_budget} 张图片超出数量/大小上限未提取")
        if self.skipped_page_budget:
            parts.append(f"{self.skipped_page_budget} 张超出单页上限未提取")
        if not parts:
            return ""
        return "[" + ",".join(parts) + "]"


@dataclass(frozen=True, slots=True)
class DocxRich:
    """Marker-annotated body paragraphs plus the images they reference."""

    paragraphs: tuple[str, ...] = ()
    collection: ImageCollection = field(default_factory=ImageCollection)


@dataclass(frozen=True, slots=True)
class PptxRich:
    """Per-slide text with image markers, plus the slide images."""

    slides: tuple[str, ...] = ()
    collection: ImageCollection = field(default_factory=ImageCollection)


@dataclass(frozen=True, slots=True)
class PdfImages:
    """PDF page images with the page each first appeared on."""

    collection: ImageCollection = field(default_factory=ImageCollection)
    page_map: tuple[tuple[int, tuple[int, ...]], ...] = ()  # (page, image indices)


def image_index_from_name(name: str) -> int:
    """``image-03.png`` → 3. Marker text and numbering stay in lockstep."""
    stem = posixpath.basename(name).rsplit(".", 1)[0]
    tail = stem.rsplit("-", 1)[-1]
    try:
        return int(tail)
    except ValueError:
        return 0


def build_marker(image: EmbeddedImage) -> str:
    """The canonical in-text marker for one image."""
    return f"[图片 {image_index_from_name(image.name)}: {image.name}]"


_MARKER_PATTERN = re.compile(r"\[图片 (\d+): ([^\]\s]+)\]")


def find_markers(text: str) -> list[tuple[int, str]]:
    """All ``(image_index, image_name)`` markers in *text*, in order."""
    return [(int(m.group(1)), m.group(2)) for m in _MARKER_PATTERN.finditer(text)]


def _local_name(tag: Any) -> str:
    if isinstance(tag, str) and "}" in tag:
        return tag.rsplit("}", 1)[-1]
    return tag if isinstance(tag, str) else ""


def _parse_xml(raw: bytes) -> Any:
    from defusedxml import ElementTree as DefusedElementTree

    return DefusedElementTree.fromstring(raw)


def fit_inline_image(ext: str, data: bytes, *, max_edge: int) -> tuple[str, bytes]:
    """Fit a raster within ``max_edge`` so one inline request stays affordable.

    Returns the (possibly re-encoded) image and its extension. An image already
    within the bound is returned untouched, byte for byte, so small figures and
    screenshots keep their original format and quality — re-encoding is reserved
    for images that would otherwise carry detail no vision model reads.

    ``max_edge=0`` disables fitting. Every failure — an undecodable candidate, a
    pixel count past :data:`MAX_INLINE_FIT_PIXELS`, a re-encode that would not
    even shrink the payload — returns the original bytes: a transport-size
    optimisation must never be the reason an attachment becomes unreadable.
    """
    if max_edge <= 0 or not data:
        return ext, data
    try:
        from PIL import Image
    except ImportError:  # pragma: no cover - Pillow ships with the image extras
        return ext, data
    try:
        with Image.open(io.BytesIO(data)) as probe:
            width, height = probe.size
            if max(width, height) <= max_edge or width * height > MAX_INLINE_FIT_PIXELS:
                return ext, data
            probe.load()
            has_alpha = probe.mode in {"RGBA", "LA"} or (
                probe.mode == "P" and "transparency" in probe.info
            )
            if has_alpha:
                # JPEG has no alpha channel. Composite onto white — a page or a
                # slide — rather than letting a transparent figure turn black.
                rgba = probe.convert("RGBA")
                fitted = Image.new("RGB", rgba.size, (255, 255, 255))
                fitted.paste(rgba, mask=rgba.split()[-1])
                rgba.close()
            else:
                fitted = probe.convert("RGB")
    except Exception:
        logger.debug("inline image fit probe failed", exc_info=True)
        return ext, data
    try:
        scale = max_edge / max(width, height)
        target = (max(1, round(width * scale)), max(1, round(height * scale)))
        buffer = io.BytesIO()
        fitted.resize(target, Image.LANCZOS).save(
            buffer, format="JPEG", quality=INLINE_IMAGE_JPEG_QUALITY, optimize=True
        )
        shrunk = buffer.getvalue()
    except Exception:
        logger.debug("inline image fit failed", exc_info=True)
        return ext, data
    finally:
        fitted.close()
    if not shrunk or len(shrunk) >= len(data):
        return ext, data
    return ".jpg", shrunk


def _normalise_ext(ext: str) -> str:
    ext = (ext or "").lower()
    if ext == ".jpeg":
        return ".jpg"
    return ext


# ---------------------------------------------------------------------------
# DOCX
# ---------------------------------------------------------------------------


def extract_docx_rich(data: bytes, *, budget: ImageBudget | None = None) -> DocxRich:
    """Walk ``word/document.xml`` once, collecting ordered paragraphs and images.

    Only the main body is mined (header/footer images are decoration). A
    relationship target referenced by several paragraphs is extracted once but
    marked at every position, so repeated/floating figures stay anchored to
    the text around them.
    """
    budget = budget or ImageBudget()
    images: list[EmbeddedImage] = []
    by_target: dict[str, EmbeddedImage] = {}
    attempted: set[str] = set()

    with zipfile.ZipFile(io.BytesIO(data)) as zf:
        names = set(zf.namelist())
        rel_targets = _office_rels(zf, "word/_rels/document.xml.rels")
        root = _parse_xml(zf.read("word/document.xml"))

        paragraphs: list[str] = []
        for node in root.iter():
            if _local_name(node.tag) != "p":
                continue
            text_parts: list[str] = []
            markers: list[str] = []
            seen_rids: set[str] = set()
            for child in node.iter():
                name = _local_name(child.tag)
                if name == "t" and child.text:
                    text_parts.append(child.text)
                elif name == "tab":
                    text_parts.append("\t")
                elif name in {"br", "cr"}:
                    text_parts.append("\n")
                elif name == "blip":
                    rid = _blip_embed_rid(child)
                    if not rid or rid in seen_rids:
                        continue
                    seen_rids.add(rid)
                    target = rel_targets.get(rid, "")
                    image = by_target.get(target)
                    if image is None and target not in attempted:
                        attempted.add(target)
                        image = _load_office_image(zf, names, target, budget, images)
                        if image is not None:
                            by_target[target] = image
                    if image is not None:
                        markers.append(build_marker(image))
            paragraph = "".join(text_parts).strip()
            if markers:
                paragraph = (paragraph + "\n" if paragraph else "") + "\n".join(markers)
            if paragraph:
                paragraphs.append(paragraph)

    return DocxRich(
        paragraphs=tuple(paragraphs),
        collection=_collection_from(budget, images),
    )


def _blip_embed_rid(blip: Any) -> str:
    """The ``r:embed`` relationship id of one ``a:blip`` (external links out)."""
    for key, value in blip.attrib.items():
        if value and _local_name(key) == "embed":
            return str(value)
    return ""


def _office_rels(zf: zipfile.ZipFile, rels_member: str) -> dict[str, str]:
    """rId → normalised archive path for internal media targets."""
    try:
        raw = zf.read(rels_member)
    except KeyError:
        return {}
    try:
        root = _parse_xml(raw)
    except Exception:
        return {}
    base = posixpath.dirname(posixpath.dirname(rels_member))  # …/word/_rels → …/word
    targets: dict[str, str] = {}
    for node in root.iter():
        if _local_name(node.tag) != "Relationship":
            continue
        if (node.get("TargetMode") or "").lower() == "external":
            continue
        rid = node.get("Id") or ""
        target = node.get("Target") or ""
        if rid and target:
            targets[rid] = posixpath.normpath(posixpath.join(base, target))
    return targets


def _load_office_image(
    zf: zipfile.ZipFile,
    names: set[str],
    target: str,
    budget: ImageBudget,
    images: list[EmbeddedImage],
) -> EmbeddedImage | None:
    if not target or target not in names:
        return None
    ext = _normalise_ext(posixpath.splitext(target)[1])
    if ext in {".emf", ".wmf"}:
        budget.register_vector_skip()
        return None
    if ext not in SUPPORTED_IMAGE_EXTENSIONS:
        budget.register_other_skip()
        return None
    try:
        data = zf.read(target)
    except (KeyError, zipfile.BadZipFile):
        return None
    image = budget.try_admit(ext, data)
    if image is not None:
        images.append(image)
    return image


def _collection_from(budget: ImageBudget, images: list[EmbeddedImage]) -> ImageCollection:
    return ImageCollection(
        images=tuple(images),
        skipped_vector=budget.skipped_vector,
        skipped_other=budget.skipped_other,
        skipped_budget=budget.skipped_budget,
        skipped_page_budget=budget.skipped_page_budget,
    )


# ---------------------------------------------------------------------------
# PPTX
# ---------------------------------------------------------------------------


def extract_pptx_rich(data: bytes, *, budget: ImageBudget | None = None) -> PptxRich:
    """Slide texts with markers plus pictures, via python-pptx when available.

    python-pptx resolves picture placeholders and grouped shapes cleanly and
    only reports images actually placed on a slide (layout/master decoration
    stays out). The raw-OOXML fallback additionally catches background fills
    when python-pptx is missing or fails.
    """
    try:
        from pptx import Presentation
    except ImportError:
        return _extract_pptx_rich_ooxml(data, budget=budget)

    try:
        prs = Presentation(io.BytesIO(data))
    except Exception:
        return _extract_pptx_rich_ooxml(data, budget=budget)

    budget = budget or ImageBudget()
    images: list[EmbeddedImage] = []
    slides: list[str] = []
    for slide in prs.slides:
        lines: list[str] = []
        _collect_shape_rich(slide.shapes, lines, budget, images)
        slides.append("\n".join(line for line in lines if line))

    return PptxRich(slides=tuple(slides), collection=_collection_from(budget, images))


def _collect_shape_rich(
    shapes: Any,
    lines: list[str],
    budget: ImageBudget,
    images: list[EmbeddedImage],
) -> None:
    for shape in shapes:
        sub_shapes = getattr(shape, "shapes", None)
        if sub_shapes is not None:
            _collect_shape_rich(sub_shapes, lines, budget, images)
            continue

        try:
            image = shape.image  # Picture only; other shapes raise or lack it
            blob = image.blob
            ext = _normalise_ext("." + (image.ext or ""))
        except Exception:
            blob, ext = b"", ""
        if blob:
            if ext in {".emf", ".wmf"}:
                budget.register_vector_skip()
            elif ext in SUPPORTED_IMAGE_EXTENSIONS:
                admitted = budget.try_admit(ext, blob)
                if admitted is not None:
                    images.append(admitted)
                    lines.append(build_marker(admitted))
            else:
                budget.register_other_skip()

        if getattr(shape, "has_table", False):
            for row in shape.table.rows:
                cells = [cell.text.strip() for cell in row.cells]
                line = "\t".join(cell for cell in cells if cell)
                if line:
                    lines.append(line)
            continue
        text = getattr(shape, "text", "")
        if text and text.strip():
            lines.append(text.strip())


def _extract_pptx_rich_ooxml(data: bytes, *, budget: ImageBudget | None = None) -> PptxRich:
    """Raw fallback: per-slide paragraph text + blip markers from slide rels."""
    budget = budget or ImageBudget()
    images: list[EmbeddedImage] = []
    slides: list[str] = []
    with zipfile.ZipFile(io.BytesIO(data)) as zf:
        names = set(zf.namelist())
        slide_members = sorted(
            (name for name in names if _SLIDE_MEMBER_PATTERN.fullmatch(name)),
            key=_natural_slide_key,
        )
        for member in slide_members:
            rel_targets = _office_rels(zf, _rels_for(member))
            lines: list[str] = []
            try:
                root = _parse_xml(zf.read(member))
            except Exception:
                slides.append("")
                continue
            for node in root.iter():
                if _local_name(node.tag) != "p":
                    continue
                text_parts: list[str] = []
                markers: list[str] = []
                seen_rids: set[str] = set()
                for child in node.iter():
                    name = _local_name(child.tag)
                    if name == "t" and child.text:
                        text_parts.append(child.text)
                    elif name == "tab":
                        text_parts.append("\t")
                    elif name in {"br", "cr"}:
                        text_parts.append("\n")
                    elif name == "blip":
                        rid = _blip_embed_rid(child)
                        if not rid or rid in seen_rids:
                            continue
                        seen_rids.add(rid)
                        image = _load_office_image(
                            zf, names, rel_targets.get(rid, ""), budget, images
                        )
                        if image is not None:
                            markers.append(build_marker(image))
                paragraph = "".join(text_parts).strip()
                if markers:
                    paragraph = (paragraph + "\n" if paragraph else "") + "\n".join(markers)
                if paragraph:
                    lines.append(paragraph)
            slides.append("\n".join(lines))

    return PptxRich(slides=tuple(slides), collection=_collection_from(budget, images))


_SLIDE_MEMBER_PATTERN = re.compile(r"ppt/slides/slide\d+\.xml")


def _natural_slide_key(name: str) -> list[int]:
    digits = re.findall(r"\d+", posixpath.basename(name))
    return [int(digits[0])] if digits else [0]


def _rels_for(member: str) -> str:
    directory, filename = posixpath.split(member)
    return posixpath.join(directory, "_rels", filename + ".rels")


# ---------------------------------------------------------------------------
# PDF
# ---------------------------------------------------------------------------


def extract_pdf_images(data: bytes, *, budget: ImageBudget | None = None) -> PdfImages:
    """Raster images per page via PyMuPDF, deduplicated across the document.

    Running headers/footers embed the same logo xref on every page. Store its
    bytes once, but map that image to every page where it appears.

    ``budget`` defaults to the chat-attachment caps (unchanged behaviour). When
    it carries a non-zero ``max_images_per_page``, a page stops admitting once
    it hits that many images and the rest of that page's candidates are counted
    as page-budget skips. Reused xrefs still count toward the per-page cap,
    but do not consume the document-wide image or byte budgets again.
    """
    try:
        import pymupdf
    except ImportError:
        return PdfImages()

    budget = budget or ImageBudget()
    images: list[EmbeddedImage] = []
    page_map: list[tuple[int, tuple[int, ...]]] = []
    xref_to_index: dict[int, int] = {}
    rejected_xrefs: set[int] = set()

    try:
        with pymupdf.open(stream=data, filetype="pdf") as doc:
            for page_number, page in enumerate(doc, 1):
                indices: list[int] = []
                admitted_on_page = 0
                seen_on_page: set[int] = set()
                for info in page.get_images(full=True):
                    xref = info[0]
                    if xref in seen_on_page:
                        continue
                    seen_on_page.add(xref)
                    if budget.max_images_per_page and (
                        admitted_on_page >= budget.max_images_per_page
                    ):
                        budget.skipped_page_budget += 1
                        continue
                    if xref in xref_to_index:
                        indices.append(xref_to_index[xref])
                        admitted_on_page += 1
                        continue
                    if xref in rejected_xrefs:
                        continue
                    rejected_xrefs.add(xref)
                    try:
                        extracted = doc.extract_image(xref)
                    except Exception:
                        continue
                    raw = extracted.get("image") or b""
                    if (
                        int(extracted.get("width") or 0) < 64
                        or int(extracted.get("height") or 0) < 64
                    ):
                        continue
                    ext = _normalise_ext("." + (extracted.get("ext") or ""))
                    if ext in {".emf", ".wmf"}:
                        budget.register_vector_skip()
                        continue
                    if ext not in SUPPORTED_IMAGE_EXTENSIONS:
                        budget.register_other_skip()
                        continue
                    admitted = budget.try_admit(ext, raw)
                    if admitted is not None:
                        images.append(admitted)
                        xref_to_index[xref] = len(images) - 1
                        indices.append(xref_to_index[xref])
                        admitted_on_page += 1
                if indices:
                    page_map.append((page_number, tuple(indices)))
    except Exception:
        logger.warning("pdf image extraction failed", exc_info=True)
        return PdfImages()

    return PdfImages(collection=_collection_from(budget, images), page_map=tuple(page_map))


__all__ = [
    "EmbeddedImage",
    "ImageBudget",
    "ImageCollection",
    "DocxRich",
    "PptxRich",
    "PdfImages",
    "MAX_IMAGES_PER_DOC",
    "MAX_INLINE_IMAGE_EDGE",
    "SUPPORTED_IMAGE_EXTENSIONS",
    "build_marker",
    "find_markers",
    "fit_inline_image",
    "image_index_from_name",
    "reading_image_budget",
    "extract_docx_rich",
    "extract_pptx_rich",
    "extract_pdf_images",
]