Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

22.9 KB
Raw Blame
"""Cut a source file into addressable units (the locator space).

One job, no I/O beyond reading the source: given a path, return the unit texts
plus what a unit *is* for that format. Everything else in the reading engine
consumes :class:`Extraction` and never looks at the original bytes again —
which is what keeps store, search, outline and export format-agnostic.

Per-format strategy, and why:

* **PDF** — PyMuPDF, one unit per physical page. Done here rather than through
  the shared text extractor because that one joins the pages into a single
  string; we need them apart. Physical pages are also the only unit that lines
  up with what the reader renders, so PDFs are the one format with a faithful
  raw view.
* **PPTX** — the shared extractor already emits ``--- Slide N ---`` separators,
  so we split on those instead of re-implementing python-pptx handling.
* **Markdown** — the shared extractor's text, cut at usable ATX headings and
  then at paragraph boundaries for long sections.
* **everything else** (EPUB, DOCX, XLSX, TXT, code, …) — the shared
  extractor's plain text, cut into fixed-size *sections* on paragraph
  boundaries.

The last bullet is a deliberate trade-off for EPUB: chapter-accurate cutting
would mean reaching into the extractor's private per-chapter helpers, coupling
this module to their internals. Since a non-PDF material is read from extracted
text anyway (no faithful raw view), a section is nearly as good a handle as a
chapter, and the seam stays clean. Chapter cutting can be added later as one
more branch here without touching a single consumer.
"""

from __future__ import annotations

from dataclasses import dataclass, field
import logging
from pathlib import Path
import re

from deeptutor.reading.models import OutlineEntry, ReadingError, RenderMode, UnitKind, UnitReference
from deeptutor.utils.document_images import (
    EmbeddedImage,
    build_marker,
    extract_docx_rich,
    extract_pdf_images,
    extract_pptx_rich,
    find_markers,
    reading_image_budget,
)

logger = logging.getLogger(__name__)

# Target size of a synthesised "section". Chosen to land near the character
# count of one dense printed page (~3k), so a section feels like a page to the
# reader and to the model's sense of "how much is this".
SECTION_TARGET_CHARS = 2800
# Never emit a section longer than this even if no paragraph break was found —
# a minified file or a single 200k-character line must still be addressable.
SECTION_HARD_CHARS = 4200

_SLIDE_SEPARATOR = re.compile(r"^--- Slide \d+ ---$", re.MULTILINE)
# Title candidates: a markdown heading, or the first non-trivial line.
_MD_HEADING = re.compile(r"^\s{0,3}(?P<marks>#{1,6})\s+(?P<title>.+?)\s*#*\s*$")
_MD_FENCE = re.compile(r"^\s{0,3}(?P<marker>`{3,}|~{3,})(?P<info>.*)$")

# Formats whose original bytes the browser can render faithfully next to the
# extracted text. Only PDF today; adding one means teaching the reader pane to
# render it, not changing this engine.
RAW_VIEW_EXTENSIONS = frozenset({".pdf"})


@dataclass(frozen=True, slots=True)
class MediaItem:
    """One embedded image pinned to the locator whose text references it."""

    name: str
    mime_type: str
    data: bytes
    locator: int


@dataclass(frozen=True, slots=True)
class Extraction:
    """The result of cutting one source file into units."""

    units: tuple[str, ...]
    unit: UnitKind
    extractor: str
    has_raw_view: bool = False
    title: str = ""
    # Only populated when the format carries its own structure (PDF bookmarks).
    # Otherwise the outline is synthesised later from unit first lines, so that
    # a material without bookmarks is still navigable by meaning.
    outline: tuple[OutlineEntry, ...] = field(default_factory=tuple)
    render_mode: RenderMode = "text"
    unit_refs: tuple[UnitReference, ...] = field(default_factory=tuple)
    # Embedded pictures (DOCX/PPTX) mapped to the unit that mentions them.
    media: tuple[MediaItem, ...] = field(default_factory=tuple)

    @property
    def char_count(self) -> int:
        return sum(len(u) for u in self.units)


def _media_for_units(
    units: tuple[str, ...], images: tuple[EmbeddedImage, ...]
) -> tuple[MediaItem, ...]:
    """Map ``[图片 N]`` markers in unit texts to their image bytes.

    A marker names the image it belongs to, so the mapping needs no position
    bookkeeping: scan each unit, look the name up, and pin the image to that
    locator. Repeated markers (the same figure floated into two sections) are
    kept once per locator; unknown names are skipped silently — the text still
    stands on its own.
    """
    by_name = {image.name: image for image in images}
    if not by_name:
        return ()
    items: list[MediaItem] = []
    seen: set[tuple[int, str]] = set()
    for locator, unit in enumerate(units, start=1):
        for _, name in find_markers(unit):
            image = by_name.get(name)
            if image is None or (locator, name) in seen:
                continue
            seen.add((locator, name))
            items.append(
                MediaItem(name=name, mime_type=image.mime_type, data=image.data, locator=locator)
            )
    return tuple(items)


def extract_material(path: str | Path, *, data: bytes | None = None) -> Extraction:
    """Cut *path* into units, dispatching on its extension.

    Raises :class:`ReadingError` when the file cannot be read at all, or when
    it yields no text — an image-only scan, for instance, which the reader
    would otherwise present as an empty document with no explanation. For an
    EPUB, ``data`` may contain already-normalized archive bytes so callers can
    extract and store exactly what they read.
    """
    source = Path(path)
    if not source.is_file():
        raise ReadingError(f"{source.name}: file not found")

    suffix = source.suffix.lower()
    if suffix == ".pdf":
        extraction = _extract_pdf(source)
    elif suffix == ".epub":
        if data is None:
            try:
                data = source.read_bytes()
            except OSError as exc:
                raise ReadingError(f"{source.name}: could not be read ({exc})") from exc
        extraction = _extract_epub(data, source.name)
    elif suffix == ".pptx":
        extraction = _extract_slides(source)
    elif suffix in {".md", ".markdown"}:
        units, outline = split_markdown_by_headings(_shared_extract(source))
        extraction = Extraction(units=units, unit="section", extractor="text", outline=outline)
    else:
        extraction = _extract_sections(source)

    if not any(unit.strip() for unit in extraction.units):
        raise ReadingError(
            f"{source.name}: no readable text could be extracted. "
            "A scanned document needs OCR before it can be read here."
        )
    return extraction


# ---------------------------------------------------------------------------
# PDF
# ---------------------------------------------------------------------------


def _extract_pdf(source: Path) -> Extraction:
    try:
        import pymupdf
    except ImportError as exc:  # pragma: no cover - pymupdf is a core dep
        raise ReadingError("PDF reading needs PyMuPDF (pip install pymupdf)") from exc

    try:
        with pymupdf.open(source) as doc:
            if doc.is_encrypted and not doc.authenticate(""):
                raise ReadingError(f"{source.name} is encrypted and cannot be read")
            units = tuple((page.get_text() or "") for page in doc)
            outline = _pdf_outline(doc, page_count=len(units))
            title = str((doc.metadata or {}).get("title") or "").strip()
    except ReadingError:
        raise
    except Exception as exc:
        raise ReadingError(f"{source.name}: failed to read PDF ({exc})") from exc

    media: tuple[MediaItem, ...] = ()
    # Every page's figures ride along as locator-pinned media so the reader can
    # attach what the question is about. Mining them is best-effort: a quirk in
    # one image must never turn a readable PDF into a failed ingest.
    units, media = _pdf_pages_with_image_markers(units, source)

    return Extraction(
        units=units,
        unit="page",
        extractor="pymupdf",
        has_raw_view=True,
        title=title,
        outline=outline,
        render_mode="pdf",
        media=media,
    )


def _pdf_pages_with_image_markers(
    units: tuple[str, ...], source: Path
) -> tuple[tuple[str, ...], tuple[MediaItem, ...]]:
    """Append the page's ``[图片 N: name]`` markers to that page's text tail.

    Markers are appended only at the very end of each unit so the character
    offsets of the page's own prose never move — stored per-character
    annotations resolve against quotes lifted from this text, and shifting them
    would silently detach every highlight. The markers name the images, which
    :func:`_media_for_units` then resolves back into locator-pinned media via
    the same mapping DOCX/PPTX already use.

    Best-effort by contract: any failure or empty result returns the units and
    no media untouched, so an image quirk never turns a readable PDF into a
    failed ingest.
    """
    try:
        pdf_images = extract_pdf_images(source.read_bytes(), budget=reading_image_budget())
    except Exception:
        logger.warning("%s: PDF image extraction failed", source.name, exc_info=True)
        return units, ()
    if not pdf_images.collection.images:
        return units, ()

    marker_by_page: dict[int, list[str]] = {}
    for page_number, indices in pdf_images.page_map:
        marker_by_page[page_number] = [
            build_marker(pdf_images.collection.images[index]) for index in indices
        ]
    targeted = tuple(
        unit + ("\n" + "\n".join(marker_by_page[i]) if i in marker_by_page else "")
        for i, unit in enumerate(units, 1)
    )
    return targeted, _media_for_units(targeted, pdf_images.collection.images)


def _extract_epub(data: bytes, filename: str) -> Extraction:
    """Preserve EPUB spine order so browser and assistant locators agree."""
    from deeptutor.utils.document_extractor import DocumentExtractionError, extract_epub_spine

    try:
        units, navigation = extract_epub_spine(data, filename)
    except (OSError, DocumentExtractionError) as exc:
        raise ReadingError(f"{filename}: failed to read EPUB ({exc})") from exc

    refs = tuple(
        UnitReference(locator=index, source_href=unit.href, title=unit.title)
        for index, unit in enumerate(units, start=1)
    )
    outline = tuple(
        OutlineEntry(
            locator=row.locator,
            title=row.title,
            level=row.level,
            source_href=row.source_href,
            source_anchor=row.source_anchor,
        )
        for row in navigation
    )
    if not outline:
        outline = tuple(
            OutlineEntry(
                locator=ref.locator,
                title=ref.title or Path(ref.source_href).stem,
                synthesised=not bool(ref.title),
            )
            for ref in refs
        )
    return Extraction(
        units=tuple(unit.text for unit in units),
        unit="chapter",
        extractor="epub-spine",
        outline=outline,
        render_mode="epub",
        unit_refs=refs,
    )


def _pdf_outline(doc: object, *, page_count: int) -> tuple[OutlineEntry, ...]:
    """Map the PDF's bookmark tree to outline rows, dropping unusable ones.

    A bookmark pointing outside the page range (some generators emit 0 or a
    stale page) is skipped rather than clamped, so the model is never sent to a
    locator that does not correspond to the heading it asked for.
    """
    try:
        toc = doc.get_toc()  # type: ignore[attr-defined]
    except Exception:
        logger.debug("PDF has no readable table of contents", exc_info=True)
        return ()

    entries: list[OutlineEntry] = []
    for row in toc or []:
        if not isinstance(row, (list, tuple)) or len(row) < 3:
            continue
        try:
            level, title, page = int(row[0]), str(row[1]).strip(), int(row[2])
        except (TypeError, ValueError):
            continue
        if not title or not (1 <= page <= page_count):
            continue
        entries.append(OutlineEntry(locator=page, title=title, level=max(1, level)))
    return tuple(entries)


# ---------------------------------------------------------------------------
# Slides
# ---------------------------------------------------------------------------


def _extract_slides(source: Path) -> Extraction:
    text, images = _slides_text_and_images(source)
    parts = [part.strip() for part in _SLIDE_SEPARATOR.split(text)]
    units = tuple(part for part in parts if part)
    if not units:
        # The extractor found text but no slide separators (legacy .ppt via the
        # raw-OOXML fallback). Treat it as flat text rather than losing it.
        return _sections_from_text(text, extractor="pptx-text")
    return Extraction(
        units=units,
        unit="slide",
        extractor="pptx",
        media=_media_for_units(units, images),
    )


def _slides_text_and_images(source: Path) -> tuple[str, tuple[EmbeddedImage, ...]]:
    """Slide text with image markers, preferring the rich extraction.

    Falls back to the shared extractor (text only) when the rich path cannot
    read the deck, so ingest never fails on a picture quirk. The reading budget
    keeps slide pictures at source resolution, like the PDF ingest above.
    """
    try:
        rich = extract_pptx_rich(source.read_bytes(), budget=reading_image_budget())
    except Exception:
        rich = None
    if rich is not None and any(slide.strip() for slide in rich.slides):
        parts = [
            f"--- Slide {index} ---\n{slide}".rstrip()
            for index, slide in enumerate(rich.slides, 1)
            if slide.strip()
        ]
        text = "\n\n".join(parts)
        note = rich.collection.summary_note()
        if note:
            text += f"\n\n{note}"
        return text, rich.collection.images
    return _shared_extract(source), ()


# ---------------------------------------------------------------------------
# Flat text → sections
# ---------------------------------------------------------------------------


def _extract_sections(source: Path) -> Extraction:
    """Flat-text formats, cut into sections on paragraph boundaries.

    DOCX goes through the rich extraction first so embedded images survive
    with their section mapping; every other format (and a failed rich parse)
    uses the shared text-only extractor. The reading budget keeps those images
    at source resolution for the reader pane, like the PDF ingest above.
    """
    images: tuple[EmbeddedImage, ...] = ()
    if source.suffix.lower() == ".docx":
        try:
            rich = extract_docx_rich(source.read_bytes(), budget=reading_image_budget())
        except Exception:
            rich = None
        if rich is not None and any(paragraph.strip() for paragraph in rich.paragraphs):
            text = "\n\n".join(rich.paragraphs)
            note = rich.collection.summary_note()
            if note:
                text += f"\n\n{note}"
            units = split_into_sections(text)
            return Extraction(
                units=units,
                unit="section",
                extractor="docx",
                media=_media_for_units(units, rich.collection.images),
            )
    return _sections_from_text(_shared_extract(source), extractor="text")


def _sections_from_text(text: str, *, extractor: str) -> Extraction:
    return Extraction(units=split_into_sections(text), unit="section", extractor=extractor)


def split_into_sections(text: str) -> tuple[str, ...]:
    """Cut flat text into sections, preferring paragraph boundaries.

    Greedy accumulation up to :data:`SECTION_TARGET_CHARS`, flushing at the
    paragraph that crosses it. A single paragraph longer than
    :data:`SECTION_HARD_CHARS` is hard-split so that pathological input (one
    enormous line) stays addressable.
    """
    normalised = text.replace("\r\n", "\n").replace("\r", "\n").strip()
    if not normalised:
        return ()

    sections: list[str] = []
    buffer: list[str] = []
    size = 0

    def flush() -> None:
        nonlocal size
        if buffer:
            joined = "\n\n".join(buffer).strip()
            if joined:
                sections.append(joined)
            buffer.clear()
            size = 0

    for paragraph in normalised.split("\n\n"):
        block = paragraph.strip()
        if not block:
            continue
        for piece in _hard_split(block, SECTION_HARD_CHARS):
            buffer.append(piece)
            size += len(piece)
            if size >= SECTION_TARGET_CHARS:
                flush()
    flush()
    return tuple(sections)


def split_markdown_by_headings(
    text: str,
) -> tuple[tuple[str, ...], tuple[OutlineEntry, ...]]:
    """Cut article markdown at headings and map every unit to its heading.

    A fetched page normally starts with the synthetic title heading added by
    the HTML extractor.  That heading alone is not meaningful article
    structure, so fewer than two usable headings deliberately falls back to
    :func:`split_into_sections` and returns no outline.  The store can then use
    its existing first-line outline rather than labelling an entire article as
    repeated continuations of the page title.

    Each heading-delimited region is still passed through the regular section
    splitter.  This preserves its hard cap for very long paragraphs, and every
    continuation gets an outline row pointing at its own locator while keeping
    the source heading's title and level.
    """
    normalised = text.replace("\r\n", "\n").replace("\r", "\n").strip()
    if not normalised:
        return (), ()

    boundaries: list[tuple[int, int, str]] = []
    # The run that opened the current fenced block ("```", "~~~~", ...). As in
    # CommonMark, only a bare run of the same character that is at least as long
    # closes it, so a ```` fence can show a ``` example and "```py" inside a
    # block is content, not a closing fence.
    fence_marker = ""
    offset = 0
    for line in normalised.splitlines(keepends=True):
        fence = _MD_FENCE.match(line.rstrip("\n"))
        if fence:
            marker, info = fence.group("marker"), fence.group("info")
            if not fence_marker:
                # A backtick fence's info string cannot contain a backtick.
                if not (marker[0] == "`" and "`" in info):
                    fence_marker = marker
                    offset += len(line)
                    continue
            elif (
                marker[0] == fence_marker[0]
                and len(marker) >= len(fence_marker)
                and not info.strip()
            ):
                fence_marker = ""
                offset += len(line)
                continue
        if not fence_marker:
            heading = _MD_HEADING.match(line.rstrip("\n"))
            if heading:
                title = _clean_heading_title(heading.group("title"))
                if title:
                    boundaries.append((offset, len(heading.group("marks")), title))
        offset += len(line)

    if len(boundaries) < 2:
        return split_into_sections(normalised), ()

    units: list[str] = []
    outline: list[OutlineEntry] = []
    for index, (start, level, title) in enumerate(boundaries):
        end = boundaries[index + 1][0] if index + 1 < len(boundaries) else len(normalised)
        # Keep prose before the first heading instead of dropping it.  In web
        # extraction this is uncommon (the title is normally first), but hand-
        # authored pages sometimes place a short deck or byline above it.
        section_start = 0 if index == 0 else start
        for piece in split_into_sections(normalised[section_start:end]):
            units.append(piece)
            outline.append(OutlineEntry(locator=len(units), title=title, level=level))
    return tuple(units), tuple(outline)


def _clean_heading_title(title: str) -> str:
    """Remove inline Markdown decoration from a heading used as a UI label."""
    clean = re.sub(r"\[([^]]*)\]\([^)]*\)", r"\1", title)
    return re.sub(r"[*`_~]", "", clean).strip()


def _hard_split(block: str, limit: int) -> list[str]:
    """Break an over-long paragraph at whitespace near *limit*, else mid-word."""
    if len(block) <= limit:
        return [block]
    pieces: list[str] = []
    remaining = block
    while len(remaining) > limit:
        window = remaining[:limit]
        cut = window.rfind(" ")
        if cut < limit // 2:
            cut = limit
        pieces.append(remaining[:cut].strip())
        remaining = remaining[cut:].strip()
    if remaining:
        pieces.append(remaining)
    return [piece for piece in pieces if piece]


def _shared_extract(source: Path) -> str:
    """Read *source* through the shared text extractor.

    Reuses the same code path chat attachments use, so a format that works as
    an attachment works as reading material — and gaining a format there gains
    it here for free. ``max_chars=None`` because the reading store keeps the
    full text on disk; the per-turn budget is enforced later, per unit read.
    ``max_bytes`` stays at the validator ceiling rather than ``None`` so a
    pathological upload still fails fast instead of being read into memory.
    """
    from deeptutor.utils.document_extractor import (
        DocumentExtractionError,
        extract_text_from_path,
    )
    from deeptutor.utils.document_validator import DocumentValidator

    try:
        return extract_text_from_path(
            source, max_bytes=DocumentValidator.MAX_FILE_SIZE, max_chars=None
        )
    except DocumentExtractionError as exc:
        raise ReadingError(f"{source.name}: {exc}") from exc
    except OSError as exc:
        raise ReadingError(f"{source.name}: could not be read ({exc})") from exc


def synthesise_outline(units: tuple[str, ...] | list[str]) -> tuple[OutlineEntry, ...]:
    """Build a fallback outline: one row per unit, labelled by its first line.

    Used for every material whose format carries no structure of its own. The
    label matters more than it looks: without it ``material_outline`` would
    return a bare count and the model would have to read units blindly to find
    anything.
    """
    entries: list[OutlineEntry] = []
    for index, unit in enumerate(units, start=1):
        entries.append(
            OutlineEntry(locator=index, title=first_line_label(unit), level=1, synthesised=True)
        )
    return tuple(entries)


def first_line_label(unit: str, *, limit: int = 90) -> str:
    """A short human label for a unit: its heading, else its first real line."""
    for raw_line in unit.splitlines():
        line = raw_line.strip()
        if len(line) < 2:
            continue
        heading = _MD_HEADING.match(line)
        if heading:
            line = heading.group("title").strip()
        if not line:
            continue
        return line if len(line) <= limit else line[: limit - 1].rstrip() + "…"
    return ""


__all__ = [
    "RAW_VIEW_EXTENSIONS",
    "SECTION_HARD_CHARS",
    "SECTION_TARGET_CHARS",
    "Extraction",
    "extract_material",
    "first_line_label",
    "split_markdown_by_headings",
    "split_into_sections",
    "synthesise_outline",
]