Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

3.9 KB
Raw Blame
"""Bound PDF text extraction for byte-backed document loaders.

RAG uploads and Zotero sync both call ``extract_text_from_bytes``. The
registry's ``PyPDFLoader.load()`` materializes text from every PDF page, so
those entry points need the same ceilings as the other extraction paths.
Pypdf still flattens the page tree before the first page is yielded, and one
``extract_text()`` call cannot be interrupted without process isolation.
"""

from __future__ import annotations

import io
import time

from langchain_core.documents import Document
from loguru import logger
from pypdf import PdfReader

from ..utilities import pdf_extraction_limits as limits
from ..utilities.pdf_stream_release import PypdfWalkReleaser
from ..utilities.resource_utils import safe_close


def load_bounded_pdf(content: bytes, source: str) -> list[Document]:
    """Return page documents, truncating at the shared extraction ceilings."""
    deadline = time.thread_time() + limits.MAX_PDF_EXTRACTION_CPU_SECONDS
    documents: list[Document] = []
    retained_chars = 0
    text_pages = 0
    truncated_by: str | None = None

    with io.BytesIO(content) as stream:
        reader = PdfReader(stream)
        try:
            # Pypdf materializes its page tree here; the limit below bounds
            # text extraction, not the construction of that tree.
            total_pages = len(reader.pages)
            releaser = PypdfWalkReleaser(reader)
            for page_number in range(
                min(total_pages, limits.MAX_PDF_EXTRACTION_PAGES)
            ):
                if time.thread_time() >= deadline:
                    truncated_by = "CPU-time"
                    break
                separator_chars = 2 if text_pages else 0
                remaining = (
                    limits.MAX_PDF_EXTRACTED_CHARS
                    - retained_chars
                    - separator_chars
                )
                if remaining <= 0:
                    truncated_by = "character"
                    break

                page = reader.pages[page_number]
                releaser.before_page()
                try:
                    page_text = (page.extract_text() or "").strip()
                finally:
                    # The reader otherwise retains decoded streams and parsed
                    # objects after each page has yielded its text.
                    releaser.after_page()

                if page_text:
                    kept_text = page_text[:remaining]
                    retained_chars += separator_chars + len(kept_text)
                    text_pages += 1
                else:
                    kept_text = ""

                documents.append(
                    Document(
                        page_content=kept_text,
                        metadata={
                            "source": source,
                            "page": page_number,
                            "total_pages": total_pages,
                        },
                    )
                )
                if len(kept_text) < len(page_text):
                    truncated_by = "character"
                    break
                if (
                    retained_chars >= limits.MAX_PDF_EXTRACTED_CHARS
                    and page_number + 1 < total_pages
                ):
                    truncated_by = "character"
                    break
            if (
                truncated_by is None
                and total_pages > limits.MAX_PDF_EXTRACTION_PAGES
            ):
                truncated_by = "page"
        finally:
            safe_close(reader, "PDF byte-loader reader")

    if truncated_by:
        for document in documents:
            document.metadata["truncated"] = True
        logger.warning(
            "PDF document-loader extraction stopped at {} ceiling "
            "after {} of {} pages",
            truncated_by,
            len(documents),
            total_pages,
        )
    return documents