~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Bound PDF text extraction for byte-backed document loaders.
RAG uploads and Zotero sync both call ``extract_text_from_bytes``. The
registry's ``PyPDFLoader.load()`` materializes text from every PDF page, so
those entry points need the same ceilings as the other extraction paths.
Pypdf still flattens the page tree before the first page is yielded, and one
``extract_text()`` call cannot be interrupted without process isolation.
"""
from __future__ import annotations
import io
import time
from langchain_core.documents import Document
from loguru import logger
from pypdf import PdfReader
from ..utilities import pdf_extraction_limits as limits
from ..utilities.pdf_stream_release import PypdfWalkReleaser
from ..utilities.resource_utils import safe_close
def load_bounded_pdf(content: bytes, source: str) -> list[Document]:
"""Return page documents, truncating at the shared extraction ceilings."""
deadline = time.thread_time() + limits.MAX_PDF_EXTRACTION_CPU_SECONDS
documents: list[Document] = []
retained_chars = 0
text_pages = 0
truncated_by: str | None = None
with io.BytesIO(content) as stream:
reader = PdfReader(stream)
try:
# Pypdf materializes its page tree here; the limit below bounds
# text extraction, not the construction of that tree.
total_pages = len(reader.pages)
releaser = PypdfWalkReleaser(reader)
for page_number in range(
min(total_pages, limits.MAX_PDF_EXTRACTION_PAGES)
):
if time.thread_time() >= deadline:
truncated_by = "CPU-time"
break
separator_chars = 2 if text_pages else 0
remaining = (
limits.MAX_PDF_EXTRACTED_CHARS
- retained_chars
- separator_chars
)
if remaining <= 0:
truncated_by = "character"
break
page = reader.pages[page_number]
releaser.before_page()
try:
page_text = (page.extract_text() or "").strip()
finally:
# The reader otherwise retains decoded streams and parsed
# objects after each page has yielded its text.
releaser.after_page()
if page_text:
kept_text = page_text[:remaining]
retained_chars += separator_chars + len(kept_text)
text_pages += 1
else:
kept_text = ""
documents.append(
Document(
page_content=kept_text,
metadata={
"source": source,
"page": page_number,
"total_pages": total_pages,
},
)
)
if len(kept_text) < len(page_text):
truncated_by = "character"
break
if (
retained_chars >= limits.MAX_PDF_EXTRACTED_CHARS
and page_number + 1 < total_pages
):
truncated_by = "character"
break
if (
truncated_by is None
and total_pages > limits.MAX_PDF_EXTRACTION_PAGES
):
truncated_by = "page"
finally:
safe_close(reader, "PDF byte-loader reader")
if truncated_by:
for document in documents:
document.metadata["truncated"] = True
logger.warning(
"PDF document-loader extraction stopped at {} ceiling "
"after {} of {} pages",
truncated_by,
len(documents),
total_pages,
)
return documents