Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix: walk pdfplumber pages lazily during extraction (#7346)

Abarnaa Sree Nada9db335490
diff --git a/src/local_deep_research/research_library/services/download_service.py b/src/local_deep_research/research_library/services/download_service.py
index c86ddb37c5..27da388288 100644
--- a/src/local_deep_research/research_library/services/download_service.py
+++ b/src/local_deep_research/research_library/services/download_service.py
@@ -2198,11 +2198,12 @@ class DownloadService:
between pages stays near that threshold.
What is NOT bounded: the budget is checked between pages only,
- so neither a single page's ``extract_text()`` nor the parser's
- page-tree construction (``pdfplumber.open``/``PdfReader`` and the
- first access to their pages, O(all pages) in time and memory for
- a file with many pages) can be interrupted, and what one page
- decodes and parses is held until the release after it.
+ so a single page's ``extract_text()`` cannot be interrupted. The
+ pdfplumber path walks pages lazily, but the PyPDF fallback may
+ eagerly materialize the page tree during ``PdfReader`` construction
+ or its first access to ``pages``. That operation can take O(all
+ pages) time and memory before the page ceiling can stop the walk.
+ What one page decodes and parses is held until the release after it.
Args:
pdf_content: Raw PDF bytes
@@ -2227,18 +2228,22 @@ class DownloadService:
PypdfWalkReleaser,
)
from ...utilities.resource_utils import safe_close
+ from ...utilities.pdf_page_walk import iter_pdfplumber_pages
# Extraction ceilings: the storage cap bounds file SIZE, not
- # extraction COMPLEXITY. These stop the walk after
- # MAX_PDF_EXTRACTION_PAGES pages, MAX_PDF_EXTRACTED_CHARS
- # characters or the shared CPU budget, whichever comes first.
- # They are consulted only once the parser has materialised
- # the page tree, and only between pages.
- with pdfplumber.open(io.BytesIO(pdf_content)) as pdf:
+ # extraction COMPLEXITY. These limits bound work and retained
+ # text during the lazy page walk. The page ceiling is checked
+ # before extracting text from the first page beyond the limit;
+ # character and CPU budgets are enforced during the walk.
+ pdf_stream = io.BytesIO(pdf_content)
+ pdf = pdfplumber.open(pdf_stream)
+ try:
text_parts = []
extracted_chars = 0
releaser = PdfplumberWalkReleaser(pdf)
- for page_number, page in enumerate(pdf.pages):
+ for page_number, page in enumerate(
+ iter_pdfplumber_pages(pdf, releaser)
+ ):
# pdfplumber keeps each page's parsed layout objects
# and text map cached on the Page until close(): about
# 25 MB per dense page, so walking to the page ceiling
@@ -2248,8 +2253,8 @@ class DownloadService:
if page_number >= MAX_PDF_EXTRACTION_PAGES:
logger.warning(
"PDF extraction stopped at page ceiling "
- f"({MAX_PDF_EXTRACTION_PAGES}); document has "
- f"{len(pdf.pages)} pages"
+ f"({MAX_PDF_EXTRACTION_PAGES}); document exceeds "
+ "the page ceiling"
)
break
if time.thread_time() >= deadline:
@@ -2268,7 +2273,6 @@ class DownloadService:
f"({MAX_PDF_EXTRACTED_CHARS})"
)
break
- releaser.before_page()
page_text = page.extract_text()
if page_text:
if len(page_text) > remaining:
@@ -2298,6 +2302,8 @@ class DownloadService:
if text_parts:
return "\n\n".join(text_parts)
+ finally:
+ safe_close(pdf_stream, "PDF stream")
# Fallback to PyPDF if pdfplumber produced no text -- but only
# within the same budget: do not start a second parse once the
diff --git a/src/local_deep_research/utilities/pdf_extraction_limits.py b/src/local_deep_research/utilities/pdf_extraction_limits.py
index d2b34ff949..846e8b8ff0 100644
--- a/src/local_deep_research/utilities/pdf_extraction_limits.py
+++ b/src/local_deep_research/utilities/pdf_extraction_limits.py
@@ -50,17 +50,16 @@ On all four paths they bound exactly four things:
What they do NOT bound:
-- page-tree materialisation, where the walk does not avoid it.
- ``pypdf`` flattens the whole page tree, and pdfplumber's ``PDF.pages``
- builds every page object, before the first page is yielded -- O(all
- pages) time and memory for a file with many pages, whatever page
- count the file declares -- and none of the ceilings above is
- consulted until that has finished. That cost is paid by every pypdf
- walk and by the download service's pdfplumber walk. The upload-path
- service walks the page tree lazily instead
- (``utilities/pdf_page_walk``) and never touches
- ``PDF.pages``, so they build at most the pages they walk (plus the
- one that shows the ceiling was reached).
+- page-tree materialisation on paths that still use eager page access.
+ ``pypdf`` flattens the whole page tree, and accessing
+ ``pdfplumber.PDF.pages`` builds every page object before the first page
+ is yielded. These operations can take O(all pages) time and memory,
+ regardless of the page ceiling. The download service and upload-path
+ extraction instead use ``utilities/pdf_page_walk`` to walk pdfplumber
+ pages lazily, building only the pages reached by the walk (plus the
+ one that establishes that more pages exist). The PyPDF fallback remains
+ eager, so its page-tree construction is not bounded by the extraction
+ ceilings.
- the cost of a single page. One page's ``extract_text()`` runs to
completion inside the library with no check point available to us; a
few kilobytes of compressed content streams can take minutes or
diff --git a/tests/research_library/services/test_download_service.py b/tests/research_library/services/test_download_service.py
index 99ff1212af..a1334aca00 100644
--- a/tests/research_library/services/test_download_service.py
+++ b/tests/research_library/services/test_download_service.py
@@ -478,6 +478,16 @@ class TestDownloadServiceTextExtraction:
return_value=mock_pdf,
)
+ def walk_mock_pages(pdf, releaser):
+ for page in mock_pdf.pages:
+ releaser.before_page()
+ yield page
+
+ mocker.patch(
+ "local_deep_research.utilities.pdf_page_walk.iter_pdfplumber_pages",
+ side_effect=walk_mock_pages,
+ )
+
from local_deep_research.research_library.services.download_service import (
DownloadService,
)
@@ -666,6 +676,16 @@ class TestPyPDFTextExtraction:
return_value=mock_pdf,
)
+ def walk_mock_pages(pdf, releaser):
+ for page in mock_pdf.pages:
+ releaser.before_page()
+ yield page
+
+ mocker.patch(
+ "local_deep_research.utilities.pdf_page_walk.iter_pdfplumber_pages",
+ side_effect=walk_mock_pages,
+ )
+
from local_deep_research.research_library.services.download_service import (
DownloadService,
)
diff --git a/tests/research_library/services/test_download_service_coverage.py b/tests/research_library/services/test_download_service_coverage.py
index a3a251352f..9dba377fbb 100644
--- a/tests/research_library/services/test_download_service_coverage.py
+++ b/tests/research_library/services/test_download_service_coverage.py
@@ -57,6 +57,23 @@ def svc():
return service
[email protected](autouse=True)
+def _walk_mocked_pdfs_through_their_pages_list(monkeypatch):
+ """Use mock page lists for mocked PDFs, but preserve lazy walking for real PDFs."""
+ import pdfplumber
+
+ from local_deep_research.utilities import pdf_page_walk
+
+ real_walk = pdf_page_walk.iter_pdfplumber_pages
+
+ def walk(pdf, releaser=None):
+ if isinstance(pdf, pdfplumber.PDF):
+ return real_walk(pdf, releaser)
+ return iter(pdf.pages)
+
+ monkeypatch.setattr(pdf_page_walk, "iter_pdfplumber_pages", walk)
+
+
# ============================================================
# _normalize_url
# ============================================================
diff --git a/tests/research_library/services/test_pdf_extraction_bounds.py b/tests/research_library/services/test_pdf_extraction_bounds.py
index 2af569a69e..31318e20c8 100644
--- a/tests/research_library/services/test_pdf_extraction_bounds.py
+++ b/tests/research_library/services/test_pdf_extraction_bounds.py
@@ -45,16 +45,27 @@ def service(mocker):
def _pdf_with_pages(mocker, pages):
- """Patch pdfplumber.open with a PDF holding *pages* mock pages."""
+ """Patch pdfplumber.open and the lazy page walker with mock pages."""
mock_pdf = MagicMock()
mock_pdf.pages = pages
mock_pdf.__enter__ = Mock(return_value=mock_pdf)
mock_pdf.__exit__ = Mock(return_value=False)
+
mocker.patch(
"local_deep_research.research_library.services.download_service."
"pdfplumber.open",
return_value=mock_pdf,
)
+
+ def walk(pdf, releaser):
+ for page in pages:
+ releaser.before_page()
+ yield page
+
+ mocker.patch(
+ "local_deep_research.utilities.pdf_page_walk.iter_pdfplumber_pages",
+ side_effect=walk,
+ )
return mock_pdf
@@ -494,11 +505,15 @@ class TestCpuTimeBudget:
side_effect=slow_open,
)
reader = _reader_with_pages(mocker, _pages_of(["fallback text"]))
+ mocker.patch(
+ "local_deep_research.utilities.pdf_page_walk.iter_pdfplumber_pages",
+ return_value=iter(pages),
+ )
text = service._extract_text_from_pdf(b"%PDF-fake")
assert not any(page.extract_text.called for page in pages), (
- "a page was walked even though pdfplumber.open() construction "
+ "a page was extracted even though pdfplumber.open() construction "
"alone spent the whole deadline"
)
assert not reader.called, (
@@ -693,6 +708,7 @@ class TestExactFitThenOneMorePage:
def test_download_service_pdfplumber_loop(self, service, mocker):
pages = _pages_of(["x" * MAX_PDF_EXTRACTED_CHARS, "y" * 100])
_pdf_with_pages(mocker, pages)
+ TestReleaserWiring._patch_page_walk(mocker, pages)
text = service._extract_text_from_pdf(b"%PDF-fake")
@@ -1356,13 +1372,12 @@ def _kept_elements(releaser):
)
-def _pdfminer_decoded(pdf):
- """Decoded bytes pdfminer still holds: cached streams and the
- streams pages hold directly."""
+def _pdfminer_decoded(pdf, pages):
+ """Decoded bytes still held by cached objects and walked pages."""
from pdfminer.pdftypes import PDFStream
streams = [obj for obj, _ in pdf.doc._cached_objs.values()]
- for page in pdf.pages:
+ for page in pages:
streams.extend(page.page_obj.contents)
return sum(
len(obj.data)
@@ -1371,6 +1386,26 @@ def _pdfminer_decoded(pdf):
)
+def _capture_pdfplumber_walk(mocker):
+ """Capture pages yielded by the lazy PDF page walker."""
+ from local_deep_research.utilities.pdf_page_walk import (
+ iter_pdfplumber_pages,
+ )
+
+ walked_pages = []
+
+ def capture_walk(pdf, releaser):
+ for page in iter_pdfplumber_pages(pdf, releaser):
+ walked_pages.append(page)
+ yield page
+
+ mocker.patch(
+ "local_deep_research.utilities.pdf_page_walk.iter_pdfplumber_pages",
+ side_effect=capture_walk,
+ )
+ return walked_pages
+
+
def _pypdf_decoded(reader):
from pypdf.generic import EncodedStreamObject
@@ -1396,6 +1431,7 @@ class TestDecodedStreamsAreReleased:
import pdfplumber
mocker.patch(_RETAINED_LIMIT, 1024 * 1024)
+ walked_pages = _capture_pdfplumber_walk(mocker)
opened = _capture(
mocker, f"{_SERVICE_MODULE}.pdfplumber.open", pdfplumber.open
)
@@ -1403,7 +1439,10 @@ class TestDecodedStreamsAreReleased:
text = service._extract_text_from_pdf(_crafted_pdf(3, _PAD))
assert text == _page_texts(3, "\n\n")
- assert _pdfminer_decoded(opened[0]) == 0, (
+ assert not hasattr(opened[0], "_pages"), (
+ "service extraction eagerly materialized pdfplumber's page list"
+ )
+ assert _pdfminer_decoded(opened[0], walked_pages) == 0, (
"pdfminer still holds decoded content streams after the walk"
)
@@ -1415,6 +1454,7 @@ class TestDecodedStreamsAreReleased:
threshold, are not touched."""
import pdfplumber
+ walked_pages = _capture_pdfplumber_walk(mocker)
opened = _capture(
mocker, f"{_SERVICE_MODULE}.pdfplumber.open", pdfplumber.open
)
@@ -1422,7 +1462,7 @@ class TestDecodedStreamsAreReleased:
text = service._extract_text_from_pdf(_crafted_pdf(3, _PAD))
assert text == _page_texts(3, "\n\n")
- assert _pdfminer_decoded(opened[0]) >= 3 * _PAD
+ assert _pdfminer_decoded(opened[0], walked_pages) >= 3 * _PAD
def test_streams_a_later_page_reuses_still_extract(self, service, mocker):
"""Rewound streams decode again when a later page reads them:
@@ -1431,6 +1471,7 @@ class TestDecodedStreamsAreReleased:
import pdfplumber
mocker.patch(_RETAINED_LIMIT, 1)
+ walked_pages = _capture_pdfplumber_walk(mocker)
opened = _capture(
mocker, f"{_SERVICE_MODULE}.pdfplumber.open", pdfplumber.open
)
@@ -1440,7 +1481,7 @@ class TestDecodedStreamsAreReleased:
)
assert text == "\n\n".join([_page_texts(3, "\n\n")] * 2)
- assert _pdfminer_decoded(opened[0]) == 0
+ assert _pdfminer_decoded(opened[0], walked_pages) == 0
def test_pypdf_fallback_releases_decoded_streams(self, service, mocker):
from pypdf import PdfReader
@@ -1592,8 +1633,7 @@ class TestWalkReleasers:
assert releaser.after_page() is True
# after_page() without a before_page() has nothing to count.
assert releaser.after_page() is False
-
- assert _pdfminer_decoded(pdf) == 0
+ assert _pdfminer_decoded(pdf, pages) == 0
for objid, (obj, _) in pre_walk.items():
assert pdf.doc._cached_objs[objid][0] is obj, (
"an object cached before the walk was dropped or replaced"
@@ -3741,6 +3781,18 @@ class TestReleaserWiring:
releaser.after_page.side_effect = lambda: events.append("after")
return mocker.patch(target, return_value=releaser)
+ @staticmethod
+ def _patch_page_walk(mocker, pages):
+ def walk(pdf, releaser):
+ for page in pages:
+ releaser.before_page()
+ yield page
+
+ return mocker.patch(
+ "local_deep_research.utilities.pdf_page_walk.iter_pdfplumber_pages",
+ side_effect=walk,
+ )
+
def test_pdfplumber_loop(self, service, mocker):
events = []
pages = _recording_pages(["first", "second"], events)
@@ -3751,6 +3803,7 @@ class TestReleaserWiring:
"PdfplumberWalkReleaser",
events,
)
+ self._patch_page_walk(mocker, pages)
assert service._extract_text_from_pdf(b"%PDF-fake") == (
"first\n\nsecond"
@@ -3781,6 +3834,7 @@ class TestReleaserWiring:
"PdfplumberWalkReleaser",
events,
)
+ self._patch_page_walk(mocker, pages)
assert service._extract_text_from_pdf(b"%PDF-fake") is None
@@ -3788,9 +3842,19 @@ class TestReleaserWiring:
def test_pypdf_fallback_loop(self, service, mocker):
events = []
- _pdf_with_pages(mocker, _pages_of([""]))
+ pdf_pages = _pages_of([""])
+ _pdf_with_pages(mocker, pdf_pages)
fallback = _recording_pages(["first", "second"], events)
reader = _reader_with_pages(mocker, fallback)
+
+ self._recording_releaser(
+ mocker,
+ "local_deep_research.utilities.pdf_stream_release."
+ "PdfplumberWalkReleaser",
+ [],
+ )
+ self._patch_page_walk(mocker, pdf_pages)
+
cls = self._recording_releaser(
mocker,
"local_deep_research.utilities.pdf_stream_release."
@@ -3801,7 +3865,6 @@ class TestReleaserWiring:
assert service._extract_text_from_pdf(b"%PDF-fake") == (
"first\n\nsecond"
)
-
cls.assert_called_once_with(reader.return_value)
assert events == [
"before",