Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): bound streamed PDF image memory (#4664)

LuckVd9688748b73ff
diff --git a/docreader/main.py b/docreader/main.py
index 30fee0e73..5504f31ec 100644
--- a/docreader/main.py
+++ b/docreader/main.py
@@ -16,6 +16,7 @@ from docreader.auth import AuthInterceptor, TLSConfigError, load_tls_credentials
from docreader import config
from docreader.config import CONFIG
from docreader.parser import Parser
+from docreader.parser.stream import ParsedDocumentStream
from docreader.preprocess import PreprocessLimits, PreprocessRunner
from docreader.proto import docreader_pb2_grpc
from docreader.parser.registry import registry
@@ -150,6 +151,17 @@ def _iter_image_refs(images: dict):
)
+def _image_ref_from_streamed_data(ref_path: str, image_data: bytes) -> ImageRef:
+ """Build one image reference from raw bytes yielded by a parser."""
+ fname, mime = _mime_for_ref(ref_path)
+ return ImageRef(
+ filename=fname,
+ original_ref=ref_path,
+ mime_type=mime,
+ image_data=image_data,
+ )
+
+
class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
def __init__(self, preprocess_runner=None):
super().__init__()
@@ -249,52 +261,106 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
return ReadResponse(error=str(e))
def ReadStream(self, request: ReadRequest, context):
- """Streaming read: yields one meta frame, then one frame per image.
-
- Each frame is a small, independent gRPC message, so documents with many
- page images (large scanned PDFs) are returned without hitting the unary
- message-size cap, and neither side has to hold the whole payload at once.
- """
+ """Yield metadata first, then images as each parser produces them."""
request_id = request.request_id or str(uuid.uuid4())
with request_id_context(request_id):
_c = to_valid_utf8_text
+ parsed_stream = None
try:
- result, source_desc = self._parse_request(request)
+ if request.url or not callable(
+ getattr(self.parser, "parse_file_stream", None)
+ ):
+ result, source_desc = self._parse_request(request)
+ parsed_stream = ParsedDocumentStream(
+ document=result,
+ image_count=len(result.images or {}),
+ )
+ else:
+ cfg = request.config
+ parser_engine = cfg.parser_engine if cfg else ""
+ engine_overrides = (
+ dict(cfg.parser_engine_overrides) if cfg else {}
+ )
+ file_type = request.file_type or os.path.splitext(
+ request.file_name
+ )[1][1:]
+ file_content = request.file_content
+ try:
+ track_request_size(len(file_content))
+ logger.info(
+ "Read(File stream): file=%s, type=%s, size=%d bytes",
+ request.file_name,
+ file_type,
+ len(file_content),
+ )
+ parsed_stream = self.parser.parse_file_stream(
+ request.file_name,
+ file_type,
+ file_content,
+ parser_engine=parser_engine,
+ engine_overrides=engine_overrides,
+ )
+ finally:
+ del file_content
+ result = parsed_stream.document
+ source_desc = request.file_name
except Exception as e:
logger.error("Error reading document: %s", e)
logger.info("Traceback: %s", traceback.format_exc())
+ if parsed_stream is not None:
+ parsed_stream.close()
+ del request
yield ReadStreamResponse(meta=ReadStreamMeta(error=str(e)))
return
+ # PDF parser streams own a temporary source file; drop the request's
+ # full bytes field while the client drains page images.
+ del request
+
if not result or not result.content:
error_msg = f"Failed to parse: {source_desc}"
logger.error(error_msg)
+ parsed_stream.close()
yield ReadStreamResponse(meta=ReadStreamMeta(error=error_msg))
return
images = result.images or {}
- image_count = len(images)
- yield ReadStreamResponse(
- meta=ReadStreamMeta(
- markdown_content=_c(result.content),
- image_dir_path="",
- metadata={k: _c(str(v)) for k, v in result.metadata.items()}
- if result.metadata
- else {},
- image_count=image_count,
- source_blocks=source_blocks_to_proto(result),
+ content_len = len(result.content)
+ sent = 0
+ try:
+ yield ReadStreamResponse(
+ meta=ReadStreamMeta(
+ markdown_content=_c(result.content),
+ image_dir_path="",
+ metadata={k: _c(str(v)) for k, v in result.metadata.items()}
+ if result.metadata
+ else {},
+ image_count=parsed_stream.image_count,
+ source_blocks=source_blocks_to_proto(result),
+ )
)
- )
- sent = 0
- for ref in _iter_image_refs(images):
- yield ReadStreamResponse(image=ref)
- sent += 1
+ for ref in _iter_image_refs(images):
+ yield ReadStreamResponse(image=ref)
+ sent += 1
+
+ for image in parsed_stream.images:
+ yield ReadStreamResponse(
+ image=_image_ref_from_streamed_data(
+ image.ref_path, image.image_data
+ )
+ )
+ sent += 1
+ except Exception:
+ logger.exception("Error streaming images for %s", source_desc)
+ raise
+ finally:
+ parsed_stream.close()
logger.info(
"ReadStream response: content_len=%d, images=%d",
- len(result.content),
+ content_len,
sent,
)
diff --git a/docreader/parser/parser.py b/docreader/parser/parser.py
index 5b9a8d2dd..d92c7dc35 100644
--- a/docreader/parser/parser.py
+++ b/docreader/parser/parser.py
@@ -3,6 +3,7 @@ from typing import Any, Optional
from docreader.models.document import Document
from docreader.parser.registry import registry
+from docreader.parser.stream import ParsedDocumentStream
from docreader.parser.web_parser import WebParser
logger = logging.getLogger(__name__)
@@ -83,6 +84,38 @@ class Parser:
logger.info("Parsed file %s, content length=%d", file_name, len(result.content))
return result
+ def parse_file_stream(
+ self,
+ file_name: str,
+ file_type: str,
+ content: bytes,
+ parser_engine: Optional[str] = None,
+ engine_overrides: Optional[dict[str, Any]] = None,
+ ) -> ParsedDocumentStream:
+ """Parse a file for ReadStream, using a parser's lazy image path when available."""
+ engine = parser_engine or ""
+ overrides = engine_overrides or {}
+ effective_file_type = detect_effective_file_type(file_type, content)
+ cls = self.registry.get_parser_class(engine, effective_file_type)
+ parser = cls(
+ file_name=file_name,
+ file_type=effective_file_type,
+ **overrides,
+ )
+
+ parse_stream = getattr(parser, "parse_stream", None)
+ if callable(parse_stream):
+ return parse_stream(content)
+
+ # Non-streaming parsers retain their established behavior. Their images
+ # are still sent one at a time by ReadStream, but parsing itself may
+ # materialize them before the first frame.
+ result = parser.parse(content)
+ return ParsedDocumentStream(
+ document=result,
+ image_count=len(result.images or {}),
+ )
+
def parse_url(
self,
url: str,
diff --git a/docreader/parser/pdf_parser.py b/docreader/parser/pdf_parser.py
index 0f6c227c6..fe7d2e6cc 100644
--- a/docreader/parser/pdf_parser.py
+++ b/docreader/parser/pdf_parser.py
@@ -17,19 +17,21 @@ self-sufficient using pypdfium2 + the Go-side OCR that already exists.
"""
import base64
-from collections import Counter
+from collections import Counter, deque
import io
import logging
import os
import re
import statistics
import threading
+from typing import Iterator
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.concurrency import parser_worker_limit
from docreader.parser.source_locator import PageGeometry, page_blocks
+from docreader.parser.stream import ParsedDocumentStream, StreamedImage
logger = logging.getLogger(__name__)
@@ -1121,6 +1123,11 @@ def _render_page_to_jpeg(page, scale: float, quality: int, max_edge: int = 0) ->
# per-page render (the dominant cost for big scanned PDFs — hours on
# CPU-constrained containers) into a near-linear speedup.
+# PDFium retains decoded page resources on some inputs until its document closes.
+# ReadStream therefore restarts the renderer every fixed-size batch; the bound
+# is independent of total page count while retaining useful parallel work.
+PDF_STREAM_RENDER_BATCH_PAGES = 16
+
# Per-worker document handle, populated by the pool initializer.
_WORKER_RENDER_DOC = None
@@ -1129,8 +1136,7 @@ def _render_pool_init(pdf_path: str) -> None:
global _WORKER_RENDER_DOC
import pypdfium2 as pdfium
- with open(pdf_path, "rb") as f:
- _WORKER_RENDER_DOC = pdfium.PdfDocument(f.read())
+ _WORKER_RENDER_DOC = pdfium.PdfDocument(pdf_path)
def _render_pool_task(args):
@@ -1212,6 +1218,191 @@ def _render_pages_parallel(
pass
+def _write_pdf_temp_file(content: bytes) -> str:
+ import tempfile
+
+ path = None
+ try:
+ with tempfile.NamedTemporaryFile(
+ prefix="docreader_stream_", suffix=".pdf", delete=False
+ ) as tmp:
+ path = tmp.name
+ tmp.write(content)
+ return path
+ except Exception:
+ if path is not None:
+ _unlink_pdf_temp_file(path)
+ raise
+
+
+def _unlink_pdf_temp_file(path: str) -> None:
+ try:
+ os.unlink(path)
+ except FileNotFoundError:
+ pass
+ except OSError:
+ logger.warning("Failed to remove temporary PDF %s", path, exc_info=True)
+
+
+def _iter_scanned_pages_from_open_pdf(
+ pdf,
+ indices: list[int],
+ base_name: str,
+ scale: float,
+ quality: int,
+ max_edge: int,
+) -> Iterator[StreamedImage]:
+ """Render one page at a time from an already-open PdfDocument."""
+ for index in indices:
+ with _PDFIUM_LOCK, parser_worker_limit(
+ "pdf_render", CONFIG.pdf_render_max_workers
+ ):
+ page = pdf[index]
+ try:
+ image_bytes = _render_page_to_jpeg(page, scale, quality, max_edge)
+ finally:
+ _close_pdfium_resource(page)
+ yield StreamedImage(
+ ref_path=f"images/{base_name}_page_{index + 1}.jpg",
+ image_data=image_bytes,
+ )
+
+
+def _iter_scanned_pages_serial_batch(
+ pdf_path: str,
+ indices: list[int],
+ base_name: str,
+ scale: float,
+ quality: int,
+ max_edge: int,
+) -> Iterator[StreamedImage]:
+ import pypdfium2 as pdfium
+
+ with _PDFIUM_LOCK:
+ pdf = pdfium.PdfDocument(pdf_path)
+ try:
+ yield from _iter_scanned_pages_from_open_pdf(
+ pdf, indices, base_name, scale, quality, max_edge
+ )
+ finally:
+ with _PDFIUM_LOCK:
+ _close_pdfium_resource(pdf)
+
+
+def _iter_scanned_pages_parallel_batch(
+ pdf_path: str,
+ indices: list[int],
+ base_name: str,
+ scale: float,
+ quality: int,
+ max_edge: int,
+ workers: int,
+ mp_context,
+) -> Iterator[StreamedImage]:
+ from concurrent.futures import ProcessPoolExecutor
+
+ max_workers = min(workers, len(indices))
+ window_size = max_workers * 2
+ pending = deque()
+ executor = None
+ sent = False
+ fallback = False
+
+ # Hold the limiter until the executor has shut down, including when the
+ # consumer cancels mid-batch. Otherwise another request could start a new
+ # pool while the old worker processes still hold their PDFium documents.
+ with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers):
+ try:
+ executor = ProcessPoolExecutor(
+ max_workers=max_workers,
+ mp_context=mp_context,
+ initializer=_render_pool_init,
+ initargs=(pdf_path,),
+ )
+ indices_iter = iter(indices)
+ for _ in range(window_size):
+ try:
+ index = next(indices_iter)
+ except StopIteration:
+ break
+ pending.append(
+ executor.submit(
+ _render_pool_task,
+ (index, scale, quality, max_edge),
+ )
+ )
+
+ while pending:
+ future = pending.popleft()
+ index, image_bytes = future.result()
+ try:
+ next_index = next(indices_iter)
+ except StopIteration:
+ pass
+ else:
+ pending.append(
+ executor.submit(
+ _render_pool_task,
+ (next_index, scale, quality, max_edge),
+ )
+ )
+ sent = True
+ yield StreamedImage(
+ ref_path=f"images/{base_name}_page_{index + 1}.jpg",
+ image_data=image_bytes,
+ )
+ except Exception:
+ if sent:
+ raise
+ logger.warning(
+ "parallel streamed PDF rendering failed before the first image; "
+ "falling back to serial rendering for this batch",
+ exc_info=True,
+ )
+ fallback = True
+ finally:
+ if executor is not None:
+ for future in pending:
+ future.cancel()
+ executor.shutdown(wait=True, cancel_futures=True)
+ pending.clear()
+
+ if fallback:
+ yield from _iter_scanned_pages_serial_batch(
+ pdf_path, indices, base_name, scale, quality, max_edge
+ )
+
+
+def _iter_scanned_pages_from_path(
+ pdf_path: str,
+ indices: list[int],
+ base_name: str,
+ scale: float,
+ quality: int,
+ max_edge: int,
+ workers: int,
+) -> Iterator[StreamedImage]:
+ """Yield images in fixed-size batches to cap PDFium and result memory."""
+ for start in range(0, len(indices), PDF_STREAM_RENDER_BATCH_PAGES):
+ batch = indices[start : start + PDF_STREAM_RENDER_BATCH_PAGES]
+ mp_context = _select_mp_context() if workers > 1 and len(batch) > 1 else None
+ if workers > 1 and len(batch) > 1 and mp_context is not None:
+ yield from _iter_scanned_pages_parallel_batch(
+ pdf_path,
+ batch,
+ base_name,
+ scale,
+ quality,
+ max_edge,
+ workers,
+ mp_context,
+ )
+ else:
+ yield from _iter_scanned_pages_serial_batch(
+ pdf_path, batch, base_name, scale, quality, max_edge
+ )
+
+
def _render_scanned_pages(
pdf, content: bytes, indices: list, scale: float, quality: int, max_edge: int
) -> dict:
@@ -1560,6 +1751,66 @@ class PDFScannedParser(BaseParser):
logger.exception("PDFScannedParser failed to parse PDF: %s", e)
raise e
+ def parse_stream(self, content: bytes) -> ParsedDocumentStream:
+ """Return metadata first and lazily render each scanned page."""
+ import pypdfium2 as pdfium
+
+ base_name = os.path.splitext(self.file_name or "document")[0]
+ with _PDFIUM_LOCK:
+ pdf = pdfium.PdfDocument(content)
+ try:
+ page_count = len(pdf)
+ finally:
+ _close_pdfium_resource(pdf)
+
+ markdown_lines = []
+ source_blocks = []
+ offset = 0
+ for index in range(page_count):
+ page_filename = f"{base_name}_page_{index + 1}.jpg"
+ ref_path = f"images/{page_filename}"
+ line = f"![{page_filename}]({ref_path})"
+ if markdown_lines:
+ offset += 2
+ markdown_lines.append(line)
+ source_blocks.append(
+ {
+ "start": offset,
+ "end": offset + len(line),
+ "locator": {"type": "pdf", "page": index + 1, "mapping": "exact"},
+ }
+ )
+ offset += len(line)
+
+ document = Document(
+ content="\n\n".join(markdown_lines),
+ metadata={"image_source_type": "scanned_pdf", "page_count": page_count},
+ source_blocks=source_blocks,
+ )
+ image_count = page_count
+ if page_count == 0:
+ return ParsedDocumentStream(document=document, image_count=0)
+
+ scale = max(1, CONFIG.pdf_render_dpi) / 72
+ quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality)
+ indices = list(range(page_count))
+
+ pdf_path = _write_pdf_temp_file(content)
+ return ParsedDocumentStream(
+ document=document,
+ image_count=image_count,
+ images=_iter_scanned_pages_from_path(
+ pdf_path,
+ indices,
+ base_name,
+ scale,
+ quality,
+ CONFIG.pdf_render_max_edge,
+ CONFIG.pdf_render_parallelism,
+ ),
+ _cleanup=lambda: _unlink_pdf_temp_file(pdf_path),
+ )
+
class PDFParser(BaseParser):
"""Per-page router between native text extraction and scanned rendering.
@@ -1622,15 +1873,77 @@ class PDFParser(BaseParser):
file_name=self.file_name, file_type=self.file_type
).parse_into_text(content)
+ def parse_stream(self, content: bytes) -> ParsedDocumentStream:
+ """Prepare PDF metadata, then render scanned pages only as consumed."""
+ if self._force_scanned:
+ stream = PDFScannedParser(
+ file_name=self.file_name, file_type=self.file_type
+ ).parse_stream(content)
+ page_count = stream.document.metadata.get("page_count", 0)
+ stream.document.metadata.update(
+ {
+ "scanned_page_count": page_count,
+ "text_page_count": 0,
+ "embedded_image_count": 0,
+ "vector_figure_count": 0,
+ }
+ )
+ return stream
+
+ try:
+ with _PDFIUM_LOCK:
+ document, scanned_indices = self._route_locked(
+ content, include_scanned_images=False
+ )
+ except Exception:
+ logger.exception(
+ "PDFParser: per-page streaming preparation failed for %s; "
+ "falling back to full image rendering",
+ self.file_name,
+ )
+ return PDFScannedParser(
+ file_name=self.file_name, file_type=self.file_type
+ ).parse_stream(content)
+
+ image_count = len(document.images or {}) + len(scanned_indices)
+ if not scanned_indices:
+ return ParsedDocumentStream(document=document, image_count=image_count)
+
+ scale = max(1, CONFIG.pdf_render_dpi) / 72
+ quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality)
+ base_name = os.path.splitext(self.file_name or "document")[0]
+
+ # Use a temporary source file for every stream mode so each fixed-size
+ # batch can close its PdfDocument before the next batch starts. This
+ # bounds PDFium's decoded-page cache independently of document length.
+ pdf_path = _write_pdf_temp_file(content)
+ return ParsedDocumentStream(
+ document=document,
+ image_count=image_count,
+ images=_iter_scanned_pages_from_path(
+ pdf_path,
+ scanned_indices,
+ base_name,
+ scale,
+ quality,
+ CONFIG.pdf_render_max_edge,
+ CONFIG.pdf_render_parallelism,
+ ),
+ _cleanup=lambda: _unlink_pdf_temp_file(pdf_path),
+ )
+
def _route(self, content: bytes) -> Document:
# Serialize all pdfium work: see _PDFIUM_LOCK. Holding it for the whole
# route (both the text pass and the render pass) is what prevents the
# concurrent-upload deadlock; the trailing markdown assembly is cheap
# pure-Python so keeping it inside the lock costs nothing meaningful.
with _PDFIUM_LOCK:
- return self._route_locked(content)
+ document, _scanned_indices = self._route_locked(content)
+ return document
- def _route_locked(self, content: bytes) -> Document:
+ def _route_locked(
+ self, content: bytes, *, include_scanned_images: bool = True
+ ) -> tuple[Document, list[int]]:
import pypdfium2 as pdfium
import pypdfium2.raw as pdfium_r
@@ -1700,7 +2013,10 @@ class PDFParser(BaseParser):
scanned_indices = [i for i, c in enumerate(classes) if c == "scanned"]
# Pass 2: render only the scanned pages (heavy work, rate-limited).
- if scanned_indices:
+ # ReadStream defers this work until after its metadata frame and
+ # renders pages incrementally; unary Read keeps the materialized
+ # response path for compatibility.
+ if scanned_indices and include_scanned_images:
with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers):
rendered = _render_scanned_pages(
pdf,
@@ -1799,9 +2115,10 @@ class PDFParser(BaseParser):
embedded_count,
len(content_text),
)
- return Document(
+ document = Document(
content=content_text,
images=images,
metadata=metadata,
source_blocks=source_blocks,
)
+ return document, scanned_indices
diff --git a/docreader/parser/stream.py b/docreader/parser/stream.py
new file mode 100644
index 000000000..7524d6bd3
--- /dev/null
+++ b/docreader/parser/stream.py
@@ -0,0 +1,51 @@
+"""Common result types for incremental document parsing."""
+
+from dataclasses import dataclass, field
+from typing import Callable, Iterator
+
+from docreader.models.document import Document
+
+
+@dataclass(frozen=True)
+class StreamedImage:
+ """One image payload emitted by a parser without retaining the full set."""
+
+ ref_path: str
+ image_data: bytes
+
+
+@dataclass
+class ParsedDocumentStream:
+ """A document header plus images that may be produced incrementally.
+
+ ``document.images`` contains any small images already required to build the
+ document. ``images`` yields the remaining payloads lazily. ``image_count``
+ is the total count across both sources and is suitable for the gRPC header.
+ Call ``close`` when the consumer stops before exhausting ``images`` so
+ parser resources and temporary files are released.
+ """
+
+ document: Document
+ image_count: int
+ images: Iterator[StreamedImage] = field(default_factory=lambda: iter(()))
+ _cleanup: Callable[[], None] | None = field(default=None, repr=False)
+ _closed: bool = field(default=False, init=False, repr=False)
+
+ def close(self) -> None:
+ """Close the lazy image iterator and release parser-owned resources."""
+ if self._closed:
+ return
+ self._closed = True
+ try:
+ close_iterator = getattr(self.images, "close", None)
+ if callable(close_iterator):
+ close_iterator()
+ finally:
+ if self._cleanup is not None:
+ self._cleanup()
+
+ def __enter__(self) -> "ParsedDocumentStream":
+ return self
+
+ def __exit__(self, exc_type, exc, traceback) -> None:
+ self.close()
diff --git a/docreader/proto/docreader.proto b/docreader/proto/docreader.proto
index 5e44ea0b7..d8d91d073 100644
--- a/docreader/proto/docreader.proto
+++ b/docreader/proto/docreader.proto
@@ -10,8 +10,9 @@ service DocReader {
// ReadStreamResponse carrying the parse metadata (markdown / metadata /
// error), then emits one message per image. This keeps every gRPC message
// small so large scanned PDFs (hundreds of page images, far exceeding the
- // unary message-size cap) can be returned without RESOURCE_EXHAUSTED and
- // with bounded memory on both ends.
+ // unary message-size cap) can be returned without RESOURCE_EXHAUSTED. Parsers
+ // that support incremental images can render and send a bounded number of
+ // pages at a time instead of retaining the full image set in the server.
rpc ReadStream(ReadRequest) returns (stream ReadStreamResponse) {}
rpc ListEngines(ListEnginesRequest) returns (ListEnginesResponse) {}
// PreprocessImage straightens and crops a photographed page so that OCR sees
diff --git a/docreader/tests/test_docreader_stream.py b/docreader/tests/test_docreader_stream.py
new file mode 100644
index 000000000..db344ff7c
--- /dev/null
+++ b/docreader/tests/test_docreader_stream.py
@@ -0,0 +1,129 @@
+import io
+import os
+import unittest
+from dataclasses import replace
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from PIL import Image
+
+import docreader.parser.pdf_parser as pdf_parser
+from docreader.main import DocReaderServicer
+from docreader.proto.docreader_pb2 import ReadConfig, ReadRequest
+
+
+def _scanned_pdf(page_count=2):
+ buffer = io.BytesIO()
+ pages = [Image.new("RGB", (64, 64), color) for color in ("white", "black")]
+ sequence = (pages * ((page_count // len(pages)) + 1))[:page_count]
+ sequence[0].save(
+ buffer,
+ format="PDF",
+ save_all=True,
+ append_images=sequence[1:],
+ )
+ return buffer.getvalue()
+
+
+class _NoopPreprocessRunner:
+ def __init__(self):
+ self.limits = SimpleNamespace(max_workers=1)
+
+ def run(self, *args, **kwargs):
+ return None
+
+ def close(self):
+ pass
+
+
+class ReadStreamPDFTest(unittest.TestCase):
+ def test_metadata_precedes_lazy_scanned_page_rendering(self):
+ config = ReadConfig()
+ config.parser_engine_overrides["pdf_force_scanned"] = "true"
+ request = ReadRequest(
+ file_content=_scanned_pdf(2),
+ file_name="scan.pdf",
+ file_type="pdf",
+ config=config,
+ )
+ service = DocReaderServicer(preprocess_runner=_NoopPreprocessRunner())
+ real_render = pdf_parser._render_page_to_jpeg
+ stream = None
+
+ try:
+ with patch.object(
+ pdf_parser,
+ "CONFIG",
+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),
+ ):
+ with patch.object(
+ pdf_parser,
+ "_render_page_to_jpeg",
+ wraps=real_render,
+ ) as render:
+ stream = service.ReadStream(request, None)
+ meta = next(stream).meta
+ self.assertEqual(meta.image_count, 2)
+ self.assertEqual(render.call_count, 0)
+
+ first = next(stream).image
+ self.assertEqual(first.filename, "scan_page_1.jpg")
+ self.assertTrue(first.image_data.startswith(b"\xff\xd8"))
+ self.assertEqual(render.call_count, 1)
+
+ second = next(stream).image
+ self.assertEqual(second.filename, "scan_page_2.jpg")
+ self.assertTrue(second.image_data.startswith(b"\xff\xd8"))
+ self.assertEqual(render.call_count, 2)
+
+ with self.assertRaises(StopIteration):
+ next(stream)
+ finally:
+ if stream is not None:
+ stream.close()
+ service.close()
+
+ def test_cancelling_after_metadata_removes_temporary_pdf(self):
+ config = ReadConfig()
+ config.parser_engine_overrides["pdf_force_scanned"] = "true"
+ request = ReadRequest(
+ file_content=_scanned_pdf(2),
+ file_name="scan.pdf",
+ file_type="pdf",
+ config=config,
+ )
+ service = DocReaderServicer(preprocess_runner=_NoopPreprocessRunner())
+ real_write = pdf_parser._write_pdf_temp_file
+ temp_paths = []
+ stream = None
+
+ def record_temp_path(content):
+ path = real_write(content)
+ temp_paths.append(path)
+ return path
+
+ try:
+ with patch.object(
+ pdf_parser,
+ "CONFIG",
+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),
+ ):
+ with patch.object(
+ pdf_parser,
+ "_write_pdf_temp_file",
+ side_effect=record_temp_path,
+ ):
+ stream = service.ReadStream(request, None)
+ self.assertEqual(next(stream).meta.image_count, 2)
+ self.assertEqual(len(temp_paths), 1)
+ self.assertTrue(os.path.exists(temp_paths[0]))
+ stream.close()
+ self.assertFalse(os.path.exists(temp_paths[0]))
+ finally:
+ if stream is not None:
+ stream.close()
+ service.close()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/docreader/tests/test_parser_routing.py b/docreader/tests/test_parser_routing.py
index 505f2a593..68ca13cc3 100644
--- a/docreader/tests/test_parser_routing.py
+++ b/docreader/tests/test_parser_routing.py
@@ -3,6 +3,7 @@ import unittest
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.parser import Parser, detect_effective_file_type
+from docreader.parser.stream import ParsedDocumentStream, StreamedImage
from docreader.parser.markitdown_parser import MarkitdownParser
from docreader.parser.pdf_parser import PDFParser
from docreader.parser.registry import BUILTIN_ENGINE, ParserEngineRegistry, registry
@@ -23,13 +24,28 @@ class _RecordingParser(BaseParser):
return Document(content="parsed")
+class _StreamingParser(_RecordingParser):
+ def parse_stream(self, content: bytes) -> ParsedDocumentStream:
+ return ParsedDocumentStream(
+ document=Document(content="streamed"),
+ image_count=1,
+ images=iter([StreamedImage("images/page.jpg", b"jpeg")]),
+ )
+
+
+class _ImageRecordingParser(_RecordingParser):
+ def parse_into_text(self, content: bytes) -> Document:
+ return Document(content="parsed", images={"images/page.jpg": "anBlZw=="})
+
+
class _RecordingRegistry:
- def __init__(self):
+ def __init__(self, parser_class=_RecordingParser):
self.requested_file_type = ""
+ self.parser_class = parser_class
def get_parser_class(self, engine: str, file_type: str):
self.requested_file_type = file_type
- return _RecordingParser
+ return self.parser_class
class ParserRoutingTest(unittest.TestCase):
@@ -53,6 +69,26 @@ class ParserRoutingTest(unittest.TestCase):
self.assertEqual("doc", registry.requested_file_type)
self.assertEqual("doc", _RecordingParser.last_file_type)
+ def test_streaming_parser_result_is_forwarded(self):
+ parser = Parser()
+ parser.registry = _RecordingRegistry(_StreamingParser)
+
+ result = parser.parse_file_stream("scan.pdf", "pdf", b"%PDF-data")
+
+ self.assertEqual(result.document.content, "streamed")
+ self.assertEqual(result.image_count, 1)
+ self.assertEqual(next(result.images).ref_path, "images/page.jpg")
+
+ def test_non_streaming_parser_keeps_materialized_image_behavior(self):
+ parser = Parser()
+ parser.registry = _RecordingRegistry(_ImageRecordingParser)
+
+ result = parser.parse_file_stream("page.png", "png", b"image-data")
+
+ self.assertEqual(result.document.content, "parsed")
+ self.assertEqual(result.image_count, 1)
+ self.assertEqual(result.document.images, {"images/page.jpg": "anBlZw=="})
+
def test_real_docx_payload_keeps_docx_parser_route(self):
self.assertEqual(
"docx",
diff --git a/docreader/tests/test_pdf_router.py b/docreader/tests/test_pdf_router.py
index c883e2f74..248214029 100644
--- a/docreader/tests/test_pdf_router.py
+++ b/docreader/tests/test_pdf_router.py
@@ -1,8 +1,13 @@
import io
+import os
import unittest
+from contextlib import contextmanager
+from dataclasses import replace
+from unittest.mock import patch
from PIL import Image
+import docreader.parser.pdf_parser as pdf_parser
from docreader.parser.pdf_parser import (
PDFParser,
_classify_page,
@@ -72,6 +77,212 @@ class RenderedPageEncodingTest(unittest.TestCase):
)
+class PDFStreamTest(unittest.TestCase):
+ def test_router_streams_scanned_pages_without_materializing_images(self):
+ content = _make_image_only_pdf(2)
+ real_render = pdf_parser._render_page_to_jpeg
+
+ with patch.object(
+ pdf_parser,
+ "CONFIG",
+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),
+ ):
+ with patch.object(
+ pdf_parser,
+ "_render_page_to_jpeg",
+ wraps=real_render,
+ ) as render:
+ stream = PDFParser(file_name="scan.pdf").parse_stream(content)
+ try:
+ self.assertEqual(stream.image_count, 2)
+ self.assertEqual(stream.document.images, {})
+ self.assertEqual(
+ stream.document.metadata["scanned_page_count"], 2
+ )
+ self.assertEqual(render.call_count, 0)
+
+ image = next(stream.images)
+ self.assertEqual(image.ref_path, "images/scan_page_1.jpg")
+ self.assertEqual(render.call_count, 1)
+ finally:
+ stream.close()
+
+ def test_scanned_pages_are_rendered_lazily_in_bounded_batches(self):
+ content = _make_image_only_pdf(40)
+ real_render = pdf_parser._render_page_to_jpeg
+
+ with patch.object(
+ pdf_parser,
+ "CONFIG",
+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),
+ ):
+ with patch.object(
+ pdf_parser,
+ "_render_page_to_jpeg",
+ wraps=real_render,
+ ) as render:
+ stream = PDFParser(
+ file_name="scan.pdf", pdf_force_scanned=True
+ ).parse_stream(content)
+ try:
+ self.assertEqual(stream.image_count, 40)
+ self.assertEqual(stream.document.images, {})
+ self.assertEqual(render.call_count, 0)
+
+ first = next(stream.images)
+ self.assertEqual(first.ref_path, "images/scan_page_1.jpg")
+ self.assertTrue(first.image_data.startswith(b"\xff\xd8"))
+ self.assertEqual(render.call_count, 1)
+
+ for _ in range(pdf_parser.PDF_STREAM_RENDER_BATCH_PAGES - 1):
+ next(stream.images)
+ self.assertEqual(
+ render.call_count, pdf_parser.PDF_STREAM_RENDER_BATCH_PAGES
+ )
+
+ next_batch = next(stream.images)
+ expected_next_page = (
+ pdf_parser.PDF_STREAM_RENDER_BATCH_PAGES + 1
+ )
+ self.assertEqual(
+ next_batch.ref_path,
+ f"images/scan_page_{expected_next_page}.jpg",
+ )
+ self.assertEqual(
+ render.call_count,
+ pdf_parser.PDF_STREAM_RENDER_BATCH_PAGES + 1,
+ )
+ finally:
+ stream.close()
+
+ def test_parallel_stream_bounds_in_flight_render_results(self):
+ tracker = {
+ "submitted": 0,
+ "pending": 0,
+ "max_pending": 0,
+ "shutdown": False,
+ }
+ events = []
+
+ @contextmanager
+ def tracking_limit(*args, **kwargs):
+ events.append("limit-enter")
+ try:
+ yield
+ finally:
+ events.append("limit-exit")
+
+ class Future:
+ def __init__(self, value):
+ self.value = value
+ self.pending = True
+
+ def result(self):
+ if self.pending:
+ self.pending = False
+ tracker["pending"] -= 1
+ return self.value
+
+ def cancel(self):
+ if self.pending:
+ self.pending = False
+ tracker["pending"] -= 1
+ return True
+
+ class Executor:
+ def __init__(self, **kwargs):
+ pass
+
+ def submit(self, _fn, args):
+ index = args[0]
+ tracker["submitted"] += 1
+ tracker["pending"] += 1
+ tracker["max_pending"] = max(
+ tracker["max_pending"], tracker["pending"]
+ )
+ return Future((index, b"jpeg"))
+
+ def shutdown(self, **kwargs):
+ tracker["shutdown"] = True
+ events.append("shutdown")
+
+ with patch.object(pdf_parser, "_select_mp_context", return_value=object()):
+ with patch("concurrent.futures.ProcessPoolExecutor", Executor):
+ with patch.object(
+ pdf_parser, "parser_worker_limit", tracking_limit
+ ):
+ images = pdf_parser._iter_scanned_pages_from_path(
+ "unused.pdf",
+ list(range(100)),
+ "scan",
+ 1.0,
+ 85,
+ 0,
+ 4,
+ )
+ try:
+ first = next(images)
+ self.assertEqual(first.ref_path, "images/scan_page_1.jpg")
+ self.assertEqual(tracker["submitted"], 9)
+ self.assertEqual(tracker["pending"], 8)
+ self.assertEqual(tracker["max_pending"], 8)
+
+ second = next(images)
+ self.assertEqual(second.ref_path, "images/scan_page_2.jpg")
+ self.assertLessEqual(tracker["pending"], 8)
+ finally:
+ images.close()
+
+ self.assertEqual(tracker["pending"], 0)
+ self.assertTrue(tracker["shutdown"])
+ self.assertLess(events.index("shutdown"), events.index("limit-exit"))
+
+ def test_unconsumed_stream_removes_temporary_pdf(self):
+ content = _make_image_only_pdf(2)
+ real_write = pdf_parser._write_pdf_temp_file
+ created_paths = []
+
+ def record_temp_path(pdf_bytes):
+ path = real_write(pdf_bytes)
+ created_paths.append(path)
+ return path
+
+ with patch.object(
+ pdf_parser,
+ "CONFIG",
+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),
+ ):
+ with patch.object(
+ pdf_parser,
+ "_write_pdf_temp_file",
+ side_effect=record_temp_path,
+ ):
+ stream = PDFParser(
+ file_name="scan.pdf", pdf_force_scanned=True
+ ).parse_stream(content)
+ temp_path = created_paths[0]
+ try:
+ self.assertTrue(os.path.exists(temp_path))
+ finally:
+ stream.close()
+ self.assertFalse(os.path.exists(temp_path))
+
+ def test_unary_pdf_parser_still_collects_scanned_images(self):
+ content = _make_image_only_pdf(2)
+ with patch.object(
+ pdf_parser,
+ "CONFIG",
+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),
+ ):
+ document = PDFParser(
+ file_name="scan.pdf", pdf_force_scanned=True
+ ).parse_into_text(content)
+
+ self.assertEqual(len(document.images), 2)
+ self.assertIn("images/scan_page_1.jpg", document.images)
+ self.assertIn("images/scan_page_2.jpg", document.images)
+
+
class StripRepeatingLinesTest(unittest.TestCase):
def test_removes_repeated_header_footer(self):
header = "ACME CONFIDENTIAL"