Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): bound streamed PDF image memory (#4664)
LuckVd9688748b73ff
diff --git a/docreader/main.py b/docreader/main.pyindex 30fee0e73..5504f31ec 100644--- a/docreader/main.py+++ b/docreader/main.py@@ -16,6 +16,7 @@ from docreader.auth import AuthInterceptor, TLSConfigError, load_tls_credentialsfrom docreader import configfrom docreader.config import CONFIGfrom docreader.parser import Parser+from docreader.parser.stream import ParsedDocumentStreamfrom docreader.preprocess import PreprocessLimits, PreprocessRunnerfrom docreader.proto import docreader_pb2_grpcfrom docreader.parser.registry import registry@@ -150,6 +151,17 @@ def _iter_image_refs(images: dict):)+def _image_ref_from_streamed_data(ref_path: str, image_data: bytes) -> ImageRef:+ """Build one image reference from raw bytes yielded by a parser."""+ fname, mime = _mime_for_ref(ref_path)+ return ImageRef(+ filename=fname,+ original_ref=ref_path,+ mime_type=mime,+ image_data=image_data,+ )++class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):def __init__(self, preprocess_runner=None):super().__init__()@@ -249,52 +261,106 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):return ReadResponse(error=str(e))def ReadStream(self, request: ReadRequest, context):- """Streaming read: yields one meta frame, then one frame per image.-- Each frame is a small, independent gRPC message, so documents with many- page images (large scanned PDFs) are returned without hitting the unary- message-size cap, and neither side has to hold the whole payload at once.- """+ """Yield metadata first, then images as each parser produces them."""request_id = request.request_id or str(uuid.uuid4())with request_id_context(request_id):_c = to_valid_utf8_text+ parsed_stream = Nonetry:- result, source_desc = self._parse_request(request)+ if request.url or not callable(+ getattr(self.parser, "parse_file_stream", None)+ ):+ result, source_desc = self._parse_request(request)+ parsed_stream = ParsedDocumentStream(+ document=result,+ image_count=len(result.images or {}),+ )+ else:+ cfg = request.config+ parser_engine = cfg.parser_engine if cfg else ""+ engine_overrides = (+ dict(cfg.parser_engine_overrides) if cfg else {}+ )+ file_type = request.file_type or os.path.splitext(+ request.file_name+ )[1][1:]+ file_content = request.file_content+ try:+ track_request_size(len(file_content))+ logger.info(+ "Read(File stream): file=%s, type=%s, size=%d bytes",+ request.file_name,+ file_type,+ len(file_content),+ )+ parsed_stream = self.parser.parse_file_stream(+ request.file_name,+ file_type,+ file_content,+ parser_engine=parser_engine,+ engine_overrides=engine_overrides,+ )+ finally:+ del file_content+ result = parsed_stream.document+ source_desc = request.file_nameexcept Exception as e:logger.error("Error reading document: %s", e)logger.info("Traceback: %s", traceback.format_exc())+ if parsed_stream is not None:+ parsed_stream.close()+ del requestyield ReadStreamResponse(meta=ReadStreamMeta(error=str(e)))return+ # PDF parser streams own a temporary source file; drop the request's+ # full bytes field while the client drains page images.+ del request+if not result or not result.content:error_msg = f"Failed to parse: {source_desc}"logger.error(error_msg)+ parsed_stream.close()yield ReadStreamResponse(meta=ReadStreamMeta(error=error_msg))returnimages = result.images or {}- image_count = len(images)- yield ReadStreamResponse(- meta=ReadStreamMeta(- markdown_content=_c(result.content),- image_dir_path="",- metadata={k: _c(str(v)) for k, v in result.metadata.items()}- if result.metadata- else {},- image_count=image_count,- source_blocks=source_blocks_to_proto(result),+ content_len = len(result.content)+ sent = 0+ try:+ yield ReadStreamResponse(+ meta=ReadStreamMeta(+ markdown_content=_c(result.content),+ image_dir_path="",+ metadata={k: _c(str(v)) for k, v in result.metadata.items()}+ if result.metadata+ else {},+ image_count=parsed_stream.image_count,+ source_blocks=source_blocks_to_proto(result),+ ))- )- sent = 0- for ref in _iter_image_refs(images):- yield ReadStreamResponse(image=ref)- sent += 1+ for ref in _iter_image_refs(images):+ yield ReadStreamResponse(image=ref)+ sent += 1++ for image in parsed_stream.images:+ yield ReadStreamResponse(+ image=_image_ref_from_streamed_data(+ image.ref_path, image.image_data+ )+ )+ sent += 1+ except Exception:+ logger.exception("Error streaming images for %s", source_desc)+ raise+ finally:+ parsed_stream.close()logger.info("ReadStream response: content_len=%d, images=%d",- len(result.content),+ content_len,sent,)diff --git a/docreader/parser/parser.py b/docreader/parser/parser.pyindex 5b9a8d2dd..d92c7dc35 100644--- a/docreader/parser/parser.py+++ b/docreader/parser/parser.py@@ -3,6 +3,7 @@ from typing import Any, Optionalfrom docreader.models.document import Documentfrom docreader.parser.registry import registry+from docreader.parser.stream import ParsedDocumentStreamfrom docreader.parser.web_parser import WebParserlogger = logging.getLogger(__name__)@@ -83,6 +84,38 @@ class Parser:logger.info("Parsed file %s, content length=%d", file_name, len(result.content))return result+ def parse_file_stream(+ self,+ file_name: str,+ file_type: str,+ content: bytes,+ parser_engine: Optional[str] = None,+ engine_overrides: Optional[dict[str, Any]] = None,+ ) -> ParsedDocumentStream:+ """Parse a file for ReadStream, using a parser's lazy image path when available."""+ engine = parser_engine or ""+ overrides = engine_overrides or {}+ effective_file_type = detect_effective_file_type(file_type, content)+ cls = self.registry.get_parser_class(engine, effective_file_type)+ parser = cls(+ file_name=file_name,+ file_type=effective_file_type,+ **overrides,+ )++ parse_stream = getattr(parser, "parse_stream", None)+ if callable(parse_stream):+ return parse_stream(content)++ # Non-streaming parsers retain their established behavior. Their images+ # are still sent one at a time by ReadStream, but parsing itself may+ # materialize them before the first frame.+ result = parser.parse(content)+ return ParsedDocumentStream(+ document=result,+ image_count=len(result.images or {}),+ )+def parse_url(self,url: str,diff --git a/docreader/parser/pdf_parser.py b/docreader/parser/pdf_parser.pyindex 0f6c227c6..fe7d2e6cc 100644--- a/docreader/parser/pdf_parser.py+++ b/docreader/parser/pdf_parser.py@@ -17,19 +17,21 @@ self-sufficient using pypdfium2 + the Go-side OCR that already exists."""import base64-from collections import Counter+from collections import Counter, dequeimport ioimport loggingimport osimport reimport statisticsimport threading+from typing import Iteratorfrom docreader.config import CONFIGfrom docreader.models.document import Documentfrom docreader.parser.base_parser import BaseParserfrom docreader.parser.concurrency import parser_worker_limitfrom docreader.parser.source_locator import PageGeometry, page_blocks+from docreader.parser.stream import ParsedDocumentStream, StreamedImagelogger = logging.getLogger(__name__)@@ -1121,6 +1123,11 @@ def _render_page_to_jpeg(page, scale: float, quality: int, max_edge: int = 0) -># per-page render (the dominant cost for big scanned PDFs — hours on# CPU-constrained containers) into a near-linear speedup.+# PDFium retains decoded page resources on some inputs until its document closes.+# ReadStream therefore restarts the renderer every fixed-size batch; the bound+# is independent of total page count while retaining useful parallel work.+PDF_STREAM_RENDER_BATCH_PAGES = 16+# Per-worker document handle, populated by the pool initializer._WORKER_RENDER_DOC = None@@ -1129,8 +1136,7 @@ def _render_pool_init(pdf_path: str) -> None:global _WORKER_RENDER_DOCimport pypdfium2 as pdfium- with open(pdf_path, "rb") as f:- _WORKER_RENDER_DOC = pdfium.PdfDocument(f.read())+ _WORKER_RENDER_DOC = pdfium.PdfDocument(pdf_path)def _render_pool_task(args):@@ -1212,6 +1218,191 @@ def _render_pages_parallel(pass+def _write_pdf_temp_file(content: bytes) -> str:+ import tempfile++ path = None+ try:+ with tempfile.NamedTemporaryFile(+ prefix="docreader_stream_", suffix=".pdf", delete=False+ ) as tmp:+ path = tmp.name+ tmp.write(content)+ return path+ except Exception:+ if path is not None:+ _unlink_pdf_temp_file(path)+ raise+++def _unlink_pdf_temp_file(path: str) -> None:+ try:+ os.unlink(path)+ except FileNotFoundError:+ pass+ except OSError:+ logger.warning("Failed to remove temporary PDF %s", path, exc_info=True)+++def _iter_scanned_pages_from_open_pdf(+ pdf,+ indices: list[int],+ base_name: str,+ scale: float,+ quality: int,+ max_edge: int,+) -> Iterator[StreamedImage]:+ """Render one page at a time from an already-open PdfDocument."""+ for index in indices:+ with _PDFIUM_LOCK, parser_worker_limit(+ "pdf_render", CONFIG.pdf_render_max_workers+ ):+ page = pdf[index]+ try:+ image_bytes = _render_page_to_jpeg(page, scale, quality, max_edge)+ finally:+ _close_pdfium_resource(page)+ yield StreamedImage(+ ref_path=f"images/{base_name}_page_{index + 1}.jpg",+ image_data=image_bytes,+ )+++def _iter_scanned_pages_serial_batch(+ pdf_path: str,+ indices: list[int],+ base_name: str,+ scale: float,+ quality: int,+ max_edge: int,+) -> Iterator[StreamedImage]:+ import pypdfium2 as pdfium++ with _PDFIUM_LOCK:+ pdf = pdfium.PdfDocument(pdf_path)+ try:+ yield from _iter_scanned_pages_from_open_pdf(+ pdf, indices, base_name, scale, quality, max_edge+ )+ finally:+ with _PDFIUM_LOCK:+ _close_pdfium_resource(pdf)+++def _iter_scanned_pages_parallel_batch(+ pdf_path: str,+ indices: list[int],+ base_name: str,+ scale: float,+ quality: int,+ max_edge: int,+ workers: int,+ mp_context,+) -> Iterator[StreamedImage]:+ from concurrent.futures import ProcessPoolExecutor++ max_workers = min(workers, len(indices))+ window_size = max_workers * 2+ pending = deque()+ executor = None+ sent = False+ fallback = False++ # Hold the limiter until the executor has shut down, including when the+ # consumer cancels mid-batch. Otherwise another request could start a new+ # pool while the old worker processes still hold their PDFium documents.+ with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers):+ try:+ executor = ProcessPoolExecutor(+ max_workers=max_workers,+ mp_context=mp_context,+ initializer=_render_pool_init,+ initargs=(pdf_path,),+ )+ indices_iter = iter(indices)+ for _ in range(window_size):+ try:+ index = next(indices_iter)+ except StopIteration:+ break+ pending.append(+ executor.submit(+ _render_pool_task,+ (index, scale, quality, max_edge),+ )+ )++ while pending:+ future = pending.popleft()+ index, image_bytes = future.result()+ try:+ next_index = next(indices_iter)+ except StopIteration:+ pass+ else:+ pending.append(+ executor.submit(+ _render_pool_task,+ (next_index, scale, quality, max_edge),+ )+ )+ sent = True+ yield StreamedImage(+ ref_path=f"images/{base_name}_page_{index + 1}.jpg",+ image_data=image_bytes,+ )+ except Exception:+ if sent:+ raise+ logger.warning(+ "parallel streamed PDF rendering failed before the first image; "+ "falling back to serial rendering for this batch",+ exc_info=True,+ )+ fallback = True+ finally:+ if executor is not None:+ for future in pending:+ future.cancel()+ executor.shutdown(wait=True, cancel_futures=True)+ pending.clear()++ if fallback:+ yield from _iter_scanned_pages_serial_batch(+ pdf_path, indices, base_name, scale, quality, max_edge+ )+++def _iter_scanned_pages_from_path(+ pdf_path: str,+ indices: list[int],+ base_name: str,+ scale: float,+ quality: int,+ max_edge: int,+ workers: int,+) -> Iterator[StreamedImage]:+ """Yield images in fixed-size batches to cap PDFium and result memory."""+ for start in range(0, len(indices), PDF_STREAM_RENDER_BATCH_PAGES):+ batch = indices[start : start + PDF_STREAM_RENDER_BATCH_PAGES]+ mp_context = _select_mp_context() if workers > 1 and len(batch) > 1 else None+ if workers > 1 and len(batch) > 1 and mp_context is not None:+ yield from _iter_scanned_pages_parallel_batch(+ pdf_path,+ batch,+ base_name,+ scale,+ quality,+ max_edge,+ workers,+ mp_context,+ )+ else:+ yield from _iter_scanned_pages_serial_batch(+ pdf_path, batch, base_name, scale, quality, max_edge+ )++def _render_scanned_pages(pdf, content: bytes, indices: list, scale: float, quality: int, max_edge: int) -> dict:@@ -1560,6 +1751,66 @@ class PDFScannedParser(BaseParser):logger.exception("PDFScannedParser failed to parse PDF: %s", e)raise e+ def parse_stream(self, content: bytes) -> ParsedDocumentStream:+ """Return metadata first and lazily render each scanned page."""+ import pypdfium2 as pdfium++ base_name = os.path.splitext(self.file_name or "document")[0]+ with _PDFIUM_LOCK:+ pdf = pdfium.PdfDocument(content)+ try:+ page_count = len(pdf)+ finally:+ _close_pdfium_resource(pdf)++ markdown_lines = []+ source_blocks = []+ offset = 0+ for index in range(page_count):+ page_filename = f"{base_name}_page_{index + 1}.jpg"+ ref_path = f"images/{page_filename}"+ line = f""+ if markdown_lines:+ offset += 2+ markdown_lines.append(line)+ source_blocks.append(+ {+ "start": offset,+ "end": offset + len(line),+ "locator": {"type": "pdf", "page": index + 1, "mapping": "exact"},+ }+ )+ offset += len(line)++ document = Document(+ content="\n\n".join(markdown_lines),+ metadata={"image_source_type": "scanned_pdf", "page_count": page_count},+ source_blocks=source_blocks,+ )+ image_count = page_count+ if page_count == 0:+ return ParsedDocumentStream(document=document, image_count=0)++ scale = max(1, CONFIG.pdf_render_dpi) / 72+ quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality)+ indices = list(range(page_count))++ pdf_path = _write_pdf_temp_file(content)+ return ParsedDocumentStream(+ document=document,+ image_count=image_count,+ images=_iter_scanned_pages_from_path(+ pdf_path,+ indices,+ base_name,+ scale,+ quality,+ CONFIG.pdf_render_max_edge,+ CONFIG.pdf_render_parallelism,+ ),+ _cleanup=lambda: _unlink_pdf_temp_file(pdf_path),+ )+class PDFParser(BaseParser):"""Per-page router between native text extraction and scanned rendering.@@ -1622,15 +1873,77 @@ class PDFParser(BaseParser):file_name=self.file_name, file_type=self.file_type).parse_into_text(content)+ def parse_stream(self, content: bytes) -> ParsedDocumentStream:+ """Prepare PDF metadata, then render scanned pages only as consumed."""+ if self._force_scanned:+ stream = PDFScannedParser(+ file_name=self.file_name, file_type=self.file_type+ ).parse_stream(content)+ page_count = stream.document.metadata.get("page_count", 0)+ stream.document.metadata.update(+ {+ "scanned_page_count": page_count,+ "text_page_count": 0,+ "embedded_image_count": 0,+ "vector_figure_count": 0,+ }+ )+ return stream++ try:+ with _PDFIUM_LOCK:+ document, scanned_indices = self._route_locked(+ content, include_scanned_images=False+ )+ except Exception:+ logger.exception(+ "PDFParser: per-page streaming preparation failed for %s; "+ "falling back to full image rendering",+ self.file_name,+ )+ return PDFScannedParser(+ file_name=self.file_name, file_type=self.file_type+ ).parse_stream(content)++ image_count = len(document.images or {}) + len(scanned_indices)+ if not scanned_indices:+ return ParsedDocumentStream(document=document, image_count=image_count)++ scale = max(1, CONFIG.pdf_render_dpi) / 72+ quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality)+ base_name = os.path.splitext(self.file_name or "document")[0]++ # Use a temporary source file for every stream mode so each fixed-size+ # batch can close its PdfDocument before the next batch starts. This+ # bounds PDFium's decoded-page cache independently of document length.+ pdf_path = _write_pdf_temp_file(content)+ return ParsedDocumentStream(+ document=document,+ image_count=image_count,+ images=_iter_scanned_pages_from_path(+ pdf_path,+ scanned_indices,+ base_name,+ scale,+ quality,+ CONFIG.pdf_render_max_edge,+ CONFIG.pdf_render_parallelism,+ ),+ _cleanup=lambda: _unlink_pdf_temp_file(pdf_path),+ )+def _route(self, content: bytes) -> Document:# Serialize all pdfium work: see _PDFIUM_LOCK. Holding it for the whole# route (both the text pass and the render pass) is what prevents the# concurrent-upload deadlock; the trailing markdown assembly is cheap# pure-Python so keeping it inside the lock costs nothing meaningful.with _PDFIUM_LOCK:- return self._route_locked(content)+ document, _scanned_indices = self._route_locked(content)+ return document- def _route_locked(self, content: bytes) -> Document:+ def _route_locked(+ self, content: bytes, *, include_scanned_images: bool = True+ ) -> tuple[Document, list[int]]:import pypdfium2 as pdfiumimport pypdfium2.raw as pdfium_r@@ -1700,7 +2013,10 @@ class PDFParser(BaseParser):scanned_indices = [i for i, c in enumerate(classes) if c == "scanned"]# Pass 2: render only the scanned pages (heavy work, rate-limited).- if scanned_indices:+ # ReadStream defers this work until after its metadata frame and+ # renders pages incrementally; unary Read keeps the materialized+ # response path for compatibility.+ if scanned_indices and include_scanned_images:with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers):rendered = _render_scanned_pages(pdf,@@ -1799,9 +2115,10 @@ class PDFParser(BaseParser):embedded_count,len(content_text),)- return Document(+ document = Document(content=content_text,images=images,metadata=metadata,source_blocks=source_blocks,)+ return document, scanned_indicesdiff --git a/docreader/parser/stream.py b/docreader/parser/stream.pynew file mode 100644index 000000000..7524d6bd3--- /dev/null+++ b/docreader/parser/stream.py@@ -0,0 +1,51 @@+"""Common result types for incremental document parsing."""++from dataclasses import dataclass, field+from typing import Callable, Iterator++from docreader.models.document import Document+++@dataclass(frozen=True)+class StreamedImage:+ """One image payload emitted by a parser without retaining the full set."""++ ref_path: str+ image_data: bytes+++@dataclass+class ParsedDocumentStream:+ """A document header plus images that may be produced incrementally.++ ``document.images`` contains any small images already required to build the+ document. ``images`` yields the remaining payloads lazily. ``image_count``+ is the total count across both sources and is suitable for the gRPC header.+ Call ``close`` when the consumer stops before exhausting ``images`` so+ parser resources and temporary files are released.+ """++ document: Document+ image_count: int+ images: Iterator[StreamedImage] = field(default_factory=lambda: iter(()))+ _cleanup: Callable[[], None] | None = field(default=None, repr=False)+ _closed: bool = field(default=False, init=False, repr=False)++ def close(self) -> None:+ """Close the lazy image iterator and release parser-owned resources."""+ if self._closed:+ return+ self._closed = True+ try:+ close_iterator = getattr(self.images, "close", None)+ if callable(close_iterator):+ close_iterator()+ finally:+ if self._cleanup is not None:+ self._cleanup()++ def __enter__(self) -> "ParsedDocumentStream":+ return self++ def __exit__(self, exc_type, exc, traceback) -> None:+ self.close()diff --git a/docreader/proto/docreader.proto b/docreader/proto/docreader.protoindex 5e44ea0b7..d8d91d073 100644--- a/docreader/proto/docreader.proto+++ b/docreader/proto/docreader.proto@@ -10,8 +10,9 @@ service DocReader {// ReadStreamResponse carrying the parse metadata (markdown / metadata /// error), then emits one message per image. This keeps every gRPC message// small so large scanned PDFs (hundreds of page images, far exceeding the- // unary message-size cap) can be returned without RESOURCE_EXHAUSTED and- // with bounded memory on both ends.+ // unary message-size cap) can be returned without RESOURCE_EXHAUSTED. Parsers+ // that support incremental images can render and send a bounded number of+ // pages at a time instead of retaining the full image set in the server.rpc ReadStream(ReadRequest) returns (stream ReadStreamResponse) {}rpc ListEngines(ListEnginesRequest) returns (ListEnginesResponse) {}// PreprocessImage straightens and crops a photographed page so that OCR seesdiff --git a/docreader/tests/test_docreader_stream.py b/docreader/tests/test_docreader_stream.pynew file mode 100644index 000000000..db344ff7c--- /dev/null+++ b/docreader/tests/test_docreader_stream.py@@ -0,0 +1,129 @@+import io+import os+import unittest+from dataclasses import replace+from types import SimpleNamespace+from unittest.mock import patch++from PIL import Image++import docreader.parser.pdf_parser as pdf_parser+from docreader.main import DocReaderServicer+from docreader.proto.docreader_pb2 import ReadConfig, ReadRequest+++def _scanned_pdf(page_count=2):+ buffer = io.BytesIO()+ pages = [Image.new("RGB", (64, 64), color) for color in ("white", "black")]+ sequence = (pages * ((page_count // len(pages)) + 1))[:page_count]+ sequence[0].save(+ buffer,+ format="PDF",+ save_all=True,+ append_images=sequence[1:],+ )+ return buffer.getvalue()+++class _NoopPreprocessRunner:+ def __init__(self):+ self.limits = SimpleNamespace(max_workers=1)++ def run(self, *args, **kwargs):+ return None++ def close(self):+ pass+++class ReadStreamPDFTest(unittest.TestCase):+ def test_metadata_precedes_lazy_scanned_page_rendering(self):+ config = ReadConfig()+ config.parser_engine_overrides["pdf_force_scanned"] = "true"+ request = ReadRequest(+ file_content=_scanned_pdf(2),+ file_name="scan.pdf",+ file_type="pdf",+ config=config,+ )+ service = DocReaderServicer(preprocess_runner=_NoopPreprocessRunner())+ real_render = pdf_parser._render_page_to_jpeg+ stream = None++ try:+ with patch.object(+ pdf_parser,+ "CONFIG",+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),+ ):+ with patch.object(+ pdf_parser,+ "_render_page_to_jpeg",+ wraps=real_render,+ ) as render:+ stream = service.ReadStream(request, None)+ meta = next(stream).meta+ self.assertEqual(meta.image_count, 2)+ self.assertEqual(render.call_count, 0)++ first = next(stream).image+ self.assertEqual(first.filename, "scan_page_1.jpg")+ self.assertTrue(first.image_data.startswith(b"\xff\xd8"))+ self.assertEqual(render.call_count, 1)++ second = next(stream).image+ self.assertEqual(second.filename, "scan_page_2.jpg")+ self.assertTrue(second.image_data.startswith(b"\xff\xd8"))+ self.assertEqual(render.call_count, 2)++ with self.assertRaises(StopIteration):+ next(stream)+ finally:+ if stream is not None:+ stream.close()+ service.close()++ def test_cancelling_after_metadata_removes_temporary_pdf(self):+ config = ReadConfig()+ config.parser_engine_overrides["pdf_force_scanned"] = "true"+ request = ReadRequest(+ file_content=_scanned_pdf(2),+ file_name="scan.pdf",+ file_type="pdf",+ config=config,+ )+ service = DocReaderServicer(preprocess_runner=_NoopPreprocessRunner())+ real_write = pdf_parser._write_pdf_temp_file+ temp_paths = []+ stream = None++ def record_temp_path(content):+ path = real_write(content)+ temp_paths.append(path)+ return path++ try:+ with patch.object(+ pdf_parser,+ "CONFIG",+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),+ ):+ with patch.object(+ pdf_parser,+ "_write_pdf_temp_file",+ side_effect=record_temp_path,+ ):+ stream = service.ReadStream(request, None)+ self.assertEqual(next(stream).meta.image_count, 2)+ self.assertEqual(len(temp_paths), 1)+ self.assertTrue(os.path.exists(temp_paths[0]))+ stream.close()+ self.assertFalse(os.path.exists(temp_paths[0]))+ finally:+ if stream is not None:+ stream.close()+ service.close()+++if __name__ == "__main__":+ unittest.main()diff --git a/docreader/tests/test_parser_routing.py b/docreader/tests/test_parser_routing.pyindex 505f2a593..68ca13cc3 100644--- a/docreader/tests/test_parser_routing.py+++ b/docreader/tests/test_parser_routing.py@@ -3,6 +3,7 @@ import unittestfrom docreader.models.document import Documentfrom docreader.parser.base_parser import BaseParserfrom docreader.parser.parser import Parser, detect_effective_file_type+from docreader.parser.stream import ParsedDocumentStream, StreamedImagefrom docreader.parser.markitdown_parser import MarkitdownParserfrom docreader.parser.pdf_parser import PDFParserfrom docreader.parser.registry import BUILTIN_ENGINE, ParserEngineRegistry, registry@@ -23,13 +24,28 @@ class _RecordingParser(BaseParser):return Document(content="parsed")+class _StreamingParser(_RecordingParser):+ def parse_stream(self, content: bytes) -> ParsedDocumentStream:+ return ParsedDocumentStream(+ document=Document(content="streamed"),+ image_count=1,+ images=iter([StreamedImage("images/page.jpg", b"jpeg")]),+ )+++class _ImageRecordingParser(_RecordingParser):+ def parse_into_text(self, content: bytes) -> Document:+ return Document(content="parsed", images={"images/page.jpg": "anBlZw=="})++class _RecordingRegistry:- def __init__(self):+ def __init__(self, parser_class=_RecordingParser):self.requested_file_type = ""+ self.parser_class = parser_classdef get_parser_class(self, engine: str, file_type: str):self.requested_file_type = file_type- return _RecordingParser+ return self.parser_classclass ParserRoutingTest(unittest.TestCase):@@ -53,6 +69,26 @@ class ParserRoutingTest(unittest.TestCase):self.assertEqual("doc", registry.requested_file_type)self.assertEqual("doc", _RecordingParser.last_file_type)+ def test_streaming_parser_result_is_forwarded(self):+ parser = Parser()+ parser.registry = _RecordingRegistry(_StreamingParser)++ result = parser.parse_file_stream("scan.pdf", "pdf", b"%PDF-data")++ self.assertEqual(result.document.content, "streamed")+ self.assertEqual(result.image_count, 1)+ self.assertEqual(next(result.images).ref_path, "images/page.jpg")++ def test_non_streaming_parser_keeps_materialized_image_behavior(self):+ parser = Parser()+ parser.registry = _RecordingRegistry(_ImageRecordingParser)++ result = parser.parse_file_stream("page.png", "png", b"image-data")++ self.assertEqual(result.document.content, "parsed")+ self.assertEqual(result.image_count, 1)+ self.assertEqual(result.document.images, {"images/page.jpg": "anBlZw=="})+def test_real_docx_payload_keeps_docx_parser_route(self):self.assertEqual("docx",diff --git a/docreader/tests/test_pdf_router.py b/docreader/tests/test_pdf_router.pyindex c883e2f74..248214029 100644--- a/docreader/tests/test_pdf_router.py+++ b/docreader/tests/test_pdf_router.py@@ -1,8 +1,13 @@import io+import osimport unittest+from contextlib import contextmanager+from dataclasses import replace+from unittest.mock import patchfrom PIL import Image+import docreader.parser.pdf_parser as pdf_parserfrom docreader.parser.pdf_parser import (PDFParser,_classify_page,@@ -72,6 +77,212 @@ class RenderedPageEncodingTest(unittest.TestCase):)+class PDFStreamTest(unittest.TestCase):+ def test_router_streams_scanned_pages_without_materializing_images(self):+ content = _make_image_only_pdf(2)+ real_render = pdf_parser._render_page_to_jpeg++ with patch.object(+ pdf_parser,+ "CONFIG",+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),+ ):+ with patch.object(+ pdf_parser,+ "_render_page_to_jpeg",+ wraps=real_render,+ ) as render:+ stream = PDFParser(file_name="scan.pdf").parse_stream(content)+ try:+ self.assertEqual(stream.image_count, 2)+ self.assertEqual(stream.document.images, {})+ self.assertEqual(+ stream.document.metadata["scanned_page_count"], 2+ )+ self.assertEqual(render.call_count, 0)++ image = next(stream.images)+ self.assertEqual(image.ref_path, "images/scan_page_1.jpg")+ self.assertEqual(render.call_count, 1)+ finally:+ stream.close()++ def test_scanned_pages_are_rendered_lazily_in_bounded_batches(self):+ content = _make_image_only_pdf(40)+ real_render = pdf_parser._render_page_to_jpeg++ with patch.object(+ pdf_parser,+ "CONFIG",+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),+ ):+ with patch.object(+ pdf_parser,+ "_render_page_to_jpeg",+ wraps=real_render,+ ) as render:+ stream = PDFParser(+ file_name="scan.pdf", pdf_force_scanned=True+ ).parse_stream(content)+ try:+ self.assertEqual(stream.image_count, 40)+ self.assertEqual(stream.document.images, {})+ self.assertEqual(render.call_count, 0)++ first = next(stream.images)+ self.assertEqual(first.ref_path, "images/scan_page_1.jpg")+ self.assertTrue(first.image_data.startswith(b"\xff\xd8"))+ self.assertEqual(render.call_count, 1)++ for _ in range(pdf_parser.PDF_STREAM_RENDER_BATCH_PAGES - 1):+ next(stream.images)+ self.assertEqual(+ render.call_count, pdf_parser.PDF_STREAM_RENDER_BATCH_PAGES+ )++ next_batch = next(stream.images)+ expected_next_page = (+ pdf_parser.PDF_STREAM_RENDER_BATCH_PAGES + 1+ )+ self.assertEqual(+ next_batch.ref_path,+ f"images/scan_page_{expected_next_page}.jpg",+ )+ self.assertEqual(+ render.call_count,+ pdf_parser.PDF_STREAM_RENDER_BATCH_PAGES + 1,+ )+ finally:+ stream.close()++ def test_parallel_stream_bounds_in_flight_render_results(self):+ tracker = {+ "submitted": 0,+ "pending": 0,+ "max_pending": 0,+ "shutdown": False,+ }+ events = []++ @contextmanager+ def tracking_limit(*args, **kwargs):+ events.append("limit-enter")+ try:+ yield+ finally:+ events.append("limit-exit")++ class Future:+ def __init__(self, value):+ self.value = value+ self.pending = True++ def result(self):+ if self.pending:+ self.pending = False+ tracker["pending"] -= 1+ return self.value++ def cancel(self):+ if self.pending:+ self.pending = False+ tracker["pending"] -= 1+ return True++ class Executor:+ def __init__(self, **kwargs):+ pass++ def submit(self, _fn, args):+ index = args[0]+ tracker["submitted"] += 1+ tracker["pending"] += 1+ tracker["max_pending"] = max(+ tracker["max_pending"], tracker["pending"]+ )+ return Future((index, b"jpeg"))++ def shutdown(self, **kwargs):+ tracker["shutdown"] = True+ events.append("shutdown")++ with patch.object(pdf_parser, "_select_mp_context", return_value=object()):+ with patch("concurrent.futures.ProcessPoolExecutor", Executor):+ with patch.object(+ pdf_parser, "parser_worker_limit", tracking_limit+ ):+ images = pdf_parser._iter_scanned_pages_from_path(+ "unused.pdf",+ list(range(100)),+ "scan",+ 1.0,+ 85,+ 0,+ 4,+ )+ try:+ first = next(images)+ self.assertEqual(first.ref_path, "images/scan_page_1.jpg")+ self.assertEqual(tracker["submitted"], 9)+ self.assertEqual(tracker["pending"], 8)+ self.assertEqual(tracker["max_pending"], 8)++ second = next(images)+ self.assertEqual(second.ref_path, "images/scan_page_2.jpg")+ self.assertLessEqual(tracker["pending"], 8)+ finally:+ images.close()++ self.assertEqual(tracker["pending"], 0)+ self.assertTrue(tracker["shutdown"])+ self.assertLess(events.index("shutdown"), events.index("limit-exit"))++ def test_unconsumed_stream_removes_temporary_pdf(self):+ content = _make_image_only_pdf(2)+ real_write = pdf_parser._write_pdf_temp_file+ created_paths = []++ def record_temp_path(pdf_bytes):+ path = real_write(pdf_bytes)+ created_paths.append(path)+ return path++ with patch.object(+ pdf_parser,+ "CONFIG",+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),+ ):+ with patch.object(+ pdf_parser,+ "_write_pdf_temp_file",+ side_effect=record_temp_path,+ ):+ stream = PDFParser(+ file_name="scan.pdf", pdf_force_scanned=True+ ).parse_stream(content)+ temp_path = created_paths[0]+ try:+ self.assertTrue(os.path.exists(temp_path))+ finally:+ stream.close()+ self.assertFalse(os.path.exists(temp_path))++ def test_unary_pdf_parser_still_collects_scanned_images(self):+ content = _make_image_only_pdf(2)+ with patch.object(+ pdf_parser,+ "CONFIG",+ replace(pdf_parser.CONFIG, pdf_render_parallelism=1),+ ):+ document = PDFParser(+ file_name="scan.pdf", pdf_force_scanned=True+ ).parse_into_text(content)++ self.assertEqual(len(document.images), 2)+ self.assertIn("images/scan_page_1.jpg", document.images)+ self.assertIn("images/scan_page_2.jpg", document.images)++class StripRepeatingLinesTest(unittest.TestCase):def test_removes_repeated_header_footer(self):header = "ACME CONFIDENTIAL"