Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): release raw PDF images after encoding (#4562)
uhhgxgb581f9521736
diff --git a/docreader/parser/pdf_parser.py b/docreader/parser/pdf_parser.pyindex 744a6350f..0f34dab1b 100644--- a/docreader/parser/pdf_parser.py+++ b/docreader/parser/pdf_parser.py@@ -1237,6 +1237,23 @@ def _render_scanned_pages(return out+def _encode_rendered_page_images(rendered: dict, base_name: str) -> dict:+ """Encode rendered pages while releasing each raw JPEG immediately.++ ``_render_scanned_pages`` returns raw JPEG bytes because the render workers+ need that representation. Keeping the raw mapping alive while building a+ second base64 mapping temporarily doubles the image memory for a scanned+ document. Pop each page before encoding it so the parser retains only the+ representation that the gRPC response needs.+ """+ encoded = {}+ for index in sorted(rendered):+ image_bytes = rendered.pop(index)+ ref_path = f"images/{base_name}_page_{index + 1}.jpg"+ encoded[ref_path] = base64.b64encode(image_bytes).decode("utf-8")+ return encoded++def _select_embedded_images(meta: list,num_text_pages: int,@@ -1491,6 +1508,7 @@ class PDFScannedParser(BaseParser):source_blocks = []offset = 0+ images.update(_encode_rendered_page_images(rendered, base_name))for i in range(page_count):page_filename = f"{base_name}_page_{i+1}.jpg"ref_path = f"images/{page_filename}"@@ -1506,7 +1524,6 @@ class PDFScannedParser(BaseParser):})offset += len(line)- images[ref_path] = base64.b64encode(rendered[i]).decode("utf-8")text = "\n\n".join(markdown_lines)return Document(@@ -1672,9 +1689,7 @@ class PDFParser(BaseParser):quality,CONFIG.pdf_render_max_edge,)- for i, img_bytes in rendered.items():- ref_path = f"images/{base_name}_page_{i+1}.jpg"- images[ref_path] = base64.b64encode(img_bytes).decode("utf-8")+ images.update(_encode_rendered_page_images(rendered, base_name))# Pass 3: extract embedded figures from native text pages so the Go# App can OCR/caption them (logos/watermarks/tiny images filtered).diff --git a/docreader/tests/test_pdf_router.py b/docreader/tests/test_pdf_router.pyindex dc283eafb..c883e2f74 100644--- a/docreader/tests/test_pdf_router.py+++ b/docreader/tests/test_pdf_router.py@@ -6,6 +6,7 @@ from PIL import Imagefrom docreader.parser.pdf_parser import (PDFParser,_classify_page,+ _encode_rendered_page_images,_filter_reading_columns,_group_lines,_is_artifact_column,@@ -55,6 +56,22 @@ class ClassifyPageTest(unittest.TestCase):self.assertEqual(_classify_page(0.0, 0), "text")+class RenderedPageEncodingTest(unittest.TestCase):+ def test_encoding_releases_raw_pages_as_it_builds_response_images(self):+ rendered = {1: b"page-two", 0: b"page-one"}++ encoded = _encode_rendered_page_images(rendered, "scan")++ self.assertEqual(rendered, {})+ self.assertEqual(+ encoded,+ {+ "images/scan_page_1.jpg": "cGFnZS1vbmU=",+ "images/scan_page_2.jpg": "cGFnZS10d28=",+ },+ )++class StripRepeatingLinesTest(unittest.TestCase):def test_removes_repeated_header_footer(self):header = "ACME CONFIDENTIAL"