Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(api): stop accumulating duplicate extracted PDF images on repeated previews (#44684)
uhhgxge08b084b197f
diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.pyindex fa9d0437bdd..5f26f16717a 100644--- a/api/core/rag/extractor/extract_processor.py+++ b/api/core/rag/extractor/extract_processor.py@@ -153,6 +153,7 @@ class ExtractProcessor:file_path,upload_file.tenant_id if upload_file else None,upload_file.created_by if upload_file else None,+ source_file_id=upload_file.id if upload_file else None,session=session,)elif file_extension in {".md", ".markdown", ".mdx"}:@@ -203,6 +204,7 @@ class ExtractProcessor:file_path,upload_file.tenant_id if upload_file else None,upload_file.created_by if upload_file else None,+ source_file_id=upload_file.id if upload_file else None,session=session,)elif file_extension in {".md", ".markdown", ".mdx"}:diff --git a/api/core/rag/extractor/pdf_extractor.py b/api/core/rag/extractor/pdf_extractor.pyindex 778514cd27d..4bf2d2b3b07 100644--- a/api/core/rag/extractor/pdf_extractor.py+++ b/api/core/rag/extractor/pdf_extractor.py@@ -1,14 +1,16 @@"""Abstract interface for document loader implementations."""import contextlib+import hashlibimport ioimport loggingimport uuidfrom collections.abc import Iterator-from typing import override+from typing import TypedDict, overrideimport pypdfium2import pypdfium2.raw as pdfium_c+from sqlalchemy import selectfrom sqlalchemy.orm import Sessionfrom configs import dify_config@@ -25,6 +27,14 @@ from models.model import UploadFilelogger = logging.getLogger(__name__)+class _PdfImageCandidate(TypedDict):+ image_bytes: bytes+ image_ext: str+ mime_type: str+ content_hash: str+ file_key: str++class PdfExtractor(BaseExtractor):"""PdfExtractor is used to extract text and images from PDF files.@@ -34,6 +44,9 @@ class PdfExtractor(BaseExtractor):tenant_id: Workspace ID.user_id: ID of the user performing the extraction.file_cache_key: Optional cache key for the extracted text.+ source_file_id: Optional id of the source upload file. When provided,+ extracted images are persisted under deterministic storage keys so+ repeated extractions of the same source reuse the same assets.session: Session used to persist extracted images."""@@ -51,6 +64,7 @@ class PdfExtractor(BaseExtractor):)MAX_MAGIC_LEN = max(len(m) for m, _, _ in IMAGE_FORMATS)_session: Session | None+ _source_file_id: str | Nonedef __init__(self,@@ -58,6 +72,7 @@ class PdfExtractor(BaseExtractor):tenant_id: str | None = None,user_id: str | None = None,file_cache_key: str | None = None,+ source_file_id: str | None = None,*,session: Session | None = None,):@@ -66,6 +81,7 @@ class PdfExtractor(BaseExtractor):self._tenant_id = tenant_idself._user_id = user_idself._file_cache_key = file_cache_key+ self._source_file_id = source_file_idself._session = session@override@@ -106,7 +122,7 @@ class PdfExtractor(BaseExtractor):content = text_page.get_text_range()text_page.close()- image_content = self._extract_images(page)+ image_content = self._extract_images(page, page_number)if image_content:content += "\n" + image_content@@ -116,13 +132,20 @@ class PdfExtractor(BaseExtractor):finally:pdf_reader.close()- def _extract_images(self, page) -> str:+ def _extract_images(self, page, page_number: int = 0) -> str:"""Extract images from a PDF page, save them to storage and database,and return markdown image links.+ When a source file id is available, images are persisted under+ deterministic keys derived from the source upload file, page, object+ index, and content hash so repeated extractions of the same unchanged+ source reuse the existing UploadFile rows and storage objects instead+ of accumulating duplicates.+Args:page: pypdfium2 page object.+ page_number: Zero-based index of the page within the PDF.Returns:Markdown string containing links to the extracted images.@@ -133,13 +156,12 @@ class PdfExtractor(BaseExtractor):# image extraction and return text only.return ""- image_content = []- upload_files = []base_url = dify_config.FILES_URL+ candidates: list[_PdfImageCandidate] = []try:image_objects = page.get_objects(filter=(pdfium_c.FPDF_PAGEOBJ_IMAGE,))- for obj in image_objects:+ for obj_index, obj in enumerate(image_objects):try:# Extract image bytesimg_byte_arr = io.BytesIO()@@ -163,37 +185,81 @@ class PdfExtractor(BaseExtractor):if not image_ext or not mime_type:continue- file_uuid = str(uuid.uuid4())- file_key = "image_files/" + self._tenant_id + "/" + file_uuid + "." + image_ext-- storage.save(file_key, img_bytes)-- # save file to db- upload_file = UploadFile(- tenant_id=self._tenant_id,- storage_type=StorageType(dify_config.STORAGE_TYPE),- key=file_key,- name=file_key,- size=len(img_bytes),- extension=image_ext,- mime_type=mime_type,- created_by=self._user_id,- created_by_role=CreatorUserRole.ACCOUNT,- created_at=naive_utc_now(),- used=True,- used_by=self._user_id,- used_at=naive_utc_now(),+ content_hash = hashlib.sha256(img_bytes).hexdigest()+ if self._source_file_id:+ # Deterministic key so retried or repeated extractions+ # reuse the same assets (mirrors ExcelExtractor).+ file_key = (+ f"image_files/{self._tenant_id}/{self._source_file_id}/"+ f"p{page_number}_{obj_index}_{content_hash}.{image_ext}"+ )+ else:+ # Without a source upload file id there is no stable+ # identity to deduplicate against; keep unique keys.+ file_key = f"image_files/{self._tenant_id}/{uuid.uuid4()}.{image_ext}"++ candidates.append(+ _PdfImageCandidate(+ image_bytes=img_bytes,+ image_ext=image_ext,+ mime_type=mime_type,+ content_hash=content_hash,+ file_key=file_key,+ ))- upload_files.append(upload_file)- image_content.append(f"")except Exception as e:logger.warning("Failed to extract image from PDF: %s", e)continueexcept Exception as e:logger.warning("Failed to get objects from PDF page: %s", e)- if upload_files:- session = self._session or db.session- session.add_all(upload_files)++ if not candidates:+ return ""++ session = self._session or db.session++ upload_files_by_key: dict[str, UploadFile] = {}+ if self._source_file_id:+ existing_upload_files = session.scalars(+ select(UploadFile).where(+ UploadFile.tenant_id == self._tenant_id,+ UploadFile.key.in_([candidate["file_key"] for candidate in candidates]),+ )+ ).all()+ upload_files_by_key = {upload_file.key: upload_file for upload_file in existing_upload_files}++ image_content = []+ new_upload_files: list[UploadFile] = []+ for candidate in candidates:+ upload_file = upload_files_by_key.get(candidate["file_key"])+ if upload_file is None:+ try:+ storage.save(candidate["file_key"], candidate["image_bytes"])+ except Exception as e:+ logger.warning("Failed to persist extracted PDF image: %s", e)+ continue+ upload_file = UploadFile(+ tenant_id=self._tenant_id,+ storage_type=StorageType(dify_config.STORAGE_TYPE),+ key=candidate["file_key"],+ name=candidate["file_key"],+ size=len(candidate["image_bytes"]),+ extension=candidate["image_ext"],+ mime_type=candidate["mime_type"],+ created_by=self._user_id,+ created_by_role=CreatorUserRole.ACCOUNT,+ created_at=naive_utc_now(),+ used=True,+ used_by=self._user_id,+ used_at=naive_utc_now(),+ hash=candidate["content_hash"],+ )+ upload_files_by_key[candidate["file_key"]] = upload_file+ new_upload_files.append(upload_file)+ image_content.append(f"")++ if new_upload_files:+ session.add_all(new_upload_files)if self._session is None:session.commit()return "\n".join(image_content)diff --git a/api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.pyindex 0b9a211c5d5..8a221fc883d 100644--- a/api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.py+++ b/api/tests/unit_tests/core/rag/extractor/test_pdf_extractor.py@@ -1,9 +1,14 @@+import hashlibimport io+from collections.abc import Iterator+from contextlib import closingfrom dataclasses import dataclassfrom unittest.mock import MagicMock, patchfrom uuid import uuid4+import pypdfium2import pytest+from PIL import Imagefrom sqlalchemy import selectfrom sqlalchemy.orm import Session@@ -157,7 +162,7 @@ def test_extract_calls_extract_images(monkeypatch: pytest.MonkeyPatch) -> None:extractor = pe.PdfExtractor(file_path="test.pdf", tenant_id=TENANT_ID, user_id=USER_ID)# Mock _extract_images to return a known string- monkeypatch.setattr(extractor, "_extract_images", lambda _page: "")+ monkeypatch.setattr(extractor, "_extract_images", lambda _page, _page_number=0: "")documents = list(extractor.extract())@@ -210,3 +215,150 @@ def test_extract_images_skipped_without_tenant_context() -> None:assert result == ""mock_page.get_objects.assert_not_called()+++def image_pdf() -> Iterator[tuple[pypdfium2.PdfDocument, bytes]]:+ """Build real PDF pages so image extraction exercises PDFium, too."""+ image_buffer = io.BytesIO()+ with Image.new("RGB", (8, 8), "red") as image:+ image.save(image_buffer, format="JPEG")+ image_bytes = image_buffer.getvalue()++ with pypdfium2.PdfDocument.new() as pdf:+ for _ in range(2):+ with closing(pdf.new_page(100, 100)) as page:+ image_object = pypdfium2.PdfImage.new(pdf)+ image_object.load_jpeg(io.BytesIO(image_bytes), inline=True)+ page.insert_obj(image_object)+ page.gen_content()+ yield pdf, image_bytes++[email protected]("sqlite_session", [(UploadFile,)], indirect=True)[email protected]("inject_session", [False, True])+def test_extract_images_with_source_file_id_reuses_existing_assets(+ mock_dependencies: _Dependencies,+ image_pdf: tuple[pypdfium2.PdfDocument, bytes],+ inject_session: bool,+) -> None:+ """Repeated extraction of the same source must not accumulate duplicate assets."""+ source_file_id = str(uuid4())+ pdf, jpeg_bytes = image_pdf++ with closing(pdf[0]) as page:+ first_extractor = pe.PdfExtractor(+ file_path="test.pdf",+ tenant_id=TENANT_ID,+ user_id=USER_ID,+ source_file_id=source_file_id,+ session=mock_dependencies.session if inject_session else None,+ )+ first_link = first_extractor._extract_images(page, 2)++ second_extractor = pe.PdfExtractor(+ file_path="test.pdf",+ tenant_id=TENANT_ID,+ user_id=USER_ID,+ source_file_id=source_file_id,+ session=mock_dependencies.session if inject_session else None,+ )+ second_link = second_extractor._extract_images(page, 2)++ rows = mock_dependencies.session.scalars(select(UploadFile)).all()+ assert len(rows) == 1+ upload_file = rows[0]+ assert f"" == first_link+ assert first_link == second_link+ assert mock_dependencies.storage.saves == [(upload_file.key, jpeg_bytes)]+ expected_hash = hashlib.sha256(jpeg_bytes).hexdigest()+ assert upload_file.key == f"image_files/{TENANT_ID}/{source_file_id}/p2_0_{expected_hash}.jpg"+ assert upload_file.hash == expected_hash++[email protected]("sqlite_session", [(UploadFile,)], indirect=True)+def test_extract_images_with_source_file_id_keeps_unique_keys_per_page(+ mock_dependencies: _Dependencies,+ image_pdf: tuple[pypdfium2.PdfDocument, bytes],+) -> None:+ """The same image content on different pages maps to distinct assets."""+ source_file_id = str(uuid4())+ pdf, jpeg_bytes = image_pdf+ extractor = pe.PdfExtractor(+ file_path="test.pdf",+ tenant_id=TENANT_ID,+ user_id=USER_ID,+ source_file_id=source_file_id,+ )+ with closing(pdf[0]) as first_page, closing(pdf[1]) as second_page:+ first_link = extractor._extract_images(first_page, 0)+ second_link = extractor._extract_images(second_page, 1)++ rows = mock_dependencies.session.scalars(select(UploadFile)).all()+ assert len(rows) == 2+ assert {row.key for row in rows} == {+ f"image_files/{TENANT_ID}/{source_file_id}/p0_0_{hashlib.sha256(jpeg_bytes).hexdigest()}.jpg",+ f"image_files/{TENANT_ID}/{source_file_id}/p1_0_{hashlib.sha256(jpeg_bytes).hexdigest()}.jpg",+ }+ assert first_link != second_link+ assert len(mock_dependencies.storage.saves) == 2++[email protected]("sqlite_session", [(UploadFile,)], indirect=True)[email protected]("inject_session", [False, True])[email protected]("fail_all", [False, True])+def test_extract_images_storage_failure_isolated_and_retryable(+ mock_dependencies: _Dependencies,+ image_pdf: tuple[pypdfium2.PdfDocument, bytes],+ monkeypatch: pytest.MonkeyPatch,+ inject_session: bool,+ fail_all: bool,+) -> None:+ """Failed saves create no rows; other images and later retries still succeed."""+ pdf, jpeg_bytes = image_pdf+ source_file_id = str(uuid4())+ extractor = pe.PdfExtractor(+ file_path="test.pdf",+ tenant_id=TENANT_ID,+ user_id=USER_ID,+ source_file_id=source_file_id,+ session=mock_dependencies.session if inject_session else None,+ )+ save = mock_dependencies.storage.save+ attempts = 0++ def fail_save(key: str, data: bytes) -> None:+ nonlocal attempts+ attempts += 1+ if fail_all or attempts == 1:+ raise OSError("Storage unavailable")+ save(key, data)++ with closing(pdf[0]) as page:+ image_object = pypdfium2.PdfImage.new(pdf)+ image_object.load_jpeg(io.BytesIO(jpeg_bytes), inline=True)+ page.insert_obj(image_object)+ page.gen_content()+ monkeypatch.setattr(mock_dependencies.storage, "save", fail_save)+ first_result = extractor._extract_images(page)++ rows = mock_dependencies.session.scalars(select(UploadFile)).all()+ assert attempts == 2+ assert len(rows) == (0 if fail_all else 1)+ assert len(mock_dependencies.storage.saves) == len(rows)+ if fail_all:+ assert first_result == ""+ else:+ assert first_result == f""+ assert "/p0_1_" in rows[0].key++ monkeypatch.setattr(mock_dependencies.storage, "save", save)+ retry_result = extractor._extract_images(page)+ assert extractor._extract_images(page) == retry_result++ rows = mock_dependencies.session.scalars(select(UploadFile)).all()+ assert len(rows) == 2+ assert len(mock_dependencies.storage.saves) == 2+ assert retry_result.count("![image]") == 2+ for row in rows:+ assert f"/files/{row.id}/file-preview" in retry_result