Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(rag): preserve already-decoded EML text (#44953)

Winorena8e39b3036bd
diff --git a/api/core/rag/extractor/unstructured/unstructured_eml_extractor.py b/api/core/rag/extractor/unstructured/unstructured_eml_extractor.py
index 1b87298587e..cf474667535 100644
--- a/api/core/rag/extractor/unstructured/unstructured_eml_extractor.py
+++ b/api/core/rag/extractor/unstructured/unstructured_eml_extractor.py
@@ -1,10 +1,6 @@
-import base64
-import contextlib
import logging
from typing import override
-from bs4 import BeautifulSoup
-
from configs import dify_config
from core.rag.extractor.extractor_base import BaseExtractor
from core.rag.models.document import Document
@@ -14,6 +10,9 @@ logger = logging.getLogger(__name__)
class UnstructuredEmailExtractor(BaseExtractor):
"""Load eml files.
+
+ Unstructured owns MIME transfer decoding and HTML partitioning; its elements
+ already contain text and must not be decoded again.
Args:
file_path: Path to the file to load.
"""
@@ -35,18 +34,6 @@ class UnstructuredEmailExtractor(BaseExtractor):
elements = partition_email(filename=self._file_path)
- # noinspection PyBroadException
- with contextlib.suppress(Exception):
- for element in elements:
- element_text = element.text.strip()
-
- padding_needed = 4 - len(element_text) % 4
- element_text += "=" * padding_needed
-
- element_decode = base64.b64decode(element_text)
- soup = BeautifulSoup(element_decode.decode("utf-8"), "html.parser")
- element.text = soup.get_text()
-
from unstructured.chunking.title import chunk_by_title
max_characters = dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH
diff --git a/api/tests/unit_tests/core/rag/extractor/unstructured/test_unstructured_doc_extractor.py b/api/tests/unit_tests/core/rag/extractor/unstructured/test_unstructured_doc_extractor.py
index 6af0d1e5919..de3847952ae 100644
--- a/api/tests/unit_tests/core/rag/extractor/unstructured/test_unstructured_doc_extractor.py
+++ b/api/tests/unit_tests/core/rag/extractor/unstructured/test_unstructured_doc_extractor.py
@@ -117,9 +117,7 @@ class TestUnstructuredMarkdownMsgXml:
class TestUnstructuredEmailAndEpub:
- def test_email_extractor_local_decodes_html_and_suppresses_decode_errors(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
+ def test_email_extractor_local_preserves_partitioned_text(self, monkeypatch: pytest.MonkeyPatch) -> None:
_register_unstructured_packages(monkeypatch)
captured: dict[str, list[Text]] = {}
@@ -140,7 +138,7 @@ class TestUnstructuredEmailAndEpub:
assert docs[0].page_content == "chunked-email"
chunk_elements = captured["elements"]
- assert "Hello Email" in chunk_elements[0].text
+ assert chunk_elements[0].text == encoded_html
assert chunk_elements[1].text == bad_base64
def test_email_extractor_with_api(self, monkeypatch: pytest.MonkeyPatch) -> None: