RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix the pipeline email parser failing on every multipart message (#19406)
JessYan16b8f9bf8a98
diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.pyindex 7a8a00cff1..dac5fca4dd 100644--- a/rag/flow/parser/parser.py+++ b/rag/flow/parser/parser.py@@ -1252,6 +1252,20 @@ class Parser(ProcessBase):self.set_output("output_format", conf["output_format"])target_fields = conf["fields"]+ def _decode_payload(payload, charset):+ """Decode a MIME payload, falling back through the encodings mislabelled+ mail actually uses. Empty or absent payloads decode to an empty string."""+ if not payload:+ return ""+ for enc in [charset, "utf-8", "gb2312", "gbk", "gb18030", "latin1"]:+ if not enc:+ continue+ try:+ return payload.decode(enc)+ except (UnicodeDecodeError, LookupError):+ continue+ return payload.decode("utf-8", errors="ignore")+_, ext = os.path.splitext(name)if ext == ".eml":# handle eml file@@ -1273,31 +1287,16 @@ class Parser(ProcessBase):body_text, body_html = [], []def _add_content(m, content_type):- def _decode_payload(payload, charset, target_list):- try:- target_list.append(payload.decode(charset))- except (UnicodeDecodeError, LookupError):- for enc in ["utf-8", "gb2312", "gbk", "gb18030", "latin1"]:- try:- target_list.append(payload.decode(enc))- break- except UnicodeDecodeError:- continue- else:- target_list.append(payload.decode("utf-8", errors="ignore"))-+ # Read the part that was handed in, not the top-level message:+ # get_payload(decode=True) on a multipart container returns None,+ # so any message with an attachment used to fail here.if content_type == "text/plain":- payload = msg.get_payload(decode=True)- charset = msg.get_content_charset() or "utf-8"- _decode_payload(payload, charset, body_text)+ body_text.append(_decode_payload(m.get_payload(decode=True), m.get_content_charset()))elif content_type == "text/html":- payload = msg.get_payload(decode=True)- charset = msg.get_content_charset() or "utf-8"- _decode_payload(payload, charset, body_html)- elif "multipart" in content_type:- if m.is_multipart():- for part in m.iter_parts():- _add_content(part, part.get_content_type())+ body_html.append(_decode_payload(m.get_payload(decode=True), m.get_content_charset()))+ elif "multipart" in content_type and m.is_multipart():+ for part in m.iter_parts():+ _add_content(part, part.get_content_type())_add_content(msg, msg.get_content_type())@@ -1312,7 +1311,12 @@ class Parser(ProcessBase):dispositions = content_disposition.strip().split(";")if dispositions[0].lower() == "attachment":filename = part.get_filename()- payload = part.get_payload(decode=True).decode(part.get_content_charset())+ # A binary attachment carries no charset, so decoding its+ # bytes as text would only push mojibake into the indexed+ # content. Keep the name and leave the payload empty.+ payload = ""+ if part.get_content_maintype() == "text":+ payload = _decode_payload(part.get_payload(decode=True), part.get_content_charset())attachments.append({"filename": filename,@@ -1349,10 +1353,18 @@ class Parser(ProcessBase):if "attachments" in target_fields:attachments = []for t in msg.attachments:+ # extract_msg exposes no charset for an attachment, so treat only+ # payloads that are valid UTF-8 as text and skip the rest.+ payload = ""+ if isinstance(t.data, bytes):+ try:+ payload = t.data.decode("utf-8")+ except UnicodeDecodeError:+ passattachments.append({"filename": t.name,- "payload": t.data.decode("utf-8"),+ "payload": payload,})email_content["attachments"] = attachmentsdiff --git a/test/unit_test/rag/flow/parser/test_email_attachments.py b/test/unit_test/rag/flow/parser/test_email_attachments.pynew file mode 100644index 0000000000..5af08b01b4--- /dev/null+++ b/test/unit_test/rag/flow/parser/test_email_attachments.py@@ -0,0 +1,220 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""`Parser._email` has two paths a single-part message never reaches: the recursive+body walk, and the attachment loop. Any `multipart/*` message exercises the first —+with or without an attachment — and an attachment exercises the second. Both are+covered here.+"""++import importlib+import sys+from email.message import EmailMessage+from types import SimpleNamespace+from unittest.mock import Mock++import pytest++EMAIL_FIELDS = ["from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"]+++def _load_parser_module(monkeypatch):+ for module_name in ("rag.flow.parser.parser", "deepdoc.parser.pdf_parser", "deepdoc.parser", "deepdoc"):+ monkeypatch.delitem(sys.modules, module_name, raising=False)+ importlib.invalidate_caches()+ importlib.import_module("deepdoc.parser.pdf_parser")+ return importlib.import_module("rag.flow.parser.parser")+++class _FakeProcess:+ def __init__(self):+ self._param = SimpleNamespace(+ setups={+ "email": {+ "suffix": ["eml", "msg"],+ "fields": EMAIL_FIELDS,+ "output_format": "json",+ }+ }+ )+ self._canvas = SimpleNamespace(_tenant_id="tenant", _language="English")+ self.callback = Mock()+ self.outputs = {}++ def set_output(self, name, value):+ self.outputs[name] = value+++def _eml(*, body="Please find the report attached.", attach=None):+ """Build a real .eml. Adding an attachment makes the message multipart/mixed."""+ msg = EmailMessage()+ msg["From"] = "[email protected]"+ msg["To"] = "[email protected]"+ msg["Subject"] = "quarterly report"+ msg.set_content(body)+ if attach is not None:+ msg.add_attachment(attach["data"], maintype=attach["maintype"], subtype=attach["subtype"], filename=attach["filename"], **attach.get("kwargs", {}))+ return msg.as_bytes()+++def _parse(parser_module, raw, name="message.eml"):+ process = _FakeProcess()+ parser_module.Parser._email(process, name, raw)+ return process.outputs["json"][0]+++def test_multipart_body_comes_from_the_part_not_the_whole_message(monkeypatch):+ """`get_payload(decode=True)` on a multipart container returns None. The body+ walk must read the part it was handed, not the top-level message."""+ parser_module = _load_parser_module(monkeypatch)+ raw = _eml(body="body survives", attach={"data": b"%PDF-1.4\n\xff\xd8bin", "maintype": "application", "subtype": "pdf", "filename": "report.pdf"})++ content = _parse(parser_module, raw)++ assert content["text"].strip() == "body survives"+++def test_single_part_body_still_parses(monkeypatch):+ parser_module = _load_parser_module(monkeypatch)++ content = _parse(parser_module, _eml(body="lone body"))++ assert content["text"].strip() == "lone body"+ assert content.get("attachments", []) == []+++ "maintype,subtype,data,filename",+ [+ ("application", "pdf", b"%PDF-1.4\n\xff\xd8bin", "report.pdf"),+ ("image", "png", b"\x89PNG\r\n\x1a\n\x00\x00\x00", "chart.png"),+ ],+)+def test_binary_attachment_keeps_its_name_and_contributes_no_text(monkeypatch, maintype, subtype, data, filename):+ """A binary attachment carries no charset, so decoding its bytes as text would+ only push mojibake into the indexed content. Keep the name, drop the payload."""+ parser_module = _load_parser_module(monkeypatch)+ raw = _eml(attach={"data": data, "maintype": maintype, "subtype": subtype, "filename": filename})++ content = _parse(parser_module, raw)++ assert [a["filename"] for a in content["attachments"]] == [filename]+ assert content["attachments"][0]["payload"] == ""+++def test_utf8_text_attachment_is_decoded(monkeypatch):+ parser_module = _load_parser_module(monkeypatch)+ raw = _eml(attach={"data": b"line one\nline two\n", "maintype": "text", "subtype": "plain", "filename": "notes.txt"})++ content = _parse(parser_module, raw)++ assert content["attachments"][0]["filename"] == "notes.txt"+ assert "line one" in content["attachments"][0]["payload"]+++def test_gbk_text_attachment_goes_through_the_fallback_chain(monkeypatch):+ """A charset the payload does not actually honour must fall back rather than raise."""+ parser_module = _load_parser_module(monkeypatch)+ raw = _eml(+ attach={+ "data": "中文内容".encode("gbk"),+ "maintype": "text",+ "subtype": "plain",+ "filename": "chinese.txt",+ "kwargs": {"params": {"charset": "utf-8"}},+ }+ )++ content = _parse(parser_module, raw)++ assert content["attachments"][0]["filename"] == "chinese.txt"+ assert content["attachments"][0]["payload"] == "中文内容"+++def test_msg_binary_attachment_keeps_its_name_and_contributes_no_text(monkeypatch):+ """The .msg branch hardcodes utf-8, which raises on the same inputs."""+ parser_module = _load_parser_module(monkeypatch)+ fake_extract_msg = SimpleNamespace(+ Message=Mock(+ return_value=SimpleNamespace(+ sender="[email protected]",+ to="[email protected]",+ cc=None,+ bcc=None,+ date="Mon, 8 Sep 2026 10:00:00 +0800",+ subject="quarterly report",+ body="body survives",+ htmlBody=None,+ messageId="<[email protected]>",+ inReplyTo=None,+ attachments=[SimpleNamespace(name="report.pdf", data=b"%PDF-1.4\n\xff\xd8bin")],+ )+ )+ )+ monkeypatch.setitem(sys.modules, "extract_msg", fake_extract_msg)++ content = _parse(parser_module, b"ole2-bytes", name="message.msg")++ assert content["text"] == "body survives"+ assert [a["filename"] for a in content["attachments"]] == ["report.pdf"]+ assert content["attachments"][0]["payload"] == ""+++def _multipart_alternative(*, plain="plain body", html="<p>html body</p>", attach=None):+ """A plain+HTML message. This is `multipart/alternative` on its own, and+ `multipart/mixed` wrapping an alternative once an attachment is added."""+ msg = EmailMessage()+ msg["From"] = "[email protected]"+ msg["To"] = "[email protected]"+ msg["Subject"] = "newsletter"+ msg.set_content(plain)+ msg.add_alternative(html, subtype="html")+ if attach is not None:+ msg.add_attachment(attach["data"], maintype=attach["maintype"], subtype=attach["subtype"], filename=attach["filename"])+ return msg.as_bytes()+++def test_multipart_alternative_without_attachment_recovers_both_bodies(monkeypatch):+ """The wrong-object read fired on any multipart container, so a plain+HTML mail+ with no attachment at all was already failing."""+ parser_module = _load_parser_module(monkeypatch)++ content = _parse(parser_module, _multipart_alternative())++ assert content["text"].strip() == "plain body"+ assert "html body" in content["text_html"]+++def test_nested_multipart_walks_into_the_alternative_and_the_attachment(monkeypatch):+ """`multipart/mixed` wrapping a `multipart/alternative` plus a binary attachment:+ the walk has to recurse twice, and both leaf branches have to read their own part."""+ parser_module = _load_parser_module(monkeypatch)+ raw = _multipart_alternative(attach={"data": b"%PDF-1.4\n\xff\xd8bin", "maintype": "application", "subtype": "pdf", "filename": "report.pdf"})++ content = _parse(parser_module, raw)++ assert content["text"].strip() == "plain body"+ assert "html body" in content["text_html"]+ assert [a["filename"] for a in content["attachments"]] == ["report.pdf"]+ assert content["attachments"][0]["payload"] == ""