Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): clean .eml text output for chunking (#18592)

euvree79b9bdc0123
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go
index 14bede0c39..b5995e8b8b 100644
--- a/internal/parser/parser/email_parser.go
+++ b/internal/parser/parser/email_parser.go
@@ -32,8 +32,13 @@ import (
"time"
"github.com/AkmalOt/gomsg"
+ "golang.org/x/net/html"
"golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/charmap"
+ "golang.org/x/text/encoding/japanese"
+ "golang.org/x/text/encoding/korean"
"golang.org/x/text/encoding/simplifiedchinese"
+ "golang.org/x/text/encoding/traditionalchinese"
"golang.org/x/text/transform"
"ragflow/internal/utility"
@@ -149,8 +154,21 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by
// Text output: flatten fields into a single string.
var sb strings.Builder
for k, v := range content {
+ // The metadata map (every non-basic header: Received chains,
+ // DKIM/ARC signatures, …) stays available in the JSON output, but
+ // flattening it into chunkable text buries the message under
+ // transport noise, so it is excluded from the text output.
+ if k == "metadata" {
+ continue
+ }
switch val := v.(type) {
case string:
+ if k == "text_html" {
+ // Text output feeds the chunker directly; emit the
+ // visible text of the HTML part instead of raw markup,
+ // which otherwise lands in chunks as tag soup.
+ val = htmlBodyToText(val)
+ }
sb.WriteString(k)
sb.WriteString(":")
sb.WriteString(val)
@@ -198,6 +216,59 @@ func targetFieldsSet(fields []string) map[string]bool {
return m
}
+// -- header decoding (RFC 2047) --
+
+// charsetEncoding maps a MIME charset label to its decoder, covering the
+// charsets common in real-world mail beyond utf-8: the Chinese families
+// (gb2312/gbk/gb18030, big5), shift_jis, euc-kr, and latin-1. It is shared by
+// the RFC 2047 header decoder and decodeMailPayload so headers and bodies
+// resolve a charset label identically.
+//
+// The "gb2312" label maps to GBK, not HZGB2312: mail labeled gb2312 carries
+// plain 8-bit GB2312 bytes, while HZGB2312 only decodes the 7-bit HZ escape
+// form used under the distinct "hz-gb-2312" label.
+func charsetEncoding(charset string) (encoding.Encoding, bool) {
+ switch strings.ToLower(strings.TrimSpace(charset)) {
+ case "gb2312", "gbk":
+ return simplifiedchinese.GBK, true
+ case "gb18030":
+ return simplifiedchinese.GB18030, true
+ case "hz-gb-2312":
+ return simplifiedchinese.HZGB2312, true
+ case "big5":
+ return traditionalchinese.Big5, true
+ case "shift_jis", "shift-jis", "sjis":
+ return japanese.ShiftJIS, true
+ case "euc-kr":
+ return korean.EUCKR, true
+ case "iso-8859-1", "iso8859-1", "latin1":
+ return charmap.ISO8859_1, true
+ }
+ return nil, false
+}
+
+// rfc2047Decoder decodes RFC 2047 encoded-words (e.g. "=?utf-8?B?...?=") in
+// header values. utf-8 is handled natively by mime; the CharsetReader covers
+// the non-UTF-8 charsets via charsetEncoding.
+var rfc2047Decoder = &mime.WordDecoder{
+ CharsetReader: func(charset string, input io.Reader) (io.Reader, error) {
+ if enc, ok := charsetEncoding(charset); ok {
+ return transform.NewReader(input, enc.NewDecoder()), nil
+ }
+ return nil, fmt.Errorf("email: unsupported header charset %q", charset)
+ },
+}
+
+// decodeHeaderWord decodes any RFC 2047 encoded-words in a header value,
+// leaving undecodable values untouched.
+func decodeHeaderWord(val string) string {
+ decoded, err := rfc2047Decoder.DecodeHeader(val)
+ if err != nil {
+ return val
+ }
+ return decoded
+}
+
// -- .eml parsing (RFC 5322 with multipart support) --
func parseEML(r io.Reader, fields []string) map[string]any {
@@ -210,11 +281,18 @@ func parseEML(r io.Reader, fields []string) map[string]any {
return content
}
- // Headers.
+ // Headers. net/mail does not decode RFC 2047 encoded-words, so decode
+ // each value explicitly; otherwise a non-ASCII subject/from arrives as
+ // an unreadable "=?utf-8?B?...?=" blob that chunk delimiters then shred
+ // at the "?" characters.
meta := map[string]any{}
for key, vals := range msg.Header {
keyLower := strings.ToLower(key)
- val := strings.Join(vals, ", ")
+ decoded := make([]string, len(vals))
+ for i, v := range vals {
+ decoded[i] = decodeHeaderWord(v)
+ }
+ val := strings.Join(decoded, ", ")
switch keyLower {
case "from", "to", "cc", "bcc", "date", "subject":
if target[keyLower] {
@@ -401,6 +479,73 @@ func attachmentFilename(part *multipart.Part) string {
return "attachment.bin"
}
+// -- HTML body → visible text (text output only) --
+
+// htmlBodyToText flattens an email HTML body to its visible text. Unlike the
+// structured HTML parser (which keeps table markup for the dedicated HTML
+// chunk method), email bodies use tables for layout, so this walker descends
+// into tables and emits cell text instead. <head>/<script>/<style> subtrees
+// are skipped; whitespace folding is shared with the standalone HTML parser
+// via leafWriter.
+func htmlBodyToText(body string) string {
+ if strings.TrimSpace(body) == "" {
+ return ""
+ }
+ doc, err := html.Parse(strings.NewReader(body))
+ if err != nil {
+ return body
+ }
+ var b bytes.Buffer
+ w := &leafWriter{b: &b, lineStart: true}
+ walkHTMLBodyText(doc, w)
+ return strings.TrimSpace(b.String())
+}
+
+// isEmailBlockTag reports whether tag delimits a line in flattened email
+// body text: the standard block tags plus table tags, since email uses
+// tables for layout and their cells/rows must not fuse either.
+func isEmailBlockTag(tag string) bool {
+ return isBlockTag(tag) || tag == "table" || tag == "td" || tag == "th"
+}
+
+func walkHTMLBodyText(n *html.Node, w *leafWriter) {
+ switch n.Type {
+ case html.TextNode:
+ w.writeText(n.Data)
+ case html.DocumentNode:
+ // html.Parse always wraps the fragment in a Document node holding an
+ // <html><body> skeleton; the walker relies on descending through those
+ // transparent containers (head/script/style subtrees are skipped by the
+ // ElementNode case), so without descending here the whole body would
+ // be dropped.
+ for child := n.FirstChild; child != nil; child = child.NextSibling {
+ walkHTMLBodyText(child, w)
+ }
+ case html.ElementNode:
+ switch n.Data {
+ case "head", "script", "style", "noscript":
+ return
+ case "br":
+ w.hardBreak()
+ return
+ }
+ // A nested block starts on a new line when text already precedes
+ // it, so outer text and inner block text stay separate
+ // ("<div>a<p>b</p></div>" flattens to "a\nb", not "ab").
+ if isEmailBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
+ w.hardBreak()
+ }
+ for child := n.FirstChild; child != nil; child = child.NextSibling {
+ walkHTMLBodyText(child, w)
+ }
+ // Block-level tags and table cells/rows end the current line so
+ // adjacent block/cell text does not fuse together.
+ if isEmailBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
+ w.hardBreak()
+ }
+ }
+}
+
// decodeMailPayload attempts multiple charset decodings.
// Mirrors Python's _decode_payload with fallback chain:
// utf-8 → gb2312 → gbk → gb18030 → latin1 → utf-8 (ignore).
@@ -440,18 +585,9 @@ func decodeWithCharset(payload []byte, charset string) (string, error) {
return s, nil
}
return "", fmt.Errorf("invalid utf-8")
- case "latin1", "iso-8859-1", "iso8859-1":
- runes := make([]rune, len(payload))
- for i, b := range payload {
- runes[i] = rune(b)
- }
- return string(runes), nil
- case "gb2312":
- return decodeTransform(payload, simplifiedchinese.HZGB2312.NewDecoder())
- case "gbk":
- return decodeTransform(payload, simplifiedchinese.GBK.NewDecoder())
- case "gb18030":
- return decodeTransform(payload, simplifiedchinese.GB18030.NewDecoder())
+ }
+ if enc, ok := charsetEncoding(charset); ok {
+ return decodeTransform(payload, enc.NewDecoder())
}
// Unknown charset: treat as latin-1.
runes := make([]rune, len(payload))
diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go
index dd9436aa8a..02d575a1ee 100644
--- a/internal/parser/parser/email_parser_test.go
+++ b/internal/parser/parser/email_parser_test.go
@@ -116,6 +116,102 @@ func TestEmailParser_EmlText(t *testing.T) {
}
}
+// TestEmailParser_EmlTextChunkText covers the KB chunk-quality bug behind
+// Feishu issue om_x100b675dcd7a18a4c2cbfff662a404e: the text output of an
+// .eml with an RFC 2047 encoded subject, transport headers (DKIM/ARC/
+// Received) and an HTML part used to carry (1) the flattened metadata dump,
+// (2) the raw "=?utf-8?B?...?=" subject blob and (3) shredded HTML markup —
+// all of which ended up in chunks. The text output must instead skip the
+// metadata dump, decode the subject, and flatten the HTML part to its
+// visible text. The JSON output keeps metadata and raw text_html unchanged.
+func TestEmailParser_EmlTextChunkText(t *testing.T) {
+ ctx := t.Context()
+ // "=?utf-8?B?5bim6ZmE5Lu255qEZW1haWw=?=" decodes to "带附件的email";
+ // "=?utf-8?B?5a6J?=" decodes to "安".
+ raw := strings.Join([]string{
+ "From: =?utf-8?B?5a6J?= <[email protected]>",
+ "Subject: =?utf-8?B?5bim6ZmE5Lu255qEZW1haWw=?=",
+ "DKIM-Signature: v=1; a=rsa-sha256; b=abc123",
+ "ARC-Seal: i=1; cv=none",
+ "Received: from mail.example.com by mx.example.com",
+ "MIME-Version: 1.0",
+ "Content-Type: multipart/alternative; boundary=altbound",
+ "",
+ "--altbound",
+ "Content-Type: text/plain; charset=utf-8",
+ "",
+ "带附件的email",
+ "--altbound",
+ "Content-Type: text/html; charset=utf-8",
+ "",
+ `<div><img style="width:100%" src="http://example.com/x.png"><div>带附件的email</div></div>`,
+ "--altbound--",
+ }, "\r\n")
+
+ p := NewEmailParser()
+ p.ConfigureFromSetup(map[string]any{
+ "output_format": "text",
+ "fields": []string{"from", "to", "subject", "body"},
+ })
+
+ result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
+ if result.Err != nil {
+ t.Fatalf("unexpected error: %v", result.Err)
+ }
+
+ // (1) No transport-header dump.
+ for _, noise := range []string{"metadata:", "dkim-signature", "arc-seal", "received:"} {
+ if strings.Contains(strings.ToLower(result.Text), noise) {
+ t.Errorf("text output contains header noise %q: %q", noise, result.Text)
+ }
+ }
+ // (2) Subject/from decoded from RFC 2047.
+ if !strings.Contains(result.Text, "subject:带附件的email") {
+ t.Errorf("subject not decoded: %q", result.Text)
+ }
+ if !strings.Contains(result.Text, "from:安 <[email protected]>") {
+ t.Errorf("from not decoded: %q", result.Text)
+ }
+ if strings.Contains(result.Text, "=?utf-8?") {
+ t.Errorf("raw encoded-word leaked into text: %q", result.Text)
+ }
+ // (3) HTML part flattened to visible text, no markup.
+ for _, tag := range []string{"<div", "<img", "style="} {
+ if strings.Contains(result.Text, tag) {
+ t.Errorf("raw HTML markup %q leaked into text: %q", tag, result.Text)
+ }
+ }
+ // HTML part must actually contribute its visible text: subject + plain
+ // body + html body = exactly 3 occurrences of the body string.
+ if c := strings.Count(result.Text, "带附件的email"); c != 3 {
+ t.Errorf("expected subject+plain+html body text, got %d occurrences in: %q", c, result.Text)
+ }
+ if !strings.Contains(result.Text, "text_html:带附件的email") {
+ t.Errorf("html body text missing from text_html: %q", result.Text)
+ }
+
+ // JSON output keeps the full metadata and raw text_html.
+ p.ConfigureFromSetup(map[string]any{
+ "output_format": "json",
+ "fields": []string{"from", "to", "subject", "body"},
+ })
+ jres := p.ParseWithResult(ctx, "test.eml", []byte(raw))
+ if jres.Err != nil {
+ t.Fatalf("unexpected error: %v", jres.Err)
+ }
+ item := jres.JSON[0]
+ if _, ok := item["metadata"].(map[string]any); !ok {
+ t.Errorf("json output must keep metadata: %#v", item)
+ }
+ if v, _ := item["text_html"].(string); !strings.Contains(v, "<img") {
+ t.Errorf("json output must keep raw text_html: %q", v)
+ }
+ if v, _ := item["subject"].(string); v != "带附件的email" {
+ t.Errorf("json subject not decoded: %q", v)
+ }
+}
+
// TestEmailParser_MsgSupported parses a real Outlook .msg fixture and verifies
// the Go output aligns with the Python flow parser _email() .msg branch
// (rag/flow/parser/parser.py). Replaces the old "MsgNotSupported" test now that
@@ -235,6 +331,43 @@ func TestEmailParser_MsgMetadataAlwaysPresent(t *testing.T) {
}
}
+// TestEmailParser_MsgTextOutputExcludesMetadata verifies the .msg text
+// output: .msg content flows through the same flatten loop as .eml, so the
+// unconditionally emitted metadata map stays out of the chunkable text
+// (while remaining available in the JSON output) and the basic fields are
+// still flattened.
+func TestEmailParser_MsgTextOutputExcludesMetadata(t *testing.T) {
+ ctx := t.Context()
+ data, err := os.ReadFile("testdata/sample.msg")
+ if err != nil {
+ t.Fatalf("read fixture: %v", err)
+ }
+
+ p := NewEmailParser() // text output; default fields
+ p.ConfigureFromSetup(map[string]any{
+ "fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},
+ })
+ result := p.ParseWithResult(ctx, "sample.msg", data)
+ if result.Err != nil {
+ t.Fatalf("unexpected error: %v", result.Err)
+ }
+ if result.OutputFormat != "text" {
+ t.Fatalf("output format = %q, want text", result.OutputFormat)
+ }
+
+ if !strings.Contains(result.Text, "subject:asdf") {
+ t.Errorf("text output missing subject, got %q", result.Text)
+ }
+ if !strings.Contains(result.Text, "from:<[email protected]>") {
+ t.Errorf("text output missing from, got %q", result.Text)
+ }
+ for _, leaked := range []string{"metadata", "message_id", "in_reply_to"} {
+ if strings.Contains(result.Text, leaked) {
+ t.Errorf("text output must not contain %q, got %q", leaked, result.Text)
+ }
+ }
+}
+
func TestEmailParser_Base64Attachment(t *testing.T) {
ctx := t.Context()
attachmentContent := "Hello! This is the decoded content of the attachment."
@@ -900,11 +1033,11 @@ func TestEmailParser_NestedEMLAttachmentRechunk(t *testing.T) {
t.Errorf("[json] nested body not found as separate item: %#v", result.JSON)
}
} else {
- // Text mode: the outer email's own metadata is flattened once;
- // the nested email must NOT add a second "metadata:{" segment.
- if c := strings.Count(result.Text, "metadata:{"); c != 1 {
- t.Errorf("[text] expected exactly 1 metadata:{ segment (outer only), got %d in: %q",
- c, result.Text)
+ // Text mode: metadata (transport headers) is never flattened
+ // into chunkable text — not for the outer email, and not for
+ // the nested re-parse (which always requests JSON output).
+ if strings.Contains(result.Text, "metadata:") {
+ t.Errorf("[text] metadata must not leak into text output: %q", result.Text)
}
}
}
@@ -1064,3 +1197,128 @@ func TestReadMailBody_AttachmentPreservesRaw(t *testing.T) {
t.Errorf("payload = %q, want 中文 (charset-decoded)", payload)
}
}
+
+// TestDecodeHeaderWord_Charsets verifies the charset routing behind RFC 2047
+// encoded-word decoding. The "gb2312" label must decode plain 8-bit
+// GB2312/GBK bytes — not HZ escape sequences — because real-world mail
+// labeled gb2312 carries plain bytes. HZ decoding stays available under its
+// own "hz-gb-2312" label. An unsupported charset leaves the value untouched.
+func TestDecodeHeaderWord_Charsets(t *testing.T) {
+ encodedWord := func(charset string, raw []byte) string {
+ return "=?" + charset + "?B?" + base64.StdEncoding.EncodeToString(raw) + "?="
+ }
+
+ tests := []struct {
+ name string
+ header string
+ want string
+ }{
+ {"gb2312 label decodes plain GB2312 bytes", encodedWord("gb2312", []byte{0xD6, 0xD0, 0xCE, 0xC4}), "中文"},
+ {"gbk", encodedWord("gbk", []byte{0xD6, 0xD0, 0xCE, 0xC4}), "中文"},
+ {"gb18030", encodedWord("gb18030", []byte{0xD6, 0xD0, 0xCE, 0xC4}), "中文"},
+ {"big5", encodedWord("big5", []byte{0xA4, 0xA4, 0xA4, 0xE5}), "中文"},
+ {"shift_jis", encodedWord("shift_jis", []byte{0x93, 0xFA, 0x96, 0x7B}), "日本"},
+ {"euc-kr", encodedWord("euc-kr", []byte{0xC7, 0xD1, 0xB1, 0xB9}), "한국"},
+ {"iso-8859-1", encodedWord("iso-8859-1", []byte{'c', 'a', 'f', 0xE9}), "café"},
+ {"hz-gb-2312 keeps HZ escape decoding", encodedWord("hz-gb-2312", []byte("~{VPND~}")), "中文"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := decodeHeaderWord(tt.header); got != tt.want {
+ t.Errorf("decodeHeaderWord(%q) = %q, want %q", tt.header, got, tt.want)
+ }
+ })
+ }
+
+ unsupported := encodedWord("x-unsupported-charset", []byte{0x01, 0x02})
+ if got := decodeHeaderWord(unsupported); got != unsupported {
+ t.Errorf("unsupported charset: decodeHeaderWord(%q) = %q, want value untouched", unsupported, got)
+ }
+}
+
+// TestParseEML_GB2312EncodedSubject is the end-to-end regression for the
+// charset fix: a subject labeled gb2312 carrying plain 8-bit bytes must
+// arrive decoded, not as the raw "=?gb2312?B?...?=" blob.
+func TestParseEML_GB2312EncodedSubject(t *testing.T) {
+ raw := strings.Join([]string{
+ "Subject: =?gb2312?B?" + base64.StdEncoding.EncodeToString([]byte{0xD6, 0xD0, 0xCE, 0xC4}) + "?=",
+ "Content-Type: text/plain; charset=utf-8",
+ "",
+ "body",
+ }, "\r\n")
+
+ content := parseEML(strings.NewReader(raw), []string{"subject"})
+ if content["subject"] != "中文" {
+ t.Errorf("subject = %q, want 中文", content["subject"])
+ }
+}
+
+// TestDecodeMailPayload_DeclaredCharsets verifies the body-side charset
+// resolution: a gb2312-labeled body carries plain 8-bit bytes (routed to
+// GBK like the header path, not HZGB2312), HZ-escaped content decodes only
+// under its own "hz-gb-2312" label, and a declared charset beyond the
+// fallback chain (big5) decodes via charsetEncoding instead of degrading to
+// latin-1 mojibake.
+func TestDecodeMailPayload_DeclaredCharsets(t *testing.T) {
+ if got := decodeMailPayload([]byte{0xD6, 0xD0, 0xCE, 0xC4}, "gb2312"); got != "中文" {
+ t.Errorf("gb2312 body = %q, want 中文", got)
+ }
+ if got := decodeMailPayload([]byte("~{VPND~}"), "hz-gb-2312"); got != "中文" {
+ t.Errorf("hz-gb-2312 body = %q, want 中文", got)
+ }
+ if got := decodeMailPayload([]byte{0xA4, 0xA4, 0xA4, 0xE5}, "big5"); got != "中文" {
+ t.Errorf("big5 body = %q, want 中文", got)
+ }
+}
+
+// TestReadMailBody_DeclaredBodyCharsets locks the body-path charset routing
+// exercised end to end by readMailBody (readMailBody → decodeMailPayload →
+// decodeWithCharset), beyond the direct decodeMailPayload unit above: a
+// gb2312-declared body part decodes its plain 8-bit bytes, and HZ-escaped
+// content decodes under its own "hz-gb-2312" label.
+func TestReadMailBody_DeclaredBodyCharsets(t *testing.T) {
+ tests := []struct {
+ name string
+ charset string
+ body []byte
+ }{
+ {"gb2312 body decodes plain bytes", "gb2312", []byte{0xD6, 0xD0, 0xCE, 0xC4}},
+ {"hz-gb-2312 body decodes HZ escapes", "hz-gb-2312", []byte("~{VPND~}")},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ text, _, _ := readMailBody(strings.NewReader(string(tt.body)), "text/plain; charset="+tt.charset, false)
+ if text != "中文" {
+ t.Errorf("readMailBody(charset=%s) = %q, want 中文", tt.charset, text)
+ }
+ })
+ }
+}
+
+// TestHTMLBodyToText verifies the visible-text flattening of email HTML
+// bodies, above all that text preceding a nested block-level element starts
+// a new line instead of fusing with the block's own text.
+func TestHTMLBodyToText(t *testing.T) {
+ tests := []struct {
+ name string
+ body string
+ want string
+ }{
+ {"text before nested block", `<div>prefix<p>paragraph</p></div>`, "prefix\nparagraph"},
+ {"nested divs", `<div>outer<div>inner</div></div>`, "outer\ninner"},
+ {"sibling blocks", `<p>one</p><p>two</p>`, "one\ntwo"},
+ {"layout table cells", `<table><tr><td>a</td><td>b</td></tr></table>`, "a\nb"},
+ {"multi-row layout table stays line-separated", `<p>intro</p><table><tr><td>a</td><td>b</td></tr><tr><td>c</td></tr></table><p>outro</p>`, "intro\na\nb\nc\noutro"},
+ {"inline tags stay on one line", `<p>Hello <b>world</b></p>`, "Hello world"},
+ {"br forces a break", `one<br>two`, "one\ntwo"},
+ {"head script style skipped", `<html><head><title>t</title><style>.x{color:red}</style></head><body><p>visible</p><script>var x=1;</script></body></html>`, "visible"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := htmlBodyToText(tt.body); got != tt.want {
+ t.Errorf("htmlBodyToText(%q) = %q, want %q", tt.body, got, tt.want)
+ }
+ })
+ }
+}