RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): clean .eml text output for chunking (#18592)
euvree79b9bdc0123
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex 14bede0c39..b5995e8b8b 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -32,8 +32,13 @@ import ("time""github.com/AkmalOt/gomsg"+ "golang.org/x/net/html""golang.org/x/text/encoding"+ "golang.org/x/text/encoding/charmap"+ "golang.org/x/text/encoding/japanese"+ "golang.org/x/text/encoding/korean""golang.org/x/text/encoding/simplifiedchinese"+ "golang.org/x/text/encoding/traditionalchinese""golang.org/x/text/transform""ragflow/internal/utility"@@ -149,8 +154,21 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by// Text output: flatten fields into a single string.var sb strings.Builderfor k, v := range content {+ // The metadata map (every non-basic header: Received chains,+ // DKIM/ARC signatures, …) stays available in the JSON output, but+ // flattening it into chunkable text buries the message under+ // transport noise, so it is excluded from the text output.+ if k == "metadata" {+ continue+ }switch val := v.(type) {case string:+ if k == "text_html" {+ // Text output feeds the chunker directly; emit the+ // visible text of the HTML part instead of raw markup,+ // which otherwise lands in chunks as tag soup.+ val = htmlBodyToText(val)+ }sb.WriteString(k)sb.WriteString(":")sb.WriteString(val)@@ -198,6 +216,59 @@ func targetFieldsSet(fields []string) map[string]bool {return m}+// -- header decoding (RFC 2047) --++// charsetEncoding maps a MIME charset label to its decoder, covering the+// charsets common in real-world mail beyond utf-8: the Chinese families+// (gb2312/gbk/gb18030, big5), shift_jis, euc-kr, and latin-1. It is shared by+// the RFC 2047 header decoder and decodeMailPayload so headers and bodies+// resolve a charset label identically.+//+// The "gb2312" label maps to GBK, not HZGB2312: mail labeled gb2312 carries+// plain 8-bit GB2312 bytes, while HZGB2312 only decodes the 7-bit HZ escape+// form used under the distinct "hz-gb-2312" label.+func charsetEncoding(charset string) (encoding.Encoding, bool) {+ switch strings.ToLower(strings.TrimSpace(charset)) {+ case "gb2312", "gbk":+ return simplifiedchinese.GBK, true+ case "gb18030":+ return simplifiedchinese.GB18030, true+ case "hz-gb-2312":+ return simplifiedchinese.HZGB2312, true+ case "big5":+ return traditionalchinese.Big5, true+ case "shift_jis", "shift-jis", "sjis":+ return japanese.ShiftJIS, true+ case "euc-kr":+ return korean.EUCKR, true+ case "iso-8859-1", "iso8859-1", "latin1":+ return charmap.ISO8859_1, true+ }+ return nil, false+}++// rfc2047Decoder decodes RFC 2047 encoded-words (e.g. "=?utf-8?B?...?=") in+// header values. utf-8 is handled natively by mime; the CharsetReader covers+// the non-UTF-8 charsets via charsetEncoding.+var rfc2047Decoder = &mime.WordDecoder{+ CharsetReader: func(charset string, input io.Reader) (io.Reader, error) {+ if enc, ok := charsetEncoding(charset); ok {+ return transform.NewReader(input, enc.NewDecoder()), nil+ }+ return nil, fmt.Errorf("email: unsupported header charset %q", charset)+ },+}++// decodeHeaderWord decodes any RFC 2047 encoded-words in a header value,+// leaving undecodable values untouched.+func decodeHeaderWord(val string) string {+ decoded, err := rfc2047Decoder.DecodeHeader(val)+ if err != nil {+ return val+ }+ return decoded+}+// -- .eml parsing (RFC 5322 with multipart support) --func parseEML(r io.Reader, fields []string) map[string]any {@@ -210,11 +281,18 @@ func parseEML(r io.Reader, fields []string) map[string]any {return content}- // Headers.+ // Headers. net/mail does not decode RFC 2047 encoded-words, so decode+ // each value explicitly; otherwise a non-ASCII subject/from arrives as+ // an unreadable "=?utf-8?B?...?=" blob that chunk delimiters then shred+ // at the "?" characters.meta := map[string]any{}for key, vals := range msg.Header {keyLower := strings.ToLower(key)- val := strings.Join(vals, ", ")+ decoded := make([]string, len(vals))+ for i, v := range vals {+ decoded[i] = decodeHeaderWord(v)+ }+ val := strings.Join(decoded, ", ")switch keyLower {case "from", "to", "cc", "bcc", "date", "subject":if target[keyLower] {@@ -401,6 +479,73 @@ func attachmentFilename(part *multipart.Part) string {return "attachment.bin"}+// -- HTML body → visible text (text output only) --++// htmlBodyToText flattens an email HTML body to its visible text. Unlike the+// structured HTML parser (which keeps table markup for the dedicated HTML+// chunk method), email bodies use tables for layout, so this walker descends+// into tables and emits cell text instead. <head>/<script>/<style> subtrees+// are skipped; whitespace folding is shared with the standalone HTML parser+// via leafWriter.+func htmlBodyToText(body string) string {+ if strings.TrimSpace(body) == "" {+ return ""+ }+ doc, err := html.Parse(strings.NewReader(body))+ if err != nil {+ return body+ }+ var b bytes.Buffer+ w := &leafWriter{b: &b, lineStart: true}+ walkHTMLBodyText(doc, w)+ return strings.TrimSpace(b.String())+}++// isEmailBlockTag reports whether tag delimits a line in flattened email+// body text: the standard block tags plus table tags, since email uses+// tables for layout and their cells/rows must not fuse either.+func isEmailBlockTag(tag string) bool {+ return isBlockTag(tag) || tag == "table" || tag == "td" || tag == "th"+}++func walkHTMLBodyText(n *html.Node, w *leafWriter) {+ switch n.Type {+ case html.TextNode:+ w.writeText(n.Data)+ case html.DocumentNode:+ // html.Parse always wraps the fragment in a Document node holding an+ // <html><body> skeleton; the walker relies on descending through those+ // transparent containers (head/script/style subtrees are skipped by the+ // ElementNode case), so without descending here the whole body would+ // be dropped.+ for child := n.FirstChild; child != nil; child = child.NextSibling {+ walkHTMLBodyText(child, w)+ }+ case html.ElementNode:+ switch n.Data {+ case "head", "script", "style", "noscript":+ return+ case "br":+ w.hardBreak()+ return+ }+ // A nested block starts on a new line when text already precedes+ // it, so outer text and inner block text stay separate+ // ("<div>a<p>b</p></div>" flattens to "a\nb", not "ab").+ if isEmailBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {+ w.hardBreak()+ }+ for child := n.FirstChild; child != nil; child = child.NextSibling {+ walkHTMLBodyText(child, w)+ }+ // Block-level tags and table cells/rows end the current line so+ // adjacent block/cell text does not fuse together.+ if isEmailBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {+ w.hardBreak()+ }+ }+}+// decodeMailPayload attempts multiple charset decodings.// Mirrors Python's _decode_payload with fallback chain:// utf-8 → gb2312 → gbk → gb18030 → latin1 → utf-8 (ignore).@@ -440,18 +585,9 @@ func decodeWithCharset(payload []byte, charset string) (string, error) {return s, nil}return "", fmt.Errorf("invalid utf-8")- case "latin1", "iso-8859-1", "iso8859-1":- runes := make([]rune, len(payload))- for i, b := range payload {- runes[i] = rune(b)- }- return string(runes), nil- case "gb2312":- return decodeTransform(payload, simplifiedchinese.HZGB2312.NewDecoder())- case "gbk":- return decodeTransform(payload, simplifiedchinese.GBK.NewDecoder())- case "gb18030":- return decodeTransform(payload, simplifiedchinese.GB18030.NewDecoder())+ }+ if enc, ok := charsetEncoding(charset); ok {+ return decodeTransform(payload, enc.NewDecoder())}// Unknown charset: treat as latin-1.runes := make([]rune, len(payload))diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.goindex dd9436aa8a..02d575a1ee 100644--- a/internal/parser/parser/email_parser_test.go+++ b/internal/parser/parser/email_parser_test.go@@ -116,6 +116,102 @@ func TestEmailParser_EmlText(t *testing.T) {}}+// TestEmailParser_EmlTextChunkText covers the KB chunk-quality bug behind+// Feishu issue om_x100b675dcd7a18a4c2cbfff662a404e: the text output of an+// .eml with an RFC 2047 encoded subject, transport headers (DKIM/ARC/+// Received) and an HTML part used to carry (1) the flattened metadata dump,+// (2) the raw "=?utf-8?B?...?=" subject blob and (3) shredded HTML markup —+// all of which ended up in chunks. The text output must instead skip the+// metadata dump, decode the subject, and flatten the HTML part to its+// visible text. The JSON output keeps metadata and raw text_html unchanged.+func TestEmailParser_EmlTextChunkText(t *testing.T) {+ ctx := t.Context()+ // "=?utf-8?B?5bim6ZmE5Lu255qEZW1haWw=?=" decodes to "带附件的email";+ // "=?utf-8?B?5a6J?=" decodes to "安".+ raw := strings.Join([]string{+ "From: =?utf-8?B?5a6J?= <[email protected]>",+ "To: [email protected]",+ "Subject: =?utf-8?B?5bim6ZmE5Lu255qEZW1haWw=?=",+ "DKIM-Signature: v=1; a=rsa-sha256; b=abc123",+ "ARC-Seal: i=1; cv=none",+ "Received: from mail.example.com by mx.example.com",+ "MIME-Version: 1.0",+ "Content-Type: multipart/alternative; boundary=altbound",+ "",+ "--altbound",+ "Content-Type: text/plain; charset=utf-8",+ "",+ "带附件的email",+ "--altbound",+ "Content-Type: text/html; charset=utf-8",+ "",+ `<div><img style="width:100%" src="http://example.com/x.png"><div>带附件的email</div></div>`,+ "--altbound--",+ }, "\r\n")++ p := NewEmailParser()+ p.ConfigureFromSetup(map[string]any{+ "output_format": "text",+ "fields": []string{"from", "to", "subject", "body"},+ })++ result := p.ParseWithResult(ctx, "test.eml", []byte(raw))+ if result.Err != nil {+ t.Fatalf("unexpected error: %v", result.Err)+ }++ // (1) No transport-header dump.+ for _, noise := range []string{"metadata:", "dkim-signature", "arc-seal", "received:"} {+ if strings.Contains(strings.ToLower(result.Text), noise) {+ t.Errorf("text output contains header noise %q: %q", noise, result.Text)+ }+ }+ // (2) Subject/from decoded from RFC 2047.+ if !strings.Contains(result.Text, "subject:带附件的email") {+ t.Errorf("subject not decoded: %q", result.Text)+ }+ if !strings.Contains(result.Text, "from:安 <[email protected]>") {+ t.Errorf("from not decoded: %q", result.Text)+ }+ if strings.Contains(result.Text, "=?utf-8?") {+ t.Errorf("raw encoded-word leaked into text: %q", result.Text)+ }+ // (3) HTML part flattened to visible text, no markup.+ for _, tag := range []string{"<div", "<img", "style="} {+ if strings.Contains(result.Text, tag) {+ t.Errorf("raw HTML markup %q leaked into text: %q", tag, result.Text)+ }+ }+ // HTML part must actually contribute its visible text: subject + plain+ // body + html body = exactly 3 occurrences of the body string.+ if c := strings.Count(result.Text, "带附件的email"); c != 3 {+ t.Errorf("expected subject+plain+html body text, got %d occurrences in: %q", c, result.Text)+ }+ if !strings.Contains(result.Text, "text_html:带附件的email") {+ t.Errorf("html body text missing from text_html: %q", result.Text)+ }++ // JSON output keeps the full metadata and raw text_html.+ p.ConfigureFromSetup(map[string]any{+ "output_format": "json",+ "fields": []string{"from", "to", "subject", "body"},+ })+ jres := p.ParseWithResult(ctx, "test.eml", []byte(raw))+ if jres.Err != nil {+ t.Fatalf("unexpected error: %v", jres.Err)+ }+ item := jres.JSON[0]+ if _, ok := item["metadata"].(map[string]any); !ok {+ t.Errorf("json output must keep metadata: %#v", item)+ }+ if v, _ := item["text_html"].(string); !strings.Contains(v, "<img") {+ t.Errorf("json output must keep raw text_html: %q", v)+ }+ if v, _ := item["subject"].(string); v != "带附件的email" {+ t.Errorf("json subject not decoded: %q", v)+ }+}+// TestEmailParser_MsgSupported parses a real Outlook .msg fixture and verifies// the Go output aligns with the Python flow parser _email() .msg branch// (rag/flow/parser/parser.py). Replaces the old "MsgNotSupported" test now that@@ -235,6 +331,43 @@ func TestEmailParser_MsgMetadataAlwaysPresent(t *testing.T) {}}+// TestEmailParser_MsgTextOutputExcludesMetadata verifies the .msg text+// output: .msg content flows through the same flatten loop as .eml, so the+// unconditionally emitted metadata map stays out of the chunkable text+// (while remaining available in the JSON output) and the basic fields are+// still flattened.+func TestEmailParser_MsgTextOutputExcludesMetadata(t *testing.T) {+ ctx := t.Context()+ data, err := os.ReadFile("testdata/sample.msg")+ if err != nil {+ t.Fatalf("read fixture: %v", err)+ }++ p := NewEmailParser() // text output; default fields+ p.ConfigureFromSetup(map[string]any{+ "fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},+ })+ result := p.ParseWithResult(ctx, "sample.msg", data)+ if result.Err != nil {+ t.Fatalf("unexpected error: %v", result.Err)+ }+ if result.OutputFormat != "text" {+ t.Fatalf("output format = %q, want text", result.OutputFormat)+ }++ if !strings.Contains(result.Text, "subject:asdf") {+ t.Errorf("text output missing subject, got %q", result.Text)+ }+ if !strings.Contains(result.Text, "from:<[email protected]>") {+ t.Errorf("text output missing from, got %q", result.Text)+ }+ for _, leaked := range []string{"metadata", "message_id", "in_reply_to"} {+ if strings.Contains(result.Text, leaked) {+ t.Errorf("text output must not contain %q, got %q", leaked, result.Text)+ }+ }+}+func TestEmailParser_Base64Attachment(t *testing.T) {ctx := t.Context()attachmentContent := "Hello! This is the decoded content of the attachment."@@ -900,11 +1033,11 @@ func TestEmailParser_NestedEMLAttachmentRechunk(t *testing.T) {t.Errorf("[json] nested body not found as separate item: %#v", result.JSON)}} else {- // Text mode: the outer email's own metadata is flattened once;- // the nested email must NOT add a second "metadata:{" segment.- if c := strings.Count(result.Text, "metadata:{"); c != 1 {- t.Errorf("[text] expected exactly 1 metadata:{ segment (outer only), got %d in: %q",- c, result.Text)+ // Text mode: metadata (transport headers) is never flattened+ // into chunkable text — not for the outer email, and not for+ // the nested re-parse (which always requests JSON output).+ if strings.Contains(result.Text, "metadata:") {+ t.Errorf("[text] metadata must not leak into text output: %q", result.Text)}}}@@ -1064,3 +1197,128 @@ func TestReadMailBody_AttachmentPreservesRaw(t *testing.T) {t.Errorf("payload = %q, want 中文 (charset-decoded)", payload)}}++// TestDecodeHeaderWord_Charsets verifies the charset routing behind RFC 2047+// encoded-word decoding. The "gb2312" label must decode plain 8-bit+// GB2312/GBK bytes — not HZ escape sequences — because real-world mail+// labeled gb2312 carries plain bytes. HZ decoding stays available under its+// own "hz-gb-2312" label. An unsupported charset leaves the value untouched.+func TestDecodeHeaderWord_Charsets(t *testing.T) {+ encodedWord := func(charset string, raw []byte) string {+ return "=?" + charset + "?B?" + base64.StdEncoding.EncodeToString(raw) + "?="+ }++ tests := []struct {+ name string+ header string+ want string+ }{+ {"gb2312 label decodes plain GB2312 bytes", encodedWord("gb2312", []byte{0xD6, 0xD0, 0xCE, 0xC4}), "中文"},+ {"gbk", encodedWord("gbk", []byte{0xD6, 0xD0, 0xCE, 0xC4}), "中文"},+ {"gb18030", encodedWord("gb18030", []byte{0xD6, 0xD0, 0xCE, 0xC4}), "中文"},+ {"big5", encodedWord("big5", []byte{0xA4, 0xA4, 0xA4, 0xE5}), "中文"},+ {"shift_jis", encodedWord("shift_jis", []byte{0x93, 0xFA, 0x96, 0x7B}), "日本"},+ {"euc-kr", encodedWord("euc-kr", []byte{0xC7, 0xD1, 0xB1, 0xB9}), "한국"},+ {"iso-8859-1", encodedWord("iso-8859-1", []byte{'c', 'a', 'f', 0xE9}), "café"},+ {"hz-gb-2312 keeps HZ escape decoding", encodedWord("hz-gb-2312", []byte("~{VPND~}")), "中文"},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := decodeHeaderWord(tt.header); got != tt.want {+ t.Errorf("decodeHeaderWord(%q) = %q, want %q", tt.header, got, tt.want)+ }+ })+ }++ unsupported := encodedWord("x-unsupported-charset", []byte{0x01, 0x02})+ if got := decodeHeaderWord(unsupported); got != unsupported {+ t.Errorf("unsupported charset: decodeHeaderWord(%q) = %q, want value untouched", unsupported, got)+ }+}++// TestParseEML_GB2312EncodedSubject is the end-to-end regression for the+// charset fix: a subject labeled gb2312 carrying plain 8-bit bytes must+// arrive decoded, not as the raw "=?gb2312?B?...?=" blob.+func TestParseEML_GB2312EncodedSubject(t *testing.T) {+ raw := strings.Join([]string{+ "From: [email protected]",+ "Subject: =?gb2312?B?" + base64.StdEncoding.EncodeToString([]byte{0xD6, 0xD0, 0xCE, 0xC4}) + "?=",+ "Content-Type: text/plain; charset=utf-8",+ "",+ "body",+ }, "\r\n")++ content := parseEML(strings.NewReader(raw), []string{"subject"})+ if content["subject"] != "中文" {+ t.Errorf("subject = %q, want 中文", content["subject"])+ }+}++// TestDecodeMailPayload_DeclaredCharsets verifies the body-side charset+// resolution: a gb2312-labeled body carries plain 8-bit bytes (routed to+// GBK like the header path, not HZGB2312), HZ-escaped content decodes only+// under its own "hz-gb-2312" label, and a declared charset beyond the+// fallback chain (big5) decodes via charsetEncoding instead of degrading to+// latin-1 mojibake.+func TestDecodeMailPayload_DeclaredCharsets(t *testing.T) {+ if got := decodeMailPayload([]byte{0xD6, 0xD0, 0xCE, 0xC4}, "gb2312"); got != "中文" {+ t.Errorf("gb2312 body = %q, want 中文", got)+ }+ if got := decodeMailPayload([]byte("~{VPND~}"), "hz-gb-2312"); got != "中文" {+ t.Errorf("hz-gb-2312 body = %q, want 中文", got)+ }+ if got := decodeMailPayload([]byte{0xA4, 0xA4, 0xA4, 0xE5}, "big5"); got != "中文" {+ t.Errorf("big5 body = %q, want 中文", got)+ }+}++// TestReadMailBody_DeclaredBodyCharsets locks the body-path charset routing+// exercised end to end by readMailBody (readMailBody → decodeMailPayload →+// decodeWithCharset), beyond the direct decodeMailPayload unit above: a+// gb2312-declared body part decodes its plain 8-bit bytes, and HZ-escaped+// content decodes under its own "hz-gb-2312" label.+func TestReadMailBody_DeclaredBodyCharsets(t *testing.T) {+ tests := []struct {+ name string+ charset string+ body []byte+ }{+ {"gb2312 body decodes plain bytes", "gb2312", []byte{0xD6, 0xD0, 0xCE, 0xC4}},+ {"hz-gb-2312 body decodes HZ escapes", "hz-gb-2312", []byte("~{VPND~}")},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ text, _, _ := readMailBody(strings.NewReader(string(tt.body)), "text/plain; charset="+tt.charset, false)+ if text != "中文" {+ t.Errorf("readMailBody(charset=%s) = %q, want 中文", tt.charset, text)+ }+ })+ }+}++// TestHTMLBodyToText verifies the visible-text flattening of email HTML+// bodies, above all that text preceding a nested block-level element starts+// a new line instead of fusing with the block's own text.+func TestHTMLBodyToText(t *testing.T) {+ tests := []struct {+ name string+ body string+ want string+ }{+ {"text before nested block", `<div>prefix<p>paragraph</p></div>`, "prefix\nparagraph"},+ {"nested divs", `<div>outer<div>inner</div></div>`, "outer\ninner"},+ {"sibling blocks", `<p>one</p><p>two</p>`, "one\ntwo"},+ {"layout table cells", `<table><tr><td>a</td><td>b</td></tr></table>`, "a\nb"},+ {"multi-row layout table stays line-separated", `<p>intro</p><table><tr><td>a</td><td>b</td></tr><tr><td>c</td></tr></table><p>outro</p>`, "intro\na\nb\nc\noutro"},+ {"inline tags stay on one line", `<p>Hello <b>world</b></p>`, "Hello world"},+ {"br forces a break", `one<br>two`, "one\ntwo"},+ {"head script style skipped", `<html><head><title>t</title><style>.x{color:red}</style></head><body><p>visible</p><script>var x=1;</script></body></html>`, "visible"},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := htmlBodyToText(tt.body); got != tt.want {+ t.Errorf("htmlBodyToText(%q) = %q, want %q", tt.body, got, tt.want)+ }+ })+ }+}