RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Decode a single-part email body by its Content-Transfer-Encoding (#19178)
Yash Raj Pandey05168a37fa41
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex 70923ae4f9..99829273ef 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -284,7 +284,8 @@ func parseEML(r io.Reader, fields []string) map[string]any {// fields while attachments are still extracted when "attachments" is.if target["body"] || needAttachments {contentType := msg.Header.Get("Content-Type")- bodyText, bodyHTML, attachments := readMailBody(msg.Body, contentType, needAttachments)+ cte := msg.Header.Get("Content-Transfer-Encoding")+ bodyText, bodyHTML, attachments := readMailBody(msg.Body, contentType, cte, needAttachments)// Always emit text/text_html when "body" is requested, to match the// Python flow parser contract (rag/flow/parser/parser.py:_email),// which sets both unconditionally (empty string for a missing part)@@ -306,7 +307,7 @@ func parseEML(r io.Reader, fields []string) map[string]any {// types. Returns (textBody, htmlBody, attachments).// When collectAttachments is true, non-text parts with Content-Disposition// starting with "attachment" are collected.-func readMailBody(body io.Reader, contentType string, collectAttachments bool) (string, string, []map[string]any) {+func readMailBody(body io.Reader, contentType, cte string, collectAttachments bool) (string, string, []map[string]any) {var attachments []map[string]anymediaType, params, err := mime.ParseMediaType(contentType)@@ -316,6 +317,7 @@ func readMailBody(body io.Reader, contentType string, collectAttachments bool) (if !strings.HasPrefix(mediaType, "multipart/") {raw, _ := io.ReadAll(body)+ raw = decodeCTE(raw, cte)decoded := decodeMailPayload(raw, params["charset"])if mediaType == "text/html" {return "", decoded, attachments@@ -326,7 +328,7 @@ func readMailBody(body io.Reader, contentType string, collectAttachments bool) (boundary := params["boundary"]if boundary == "" {raw, _ := io.ReadAll(body)- return decodeMailPayload(raw, ""), "", attachments+ return decodeMailPayload(decodeCTE(raw, cte), ""), "", attachments}mr := multipart.NewReader(body, boundary)@@ -343,7 +345,9 @@ func readMailBody(body io.Reader, contentType string, collectAttachments bool) (partMedia, partParams, _ := mime.ParseMediaType(partCT)if strings.HasPrefix(partMedia, "multipart/") {- t, h, nestedAttachments := readMailBody(part, partCT, collectAttachments)+ // RFC 2045 6.4 forbids base64 and quoted-printable on a multipart+ // entity, so a nested container is never CTE-decoded.+ t, h, nestedAttachments := readMailBody(part, partCT, "", collectAttachments)if t != "" {textParts = append(textParts, t)}diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.goindex e58763ec45..3aa93711aa 100644--- a/internal/parser/parser/email_parser_test.go+++ b/internal/parser/parser/email_parser_test.go@@ -1191,7 +1191,7 @@ func TestReadMailBody_AttachmentPreservesRaw(t *testing.T) {ap.Write([]byte(base64.StdEncoding.EncodeToString(gbk)))mw.Close()- _, _, attachments := readMailBody(strings.NewReader(buf.String()), "multipart/mixed; boundary="+mw.Boundary(), true)+ _, _, attachments := readMailBody(strings.NewReader(buf.String()), "multipart/mixed; boundary="+mw.Boundary(), "", true)if len(attachments) != 1 {t.Fatalf("expected 1 attachment, got %d: %#v", len(attachments), attachments)}@@ -1294,7 +1294,7 @@ func TestReadMailBody_DeclaredBodyCharsets(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- text, _, _ := readMailBody(strings.NewReader(string(tt.body)), "text/plain; charset="+tt.charset, false)+ text, _, _ := readMailBody(strings.NewReader(string(tt.body)), "text/plain; charset="+tt.charset, "", false)if text != "中文" {t.Errorf("readMailBody(charset=%s) = %q, want 中文", tt.charset, text)}@@ -1302,6 +1302,104 @@ func TestReadMailBody_DeclaredBodyCharsets(t *testing.T) {}}+// TestParseEML_SinglePartTransferEncoding verifies that a single-part body+// is decoded by its Content-Transfer-Encoding, as a multipart part already is.+func TestParseEML_SinglePartTransferEncoding(t *testing.T) {+ const plain = "Quarterly revenue was 12.5 million.\n"+ b64 := base64.StdEncoding.EncodeToString([]byte(plain))++ const html = "<p>Quarterly revenue</p>"+ htmlB64 := base64.StdEncoding.EncodeToString([]byte(html))++ tests := []struct {+ name string+ headers string+ body string+ wantText string+ wantHTML string+ // encoded is the payload as it appears on the wire. The text output+ // feeds the chunker, so it must never carry these bytes.+ encoded string+ // wantInText is the visible text the flattened output must carry.+ wantInText string+ }{+ {+ name: "base64 text/plain",+ headers: "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64",+ body: b64,+ wantText: plain,+ encoded: b64,+ wantInText: "Quarterly revenue was 12.5 million.",+ },+ {+ name: "quoted-printable text/plain",+ headers: "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: quoted-printable",+ body: "Quarterly revenue was 12=2E5 million=2E",+ wantText: "Quarterly revenue was 12.5 million.",+ encoded: "12=2E5",+ wantInText: "Quarterly revenue was 12.5 million.",+ },+ {+ name: "base64 text/html",+ headers: "Content-Type: text/html; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64",+ body: htmlB64,+ wantHTML: html,+ encoded: htmlB64,+ wantInText: "Quarterly revenue",+ },+ {+ // Positive control: the multipart path already decodes the same+ // body, so the two paths must agree.+ name: "base64 in multipart/alternative",+ headers: "Content-Type: multipart/alternative; boundary=\"BB\"",+ body: "--BB\r\nContent-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n" + b64 + "\r\n--BB--",+ wantText: plain,+ encoded: b64,+ wantInText: "Quarterly revenue was 12.5 million.",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ raw := "From: [email protected]\r\nTo: [email protected]\r\n" ++ "Subject: Q3\r\n" + tt.headers + "\r\n\r\n" + tt.body + "\r\n"++ p := NewEmailParser()+ p.ConfigureFromSetup(map[string]any{"output_format": "json"})+ result := p.ParseWithResult(t.Context(), "test.eml", []byte(raw))+ if result.Err != nil {+ t.Fatalf("unexpected error: %v", result.Err)+ }+ item := result.JSON[0]++ text, _ := item["text"].(string)+ if strings.TrimRight(text, "\r\n") != strings.TrimRight(tt.wantText, "\r\n") {+ t.Errorf("text = %q, want %q", text, tt.wantText)+ }+ gotHTML, _ := item["text_html"].(string)+ if strings.TrimRight(gotHTML, "\r\n") != strings.TrimRight(tt.wantHTML, "\r\n") {+ t.Errorf("text_html = %q, want %q", gotHTML, tt.wantHTML)+ }++ // The text output is what reaches the chunker, and the JSON+ // output returns before it is built, so assert it separately.+ pt := NewEmailParser()+ pt.ConfigureFromSetup(map[string]any{"output_format": "text"})+ textResult := pt.ParseWithResult(t.Context(), "test.eml", []byte(raw))+ if textResult.Err != nil {+ t.Fatalf("unexpected error on the text path: %v", textResult.Err)+ }+ flat := textResult.Text+ if !strings.Contains(flat, tt.wantInText) {+ t.Errorf("text output = %q, want it to contain %q", flat, tt.wantInText)+ }+ if strings.Contains(flat, tt.encoded) {+ t.Errorf("text output still carries the encoded payload %q: %q", tt.encoded, flat)+ }+ })+ }+}+// TestHTMLBodyToText verifies the visible-text flattening of email HTML// bodies, above all that text preceding a nested block-level element starts// a new line instead of fusing with the block's own text.