Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): write an email's text output in a fixed field order (#21100)

Lukasfbae305f3eb1
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go
index e34d3c60a8..c6eec770b3 100644
--- a/internal/parser/parser/email_parser.go
+++ b/internal/parser/parser/email_parser.go
@@ -27,6 +27,7 @@ import (
"mime/quotedprintable"
"net/mail"
"path/filepath"
+ "slices"
"strings"
"time"
@@ -140,7 +141,8 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by
// Text representation: flatten fields into a single string for explicit
// text output and callers that need a display representation.
var sb strings.Builder
- for k, v := range content {
+ for _, k := range emailTextKeys(content) {
+ v := content[k]
// The metadata map (every non-basic header: Received chains,
// DKIM/ARC signatures, …) stays available in the JSON output, but
// flattening it into chunkable text buries the message under
@@ -209,6 +211,30 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by
}
}
+// emailTextFields is the order the text output writes an email's fields in:
+// the headers in the order searchableEmailHeaders gives them, then the body.
+var emailTextFields = []string{"from", "to", "cc", "bcc", "date", "subject", "text", "text_html"}
+
+// emailTextKeys returns content's keys in emailTextFields order, followed by
+// any other key in sorted order. Ranging over the map instead wrote the
+// fields in a different order on every parse of the same message.
+func emailTextKeys(content map[string]any) []string {
+ keys := make([]string, 0, len(content))
+ for _, k := range emailTextFields {
+ if _, ok := content[k]; ok {
+ keys = append(keys, k)
+ }
+ }
+ var rest []string
+ for k := range content {
+ if !slices.Contains(emailTextFields, k) {
+ rest = append(rest, k)
+ }
+ }
+ slices.Sort(rest)
+ return append(keys, rest...)
+}
+
// searchableEmailHeaders returns the user-facing headers as a separate JSON
// text item. The main email item keeps its structured fields and body; the
// chunker indexes only item text, so headers need their own text item.
diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go
index aafde00560..dc6df40941 100644
--- a/internal/parser/parser/email_parser_test.go
+++ b/internal/parser/parser/email_parser_test.go
@@ -111,6 +111,44 @@ func TestEmailParser_EmlJSON(t *testing.T) {
}
}
+// TestEmailParser_EmlTextFieldOrder pins the order of the text output: the
+// headers first, in the order of the JSON header item, then the body. The
+// fields used to come out in Go's map order, a different one on every parse.
+func TestEmailParser_EmlTextFieldOrder(t *testing.T) {
+ raw := strings.Join([]string{
+ "Subject: Quarterly numbers",
+ "Date: Mon, 1 Sep 2026 10:00:00 +0000",
+ "Content-Type: text/plain; charset=utf-8",
+ "",
+ "Revenue grew.",
+ }, "\r\n")
+ want := strings.Join([]string{
+ "date:Mon, 1 Sep 2026 10:00:00 +0000",
+ "subject:Quarterly numbers",
+ "text:Revenue grew.",
+ "text_html:",
+ "",
+ }, "\n")
+
+ for i := 0; i < 20; i++ {
+ p := NewEmailParser()
+ p.ConfigureFromSetup(map[string]any{"output_format": "text"})
+ result := p.ParseWithResult(t.Context(), "order.eml", []byte(raw))
+ if result.Err != nil {
+ t.Fatalf("unexpected error: %v", result.Err)
+ }
+ if result.Text != want {
+ t.Fatalf("parse %d: text output\n got: %q\nwant: %q", i, result.Text, want)
+ }
+ }
+}
+
func TestEmailParser_InvalidEMLReturnsError(t *testing.T) {
p := NewEmailParser()
result := p.ParseWithResult(t.Context(), "broken.eml", []byte("not a valid RFC 5322 header"))