RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): write an email's text output in a fixed field order (#21100)
Lukasfbae305f3eb1
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex e34d3c60a8..c6eec770b3 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -27,6 +27,7 @@ import ("mime/quotedprintable""net/mail""path/filepath"+ "slices""strings""time"@@ -140,7 +141,8 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by// Text representation: flatten fields into a single string for explicit// text output and callers that need a display representation.var sb strings.Builder- for k, v := range content {+ for _, k := range emailTextKeys(content) {+ v := content[k]// The metadata map (every non-basic header: Received chains,// DKIM/ARC signatures, …) stays available in the JSON output, but// flattening it into chunkable text buries the message under@@ -209,6 +211,30 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by}}+// emailTextFields is the order the text output writes an email's fields in:+// the headers in the order searchableEmailHeaders gives them, then the body.+var emailTextFields = []string{"from", "to", "cc", "bcc", "date", "subject", "text", "text_html"}++// emailTextKeys returns content's keys in emailTextFields order, followed by+// any other key in sorted order. Ranging over the map instead wrote the+// fields in a different order on every parse of the same message.+func emailTextKeys(content map[string]any) []string {+ keys := make([]string, 0, len(content))+ for _, k := range emailTextFields {+ if _, ok := content[k]; ok {+ keys = append(keys, k)+ }+ }+ var rest []string+ for k := range content {+ if !slices.Contains(emailTextFields, k) {+ rest = append(rest, k)+ }+ }+ slices.Sort(rest)+ return append(keys, rest...)+}+// searchableEmailHeaders returns the user-facing headers as a separate JSON// text item. The main email item keeps its structured fields and body; the// chunker indexes only item text, so headers need their own text item.diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.goindex aafde00560..dc6df40941 100644--- a/internal/parser/parser/email_parser_test.go+++ b/internal/parser/parser/email_parser_test.go@@ -111,6 +111,44 @@ func TestEmailParser_EmlJSON(t *testing.T) {}}+// TestEmailParser_EmlTextFieldOrder pins the order of the text output: the+// headers first, in the order of the JSON header item, then the body. The+// fields used to come out in Go's map order, a different one on every parse.+func TestEmailParser_EmlTextFieldOrder(t *testing.T) {+ raw := strings.Join([]string{+ "From: [email protected]",+ "To: [email protected]",+ "Cc: [email protected]",+ "Subject: Quarterly numbers",+ "Date: Mon, 1 Sep 2026 10:00:00 +0000",+ "Content-Type: text/plain; charset=utf-8",+ "",+ "Revenue grew.",+ }, "\r\n")+ want := strings.Join([]string{+ "from:[email protected]",+ "to:[email protected]",+ "cc:[email protected]",+ "date:Mon, 1 Sep 2026 10:00:00 +0000",+ "subject:Quarterly numbers",+ "text:Revenue grew.",+ "text_html:",+ "",+ }, "\n")++ for i := 0; i < 20; i++ {+ p := NewEmailParser()+ p.ConfigureFromSetup(map[string]any{"output_format": "text"})+ result := p.ParseWithResult(t.Context(), "order.eml", []byte(raw))+ if result.Err != nil {+ t.Fatalf("unexpected error: %v", result.Err)+ }+ if result.Text != want {+ t.Fatalf("parse %d: text output\n got: %q\nwant: %q", i, result.Text, want)+ }+ }+}+func TestEmailParser_InvalidEMLReturnsError(t *testing.T) {p := NewEmailParser()result := p.ParseWithResult(t.Context(), "broken.eml", []byte("not a valid RFC 5322 header"))