Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Remove dead ParseResult.HTML field and redundant EmailParser json-path Text (#20992)

xugangqiang66f15c779d01
diff --git a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
index 719e6c7a02..946983df85 100644
--- a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
+++ b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
@@ -49,8 +49,6 @@ func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {
switch res.OutputFormat {
case "json":
inputs["json"] = res.JSON
- case "html":
- inputs["html"] = res.HTML
}
comp, err := NewQAChunker(nil)
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 6d45b28cc9..579873cc79 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -614,9 +614,6 @@ func resolveParserNormalizationSource(dispatched parser.ParseResult) string {
if dispatched.Markdown != "" {
return "markdown"
}
- if dispatched.HTML != "" {
- return "html"
- }
if dispatched.Text != "" {
return "text"
}
diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go
index 3cc586e942..1d082e6818 100644
--- a/internal/ingestion/component/parser_dispatch.go
+++ b/internal/ingestion/component/parser_dispatch.go
@@ -348,18 +348,6 @@ func normalizeParserJSON(ctx context.Context, filename string, dispatched parser
if dispatched.Markdown != "" {
return parseMarkdownToJSONItems(ctx, filename, dispatched.Markdown)
}
- if dispatched.HTML != "" {
- res := parser.NewHTMLParser().ParseWithResult(ctx, filename, []byte(dispatched.HTML))
- if res.Err == nil && len(res.JSON) > 0 {
- return res.JSON
- }
- if res.Err != nil {
- warnParserNormalizationFallback(filename, "html", res.Err)
- } else {
- warnParserNormalizationFallback(filename, "html", fmt.Errorf("parser returned no JSON items"))
- }
- return []map[string]any{parser.NewTextJSONItem(dispatched.HTML)}
- }
if dispatched.Text != "" {
return []map[string]any{parser.NewTextJSONItem(dispatched.Text)}
}
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 61c1cc0b75..470b5a5384 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -108,14 +108,6 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {
},
wantText: "Title",
},
- {
- name: "html",
- dispatched: parser.ParseResult{
- OutputFormat: "html",
- HTML: "<h1>Title</h1><p>Body</p>",
- },
- wantText: "Title",
- },
{
name: "text",
dispatched: parser.ParseResult{
@@ -158,6 +150,22 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {
}
}
+// TestNormalizeParserJSON_EmptyPayloadYieldsEmpty pins that a result with no
+// populated payload family normalizes to an empty JSON slice rather than
+// injecting a bogus item. HTML is not a supported backend output format
+// (PR #19467 unified the component to JSON-only; no Go parser produces an
+// HTML payload), so there is no HTML normalization branch to fall through
+// to — this guards that regression.
+func TestNormalizeParserJSON_EmptyPayloadYieldsEmpty(t *testing.T) {
+ r := parser.ParseResult{
+ OutputFormat: "json",
+ }
+ items := normalizeParserJSON(t.Context(), "doc.txt", r)
+ if len(items) != 0 {
+ t.Fatalf("normalizeParserJSON(empty) = %d items, want 0 (no payload → empty)", len(items))
+ }
+}
+
func TestBuildParserOutputsFallsBackWhenJSONIsEmpty(t *testing.T) {
out := buildParserOutputs(t.Context(), parser.ParseResult{
OutputFormat: "json",
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go
index fe2c10430c..ccfad86dbb 100644
--- a/internal/parser/parser/email_parser.go
+++ b/internal/parser/parser/email_parser.go
@@ -206,7 +206,6 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by
OutputFormat: "json",
File: map[string]any{"name": filename},
JSON: items,
- Text: text,
}
}
diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go
index acc1d07e4c..1ffd9012a5 100644
--- a/internal/parser/parser/email_parser_test.go
+++ b/internal/parser/parser/email_parser_test.go
@@ -399,15 +399,27 @@ func TestEmailParser_MsgTextOutputExcludesMetadata(t *testing.T) {
t.Fatalf("expected non-empty JSON items")
}
- if !strings.Contains(result.Text, "subject:asdf") {
- t.Errorf("text output missing subject, got %q", result.Text)
+ // The json path no longer duplicates the flattened body into
+ // result.Text; the searchable text lives in the JSON items instead
+ // (the header text item from searchableEmailHeaders). Assert on the
+ // concatenated item text so the test survives the json-path Text removal.
+ var sb strings.Builder
+ for _, it := range result.JSON {
+ if txt, _ := it["text"].(string); txt != "" {
+ sb.WriteString(txt)
+ sb.WriteString("\n")
+ }
+ }
+ joined := sb.String()
+ if !strings.Contains(joined, "subject:asdf") {
+ t.Errorf("json output missing subject in item text, got %q", joined)
}
- if !strings.Contains(result.Text, "from:<[email protected]>") {
- t.Errorf("text output missing from, got %q", result.Text)
+ if !strings.Contains(joined, "from:<[email protected]>") {
+ t.Errorf("json output missing from in item text, got %q", joined)
}
for _, leaked := range []string{"metadata", "message_id", "in_reply_to"} {
- if strings.Contains(result.Text, leaked) {
- t.Errorf("text output must not contain %q, got %q", leaked, result.Text)
+ if strings.Contains(joined, leaked) {
+ t.Errorf("json item text must not contain %q, got %q", leaked, joined)
}
}
}
@@ -1464,3 +1476,59 @@ func TestHTMLBodyToText(t *testing.T) {
})
}
}
+
+// TestEmailParserJSONPathOmitsDuplicateText verifies that the JSON output
+// path no longer duplicates the flattened email body into result.Text.
+// result.Text is only populated on the "text" output_format branch; on the
+// JSON branch the searchable text lives in the JSON items (body + the
+// header text item from searchableEmailHeaders), so the redundant json-path
+// Text is intentionally empty. This guards the removal of the json-path
+// Text assignment in parseEmail.
+func TestEmailParserJSONPathOmitsDuplicateText(t *testing.T) {
+ p := NewEmailParser() // default output_format -> "json"
+ p.ConfigureFromSetup(map[string]any{
+ "fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},
+ })
+ data, err := os.ReadFile("testdata/sample.msg")
+ if err != nil {
+ t.Fatalf("read fixture: %v", err)
+ }
+ res := p.ParseWithResult(t.Context(), "sample.msg", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if res.OutputFormat != "json" {
+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ }
+ if len(res.JSON) == 0 {
+ t.Fatal("JSON: want non-empty items")
+ }
+ if res.Text != "" {
+ t.Errorf("JSON-path Email result.Text = %q, want empty (text lives in JSON items)", res.Text)
+ }
+}
+
+// TestEmailParserTextPathKeepsText guards that the explicit "text"
+// output_format branch still populates result.Text — the only place the
+// field is read downstream (sys.files parseResultText, document preview,
+// and nested attachment re-parse).
+func TestEmailParserTextPathKeepsText(t *testing.T) {
+ p := NewEmailParser()
+ p.ConfigureFromSetup(map[string]any{"output_format": "text"})
+ const eml = "From: [email protected]\r\n" +
+ "To: [email protected]\r\n" +
+ "Subject: Hi\r\n" +
+ "Date: Mon, 01 Jan 2024 00:00:00 +0000\r\n" +
+ "\r\n" +
+ "Hello body.\r\n"
+ res := p.ParseWithResult(t.Context(), "msg.eml", []byte(eml))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if res.OutputFormat != "text" {
+ t.Fatalf("OutputFormat = %q, want text", res.OutputFormat)
+ }
+ if res.Text == "" {
+ t.Error("text-path Email result.Text empty, want body text")
+ }
+}
diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.go
index 655bfd88e7..09ba17d6ff 100644
--- a/internal/parser/parser/parse_result.go
+++ b/internal/parser/parser/parse_result.go
@@ -65,7 +65,7 @@ type ParseResult struct {
// Shape depends on the parser family: PDF emits
// `[]map[string]any` with `text` + `doc_type_kwd` keys (and
// optional `image` / `layout` / `positions` fields);
- // Markdown / HTML / text emit normalized
+ // Markdown / text emit normalized
// `{text, doc_type_kwd}` items; image emits an image payload and, after
// ingestion enhancement, its VLM description.
JSON []map[string]any
@@ -76,9 +76,6 @@ type ParseResult struct {
// Text is a backend response awaiting normalization.
Text string
- // HTML is a backend response awaiting normalization.
- HTML string
-
// Err is the failure reason. On non-nil Err, all payload
// fields are zero values.
Err error
diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.go
index f10dfad3c1..10d6caf0d1 100644
--- a/internal/parser/parser/parse_result_test.go
+++ b/internal/parser/parser/parse_result_test.go
@@ -94,14 +94,6 @@ func TestParseResult_Contract(t *testing.T) {
},
wantFmt: "text",
},
- {
- name: "html family only",
- in: ParseResult{
- OutputFormat: "html",
- HTML: "<p>x</p>",
- },
- wantFmt: "html",
- },
{
name: "failure clears payload",
in: ParseResult{Err: errSentinel},
@@ -117,7 +109,7 @@ func TestParseResult_Contract(t *testing.T) {
if tc.in.OutputFormat != "" {
t.Errorf("OutputFormat on failure = %q, want empty", tc.in.OutputFormat)
}
- if tc.in.JSON != nil || tc.in.Markdown != "" || tc.in.Text != "" || tc.in.HTML != "" {
+ if tc.in.JSON != nil || tc.in.Markdown != "" || tc.in.Text != "" {
t.Errorf("payload fields must be zero on failure; got %+v", tc.in)
}
return
@@ -215,13 +207,11 @@ func payloadFamily(r ParseResult) string {
return "markdown"
case "text":
return "text"
- case "html":
- return "html"
}
return ""
}
-var allPayloadFamilies = []string{"json", "markdown", "text", "html"}
+var allPayloadFamilies = []string{"json", "markdown", "text"}
func isPayloadZero(r ParseResult, family string) bool {
switch family {
@@ -231,8 +221,6 @@ func isPayloadZero(r ParseResult, family string) bool {
return r.Markdown == ""
case "text":
return r.Text == ""
- case "html":
- return r.HTML == ""
}
return false
}
diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.go
index 8287eed373..60cde67bd4 100644
--- a/internal/service/file/file_content.go
+++ b/internal/service/file/file_content.go
@@ -231,8 +231,6 @@ func parseResultText(res parser.ParseResult) (string, error) {
return res.Text, nil
case "markdown":
return res.Markdown, nil
- case "html":
- return res.HTML, nil
case "json":
if len(res.JSON) > 0 {
parts := make([]string, 0, len(res.JSON))
@@ -256,7 +254,7 @@ func parseResultText(res parser.ParseResult) (string, error) {
// Some legacy parsers mark the result as JSON while only populating a
// rendered companion field. Preserve that content instead of returning
// an empty sys.files value.
- for _, fallback := range []string{res.Markdown, res.HTML, res.Text} {
+ for _, fallback := range []string{res.Markdown, res.Text} {
if fallback != "" {
return fallback, nil
}