RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Remove dead ParseResult.HTML field and redundant EmailParser json-path Text (#20992)
xugangqiang66f15c779d01
diff --git a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go b/internal/ingestion/component/chunker/xlsx_qa_regression_test.goindex 719e6c7a02..946983df85 100644--- a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go+++ b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go@@ -49,8 +49,6 @@ func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {switch res.OutputFormat {case "json":inputs["json"] = res.JSON- case "html":- inputs["html"] = res.HTML}comp, err := NewQAChunker(nil)diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex 6d45b28cc9..579873cc79 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -614,9 +614,6 @@ func resolveParserNormalizationSource(dispatched parser.ParseResult) string {if dispatched.Markdown != "" {return "markdown"}- if dispatched.HTML != "" {- return "html"- }if dispatched.Text != "" {return "text"}diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.goindex 3cc586e942..1d082e6818 100644--- a/internal/ingestion/component/parser_dispatch.go+++ b/internal/ingestion/component/parser_dispatch.go@@ -348,18 +348,6 @@ func normalizeParserJSON(ctx context.Context, filename string, dispatched parserif dispatched.Markdown != "" {return parseMarkdownToJSONItems(ctx, filename, dispatched.Markdown)}- if dispatched.HTML != "" {- res := parser.NewHTMLParser().ParseWithResult(ctx, filename, []byte(dispatched.HTML))- if res.Err == nil && len(res.JSON) > 0 {- return res.JSON- }- if res.Err != nil {- warnParserNormalizationFallback(filename, "html", res.Err)- } else {- warnParserNormalizationFallback(filename, "html", fmt.Errorf("parser returned no JSON items"))- }- return []map[string]any{parser.NewTextJSONItem(dispatched.HTML)}- }if dispatched.Text != "" {return []map[string]any{parser.NewTextJSONItem(dispatched.Text)}}diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex 61c1cc0b75..470b5a5384 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -108,14 +108,6 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {},wantText: "Title",},- {- name: "html",- dispatched: parser.ParseResult{- OutputFormat: "html",- HTML: "<h1>Title</h1><p>Body</p>",- },- wantText: "Title",- },{name: "text",dispatched: parser.ParseResult{@@ -158,6 +150,22 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {}}+// TestNormalizeParserJSON_EmptyPayloadYieldsEmpty pins that a result with no+// populated payload family normalizes to an empty JSON slice rather than+// injecting a bogus item. HTML is not a supported backend output format+// (PR #19467 unified the component to JSON-only; no Go parser produces an+// HTML payload), so there is no HTML normalization branch to fall through+// to — this guards that regression.+func TestNormalizeParserJSON_EmptyPayloadYieldsEmpty(t *testing.T) {+ r := parser.ParseResult{+ OutputFormat: "json",+ }+ items := normalizeParserJSON(t.Context(), "doc.txt", r)+ if len(items) != 0 {+ t.Fatalf("normalizeParserJSON(empty) = %d items, want 0 (no payload → empty)", len(items))+ }+}+func TestBuildParserOutputsFallsBackWhenJSONIsEmpty(t *testing.T) {out := buildParserOutputs(t.Context(), parser.ParseResult{OutputFormat: "json",diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex fe2c10430c..ccfad86dbb 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -206,7 +206,6 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []byOutputFormat: "json",File: map[string]any{"name": filename},JSON: items,- Text: text,}}diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.goindex acc1d07e4c..1ffd9012a5 100644--- a/internal/parser/parser/email_parser_test.go+++ b/internal/parser/parser/email_parser_test.go@@ -399,15 +399,27 @@ func TestEmailParser_MsgTextOutputExcludesMetadata(t *testing.T) {t.Fatalf("expected non-empty JSON items")}- if !strings.Contains(result.Text, "subject:asdf") {- t.Errorf("text output missing subject, got %q", result.Text)+ // The json path no longer duplicates the flattened body into+ // result.Text; the searchable text lives in the JSON items instead+ // (the header text item from searchableEmailHeaders). Assert on the+ // concatenated item text so the test survives the json-path Text removal.+ var sb strings.Builder+ for _, it := range result.JSON {+ if txt, _ := it["text"].(string); txt != "" {+ sb.WriteString(txt)+ sb.WriteString("\n")+ }+ }+ joined := sb.String()+ if !strings.Contains(joined, "subject:asdf") {+ t.Errorf("json output missing subject in item text, got %q", joined)}- if !strings.Contains(result.Text, "from:<[email protected]>") {- t.Errorf("text output missing from, got %q", result.Text)+ if !strings.Contains(joined, "from:<[email protected]>") {+ t.Errorf("json output missing from in item text, got %q", joined)}for _, leaked := range []string{"metadata", "message_id", "in_reply_to"} {- if strings.Contains(result.Text, leaked) {- t.Errorf("text output must not contain %q, got %q", leaked, result.Text)+ if strings.Contains(joined, leaked) {+ t.Errorf("json item text must not contain %q, got %q", leaked, joined)}}}@@ -1464,3 +1476,59 @@ func TestHTMLBodyToText(t *testing.T) {})}}++// TestEmailParserJSONPathOmitsDuplicateText verifies that the JSON output+// path no longer duplicates the flattened email body into result.Text.+// result.Text is only populated on the "text" output_format branch; on the+// JSON branch the searchable text lives in the JSON items (body + the+// header text item from searchableEmailHeaders), so the redundant json-path+// Text is intentionally empty. This guards the removal of the json-path+// Text assignment in parseEmail.+func TestEmailParserJSONPathOmitsDuplicateText(t *testing.T) {+ p := NewEmailParser() // default output_format -> "json"+ p.ConfigureFromSetup(map[string]any{+ "fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},+ })+ data, err := os.ReadFile("testdata/sample.msg")+ if err != nil {+ t.Fatalf("read fixture: %v", err)+ }+ res := p.ParseWithResult(t.Context(), "sample.msg", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if res.OutputFormat != "json" {+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)+ }+ if len(res.JSON) == 0 {+ t.Fatal("JSON: want non-empty items")+ }+ if res.Text != "" {+ t.Errorf("JSON-path Email result.Text = %q, want empty (text lives in JSON items)", res.Text)+ }+}++// TestEmailParserTextPathKeepsText guards that the explicit "text"+// output_format branch still populates result.Text — the only place the+// field is read downstream (sys.files parseResultText, document preview,+// and nested attachment re-parse).+func TestEmailParserTextPathKeepsText(t *testing.T) {+ p := NewEmailParser()+ p.ConfigureFromSetup(map[string]any{"output_format": "text"})+ const eml = "From: [email protected]\r\n" ++ "To: [email protected]\r\n" ++ "Subject: Hi\r\n" ++ "Date: Mon, 01 Jan 2024 00:00:00 +0000\r\n" ++ "\r\n" ++ "Hello body.\r\n"+ res := p.ParseWithResult(t.Context(), "msg.eml", []byte(eml))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if res.OutputFormat != "text" {+ t.Fatalf("OutputFormat = %q, want text", res.OutputFormat)+ }+ if res.Text == "" {+ t.Error("text-path Email result.Text empty, want body text")+ }+}diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.goindex 655bfd88e7..09ba17d6ff 100644--- a/internal/parser/parser/parse_result.go+++ b/internal/parser/parser/parse_result.go@@ -65,7 +65,7 @@ type ParseResult struct {// Shape depends on the parser family: PDF emits// `[]map[string]any` with `text` + `doc_type_kwd` keys (and// optional `image` / `layout` / `positions` fields);- // Markdown / HTML / text emit normalized+ // Markdown / text emit normalized// `{text, doc_type_kwd}` items; image emits an image payload and, after// ingestion enhancement, its VLM description.JSON []map[string]any@@ -76,9 +76,6 @@ type ParseResult struct {// Text is a backend response awaiting normalization.Text string- // HTML is a backend response awaiting normalization.- HTML string-// Err is the failure reason. On non-nil Err, all payload// fields are zero values.Err errordiff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.goindex f10dfad3c1..10d6caf0d1 100644--- a/internal/parser/parser/parse_result_test.go+++ b/internal/parser/parser/parse_result_test.go@@ -94,14 +94,6 @@ func TestParseResult_Contract(t *testing.T) {},wantFmt: "text",},- {- name: "html family only",- in: ParseResult{- OutputFormat: "html",- HTML: "<p>x</p>",- },- wantFmt: "html",- },{name: "failure clears payload",in: ParseResult{Err: errSentinel},@@ -117,7 +109,7 @@ func TestParseResult_Contract(t *testing.T) {if tc.in.OutputFormat != "" {t.Errorf("OutputFormat on failure = %q, want empty", tc.in.OutputFormat)}- if tc.in.JSON != nil || tc.in.Markdown != "" || tc.in.Text != "" || tc.in.HTML != "" {+ if tc.in.JSON != nil || tc.in.Markdown != "" || tc.in.Text != "" {t.Errorf("payload fields must be zero on failure; got %+v", tc.in)}return@@ -215,13 +207,11 @@ func payloadFamily(r ParseResult) string {return "markdown"case "text":return "text"- case "html":- return "html"}return ""}-var allPayloadFamilies = []string{"json", "markdown", "text", "html"}+var allPayloadFamilies = []string{"json", "markdown", "text"}func isPayloadZero(r ParseResult, family string) bool {switch family {@@ -231,8 +221,6 @@ func isPayloadZero(r ParseResult, family string) bool {return r.Markdown == ""case "text":return r.Text == ""- case "html":- return r.HTML == ""}return false}diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.goindex 8287eed373..60cde67bd4 100644--- a/internal/service/file/file_content.go+++ b/internal/service/file/file_content.go@@ -231,8 +231,6 @@ func parseResultText(res parser.ParseResult) (string, error) {return res.Text, nilcase "markdown":return res.Markdown, nil- case "html":- return res.HTML, nilcase "json":if len(res.JSON) > 0 {parts := make([]string, 0, len(res.JSON))@@ -256,7 +254,7 @@ func parseResultText(res parser.ParseResult) (string, error) {// Some legacy parsers mark the result as JSON while only populating a// rendered companion field. Preserve that content instead of returning// an empty sys.files value.- for _, fallback := range []string{res.Markdown, res.HTML, res.Text} {+ for _, fallback := range []string{res.Markdown, res.Text} {if fallback != "" {return fallback, nil}