RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): remove dead ParseResult.Text field and text output path (#21334)
xugangqiangfa0bae42b037
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex fbabfd8449..279db84f74 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -619,9 +619,6 @@ func resolveParserNormalizationSource(dispatched parser.ParseResult) string {if dispatched.Markdown != "" {return "markdown"}- if dispatched.Text != "" {- return "text"- }return "raw"}diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.goindex c3e8fbe49f..85019f7498 100644--- a/internal/ingestion/component/parser_dispatch.go+++ b/internal/ingestion/component/parser_dispatch.go@@ -357,9 +357,6 @@ func normalizeParserJSON(ctx context.Context, filename string, dispatched parserif dispatched.Markdown != "" {return parseMarkdownToJSONItems(ctx, filename, dispatched.Markdown)}- if dispatched.Text != "" {- return []parser.DocItem{parser.NewTextJSONItem(dispatched.Text)}- }return []parser.DocItem{}}diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex 203ee4e489..62070b38ab 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -95,7 +95,7 @@ func requireJSONText(t *testing.T, out map[string]any, want string) {t.Fatalf("json payload does not contain %q: %#v", want, items)}-func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {+func TestBuildParserOutputsNormalizesMarkdownToJSON(t *testing.T) {tests := []struct {name stringdispatched parser.ParseResult@@ -109,14 +109,6 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {},wantText: "Title",},- {- name: "text",- dispatched: parser.ParseResult{- OutputFormat: "text",- Text: "plain body",- },- wantText: "plain body",- },}for _, tt := range tests {diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.goindex 6c4a81d785..1aff00bfac 100644--- a/internal/parser/parser/doc_parser.go+++ b/internal/parser/parser/doc_parser.go@@ -27,14 +27,10 @@ import (// docExtract is the native office_oxide extraction seam for the DOC family.var docExtract = deepdocoffice.OpenAndExtract-type DOCParser struct {- OutputFormat string-}+type DOCParser struct{}func NewDOCParser() *DOCParser {- return &DOCParser{- OutputFormat: "json",- }+ return &DOCParser{}}func (p *DOCParser) String() string {@@ -42,12 +38,9 @@ func (p *DOCParser) String() string {}func (p *DOCParser) ConfigureFromSetup(setup map[string]any) {- if p == nil || setup == nil {- return- }- if v, ok := setup["output_format"].(string); ok && v != "" {- p.OutputFormat = v- }+ // DOC always emits JSON (the DSL hardcodes output_format to "json" via+ // normalizeParserOutputFormats); the legacy text/markdown output paths were+ // removed with the dead ParseResult.Text field.}// ParseWithResult extracts text for the DOC family and converts it to the@@ -61,42 +54,17 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [return ParseResult{Err: fmt.Errorf("doc extract: %w", err)}}- text, mdPayload, err := extractDocText(irJSON, mdText, plainText)+ text, _, err := extractDocText(irJSON, mdText, plainText)if err != nil {return ParseResult{Err: fmt.Errorf("doc extract: %w", err)}}- outFmt := p.OutputFormat- if outFmt == "" {- outFmt = "json"- }-- markdownPayload := text- if strings.TrimSpace(mdPayload) != "" {- markdownPayload = mdPayload- }-- res := ParseResult{- OutputFormat: outFmt,- File: map[string]any{"name": filename, "format": "doc"},- Text: text,- Markdown: markdownPayload,- }-- if strings.EqualFold(outFmt, "markdown") {- return res- }- if strings.EqualFold(outFmt, "text") {- return res- }-- // JSON path: prefer the structured IR, which office_oxide recovers table- // (and list) structure for. Emitting structured sections lets the chunker- // render tables as HTML (#20918) instead of losing them to plain text — the- // legacy path flattened the whole document (tables included) into a single- // text item. Fall back to the flattened text only when the IR yields no- // usable sections, so a document whose IR is empty/unusable still keeps its- // content (PlainText / ToMarkdown) rather than silently producing 0 chunks.+ // The DSL hardcodes output_format to "json" (normalizeParserOutputFormats),+ // so the DOC parser always emits structured JSON items. Prefer the IR+ // sections, which recover table/list structure for the chunker's HTML-table+ // rendering (#20918); fall back to the flattened text only when the IR yields+ // no usable sections, so a document whose IR is empty/unusable still keeps+ // its content rather than silently producing 0 chunks.if strings.TrimSpace(irJSON) != "" {budget := newEmbeddedMediaBudget()if sections := buildDOCXJSONSections(irJSON, budget); len(sections) > 0 {diff --git a/internal/parser/parser/doc_parser_doc_table_test.go b/internal/parser/parser/doc_parser_doc_table_test.goindex 773bba7e69..77359b58f4 100644--- a/internal/parser/parser/doc_parser_doc_table_test.go+++ b/internal/parser/parser/doc_parser_doc_table_test.go@@ -89,24 +89,53 @@ func TestDOCParser_TableEmittedAsStructuredItem(t *testing.T) {// TestDOCParser_NoTableDocStaysTextItem pins the fallback for a .doc whose IR// yields no structured sections: the parser must still emit the flattened text// as a single text item (never silently drop content to 0 chunks). The fixture-// is the same table.doc driven through the text output format, which bypasses-// the IR sections path and must keep the legacy behavior intact.-func TestDOCParser_TextOutputFormatStaysText(t *testing.T) {+// is the same table.doc; with the legacy text output path removed the parser+// only emits JSON, so the fallback is exercised through the JSON item below.+func TestDOCParser_NoTableDocStaysTextItem(t *testing.T) {data, err := os.ReadFile(filepath.Join("testdata", "table.doc"))if err != nil {t.Fatalf("read fixture: %v", err)}p := NewDOCParser()- p.OutputFormat = "text"res := p.ParseWithResult(context.Background(), "table.doc", data)if res.Err != nil {t.Fatalf("ParseWithResult error: %v", res.Err)}- if res.OutputFormat != "text" {- t.Fatalf("OutputFormat = %q, want text", res.OutputFormat)+ if res.OutputFormat != "json" {+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)+ }+ if len(res.JSON) == 0 {+ t.Fatalf("doc with no structured sections must still emit a text item, got empty JSON")+ }+ gotText := false+ for _, it := range res.JSON {+ if txt, ok := it.ToMap()["text"].(string); ok && strings.TrimSpace(txt) != "" {+ gotText = true+ }+ }+ if !gotText {+ t.Fatalf("doc fallback produced no non-empty text item: %#v", res.JSON)}- if strings.TrimSpace(res.Text) == "" {- t.Fatalf("text output is empty: %#v", res)+}++// TestDOCParserTextOutputRemoved verifies the DOC parser has no "text" output+// path. The parser only emits JSON regardless of any output_format request.+func TestDOCParserTextOutputRemoved(t *testing.T) {+ data, err := os.ReadFile(filepath.Join("testdata", "table.doc"))+ if err != nil {+ t.Fatalf("read fixture: %v", err)+ }++ p := NewDOCParser()+ res := p.ParseWithResult(context.Background(), "table.doc", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult error: %v", res.Err)+ }+ if res.OutputFormat != "json" {+ t.Fatalf("DOC must ignore text output_format and emit json, got OutputFormat=%q", res.OutputFormat)+ }+ if len(res.JSON) == 0 {+ t.Fatalf("DOC json output must contain items, got empty")}}diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.goindex c3f0981013..cd0fa55904 100644--- a/internal/parser/parser/doc_parser_synth_test.go+++ b/internal/parser/parser/doc_parser_synth_test.go@@ -157,11 +157,4 @@ func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {t.Errorf("parsed JSON missing %q; got:\n%s", want, got)}}-- pText := NewDOCParser()- pText.ConfigureFromSetup(map[string]any{"output_format": "text"})- prText := pText.ParseWithResult(context.Background(), "synthetic.doc", data)- if prText.OutputFormat != "text" {- t.Errorf("configured text OutputFormat = %q, want text", prText.OutputFormat)- }}diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex 3d5286bd19..a14cc293a0 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -27,13 +27,11 @@ import ("mime/quotedprintable""net/mail""path/filepath"- "slices""strings""time""github.com/AkmalOt/gomsg""go.uber.org/zap"- "golang.org/x/net/html""golang.org/x/text/transform""ragflow/internal/common"@@ -44,23 +42,20 @@ import (// into structured JSON or plain-text output. Mirrors Python's _email()// method in rag/flow/parser/parser.py, whose .msg branch uses extract_msg.type EmailParser struct {- fields []string- outputFormat string+ fields []string}func NewEmailParser() *EmailParser {- return &EmailParser{- outputFormat: "json",- }+ return &EmailParser{}}func (p *EmailParser) ConfigureFromSetup(setup map[string]any) {if p == nil || setup == nil {return}- if v, ok := setup["output_format"].(string); ok && v != "" {- p.outputFormat = v- }+ // Email always emits JSON (the DSL hardcodes output_format to "json" via+ // normalizeParserOutputFormats); the legacy text output path was removed with+ // the dead ParseResult.Text field.if raw, ok := setup["fields"]; ok {switch list := raw.(type) {case []string:@@ -121,83 +116,18 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by}}- outputFormat := p.outputFormat- if outputFormat == "" {- outputFormat = "json"- }-// Re-chunk attachments so their content becomes retrievable within the// same document (user-oriented; mirrors Python legacy rag/app/email.py// naive_chunk). Each attachment is re-parsed by its file extension via// the shared parser registry; a single unparseable/skipped attachment// never breaks the whole email.- extraItems, attachmentText := p.rechunkEmailAttachments(ctx, content, depth)+ extraItems := p.rechunkEmailAttachments(ctx, content, depth)// Attachments have been consumed by rechunkEmailAttachments, which// re-parses each one by extension into a searchable item. Keeping their// raw payloads in the parent item would only bloat the parser result.delete(content, "attachments")- // Text representation: flatten fields into a single string for explicit- // text output and callers that need a display representation.- var sb strings.Builder- for _, k := range emailTextKeys(content) {- v := content[k]- // The metadata map (every non-basic header: Received chains,- // DKIM/ARC signatures, …) stays available in the JSON output, but- // flattening it into chunkable text buries the message under- // transport noise, so it is excluded from the text output.- if k == "metadata" {- continue- }- switch val := v.(type) {- case string:- if k == "text_html" {- // Text output feeds the chunker directly; emit the- // visible text of the HTML part instead of raw markup,- // which otherwise lands in chunks as tag soup.- val = htmlBodyToText(val)- }- sb.WriteString(k)- sb.WriteString(":")- sb.WriteString(val)- sb.WriteString("\n")- case map[string]any:- sb.WriteString(k)- sb.WriteString(":{")- for mk, mv := range val {- if ms, ok := mv.(string); ok {- sb.WriteString(mk)- sb.WriteString(":")- sb.WriteString(ms)- sb.WriteString(", ")- }- }- sb.WriteString("}\n")- case []string:- sb.WriteString(strings.Join(val, "\n"))- }- }- // Attachment text (re-parsed by extension) replaces the old crude- // "filename:payload" flatten, so binary attachments no longer leak- // mojibake into the searchable (indexed) text. The raw attachment- // payloads themselves are dropped after rechunk (see parseEmail), so the- // JSON output path carries only the re-parsed attachment text, never the- // original payload bytes.- if attachmentText != "" {- sb.WriteString(attachmentText)- sb.WriteString("\n")- }- text := sb.String()-- if outputFormat == "text" {- return ParseResult{- OutputFormat: "text",- File: map[string]any{"name": filename},- Text: text,- }- }-content["doc_type_kwd"] = "text"items := []map[string]any{content}if headerText := searchableEmailHeaders(content); headerText != "" {@@ -211,30 +141,6 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by}}-// emailTextFields is the order the text output writes an email's fields in:-// the headers in the order searchableEmailHeaders gives them, then the body.-var emailTextFields = []string{"from", "to", "cc", "bcc", "date", "subject", "text", "text_html"}--// emailTextKeys returns content's keys in emailTextFields order, followed by-// any other key in sorted order. Ranging over the map instead wrote the-// fields in a different order on every parse of the same message.-func emailTextKeys(content map[string]any) []string {- keys := make([]string, 0, len(content))- for _, k := range emailTextFields {- if _, ok := content[k]; ok {- keys = append(keys, k)- }- }- var rest []string- for k := range content {- if !slices.Contains(emailTextFields, k) {- rest = append(rest, k)- }- }- slices.Sort(rest)- return append(keys, rest...)-}-// searchableEmailHeaders returns the user-facing headers as a separate JSON// text item. The main email item keeps its structured fields and body; the// chunker indexes only item text, so headers need their own text item.@@ -502,71 +408,6 @@ func attachmentFilename(part *multipart.Part) string {// -- HTML body → visible text (text output only) ---// htmlBodyToText flattens an email HTML body to its visible text. Unlike the-// structured HTML parser (which keeps table markup for the dedicated HTML-// chunk method), email bodies use tables for layout, so this walker descends-// into tables and emits cell text instead. <head>/<script>/<style> subtrees-// are skipped; whitespace folding is shared with the standalone HTML parser-// via leafWriter.-func htmlBodyToText(body string) string {- if strings.TrimSpace(body) == "" {- return ""- }- doc, err := html.Parse(strings.NewReader(body))- if err != nil {- return body- }- var b bytes.Buffer- w := &leafWriter{b: &b, lineStart: true}- walkHTMLBodyText(doc, w)- return strings.TrimSpace(b.String())-}--// isEmailBlockTag reports whether tag delimits a line in flattened email-// body text: the standard block tags plus table tags, since email uses-// tables for layout and their cells/rows must not fuse either.-func isEmailBlockTag(tag string) bool {- return isBlockTag(tag) || tag == "table" || tag == "td" || tag == "th"-}--func walkHTMLBodyText(n *html.Node, w *leafWriter) {- switch n.Type {- case html.TextNode:- w.writeText(n.Data)- case html.DocumentNode:- // html.Parse always wraps the fragment in a Document node holding an- // <html><body> skeleton; the walker relies on descending through those- // transparent containers (head/script/style subtrees are skipped by the- // ElementNode case), so without descending here the whole body would- // be dropped.- for child := n.FirstChild; child != nil; child = child.NextSibling {- walkHTMLBodyText(child, w)- }- case html.ElementNode:- switch n.Data {- case "head", "script", "style", "noscript", "template":- return- case "br":- w.hardBreak()- return- }- // A nested block starts on a new line when text already precedes- // it, so outer text and inner block text stay separate- // ("<div>a<p>b</p></div>" flattens to "a\nb", not "ab").- if isEmailBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {- w.hardBreak()- }- for child := n.FirstChild; child != nil; child = child.NextSibling {- walkHTMLBodyText(child, w)- }- // Block-level tags and table cells/rows end the current line so- // adjacent block/cell text does not fuse together.- if isEmailBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {- w.hardBreak()- }- }-}-// decodeMailPayload attempts charset decoding using the unified DecodeToUTF8 helper.func decodeMailPayload(payload []byte, charset string) string {if len(payload) == 0 {@@ -786,23 +627,22 @@ func recoverParse(fn func() ParseResult) (res ParseResult, panicked bool) {// avoid unbounded recursion through attachments-of-attachments.const maxRechunkPayloadBytes = 32 << 20 // 32 MiB-func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[string]any, depth int) ([]map[string]any, string) {+func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[string]any, depth int) []map[string]any {if depth > 0 {- return nil, ""+ return nil}// Honor task cancellation so a long attachment list does not keep// re-parsing after the ingestion task has been stopped/aborted.if ctx.Err() != nil {- return nil, ""+ return nil}raw, ok := content["attachments"].([]map[string]any)if !ok || len(raw) == 0 {- return nil, ""+ return nil}var extra []map[string]any- var sb strings.Builderfor _, att := range raw {fn, _ := att["filename"].(string)payload, _ := att["payload"].(string)@@ -842,8 +682,7 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[s// path would otherwise also flatten metadata into the result).ep := NewEmailParser()ep.ConfigureFromSetup(map[string]any{- "fields": p.fields,- "output_format": "json",+ "fields": p.fields,})res, panicked = recoverParse(func() ParseResult {return ep.parseEmail(ctx, fn, []byte(payload), depth+1)@@ -877,10 +716,6 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[s}}}- } else if res.Text != "" {- if t := strings.TrimSpace(res.Text); t != "" {- texts = append(texts, t)- }}for _, t := range texts {@@ -888,11 +723,7 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[s"text": t,"doc_type_kwd": "text",})- if sb.Len() > 0 {- sb.WriteString("\n")- }- sb.WriteString(t)}}- return extra, sb.String()+ return extra}diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.goindex 41c68a360b..2299d23cee 100644--- a/internal/parser/parser/email_parser_test.go+++ b/internal/parser/parser/email_parser_test.go@@ -111,44 +111,6 @@ func TestEmailParser_EmlJSON(t *testing.T) {}}-// TestEmailParser_EmlTextFieldOrder pins the order of the text output: the-// headers first, in the order of the JSON header item, then the body. The-// fields used to come out in Go's map order, a different one on every parse.-func TestEmailParser_EmlTextFieldOrder(t *testing.T) {- raw := strings.Join([]string{- "From: [email protected]",- "To: [email protected]",- "Cc: [email protected]",- "Subject: Quarterly numbers",- "Date: Mon, 1 Sep 2026 10:00:00 +0000",- "Content-Type: text/plain; charset=utf-8",- "",- "Revenue grew.",- }, "\r\n")- want := strings.Join([]string{- "from:[email protected]",- "to:[email protected]",- "cc:[email protected]",- "date:Mon, 1 Sep 2026 10:00:00 +0000",- "subject:Quarterly numbers",- "text:Revenue grew.",- "text_html:",- "",- }, "\n")-- for i := 0; i < 20; i++ {- p := NewEmailParser()- p.ConfigureFromSetup(map[string]any{"output_format": "text"})- result := p.ParseWithResult(t.Context(), "order.eml", []byte(raw))- if result.Err != nil {- t.Fatalf("unexpected error: %v", result.Err)- }- if result.Text != want {- t.Fatalf("parse %d: text output\n got: %q\nwant: %q", i, result.Text, want)- }- }-}-func TestEmailParser_InvalidEMLReturnsError(t *testing.T) {p := NewEmailParser()result := p.ParseWithResult(t.Context(), "broken.eml", []byte("not a valid RFC 5322 header"))@@ -157,46 +119,15 @@ func TestEmailParser_InvalidEMLReturnsError(t *testing.T) {}}-func TestEmailParser_EmlText(t *testing.T) {- ctx := t.Context()- raw := strings.Join([]string{- "From: [email protected]",- "To: [email protected]",- "Subject: Hello",- "Content-Type: text/plain; charset=utf-8",- "",- "Hello, world!",- }, "\r\n")-- p := NewEmailParser()- p.ConfigureFromSetup(map[string]any{- "output_format": "text",- "fields": []string{"from", "to", "subject", "body"},- })-- result := p.ParseWithResult(ctx, "test.eml", []byte(raw))- if result.Err != nil {- t.Fatalf("unexpected error: %v", result.Err)- }- if result.OutputFormat != "text" {- t.Fatalf("expected output_format text, got %q", result.OutputFormat)- }- if !strings.Contains(result.Text, "Hello, world!") {- t.Errorf("text missing body: %q", result.Text)- }- if !strings.Contains(result.Text, "[email protected]") {- t.Errorf("text missing from: %q", result.Text)- }-}-// TestEmailParser_EmlTextChunkText covers the KB chunk-quality bug behind-// Feishu issue om_x100b675dcd7a18a4c2cbfff662a404e: the text output of an-// .eml with an RFC 2047 encoded subject, transport headers (DKIM/ARC/-// Received) and an HTML part used to carry (1) the flattened metadata dump,-// (2) the raw "=?utf-8?B?...?=" subject blob and (3) shredded HTML markup —-// all of which ended up in chunks. The text output must instead skip the-// metadata dump, decode the subject, and flatten the HTML part to its-// visible text. The JSON output keeps metadata and raw text_html unchanged.+// Feishu issue om_x100b675dcd7a18a4c2cbfff662a404e: an .eml with an RFC 2047+// encoded subject, transport headers (DKIM/ARC/Received) and an HTML part used+// to carry (1) the flattened metadata dump, (2) the raw "=?utf-8?B?...?="+// subject blob and (3) shredded HTML markup — all of which ended up in chunks.+// The chunkable body text (the JSON item's "text" field) must instead skip the+// metadata/transport-header dump, decode the subject, and flatten the HTML part+// to its visible text. The JSON output keeps metadata and raw text_html+// unchanged.func TestEmailParser_EmlTextChunkText(t *testing.T) {ctx := t.Context()// "=?utf-8?B?5bim6ZmE5Lu255qEZW1haWw=?=" decodes to "带附件的email";@@ -224,65 +155,59 @@ func TestEmailParser_EmlTextChunkText(t *testing.T) {p := NewEmailParser()p.ConfigureFromSetup(map[string]any{- "output_format": "text",- "fields": []string{"from", "to", "subject", "body"},+ "fields": []string{"from", "to", "subject", "body"},})result := p.ParseWithResult(ctx, "test.eml", []byte(raw))if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}+ if result.OutputFormat != "json" {+ t.Fatalf("expected output_format json, got %q", result.OutputFormat)+ }++ item := result.JSON[0].ToMap()+ bodyText, _ := item["text"].(string)- // (1) No transport-header dump.+ // (1) No transport-header dump in the chunkable body text.for _, noise := range []string{"metadata:", "dkim-signature", "arc-seal", "received:"} {- if strings.Contains(strings.ToLower(result.Text), noise) {- t.Errorf("text output contains header noise %q: %q", noise, result.Text)+ if strings.Contains(strings.ToLower(bodyText), noise) {+ t.Errorf("body text contains header noise %q: %q", noise, bodyText)}}- // (2) Subject/from decoded from RFC 2047.- if !strings.Contains(result.Text, "subject:带附件的email") {- t.Errorf("subject not decoded: %q", result.Text)+ // (2) Subject/from decoded from RFC 2047 into the item fields.+ if v, _ := item["subject"].(string); v != "带附件的email" {+ t.Errorf("subject not decoded: %q", v)}- if !strings.Contains(result.Text, "from:安 <[email protected]>") {- t.Errorf("from not decoded: %q", result.Text)+ if v, _ := item["from"].(string); v != "安 <[email protected]>" {+ t.Errorf("from not decoded: %q", v)}- if strings.Contains(result.Text, "=?utf-8?") {- t.Errorf("raw encoded-word leaked into text: %q", result.Text)+ if strings.Contains(bodyText, "=?utf-8?") {+ t.Errorf("raw encoded-word leaked into text: %q", bodyText)}// (3) HTML part flattened to visible text, no markup.for _, tag := range []string{"<div", "<img", "style="} {- if strings.Contains(result.Text, tag) {- t.Errorf("raw HTML markup %q leaked into text: %q", tag, result.Text)+ if strings.Contains(bodyText, tag) {+ t.Errorf("raw HTML markup %q leaked into text: %q", tag, bodyText)}}- // HTML part must actually contribute its visible text: subject + plain- // body + html body = exactly 3 occurrences of the body string.- if c := strings.Count(result.Text, "带附件的email"); c != 3 {- t.Errorf("expected subject+plain+html body text, got %d occurrences in: %q", c, result.Text)+ // The plain body text must be present exactly once in the body text.+ if c := strings.Count(bodyText, "带附件的email"); c != 1 {+ t.Errorf("expected plain body text once, got %d occurrences in: %q", c, bodyText)}- if !strings.Contains(result.Text, "text_html:带附件的email") {- t.Errorf("html body text missing from text_html: %q", result.Text)+ // The HTML part is kept separately as raw text_html (the chunkable body+ // text never carries shredded HTML markup).+ if v, _ := item["text_html"].(string); !strings.Contains(v, "带附件的email") {+ t.Errorf("html body text missing from text_html: %q", v)}- // JSON output keeps the full metadata and raw text_html.- p.ConfigureFromSetup(map[string]any{- "output_format": "json",- "fields": []string{"from", "to", "subject", "body"},- })- jres := p.ParseWithResult(ctx, "test.eml", []byte(raw))- if jres.Err != nil {- t.Fatalf("unexpected error: %v", jres.Err)- }- item := jres.JSON[0].ToMap()+ // (4) The JSON output keeps the full metadata and raw text_html.if _, ok := item["metadata"].(map[string]any); !ok {t.Errorf("json output must keep metadata: %#v", item)}if v, _ := item["text_html"].(string); !strings.Contains(v, "<img") {t.Errorf("json output must keep raw text_html: %q", v)}- if v, _ := item["subject"].(string); v != "带附件的email" {- t.Errorf("json subject not decoded: %q", v)- }}// TestEmailParser_MsgSupported parses a real Outlook .msg fixture and verifies@@ -814,50 +739,6 @@ func TestEmailParser_AttachmentSearchableJSON(t *testing.T) {}}-// TestEmailParser_AttachmentSearchableText is the text-output equivalent:-// the re-parsed attachment text must be present in result.Text.-func TestEmailParser_AttachmentSearchableText(t *testing.T) {- ctx := t.Context()- attachmentContent := "QUOTE: the quick brown fox jumps over the lazy dog."- encoded := base64.StdEncoding.EncodeToString([]byte(attachmentContent))- boundary := "attachboundary"- raw := strings.Join([]string{- "From: [email protected]",- "To: [email protected]",- "Subject: Attachment Test",- "Content-Type: multipart/mixed; boundary=" + boundary,- "",- "--" + boundary,- "Content-Type: text/plain; charset=utf-8",- "",- "Email body text.",- "--" + boundary,- "Content-Type: text/plain; charset=utf-8",- "Content-Disposition: attachment; filename=\"note.txt\"",- "Content-Transfer-Encoding: base64",- "",- encoded,- "--" + boundary + "--",- }, "\r\n")-- p := NewEmailParser()- p.ConfigureFromSetup(map[string]any{- "output_format": "text",- "fields": []string{"from", "body", "attachments"},- })-- result := p.ParseWithResult(ctx, "test.eml", []byte(raw))- if result.Err != nil {- t.Fatalf("unexpected error: %v", result.Err)- }- if !strings.Contains(result.Text, "quick brown fox") {- t.Errorf("attachment text missing from text output: %q", result.Text)- }- if !strings.Contains(result.Text, "Email body text") {- t.Errorf("email body missing from text output: %q", result.Text)- }-}-// TestEmailParser_AttachmentEmptyPayloadSkipped verifies error isolation:// an attachment with an empty payload (the legacy .msg nested-attachment// case, where gomsg exposes no raw bytes) is skipped without breaking the@@ -1076,65 +957,47 @@ func TestEmailParser_NestedEMLAttachmentRechunk(t *testing.T) {"--" + boundary + "--",}, "\r\n")- for _, format := range []string{"json", "text"} {- p := NewEmailParser()- p.ConfigureFromSetup(map[string]any{- "output_format": format,- "fields": []string{"from", "to", "subject", "body", "attachments", "metadata"},- })+ p := NewEmailParser()+ p.ConfigureFromSetup(map[string]any{+ "fields": []string{"from", "to", "subject", "body", "attachments", "metadata"},+ })- result := p.ParseWithResult(ctx, "outer.eml", []byte(raw))- if result.Err != nil {- t.Fatalf("[%s] unexpected error: %v", format, result.Err)- }+ result := p.ParseWithResult(ctx, "outer.eml", []byte(raw))+ if result.Err != nil {+ t.Fatalf("unexpected error: %v", result.Err)+ }- // Collect all indexed text for this output mode (JSON populates- // result.JSON; text populates result.Text).- var indexed strings.Builder- if format == "json" {- for _, it := range ItemsToMaps(result.JSON) {- if txt, ok := it["text"].(string); ok {- indexed.WriteString(txt)- indexed.WriteString("\n")- }- }- } else {- indexed.WriteString(result.Text)+ // Collect all indexed text across JSON items.+ var indexed strings.Builder+ for _, it := range ItemsToMaps(result.JSON) {+ if txt, ok := it["text"].(string); ok {+ indexed.WriteString(txt)+ indexed.WriteString("\n")}+ }- // The nested email body must be retrievable.- if !strings.Contains(indexed.String(), "INNER BODY SECRET") {- t.Errorf("[%s] nested body not indexed: %q", format, indexed.String())- }- if !strings.Contains(indexed.String(), "OUTER BODY VISIBLE") {- t.Errorf("[%s] outer body missing: %q", format, indexed.String())- }+ // The nested email body must be retrievable.+ if !strings.Contains(indexed.String(), "INNER BODY SECRET") {+ t.Errorf("nested body not indexed: %q", indexed.String())+ }+ if !strings.Contains(indexed.String(), "OUTER BODY VISIBLE") {+ t.Errorf("outer body missing: %q", indexed.String())+ }- if format == "json" {- // The nested email must be a distinct, clean item, carrying no- // metadata key (the old bug leaked a "metadata:{...}" string as- // its text).- found := false- for _, it := range ItemsToMaps(result.JSON) {- if txt, ok := it["text"].(string); ok && strings.Contains(txt, "INNER BODY SECRET") {- found = true- if _, hasMeta := it["metadata"]; hasMeta {- t.Errorf("[json] nested item must not carry metadata key: %#v", it)- }- }- }- if !found {- t.Errorf("[json] nested body not found as separate item: %#v", result.JSON)- }- } else {- // Text mode: metadata (transport headers) is never flattened- // into chunkable text — not for the outer email, and not for- // the nested re-parse (which always requests JSON output).- if strings.Contains(result.Text, "metadata:") {- t.Errorf("[text] metadata must not leak into text output: %q", result.Text)+ // The nested email must be a distinct, clean item, carrying no metadata+ // key (the old bug leaked a "metadata:{...}" string as its text).+ found := false+ for _, it := range ItemsToMaps(result.JSON) {+ if txt, ok := it["text"].(string); ok && strings.Contains(txt, "INNER BODY SECRET") {+ found = true+ if _, hasMeta := it["metadata"]; hasMeta {+ t.Errorf("nested item must not carry metadata key: %#v", it)}}}+ if !found {+ t.Errorf("nested body not found as separate item: %#v", result.JSON)+ }}// TestEmailParser_RechunkAttachmentTextPath exercises rechunkEmailAttachments@@ -1152,7 +1015,7 @@ func TestEmailParser_RechunkAttachmentTextPath(t *testing.T) {}p := NewEmailParser()- extra, text := p.rechunkEmailAttachments(ctx, content, 0)+ extra := p.rechunkEmailAttachments(ctx, content, 0)if len(extra) != 1 {t.Fatalf("expected 1 indexed attachment, got %d: %#v", len(extra), extra)@@ -1160,11 +1023,12 @@ func TestEmailParser_RechunkAttachmentTextPath(t *testing.T) {if v, _ := extra[0]["text"].(string); v != "hello from attachment" {t.Errorf("text = %q, want hello from attachment", v)}- if !strings.Contains(text, "hello from attachment") {- t.Errorf("indexed text missing attachment: %q", text)- }- if strings.Contains(text, "<binary bytes>") {- t.Errorf("binary payload leaked into indexed text: %q", text)+ // The flat text return was removed; verify the binary attachment is not+ // indexed as a separate text item (only the .txt attachment should appear).+ for _, item := range extra {+ if s, _ := item["text"].(string); strings.Contains(s, "<binary bytes>") {+ t.Errorf("binary payload leaked into indexed text: %q", s)+ }}}@@ -1192,9 +1056,9 @@ func TestRechunkEmailAttachments_ContextCancelled(t *testing.T) {{"filename": "note.txt", "payload": "must not be re-chunked"},},}- extra, text := NewEmailParser().rechunkEmailAttachments(ctx, content, 0)- if len(extra) != 0 || text != "" {- t.Errorf("cancelled context should short-circuit re-chunk: extra=%#v text=%q", extra, text)+ extra := NewEmailParser().rechunkEmailAttachments(ctx, content, 0)+ if len(extra) != 0 {+ t.Errorf("cancelled context should short-circuit re-chunk: extra=%#v", extra)}}@@ -1209,7 +1073,7 @@ func TestRecoverParse_IsolatesPanic(t *testing.T) {if !panicked {t.Error("expected panicked=true")}- if res.Err != nil || res.JSON != nil || res.Text != "" {+ if res.Err != nil || res.JSON != nil {t.Errorf("panic should yield a zero ParseResult, got %#v", res)}}@@ -1217,14 +1081,14 @@ func TestRecoverParse_IsolatesPanic(t *testing.T) {// TestRecoverParse_PassesThrough verifies a normal parser result is returned// unchanged with panicked=false.func TestRecoverParse_PassesThrough(t *testing.T) {- want := ParseResult{OutputFormat: "text", Text: "hello"}+ want := ParseResult{OutputFormat: "json", JSON: []DocItem{NewTextJSONItem("hello")}}res, panicked := recoverParse(func() ParseResult {return want})if panicked {t.Error("expected panicked=false for a normal result")}- if res.Text != "hello" {+ if txt, ok := res.JSON[0].ToMap()["text"].(string); !ok || txt != "hello" {t.Errorf("result not passed through: %#v", res)}}@@ -1236,25 +1100,22 @@ func TestEmailParser_RechunkPrefersRaw(t *testing.T) {ctx := t.Context()p := NewEmailParser()- withRaw, textRaw := p.rechunkEmailAttachments(ctx, map[string]any{+ withRaw := p.rechunkEmailAttachments(ctx, map[string]any{"attachments": []map[string]any{{"filename": "note.txt", "payload": "WORLD", "raw": "HELLO"},},}, 0)if len(withRaw) != 1 || withRaw[0]["text"] != "HELLO" {- t.Fatalf("raw not preferred: extra=%#v text=%q", withRaw, textRaw)- }- if strings.Contains(textRaw, "WORLD") {- t.Errorf("re-chunk used payload instead of raw: %q", textRaw)+ t.Fatalf("raw not preferred: extra=%#v", withRaw)}- fallback, textFallback := p.rechunkEmailAttachments(ctx, map[string]any{+ fallback := p.rechunkEmailAttachments(ctx, map[string]any{"attachments": []map[string]any{{"filename": "note.txt", "payload": "WORLD"},},}, 0)if len(fallback) != 1 || fallback[0]["text"] != "WORLD" {- t.Fatalf("payload fallback broken: extra=%#v text=%q", fallback, textFallback)+ t.Fatalf("payload fallback broken: extra=%#v", fallback)}}@@ -1405,45 +1266,32 @@ func TestParseEML_SinglePartTransferEncoding(t *testing.T) {body stringwantText stringwantHTML string- // encoded is the payload as it appears on the wire. The text output- // feeds the chunker, so it must never carry these bytes.- encoded string- // wantInText is the visible text the flattened output must carry.- wantInText string}{{- name: "base64 text/plain",- headers: "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64",- body: b64,- wantText: plain,- encoded: b64,- wantInText: "Quarterly revenue was 12.5 million.",+ name: "base64 text/plain",+ headers: "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64",+ body: b64,+ wantText: plain,},{- name: "quoted-printable text/plain",- headers: "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: quoted-printable",- body: "Quarterly revenue was 12=2E5 million=2E",- wantText: "Quarterly revenue was 12.5 million.",- encoded: "12=2E5",- wantInText: "Quarterly revenue was 12.5 million.",+ name: "quoted-printable text/plain",+ headers: "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: quoted-printable",+ body: "Quarterly revenue was 12=2E5 million=2E",+ wantText: "Quarterly revenue was 12.5 million.",},{- name: "base64 text/html",- headers: "Content-Type: text/html; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64",- body: htmlB64,- wantHTML: html,- encoded: htmlB64,- wantInText: "Quarterly revenue",+ name: "base64 text/html",+ headers: "Content-Type: text/html; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64",+ body: htmlB64,+ wantHTML: html,},{// Positive control: the multipart path already decodes the same// body, so the two paths must agree.- name: "base64 in multipart/alternative",- headers: "Content-Type: multipart/alternative; boundary=\"BB\"",- body: "--BB\r\nContent-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n" + b64 + "\r\n--BB--",- wantText: plain,- encoded: b64,- wantInText: "Quarterly revenue was 12.5 million.",+ name: "base64 in multipart/alternative",+ headers: "Content-Type: multipart/alternative; boundary=\"BB\"",+ body: "--BB\r\nContent-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n" + b64 + "\r\n--BB--",+ wantText: plain,},}@@ -1468,64 +1316,18 @@ func TestParseEML_SinglePartTransferEncoding(t *testing.T) {if strings.TrimRight(gotHTML, "\r\n") != strings.TrimRight(tt.wantHTML, "\r\n") {t.Errorf("text_html = %q, want %q", gotHTML, tt.wantHTML)}-- // The text output is what reaches the chunker, and the JSON- // output returns before it is built, so assert it separately.- pt := NewEmailParser()- pt.ConfigureFromSetup(map[string]any{"output_format": "text"})- textResult := pt.ParseWithResult(t.Context(), "test.eml", []byte(raw))- if textResult.Err != nil {- t.Fatalf("unexpected error on the text path: %v", textResult.Err)- }- flat := textResult.Text- if !strings.Contains(flat, tt.wantInText) {- t.Errorf("text output = %q, want it to contain %q", flat, tt.wantInText)- }- if strings.Contains(flat, tt.encoded) {- t.Errorf("text output still carries the encoded payload %q: %q", tt.encoded, flat)- }})}}-// TestHTMLBodyToText verifies the visible-text flattening of email HTML-// bodies, above all that text preceding a nested block-level element starts-// a new line instead of fusing with the block's own text.-func TestHTMLBodyToText(t *testing.T) {- tests := []struct {- name string- body string- want string- }{- {"text before nested block", `<div>prefix<p>paragraph</p></div>`, "prefix\nparagraph"},- {"nested divs", `<div>outer<div>inner</div></div>`, "outer\ninner"},- {"sibling blocks", `<p>one</p><p>two</p>`, "one\ntwo"},- {"layout table cells", `<table><tr><td>a</td><td>b</td></tr></table>`, "a\nb"},- {"multi-row layout table stays line-separated", `<p>intro</p><table><tr><td>a</td><td>b</td></tr><tr><td>c</td></tr></table><p>outro</p>`, "intro\na\nb\nc\noutro"},- {"inline tags stay on one line", `<p>Hello <b>world</b></p>`, "Hello world"},- {"definition list and rule", `<div>A<dl><dt>Term</dt><dd>Def</dd></dl>B<hr>C</div>`, "A\nTerm\nDef\nB\nC"},- {"br forces a break", `one<br>two`, "one\ntwo"},- {"head script style skipped", `<html><head><title>t</title><style>.x{color:red}</style></head><body><p>visible</p><script>var x=1;</script></body></html>`, "visible"},- {"template content skipped", `<p>visible</p><template><p>Hello {{name}}</p></template>`, "visible"},- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- if got := htmlBodyToText(tt.body); got != tt.want {- t.Errorf("htmlBodyToText(%q) = %q, want %q", tt.body, got, tt.want)- }- })- }-}--// TestEmailParserJSONPathOmitsDuplicateText verifies that the JSON output-// path no longer duplicates the flattened email body into result.Text.-// result.Text is only populated on the "text" output_format branch; on the-// JSON branch the searchable text lives in the JSON items (body + the-// header text item from searchableEmailHeaders), so the redundant json-path-// Text is intentionally empty. This guards the removal of the json-path-// Text assignment in parseEmail.-func TestEmailParserJSONPathOmitsDuplicateText(t *testing.T) {- p := NewEmailParser() // default output_format -> "json"+// TestEmailParserJSONOmitsDuplicateHeaderText verifies the JSON output keeps+// the decoded body text without duplicating the flattened header text that the+// legacy single-string path used to carry. The body text lives in the body+// item's "text" field; the header fields (from/to/subject/...) are separate+// top-level keys, so the body text must not re-contain a "from:..." header+// line.+func TestEmailParserJSONOmitsDuplicateHeaderText(t *testing.T) {+ p := NewEmailParser()p.ConfigureFromSetup(map[string]any{"fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},})@@ -1543,32 +1345,40 @@ func TestEmailParserJSONPathOmitsDuplicateText(t *testing.T) {if len(res.JSON) == 0 {t.Fatal("JSON: want non-empty items")}- if res.Text != "" {- t.Errorf("JSON-path Email result.Text = %q, want empty (text lives in JSON items)", res.Text)+ bodyText, _ := res.JSON[0].ToMap()["text"].(string)+ if strings.Contains(bodyText, "from:") || strings.Contains(bodyText, "subject:") {+ t.Errorf("JSON body text must not duplicate flattened header lines: %q", bodyText)}}-// TestEmailParserTextPathKeepsText guards that the explicit "text"-// output_format branch still populates result.Text — the only place the-// field is read downstream (sys.files parseResultText, document preview,-// and nested attachment re-parse).-func TestEmailParserTextPathKeepsText(t *testing.T) {+// TestEmailParserTextOutputRemoved verifies the Email parser has no "text"+// output path. A setup requesting output_format "text" must be ignored and the+// result must be JSON, never a Text payload.+func TestEmailParserTextOutputRemoved(t *testing.T) {+ ctx := context.Background()+ raw := strings.Join([]string{+ "From: [email protected]",+ "To: [email protected]",+ "Subject: Hello",+ "Content-Type: text/plain; charset=utf-8",+ "",+ "Hello, world!",+ }, "\r\n")+p := NewEmailParser()- p.ConfigureFromSetup(map[string]any{"output_format": "text"})- const eml = "From: [email protected]\r\n" +- "To: [email protected]\r\n" +- "Subject: Hi\r\n" +- "Date: Mon, 01 Jan 2024 00:00:00 +0000\r\n" +- "\r\n" +- "Hello body.\r\n"- res := p.ParseWithResult(t.Context(), "msg.eml", []byte(eml))- if res.Err != nil {- t.Fatalf("ParseWithResult: %v", res.Err)+ p.ConfigureFromSetup(map[string]any{+ "output_format": "text",+ "fields": []string{"from", "to", "subject", "body"},+ })++ result := p.ParseWithResult(ctx, "test.eml", []byte(raw))+ if result.Err != nil {+ t.Fatalf("unexpected error: %v", result.Err)}- if res.OutputFormat != "text" {- t.Fatalf("OutputFormat = %q, want text", res.OutputFormat)+ if result.OutputFormat != "json" {+ t.Fatalf("email must ignore text output_format and emit json, got OutputFormat=%q", result.OutputFormat)}- if res.Text == "" {- t.Error("text-path Email result.Text empty, want body text")+ if len(result.JSON) == 0 {+ t.Fatalf("email json output must contain items, got empty")}}diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.goindex 3398e8997b..5799c4d24c 100644--- a/internal/parser/parser/parse_result.go+++ b/internal/parser/parser/parse_result.go@@ -71,12 +71,11 @@ type ParseResult struct {// and, after ingestion enhancement, its VLM description.JSON []DocItem- // Markdown is a backend response awaiting normalization.+ // Markdown is a backend response awaiting normalization (PDF markdown+ // engines only; the DSL hardcodes output_format to "json" for all other+ // families via normalizeParserOutputFormats).Markdown string- // Text is a backend response awaiting normalization.- Text string-// Err is the failure reason. On non-nil Err, all payload// fields are zero values.Err errordiff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.goindex 073084b577..96f920a9eb 100644--- a/internal/parser/parser/parse_result_test.go+++ b/internal/parser/parser/parse_result_test.go@@ -84,14 +84,6 @@ func TestParseResult_Contract(t *testing.T) {},wantFmt: "markdown",},- {- name: "text family only",- in: ParseResult{- OutputFormat: "text",- Text: "raw text",- },- wantFmt: "text",- },{name: "failure clears payload",in: ParseResult{Err: errSentinel},@@ -107,7 +99,7 @@ func TestParseResult_Contract(t *testing.T) {if tc.in.OutputFormat != "" {t.Errorf("OutputFormat on failure = %q, want empty", tc.in.OutputFormat)}- if tc.in.JSON != nil || tc.in.Markdown != "" || tc.in.Text != "" {+ if tc.in.JSON != nil || tc.in.Markdown != "" {t.Errorf("payload fields must be zero on failure; got %+v", tc.in)}return@@ -203,13 +195,11 @@ func payloadFamily(r ParseResult) string {return "json"case "markdown":return "markdown"- case "text":- return "text"}return ""}-var allPayloadFamilies = []string{"json", "markdown", "text"}+var allPayloadFamilies = []string{"json", "markdown"}func isPayloadZero(r ParseResult, family string) bool {switch family {@@ -217,8 +207,6 @@ func isPayloadZero(r ParseResult, family string) bool {return r.JSON == nilcase "markdown":return r.Markdown == ""- case "text":- return r.Text == ""}return false}diff --git a/internal/parser/parser/pdf_result.go b/internal/parser/parser/pdf_result.goindex 6933b073e7..5b91414fc0 100644--- a/internal/parser/parser/pdf_result.go+++ b/internal/parser/parser/pdf_result.go@@ -55,8 +55,6 @@ func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, resultresult.JSON = parsed.JSON}}- case result.Text != "":- result.JSON = []DocItem{NewTextJSONItem(result.Text)}}}items := make([]DocItem, 0, len(result.JSON))@@ -139,7 +137,7 @@ func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, resultzap.Int("dropped_rows", droppedPositions))}result.JSON = items- result.OutputFormat, result.Markdown, result.Text = "json", "", ""+ result.OutputFormat, result.Markdown = "json", ""result.File = ensurePDFResultFile(result.File, filename)return result}diff --git a/internal/parser/parser/rechunk_singlereturn_test.go b/internal/parser/parser/rechunk_singlereturn_test.gonew file mode 100644index 0000000000..81652f334c--- /dev/null+++ b/internal/parser/parser/rechunk_singlereturn_test.go@@ -0,0 +1,69 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package parser++import (+ "context"+ "strings"+ "testing"+)++// TestRechunkEmailAttachmentsSingleReturn guards the dead-code cleanup of+// rechunkEmailAttachments: it must return only the re-parsed attachment items+// ([]map[string]any) and must no longer build/return a discarded flat-text+// string. If the second return value is reintroduced, this test fails to+// compile.+func TestRechunkEmailAttachmentsSingleReturn(t *testing.T) {+ p := NewEmailParser()+ ctx := context.Background()++ // depth>0 must stop recursion immediately.+ if got := p.rechunkEmailAttachments(ctx, map[string]any{}, 1); got != nil {+ t.Fatalf("depth>0 should return nil, got %d items", len(got))+ }++ // No attachments -> empty, never a nil-panic on a removed return value.+ if got := p.rechunkEmailAttachments(ctx, map[string]any{}, 0); len(got) != 0 {+ t.Fatalf("no attachments should yield 0 items, got %d", len(got))+ }++ // A non-text (image) attachment is skipped, not re-parsed.+ image := map[string]any{+ "attachments": []map[string]any{+ {"filename": "pic.png", "payload": "binary", "raw": "binary"},+ },+ }+ if got := p.rechunkEmailAttachments(ctx, image, 0); len(got) != 0 {+ t.Fatalf("image attachment should be skipped, got %d items", len(got))+ }++ // A .txt attachment re-parses into a searchable text item.+ txt := map[string]any{+ "attachments": []map[string]any{+ {"filename": "note.txt", "payload": "hello attachment"},+ },+ }+ got := p.rechunkEmailAttachments(ctx, txt, 0)+ if len(got) != 1 {+ t.Fatalf("txt attachment should yield 1 item, got %d", len(got))+ }+ if got[0]["doc_type_kwd"] != "text" {+ t.Errorf("item doc_type_kwd = %v, want text", got[0]["doc_type_kwd"])+ }+ if text, ok := got[0]["text"].(string); !ok || !strings.Contains(text, "hello attachment") {+ t.Errorf("item text = %v, want it to contain %q", got[0]["text"], "hello attachment")+ }+}diff --git a/internal/service/document/document_artifact.go b/internal/service/document/document_artifact.goindex 4144a6ecb6..5d64eec8c0 100644--- a/internal/service/document/document_artifact.go+++ b/internal/service/document/document_artifact.go@@ -314,11 +314,21 @@ func (s *DocumentService) GetDocumentPreview(ctx context.Context, userID, docID}// extractDOCPreviewText extracts plain text from a legacy .doc-// document via the office_oxide-backed DOCParser.+// document via the office_oxide-backed DOCParser. The parser only emits JSON+// payloads, so the preview text is flattened from the JSON items.func extractDOCPreviewText(ctx context.Context, filename string, data []byte) (string, error) {res := parser.NewDOCParser().ParseWithResult(ctx, filename, data)if res.Err != nil {return "", res.Err}- return res.Text, nil+ if len(res.JSON) == 0 {+ return "", fmt.Errorf("doc preview: no content extracted")+ }+ parts := make([]string, 0, len(res.JSON))+ for _, item := range res.JSON {+ if text, ok := item.ToMap()["text"].(string); ok {+ parts = append(parts, text)+ }+ }+ return strings.Join(parts, "\n"), nil}diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.goindex 7db25ec4fc..9bdf9ac38f 100644--- a/internal/service/file/file_content.go+++ b/internal/service/file/file_content.go@@ -242,8 +242,6 @@ func parseAgentUploadContent(ctx context.Context, filename string, data []byte,// text field and serializing items without one as a final fallback.func parseResultText(res parser.ParseResult) (string, error) {switch strings.ToLower(strings.TrimSpace(res.OutputFormat)) {- case "text":- return res.Text, nilcase "markdown":return res.Markdown, nilcase "json":@@ -270,10 +268,8 @@ func parseResultText(res parser.ParseResult) (string, error) {// Some legacy parsers mark the result as JSON while only populating a// rendered companion field. Preserve that content instead of returning// an empty sys.files value.- for _, fallback := range []string{res.Markdown, res.Text} {- if fallback != "" {- return fallback, nil- }+ if res.Markdown != "" {+ return res.Markdown, nil}return "", nildefault: