Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): remove dead ParseResult.Text field and text output path (#21334)

xugangqiangfa0bae42b037
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index fbabfd8449..279db84f74 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -619,9 +619,6 @@ func resolveParserNormalizationSource(dispatched parser.ParseResult) string {
if dispatched.Markdown != "" {
return "markdown"
}
- if dispatched.Text != "" {
- return "text"
- }
return "raw"
}
diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go
index c3e8fbe49f..85019f7498 100644
--- a/internal/ingestion/component/parser_dispatch.go
+++ b/internal/ingestion/component/parser_dispatch.go
@@ -357,9 +357,6 @@ func normalizeParserJSON(ctx context.Context, filename string, dispatched parser
if dispatched.Markdown != "" {
return parseMarkdownToJSONItems(ctx, filename, dispatched.Markdown)
}
- if dispatched.Text != "" {
- return []parser.DocItem{parser.NewTextJSONItem(dispatched.Text)}
- }
return []parser.DocItem{}
}
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 203ee4e489..62070b38ab 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -95,7 +95,7 @@ func requireJSONText(t *testing.T, out map[string]any, want string) {
t.Fatalf("json payload does not contain %q: %#v", want, items)
}
-func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {
+func TestBuildParserOutputsNormalizesMarkdownToJSON(t *testing.T) {
tests := []struct {
name string
dispatched parser.ParseResult
@@ -109,14 +109,6 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {
},
wantText: "Title",
},
- {
- name: "text",
- dispatched: parser.ParseResult{
- OutputFormat: "text",
- Text: "plain body",
- },
- wantText: "plain body",
- },
}
for _, tt := range tests {
diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.go
index 6c4a81d785..1aff00bfac 100644
--- a/internal/parser/parser/doc_parser.go
+++ b/internal/parser/parser/doc_parser.go
@@ -27,14 +27,10 @@ import (
// docExtract is the native office_oxide extraction seam for the DOC family.
var docExtract = deepdocoffice.OpenAndExtract
-type DOCParser struct {
- OutputFormat string
-}
+type DOCParser struct{}
func NewDOCParser() *DOCParser {
- return &DOCParser{
- OutputFormat: "json",
- }
+ return &DOCParser{}
}
func (p *DOCParser) String() string {
@@ -42,12 +38,9 @@ func (p *DOCParser) String() string {
}
func (p *DOCParser) ConfigureFromSetup(setup map[string]any) {
- if p == nil || setup == nil {
- return
- }
- if v, ok := setup["output_format"].(string); ok && v != "" {
- p.OutputFormat = v
- }
+ // DOC always emits JSON (the DSL hardcodes output_format to "json" via
+ // normalizeParserOutputFormats); the legacy text/markdown output paths were
+ // removed with the dead ParseResult.Text field.
}
// ParseWithResult extracts text for the DOC family and converts it to the
@@ -61,42 +54,17 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
return ParseResult{Err: fmt.Errorf("doc extract: %w", err)}
}
- text, mdPayload, err := extractDocText(irJSON, mdText, plainText)
+ text, _, err := extractDocText(irJSON, mdText, plainText)
if err != nil {
return ParseResult{Err: fmt.Errorf("doc extract: %w", err)}
}
- outFmt := p.OutputFormat
- if outFmt == "" {
- outFmt = "json"
- }
-
- markdownPayload := text
- if strings.TrimSpace(mdPayload) != "" {
- markdownPayload = mdPayload
- }
-
- res := ParseResult{
- OutputFormat: outFmt,
- File: map[string]any{"name": filename, "format": "doc"},
- Text: text,
- Markdown: markdownPayload,
- }
-
- if strings.EqualFold(outFmt, "markdown") {
- return res
- }
- if strings.EqualFold(outFmt, "text") {
- return res
- }
-
- // JSON path: prefer the structured IR, which office_oxide recovers table
- // (and list) structure for. Emitting structured sections lets the chunker
- // render tables as HTML (#20918) instead of losing them to plain text — the
- // legacy path flattened the whole document (tables included) into a single
- // text item. Fall back to the flattened text only when the IR yields no
- // usable sections, so a document whose IR is empty/unusable still keeps its
- // content (PlainText / ToMarkdown) rather than silently producing 0 chunks.
+ // The DSL hardcodes output_format to "json" (normalizeParserOutputFormats),
+ // so the DOC parser always emits structured JSON items. Prefer the IR
+ // sections, which recover table/list structure for the chunker's HTML-table
+ // rendering (#20918); fall back to the flattened text only when the IR yields
+ // no usable sections, so a document whose IR is empty/unusable still keeps
+ // its content rather than silently producing 0 chunks.
if strings.TrimSpace(irJSON) != "" {
budget := newEmbeddedMediaBudget()
if sections := buildDOCXJSONSections(irJSON, budget); len(sections) > 0 {
diff --git a/internal/parser/parser/doc_parser_doc_table_test.go b/internal/parser/parser/doc_parser_doc_table_test.go
index 773bba7e69..77359b58f4 100644
--- a/internal/parser/parser/doc_parser_doc_table_test.go
+++ b/internal/parser/parser/doc_parser_doc_table_test.go
@@ -89,24 +89,53 @@ func TestDOCParser_TableEmittedAsStructuredItem(t *testing.T) {
// TestDOCParser_NoTableDocStaysTextItem pins the fallback for a .doc whose IR
// yields no structured sections: the parser must still emit the flattened text
// as a single text item (never silently drop content to 0 chunks). The fixture
-// is the same table.doc driven through the text output format, which bypasses
-// the IR sections path and must keep the legacy behavior intact.
-func TestDOCParser_TextOutputFormatStaysText(t *testing.T) {
+// is the same table.doc; with the legacy text output path removed the parser
+// only emits JSON, so the fallback is exercised through the JSON item below.
+func TestDOCParser_NoTableDocStaysTextItem(t *testing.T) {
data, err := os.ReadFile(filepath.Join("testdata", "table.doc"))
if err != nil {
t.Fatalf("read fixture: %v", err)
}
p := NewDOCParser()
- p.OutputFormat = "text"
res := p.ParseWithResult(context.Background(), "table.doc", data)
if res.Err != nil {
t.Fatalf("ParseWithResult error: %v", res.Err)
}
- if res.OutputFormat != "text" {
- t.Fatalf("OutputFormat = %q, want text", res.OutputFormat)
+ if res.OutputFormat != "json" {
+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ }
+ if len(res.JSON) == 0 {
+ t.Fatalf("doc with no structured sections must still emit a text item, got empty JSON")
+ }
+ gotText := false
+ for _, it := range res.JSON {
+ if txt, ok := it.ToMap()["text"].(string); ok && strings.TrimSpace(txt) != "" {
+ gotText = true
+ }
+ }
+ if !gotText {
+ t.Fatalf("doc fallback produced no non-empty text item: %#v", res.JSON)
}
- if strings.TrimSpace(res.Text) == "" {
- t.Fatalf("text output is empty: %#v", res)
+}
+
+// TestDOCParserTextOutputRemoved verifies the DOC parser has no "text" output
+// path. The parser only emits JSON regardless of any output_format request.
+func TestDOCParserTextOutputRemoved(t *testing.T) {
+ data, err := os.ReadFile(filepath.Join("testdata", "table.doc"))
+ if err != nil {
+ t.Fatalf("read fixture: %v", err)
+ }
+
+ p := NewDOCParser()
+ res := p.ParseWithResult(context.Background(), "table.doc", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult error: %v", res.Err)
+ }
+ if res.OutputFormat != "json" {
+ t.Fatalf("DOC must ignore text output_format and emit json, got OutputFormat=%q", res.OutputFormat)
+ }
+ if len(res.JSON) == 0 {
+ t.Fatalf("DOC json output must contain items, got empty")
}
}
diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.go
index c3f0981013..cd0fa55904 100644
--- a/internal/parser/parser/doc_parser_synth_test.go
+++ b/internal/parser/parser/doc_parser_synth_test.go
@@ -157,11 +157,4 @@ func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {
t.Errorf("parsed JSON missing %q; got:\n%s", want, got)
}
}
-
- pText := NewDOCParser()
- pText.ConfigureFromSetup(map[string]any{"output_format": "text"})
- prText := pText.ParseWithResult(context.Background(), "synthetic.doc", data)
- if prText.OutputFormat != "text" {
- t.Errorf("configured text OutputFormat = %q, want text", prText.OutputFormat)
- }
}
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go
index 3d5286bd19..a14cc293a0 100644
--- a/internal/parser/parser/email_parser.go
+++ b/internal/parser/parser/email_parser.go
@@ -27,13 +27,11 @@ import (
"mime/quotedprintable"
"net/mail"
"path/filepath"
- "slices"
"strings"
"time"
"github.com/AkmalOt/gomsg"
"go.uber.org/zap"
- "golang.org/x/net/html"
"golang.org/x/text/transform"
"ragflow/internal/common"
@@ -44,23 +42,20 @@ import (
// into structured JSON or plain-text output. Mirrors Python's _email()
// method in rag/flow/parser/parser.py, whose .msg branch uses extract_msg.
type EmailParser struct {
- fields []string
- outputFormat string
+ fields []string
}
func NewEmailParser() *EmailParser {
- return &EmailParser{
- outputFormat: "json",
- }
+ return &EmailParser{}
}
func (p *EmailParser) ConfigureFromSetup(setup map[string]any) {
if p == nil || setup == nil {
return
}
- if v, ok := setup["output_format"].(string); ok && v != "" {
- p.outputFormat = v
- }
+ // Email always emits JSON (the DSL hardcodes output_format to "json" via
+ // normalizeParserOutputFormats); the legacy text output path was removed with
+ // the dead ParseResult.Text field.
if raw, ok := setup["fields"]; ok {
switch list := raw.(type) {
case []string:
@@ -121,83 +116,18 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by
}
}
- outputFormat := p.outputFormat
- if outputFormat == "" {
- outputFormat = "json"
- }
-
// Re-chunk attachments so their content becomes retrievable within the
// same document (user-oriented; mirrors Python legacy rag/app/email.py
// naive_chunk). Each attachment is re-parsed by its file extension via
// the shared parser registry; a single unparseable/skipped attachment
// never breaks the whole email.
- extraItems, attachmentText := p.rechunkEmailAttachments(ctx, content, depth)
+ extraItems := p.rechunkEmailAttachments(ctx, content, depth)
// Attachments have been consumed by rechunkEmailAttachments, which
// re-parses each one by extension into a searchable item. Keeping their
// raw payloads in the parent item would only bloat the parser result.
delete(content, "attachments")
- // Text representation: flatten fields into a single string for explicit
- // text output and callers that need a display representation.
- var sb strings.Builder
- for _, k := range emailTextKeys(content) {
- v := content[k]
- // The metadata map (every non-basic header: Received chains,
- // DKIM/ARC signatures, …) stays available in the JSON output, but
- // flattening it into chunkable text buries the message under
- // transport noise, so it is excluded from the text output.
- if k == "metadata" {
- continue
- }
- switch val := v.(type) {
- case string:
- if k == "text_html" {
- // Text output feeds the chunker directly; emit the
- // visible text of the HTML part instead of raw markup,
- // which otherwise lands in chunks as tag soup.
- val = htmlBodyToText(val)
- }
- sb.WriteString(k)
- sb.WriteString(":")
- sb.WriteString(val)
- sb.WriteString("\n")
- case map[string]any:
- sb.WriteString(k)
- sb.WriteString(":{")
- for mk, mv := range val {
- if ms, ok := mv.(string); ok {
- sb.WriteString(mk)
- sb.WriteString(":")
- sb.WriteString(ms)
- sb.WriteString(", ")
- }
- }
- sb.WriteString("}\n")
- case []string:
- sb.WriteString(strings.Join(val, "\n"))
- }
- }
- // Attachment text (re-parsed by extension) replaces the old crude
- // "filename:payload" flatten, so binary attachments no longer leak
- // mojibake into the searchable (indexed) text. The raw attachment
- // payloads themselves are dropped after rechunk (see parseEmail), so the
- // JSON output path carries only the re-parsed attachment text, never the
- // original payload bytes.
- if attachmentText != "" {
- sb.WriteString(attachmentText)
- sb.WriteString("\n")
- }
- text := sb.String()
-
- if outputFormat == "text" {
- return ParseResult{
- OutputFormat: "text",
- File: map[string]any{"name": filename},
- Text: text,
- }
- }
-
content["doc_type_kwd"] = "text"
items := []map[string]any{content}
if headerText := searchableEmailHeaders(content); headerText != "" {
@@ -211,30 +141,6 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by
}
}
-// emailTextFields is the order the text output writes an email's fields in:
-// the headers in the order searchableEmailHeaders gives them, then the body.
-var emailTextFields = []string{"from", "to", "cc", "bcc", "date", "subject", "text", "text_html"}
-
-// emailTextKeys returns content's keys in emailTextFields order, followed by
-// any other key in sorted order. Ranging over the map instead wrote the
-// fields in a different order on every parse of the same message.
-func emailTextKeys(content map[string]any) []string {
- keys := make([]string, 0, len(content))
- for _, k := range emailTextFields {
- if _, ok := content[k]; ok {
- keys = append(keys, k)
- }
- }
- var rest []string
- for k := range content {
- if !slices.Contains(emailTextFields, k) {
- rest = append(rest, k)
- }
- }
- slices.Sort(rest)
- return append(keys, rest...)
-}
-
// searchableEmailHeaders returns the user-facing headers as a separate JSON
// text item. The main email item keeps its structured fields and body; the
// chunker indexes only item text, so headers need their own text item.
@@ -502,71 +408,6 @@ func attachmentFilename(part *multipart.Part) string {
// -- HTML body → visible text (text output only) --
-// htmlBodyToText flattens an email HTML body to its visible text. Unlike the
-// structured HTML parser (which keeps table markup for the dedicated HTML
-// chunk method), email bodies use tables for layout, so this walker descends
-// into tables and emits cell text instead. <head>/<script>/<style> subtrees
-// are skipped; whitespace folding is shared with the standalone HTML parser
-// via leafWriter.
-func htmlBodyToText(body string) string {
- if strings.TrimSpace(body) == "" {
- return ""
- }
- doc, err := html.Parse(strings.NewReader(body))
- if err != nil {
- return body
- }
- var b bytes.Buffer
- w := &leafWriter{b: &b, lineStart: true}
- walkHTMLBodyText(doc, w)
- return strings.TrimSpace(b.String())
-}
-
-// isEmailBlockTag reports whether tag delimits a line in flattened email
-// body text: the standard block tags plus table tags, since email uses
-// tables for layout and their cells/rows must not fuse either.
-func isEmailBlockTag(tag string) bool {
- return isBlockTag(tag) || tag == "table" || tag == "td" || tag == "th"
-}
-
-func walkHTMLBodyText(n *html.Node, w *leafWriter) {
- switch n.Type {
- case html.TextNode:
- w.writeText(n.Data)
- case html.DocumentNode:
- // html.Parse always wraps the fragment in a Document node holding an
- // <html><body> skeleton; the walker relies on descending through those
- // transparent containers (head/script/style subtrees are skipped by the
- // ElementNode case), so without descending here the whole body would
- // be dropped.
- for child := n.FirstChild; child != nil; child = child.NextSibling {
- walkHTMLBodyText(child, w)
- }
- case html.ElementNode:
- switch n.Data {
- case "head", "script", "style", "noscript", "template":
- return
- case "br":
- w.hardBreak()
- return
- }
- // A nested block starts on a new line when text already precedes
- // it, so outer text and inner block text stay separate
- // ("<div>a<p>b</p></div>" flattens to "a\nb", not "ab").
- if isEmailBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
- w.hardBreak()
- }
- for child := n.FirstChild; child != nil; child = child.NextSibling {
- walkHTMLBodyText(child, w)
- }
- // Block-level tags and table cells/rows end the current line so
- // adjacent block/cell text does not fuse together.
- if isEmailBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
- w.hardBreak()
- }
- }
-}
-
// decodeMailPayload attempts charset decoding using the unified DecodeToUTF8 helper.
func decodeMailPayload(payload []byte, charset string) string {
if len(payload) == 0 {
@@ -786,23 +627,22 @@ func recoverParse(fn func() ParseResult) (res ParseResult, panicked bool) {
// avoid unbounded recursion through attachments-of-attachments.
const maxRechunkPayloadBytes = 32 << 20 // 32 MiB
-func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[string]any, depth int) ([]map[string]any, string) {
+func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[string]any, depth int) []map[string]any {
if depth > 0 {
- return nil, ""
+ return nil
}
// Honor task cancellation so a long attachment list does not keep
// re-parsing after the ingestion task has been stopped/aborted.
if ctx.Err() != nil {
- return nil, ""
+ return nil
}
raw, ok := content["attachments"].([]map[string]any)
if !ok || len(raw) == 0 {
- return nil, ""
+ return nil
}
var extra []map[string]any
- var sb strings.Builder
for _, att := range raw {
fn, _ := att["filename"].(string)
payload, _ := att["payload"].(string)
@@ -842,8 +682,7 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[s
// path would otherwise also flatten metadata into the result).
ep := NewEmailParser()
ep.ConfigureFromSetup(map[string]any{
- "fields": p.fields,
- "output_format": "json",
+ "fields": p.fields,
})
res, panicked = recoverParse(func() ParseResult {
return ep.parseEmail(ctx, fn, []byte(payload), depth+1)
@@ -877,10 +716,6 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[s
}
}
}
- } else if res.Text != "" {
- if t := strings.TrimSpace(res.Text); t != "" {
- texts = append(texts, t)
- }
}
for _, t := range texts {
@@ -888,11 +723,7 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[s
"text": t,
"doc_type_kwd": "text",
})
- if sb.Len() > 0 {
- sb.WriteString("\n")
- }
- sb.WriteString(t)
}
}
- return extra, sb.String()
+ return extra
}
diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go
index 41c68a360b..2299d23cee 100644
--- a/internal/parser/parser/email_parser_test.go
+++ b/internal/parser/parser/email_parser_test.go
@@ -111,44 +111,6 @@ func TestEmailParser_EmlJSON(t *testing.T) {
}
}
-// TestEmailParser_EmlTextFieldOrder pins the order of the text output: the
-// headers first, in the order of the JSON header item, then the body. The
-// fields used to come out in Go's map order, a different one on every parse.
-func TestEmailParser_EmlTextFieldOrder(t *testing.T) {
- raw := strings.Join([]string{
- "Subject: Quarterly numbers",
- "Date: Mon, 1 Sep 2026 10:00:00 +0000",
- "Content-Type: text/plain; charset=utf-8",
- "",
- "Revenue grew.",
- }, "\r\n")
- want := strings.Join([]string{
- "date:Mon, 1 Sep 2026 10:00:00 +0000",
- "subject:Quarterly numbers",
- "text:Revenue grew.",
- "text_html:",
- "",
- }, "\n")
-
- for i := 0; i < 20; i++ {
- p := NewEmailParser()
- p.ConfigureFromSetup(map[string]any{"output_format": "text"})
- result := p.ParseWithResult(t.Context(), "order.eml", []byte(raw))
- if result.Err != nil {
- t.Fatalf("unexpected error: %v", result.Err)
- }
- if result.Text != want {
- t.Fatalf("parse %d: text output\n got: %q\nwant: %q", i, result.Text, want)
- }
- }
-}
-
func TestEmailParser_InvalidEMLReturnsError(t *testing.T) {
p := NewEmailParser()
result := p.ParseWithResult(t.Context(), "broken.eml", []byte("not a valid RFC 5322 header"))
@@ -157,46 +119,15 @@ func TestEmailParser_InvalidEMLReturnsError(t *testing.T) {
}
}
-func TestEmailParser_EmlText(t *testing.T) {
- ctx := t.Context()
- raw := strings.Join([]string{
- "Subject: Hello",
- "Content-Type: text/plain; charset=utf-8",
- "",
- "Hello, world!",
- }, "\r\n")
-
- p := NewEmailParser()
- p.ConfigureFromSetup(map[string]any{
- "output_format": "text",
- "fields": []string{"from", "to", "subject", "body"},
- })
-
- result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
- if result.Err != nil {
- t.Fatalf("unexpected error: %v", result.Err)
- }
- if result.OutputFormat != "text" {
- t.Fatalf("expected output_format text, got %q", result.OutputFormat)
- }
- if !strings.Contains(result.Text, "Hello, world!") {
- t.Errorf("text missing body: %q", result.Text)
- }
- if !strings.Contains(result.Text, "[email protected]") {
- t.Errorf("text missing from: %q", result.Text)
- }
-}
-
// TestEmailParser_EmlTextChunkText covers the KB chunk-quality bug behind
-// Feishu issue om_x100b675dcd7a18a4c2cbfff662a404e: the text output of an
-// .eml with an RFC 2047 encoded subject, transport headers (DKIM/ARC/
-// Received) and an HTML part used to carry (1) the flattened metadata dump,
-// (2) the raw "=?utf-8?B?...?=" subject blob and (3) shredded HTML markup —
-// all of which ended up in chunks. The text output must instead skip the
-// metadata dump, decode the subject, and flatten the HTML part to its
-// visible text. The JSON output keeps metadata and raw text_html unchanged.
+// Feishu issue om_x100b675dcd7a18a4c2cbfff662a404e: an .eml with an RFC 2047
+// encoded subject, transport headers (DKIM/ARC/Received) and an HTML part used
+// to carry (1) the flattened metadata dump, (2) the raw "=?utf-8?B?...?="
+// subject blob and (3) shredded HTML markup — all of which ended up in chunks.
+// The chunkable body text (the JSON item's "text" field) must instead skip the
+// metadata/transport-header dump, decode the subject, and flatten the HTML part
+// to its visible text. The JSON output keeps metadata and raw text_html
+// unchanged.
func TestEmailParser_EmlTextChunkText(t *testing.T) {
ctx := t.Context()
// "=?utf-8?B?5bim6ZmE5Lu255qEZW1haWw=?=" decodes to "带附件的email";
@@ -224,65 +155,59 @@ func TestEmailParser_EmlTextChunkText(t *testing.T) {
p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
- "output_format": "text",
- "fields": []string{"from", "to", "subject", "body"},
+ "fields": []string{"from", "to", "subject", "body"},
})
result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
+ if result.OutputFormat != "json" {
+ t.Fatalf("expected output_format json, got %q", result.OutputFormat)
+ }
+
+ item := result.JSON[0].ToMap()
+ bodyText, _ := item["text"].(string)
- // (1) No transport-header dump.
+ // (1) No transport-header dump in the chunkable body text.
for _, noise := range []string{"metadata:", "dkim-signature", "arc-seal", "received:"} {
- if strings.Contains(strings.ToLower(result.Text), noise) {
- t.Errorf("text output contains header noise %q: %q", noise, result.Text)
+ if strings.Contains(strings.ToLower(bodyText), noise) {
+ t.Errorf("body text contains header noise %q: %q", noise, bodyText)
}
}
- // (2) Subject/from decoded from RFC 2047.
- if !strings.Contains(result.Text, "subject:带附件的email") {
- t.Errorf("subject not decoded: %q", result.Text)
+ // (2) Subject/from decoded from RFC 2047 into the item fields.
+ if v, _ := item["subject"].(string); v != "带附件的email" {
+ t.Errorf("subject not decoded: %q", v)
}
- if !strings.Contains(result.Text, "from:安 <[email protected]>") {
- t.Errorf("from not decoded: %q", result.Text)
+ if v, _ := item["from"].(string); v != "安 <[email protected]>" {
+ t.Errorf("from not decoded: %q", v)
}
- if strings.Contains(result.Text, "=?utf-8?") {
- t.Errorf("raw encoded-word leaked into text: %q", result.Text)
+ if strings.Contains(bodyText, "=?utf-8?") {
+ t.Errorf("raw encoded-word leaked into text: %q", bodyText)
}
// (3) HTML part flattened to visible text, no markup.
for _, tag := range []string{"<div", "<img", "style="} {
- if strings.Contains(result.Text, tag) {
- t.Errorf("raw HTML markup %q leaked into text: %q", tag, result.Text)
+ if strings.Contains(bodyText, tag) {
+ t.Errorf("raw HTML markup %q leaked into text: %q", tag, bodyText)
}
}
- // HTML part must actually contribute its visible text: subject + plain
- // body + html body = exactly 3 occurrences of the body string.
- if c := strings.Count(result.Text, "带附件的email"); c != 3 {
- t.Errorf("expected subject+plain+html body text, got %d occurrences in: %q", c, result.Text)
+ // The plain body text must be present exactly once in the body text.
+ if c := strings.Count(bodyText, "带附件的email"); c != 1 {
+ t.Errorf("expected plain body text once, got %d occurrences in: %q", c, bodyText)
}
- if !strings.Contains(result.Text, "text_html:带附件的email") {
- t.Errorf("html body text missing from text_html: %q", result.Text)
+ // The HTML part is kept separately as raw text_html (the chunkable body
+ // text never carries shredded HTML markup).
+ if v, _ := item["text_html"].(string); !strings.Contains(v, "带附件的email") {
+ t.Errorf("html body text missing from text_html: %q", v)
}
- // JSON output keeps the full metadata and raw text_html.
- p.ConfigureFromSetup(map[string]any{
- "output_format": "json",
- "fields": []string{"from", "to", "subject", "body"},
- })
- jres := p.ParseWithResult(ctx, "test.eml", []byte(raw))
- if jres.Err != nil {
- t.Fatalf("unexpected error: %v", jres.Err)
- }
- item := jres.JSON[0].ToMap()
+ // (4) The JSON output keeps the full metadata and raw text_html.
if _, ok := item["metadata"].(map[string]any); !ok {
t.Errorf("json output must keep metadata: %#v", item)
}
if v, _ := item["text_html"].(string); !strings.Contains(v, "<img") {
t.Errorf("json output must keep raw text_html: %q", v)
}
- if v, _ := item["subject"].(string); v != "带附件的email" {
- t.Errorf("json subject not decoded: %q", v)
- }
}
// TestEmailParser_MsgSupported parses a real Outlook .msg fixture and verifies
@@ -814,50 +739,6 @@ func TestEmailParser_AttachmentSearchableJSON(t *testing.T) {
}
}
-// TestEmailParser_AttachmentSearchableText is the text-output equivalent:
-// the re-parsed attachment text must be present in result.Text.
-func TestEmailParser_AttachmentSearchableText(t *testing.T) {
- ctx := t.Context()
- attachmentContent := "QUOTE: the quick brown fox jumps over the lazy dog."
- encoded := base64.StdEncoding.EncodeToString([]byte(attachmentContent))
- boundary := "attachboundary"
- raw := strings.Join([]string{
- "Subject: Attachment Test",
- "Content-Type: multipart/mixed; boundary=" + boundary,
- "",
- "--" + boundary,
- "Content-Type: text/plain; charset=utf-8",
- "",
- "Email body text.",
- "--" + boundary,
- "Content-Type: text/plain; charset=utf-8",
- "Content-Disposition: attachment; filename=\"note.txt\"",
- "Content-Transfer-Encoding: base64",
- "",
- encoded,
- "--" + boundary + "--",
- }, "\r\n")
-
- p := NewEmailParser()
- p.ConfigureFromSetup(map[string]any{
- "output_format": "text",
- "fields": []string{"from", "body", "attachments"},
- })
-
- result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
- if result.Err != nil {
- t.Fatalf("unexpected error: %v", result.Err)
- }
- if !strings.Contains(result.Text, "quick brown fox") {
- t.Errorf("attachment text missing from text output: %q", result.Text)
- }
- if !strings.Contains(result.Text, "Email body text") {
- t.Errorf("email body missing from text output: %q", result.Text)
- }
-}
-
// TestEmailParser_AttachmentEmptyPayloadSkipped verifies error isolation:
// an attachment with an empty payload (the legacy .msg nested-attachment
// case, where gomsg exposes no raw bytes) is skipped without breaking the
@@ -1076,65 +957,47 @@ func TestEmailParser_NestedEMLAttachmentRechunk(t *testing.T) {
"--" + boundary + "--",
}, "\r\n")
- for _, format := range []string{"json", "text"} {
- p := NewEmailParser()
- p.ConfigureFromSetup(map[string]any{
- "output_format": format,
- "fields": []string{"from", "to", "subject", "body", "attachments", "metadata"},
- })
+ p := NewEmailParser()
+ p.ConfigureFromSetup(map[string]any{
+ "fields": []string{"from", "to", "subject", "body", "attachments", "metadata"},
+ })
- result := p.ParseWithResult(ctx, "outer.eml", []byte(raw))
- if result.Err != nil {
- t.Fatalf("[%s] unexpected error: %v", format, result.Err)
- }
+ result := p.ParseWithResult(ctx, "outer.eml", []byte(raw))
+ if result.Err != nil {
+ t.Fatalf("unexpected error: %v", result.Err)
+ }
- // Collect all indexed text for this output mode (JSON populates
- // result.JSON; text populates result.Text).
- var indexed strings.Builder
- if format == "json" {
- for _, it := range ItemsToMaps(result.JSON) {
- if txt, ok := it["text"].(string); ok {
- indexed.WriteString(txt)
- indexed.WriteString("\n")
- }
- }
- } else {
- indexed.WriteString(result.Text)
+ // Collect all indexed text across JSON items.
+ var indexed strings.Builder
+ for _, it := range ItemsToMaps(result.JSON) {
+ if txt, ok := it["text"].(string); ok {
+ indexed.WriteString(txt)
+ indexed.WriteString("\n")
}
+ }
- // The nested email body must be retrievable.
- if !strings.Contains(indexed.String(), "INNER BODY SECRET") {
- t.Errorf("[%s] nested body not indexed: %q", format, indexed.String())
- }
- if !strings.Contains(indexed.String(), "OUTER BODY VISIBLE") {
- t.Errorf("[%s] outer body missing: %q", format, indexed.String())
- }
+ // The nested email body must be retrievable.
+ if !strings.Contains(indexed.String(), "INNER BODY SECRET") {
+ t.Errorf("nested body not indexed: %q", indexed.String())
+ }
+ if !strings.Contains(indexed.String(), "OUTER BODY VISIBLE") {
+ t.Errorf("outer body missing: %q", indexed.String())
+ }
- if format == "json" {
- // The nested email must be a distinct, clean item, carrying no
- // metadata key (the old bug leaked a "metadata:{...}" string as
- // its text).
- found := false
- for _, it := range ItemsToMaps(result.JSON) {
- if txt, ok := it["text"].(string); ok && strings.Contains(txt, "INNER BODY SECRET") {
- found = true
- if _, hasMeta := it["metadata"]; hasMeta {
- t.Errorf("[json] nested item must not carry metadata key: %#v", it)
- }
- }
- }
- if !found {
- t.Errorf("[json] nested body not found as separate item: %#v", result.JSON)
- }
- } else {
- // Text mode: metadata (transport headers) is never flattened
- // into chunkable text — not for the outer email, and not for
- // the nested re-parse (which always requests JSON output).
- if strings.Contains(result.Text, "metadata:") {
- t.Errorf("[text] metadata must not leak into text output: %q", result.Text)
+ // The nested email must be a distinct, clean item, carrying no metadata
+ // key (the old bug leaked a "metadata:{...}" string as its text).
+ found := false
+ for _, it := range ItemsToMaps(result.JSON) {
+ if txt, ok := it["text"].(string); ok && strings.Contains(txt, "INNER BODY SECRET") {
+ found = true
+ if _, hasMeta := it["metadata"]; hasMeta {
+ t.Errorf("nested item must not carry metadata key: %#v", it)
}
}
}
+ if !found {
+ t.Errorf("nested body not found as separate item: %#v", result.JSON)
+ }
}
// TestEmailParser_RechunkAttachmentTextPath exercises rechunkEmailAttachments
@@ -1152,7 +1015,7 @@ func TestEmailParser_RechunkAttachmentTextPath(t *testing.T) {
}
p := NewEmailParser()
- extra, text := p.rechunkEmailAttachments(ctx, content, 0)
+ extra := p.rechunkEmailAttachments(ctx, content, 0)
if len(extra) != 1 {
t.Fatalf("expected 1 indexed attachment, got %d: %#v", len(extra), extra)
@@ -1160,11 +1023,12 @@ func TestEmailParser_RechunkAttachmentTextPath(t *testing.T) {
if v, _ := extra[0]["text"].(string); v != "hello from attachment" {
t.Errorf("text = %q, want hello from attachment", v)
}
- if !strings.Contains(text, "hello from attachment") {
- t.Errorf("indexed text missing attachment: %q", text)
- }
- if strings.Contains(text, "<binary bytes>") {
- t.Errorf("binary payload leaked into indexed text: %q", text)
+ // The flat text return was removed; verify the binary attachment is not
+ // indexed as a separate text item (only the .txt attachment should appear).
+ for _, item := range extra {
+ if s, _ := item["text"].(string); strings.Contains(s, "<binary bytes>") {
+ t.Errorf("binary payload leaked into indexed text: %q", s)
+ }
}
}
@@ -1192,9 +1056,9 @@ func TestRechunkEmailAttachments_ContextCancelled(t *testing.T) {
{"filename": "note.txt", "payload": "must not be re-chunked"},
},
}
- extra, text := NewEmailParser().rechunkEmailAttachments(ctx, content, 0)
- if len(extra) != 0 || text != "" {
- t.Errorf("cancelled context should short-circuit re-chunk: extra=%#v text=%q", extra, text)
+ extra := NewEmailParser().rechunkEmailAttachments(ctx, content, 0)
+ if len(extra) != 0 {
+ t.Errorf("cancelled context should short-circuit re-chunk: extra=%#v", extra)
}
}
@@ -1209,7 +1073,7 @@ func TestRecoverParse_IsolatesPanic(t *testing.T) {
if !panicked {
t.Error("expected panicked=true")
}
- if res.Err != nil || res.JSON != nil || res.Text != "" {
+ if res.Err != nil || res.JSON != nil {
t.Errorf("panic should yield a zero ParseResult, got %#v", res)
}
}
@@ -1217,14 +1081,14 @@ func TestRecoverParse_IsolatesPanic(t *testing.T) {
// TestRecoverParse_PassesThrough verifies a normal parser result is returned
// unchanged with panicked=false.
func TestRecoverParse_PassesThrough(t *testing.T) {
- want := ParseResult{OutputFormat: "text", Text: "hello"}
+ want := ParseResult{OutputFormat: "json", JSON: []DocItem{NewTextJSONItem("hello")}}
res, panicked := recoverParse(func() ParseResult {
return want
})
if panicked {
t.Error("expected panicked=false for a normal result")
}
- if res.Text != "hello" {
+ if txt, ok := res.JSON[0].ToMap()["text"].(string); !ok || txt != "hello" {
t.Errorf("result not passed through: %#v", res)
}
}
@@ -1236,25 +1100,22 @@ func TestEmailParser_RechunkPrefersRaw(t *testing.T) {
ctx := t.Context()
p := NewEmailParser()
- withRaw, textRaw := p.rechunkEmailAttachments(ctx, map[string]any{
+ withRaw := p.rechunkEmailAttachments(ctx, map[string]any{
"attachments": []map[string]any{
{"filename": "note.txt", "payload": "WORLD", "raw": "HELLO"},
},
}, 0)
if len(withRaw) != 1 || withRaw[0]["text"] != "HELLO" {
- t.Fatalf("raw not preferred: extra=%#v text=%q", withRaw, textRaw)
- }
- if strings.Contains(textRaw, "WORLD") {
- t.Errorf("re-chunk used payload instead of raw: %q", textRaw)
+ t.Fatalf("raw not preferred: extra=%#v", withRaw)
}
- fallback, textFallback := p.rechunkEmailAttachments(ctx, map[string]any{
+ fallback := p.rechunkEmailAttachments(ctx, map[string]any{
"attachments": []map[string]any{
{"filename": "note.txt", "payload": "WORLD"},
},
}, 0)
if len(fallback) != 1 || fallback[0]["text"] != "WORLD" {
- t.Fatalf("payload fallback broken: extra=%#v text=%q", fallback, textFallback)
+ t.Fatalf("payload fallback broken: extra=%#v", fallback)
}
}
@@ -1405,45 +1266,32 @@ func TestParseEML_SinglePartTransferEncoding(t *testing.T) {
body string
wantText string
wantHTML string
- // encoded is the payload as it appears on the wire. The text output
- // feeds the chunker, so it must never carry these bytes.
- encoded string
- // wantInText is the visible text the flattened output must carry.
- wantInText string
}{
{
- name: "base64 text/plain",
- headers: "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64",
- body: b64,
- wantText: plain,
- encoded: b64,
- wantInText: "Quarterly revenue was 12.5 million.",
+ name: "base64 text/plain",
+ headers: "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64",
+ body: b64,
+ wantText: plain,
},
{
- name: "quoted-printable text/plain",
- headers: "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: quoted-printable",
- body: "Quarterly revenue was 12=2E5 million=2E",
- wantText: "Quarterly revenue was 12.5 million.",
- encoded: "12=2E5",
- wantInText: "Quarterly revenue was 12.5 million.",
+ name: "quoted-printable text/plain",
+ headers: "Content-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: quoted-printable",
+ body: "Quarterly revenue was 12=2E5 million=2E",
+ wantText: "Quarterly revenue was 12.5 million.",
},
{
- name: "base64 text/html",
- headers: "Content-Type: text/html; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64",
- body: htmlB64,
- wantHTML: html,
- encoded: htmlB64,
- wantInText: "Quarterly revenue",
+ name: "base64 text/html",
+ headers: "Content-Type: text/html; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64",
+ body: htmlB64,
+ wantHTML: html,
},
{
// Positive control: the multipart path already decodes the same
// body, so the two paths must agree.
- name: "base64 in multipart/alternative",
- headers: "Content-Type: multipart/alternative; boundary=\"BB\"",
- body: "--BB\r\nContent-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n" + b64 + "\r\n--BB--",
- wantText: plain,
- encoded: b64,
- wantInText: "Quarterly revenue was 12.5 million.",
+ name: "base64 in multipart/alternative",
+ headers: "Content-Type: multipart/alternative; boundary=\"BB\"",
+ body: "--BB\r\nContent-Type: text/plain; charset=\"utf-8\"\r\nContent-Transfer-Encoding: base64\r\n\r\n" + b64 + "\r\n--BB--",
+ wantText: plain,
},
}
@@ -1468,64 +1316,18 @@ func TestParseEML_SinglePartTransferEncoding(t *testing.T) {
if strings.TrimRight(gotHTML, "\r\n") != strings.TrimRight(tt.wantHTML, "\r\n") {
t.Errorf("text_html = %q, want %q", gotHTML, tt.wantHTML)
}
-
- // The text output is what reaches the chunker, and the JSON
- // output returns before it is built, so assert it separately.
- pt := NewEmailParser()
- pt.ConfigureFromSetup(map[string]any{"output_format": "text"})
- textResult := pt.ParseWithResult(t.Context(), "test.eml", []byte(raw))
- if textResult.Err != nil {
- t.Fatalf("unexpected error on the text path: %v", textResult.Err)
- }
- flat := textResult.Text
- if !strings.Contains(flat, tt.wantInText) {
- t.Errorf("text output = %q, want it to contain %q", flat, tt.wantInText)
- }
- if strings.Contains(flat, tt.encoded) {
- t.Errorf("text output still carries the encoded payload %q: %q", tt.encoded, flat)
- }
})
}
}
-// TestHTMLBodyToText verifies the visible-text flattening of email HTML
-// bodies, above all that text preceding a nested block-level element starts
-// a new line instead of fusing with the block's own text.
-func TestHTMLBodyToText(t *testing.T) {
- tests := []struct {
- name string
- body string
- want string
- }{
- {"text before nested block", `<div>prefix<p>paragraph</p></div>`, "prefix\nparagraph"},
- {"nested divs", `<div>outer<div>inner</div></div>`, "outer\ninner"},
- {"sibling blocks", `<p>one</p><p>two</p>`, "one\ntwo"},
- {"layout table cells", `<table><tr><td>a</td><td>b</td></tr></table>`, "a\nb"},
- {"multi-row layout table stays line-separated", `<p>intro</p><table><tr><td>a</td><td>b</td></tr><tr><td>c</td></tr></table><p>outro</p>`, "intro\na\nb\nc\noutro"},
- {"inline tags stay on one line", `<p>Hello <b>world</b></p>`, "Hello world"},
- {"definition list and rule", `<div>A<dl><dt>Term</dt><dd>Def</dd></dl>B<hr>C</div>`, "A\nTerm\nDef\nB\nC"},
- {"br forces a break", `one<br>two`, "one\ntwo"},
- {"head script style skipped", `<html><head><title>t</title><style>.x{color:red}</style></head><body><p>visible</p><script>var x=1;</script></body></html>`, "visible"},
- {"template content skipped", `<p>visible</p><template><p>Hello {{name}}</p></template>`, "visible"},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- if got := htmlBodyToText(tt.body); got != tt.want {
- t.Errorf("htmlBodyToText(%q) = %q, want %q", tt.body, got, tt.want)
- }
- })
- }
-}
-
-// TestEmailParserJSONPathOmitsDuplicateText verifies that the JSON output
-// path no longer duplicates the flattened email body into result.Text.
-// result.Text is only populated on the "text" output_format branch; on the
-// JSON branch the searchable text lives in the JSON items (body + the
-// header text item from searchableEmailHeaders), so the redundant json-path
-// Text is intentionally empty. This guards the removal of the json-path
-// Text assignment in parseEmail.
-func TestEmailParserJSONPathOmitsDuplicateText(t *testing.T) {
- p := NewEmailParser() // default output_format -> "json"
+// TestEmailParserJSONOmitsDuplicateHeaderText verifies the JSON output keeps
+// the decoded body text without duplicating the flattened header text that the
+// legacy single-string path used to carry. The body text lives in the body
+// item's "text" field; the header fields (from/to/subject/...) are separate
+// top-level keys, so the body text must not re-contain a "from:..." header
+// line.
+func TestEmailParserJSONOmitsDuplicateHeaderText(t *testing.T) {
+ p := NewEmailParser()
p.ConfigureFromSetup(map[string]any{
"fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},
})
@@ -1543,32 +1345,40 @@ func TestEmailParserJSONPathOmitsDuplicateText(t *testing.T) {
if len(res.JSON) == 0 {
t.Fatal("JSON: want non-empty items")
}
- if res.Text != "" {
- t.Errorf("JSON-path Email result.Text = %q, want empty (text lives in JSON items)", res.Text)
+ bodyText, _ := res.JSON[0].ToMap()["text"].(string)
+ if strings.Contains(bodyText, "from:") || strings.Contains(bodyText, "subject:") {
+ t.Errorf("JSON body text must not duplicate flattened header lines: %q", bodyText)
}
}
-// TestEmailParserTextPathKeepsText guards that the explicit "text"
-// output_format branch still populates result.Text — the only place the
-// field is read downstream (sys.files parseResultText, document preview,
-// and nested attachment re-parse).
-func TestEmailParserTextPathKeepsText(t *testing.T) {
+// TestEmailParserTextOutputRemoved verifies the Email parser has no "text"
+// output path. A setup requesting output_format "text" must be ignored and the
+// result must be JSON, never a Text payload.
+func TestEmailParserTextOutputRemoved(t *testing.T) {
+ ctx := context.Background()
+ raw := strings.Join([]string{
+ "Subject: Hello",
+ "Content-Type: text/plain; charset=utf-8",
+ "",
+ "Hello, world!",
+ }, "\r\n")
+
p := NewEmailParser()
- p.ConfigureFromSetup(map[string]any{"output_format": "text"})
- const eml = "From: [email protected]\r\n" +
- "To: [email protected]\r\n" +
- "Subject: Hi\r\n" +
- "Date: Mon, 01 Jan 2024 00:00:00 +0000\r\n" +
- "\r\n" +
- "Hello body.\r\n"
- res := p.ParseWithResult(t.Context(), "msg.eml", []byte(eml))
- if res.Err != nil {
- t.Fatalf("ParseWithResult: %v", res.Err)
+ p.ConfigureFromSetup(map[string]any{
+ "output_format": "text",
+ "fields": []string{"from", "to", "subject", "body"},
+ })
+
+ result := p.ParseWithResult(ctx, "test.eml", []byte(raw))
+ if result.Err != nil {
+ t.Fatalf("unexpected error: %v", result.Err)
}
- if res.OutputFormat != "text" {
- t.Fatalf("OutputFormat = %q, want text", res.OutputFormat)
+ if result.OutputFormat != "json" {
+ t.Fatalf("email must ignore text output_format and emit json, got OutputFormat=%q", result.OutputFormat)
}
- if res.Text == "" {
- t.Error("text-path Email result.Text empty, want body text")
+ if len(result.JSON) == 0 {
+ t.Fatalf("email json output must contain items, got empty")
}
}
diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.go
index 3398e8997b..5799c4d24c 100644
--- a/internal/parser/parser/parse_result.go
+++ b/internal/parser/parser/parse_result.go
@@ -71,12 +71,11 @@ type ParseResult struct {
// and, after ingestion enhancement, its VLM description.
JSON []DocItem
- // Markdown is a backend response awaiting normalization.
+ // Markdown is a backend response awaiting normalization (PDF markdown
+ // engines only; the DSL hardcodes output_format to "json" for all other
+ // families via normalizeParserOutputFormats).
Markdown string
- // Text is a backend response awaiting normalization.
- Text string
-
// Err is the failure reason. On non-nil Err, all payload
// fields are zero values.
Err error
diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.go
index 073084b577..96f920a9eb 100644
--- a/internal/parser/parser/parse_result_test.go
+++ b/internal/parser/parser/parse_result_test.go
@@ -84,14 +84,6 @@ func TestParseResult_Contract(t *testing.T) {
},
wantFmt: "markdown",
},
- {
- name: "text family only",
- in: ParseResult{
- OutputFormat: "text",
- Text: "raw text",
- },
- wantFmt: "text",
- },
{
name: "failure clears payload",
in: ParseResult{Err: errSentinel},
@@ -107,7 +99,7 @@ func TestParseResult_Contract(t *testing.T) {
if tc.in.OutputFormat != "" {
t.Errorf("OutputFormat on failure = %q, want empty", tc.in.OutputFormat)
}
- if tc.in.JSON != nil || tc.in.Markdown != "" || tc.in.Text != "" {
+ if tc.in.JSON != nil || tc.in.Markdown != "" {
t.Errorf("payload fields must be zero on failure; got %+v", tc.in)
}
return
@@ -203,13 +195,11 @@ func payloadFamily(r ParseResult) string {
return "json"
case "markdown":
return "markdown"
- case "text":
- return "text"
}
return ""
}
-var allPayloadFamilies = []string{"json", "markdown", "text"}
+var allPayloadFamilies = []string{"json", "markdown"}
func isPayloadZero(r ParseResult, family string) bool {
switch family {
@@ -217,8 +207,6 @@ func isPayloadZero(r ParseResult, family string) bool {
return r.JSON == nil
case "markdown":
return r.Markdown == ""
- case "text":
- return r.Text == ""
}
return false
}
diff --git a/internal/parser/parser/pdf_result.go b/internal/parser/parser/pdf_result.go
index 6933b073e7..5b91414fc0 100644
--- a/internal/parser/parser/pdf_result.go
+++ b/internal/parser/parser/pdf_result.go
@@ -55,8 +55,6 @@ func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, result
result.JSON = parsed.JSON
}
}
- case result.Text != "":
- result.JSON = []DocItem{NewTextJSONItem(result.Text)}
}
}
items := make([]DocItem, 0, len(result.JSON))
@@ -139,7 +137,7 @@ func (p *PDFParser) NormalizeResult(ctx context.Context, filename string, result
zap.Int("dropped_rows", droppedPositions))
}
result.JSON = items
- result.OutputFormat, result.Markdown, result.Text = "json", "", ""
+ result.OutputFormat, result.Markdown = "json", ""
result.File = ensurePDFResultFile(result.File, filename)
return result
}
diff --git a/internal/parser/parser/rechunk_singlereturn_test.go b/internal/parser/parser/rechunk_singlereturn_test.go
new file mode 100644
index 0000000000..81652f334c
--- /dev/null
+++ b/internal/parser/parser/rechunk_singlereturn_test.go
@@ -0,0 +1,69 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+ "context"
+ "strings"
+ "testing"
+)
+
+// TestRechunkEmailAttachmentsSingleReturn guards the dead-code cleanup of
+// rechunkEmailAttachments: it must return only the re-parsed attachment items
+// ([]map[string]any) and must no longer build/return a discarded flat-text
+// string. If the second return value is reintroduced, this test fails to
+// compile.
+func TestRechunkEmailAttachmentsSingleReturn(t *testing.T) {
+ p := NewEmailParser()
+ ctx := context.Background()
+
+ // depth>0 must stop recursion immediately.
+ if got := p.rechunkEmailAttachments(ctx, map[string]any{}, 1); got != nil {
+ t.Fatalf("depth>0 should return nil, got %d items", len(got))
+ }
+
+ // No attachments -> empty, never a nil-panic on a removed return value.
+ if got := p.rechunkEmailAttachments(ctx, map[string]any{}, 0); len(got) != 0 {
+ t.Fatalf("no attachments should yield 0 items, got %d", len(got))
+ }
+
+ // A non-text (image) attachment is skipped, not re-parsed.
+ image := map[string]any{
+ "attachments": []map[string]any{
+ {"filename": "pic.png", "payload": "binary", "raw": "binary"},
+ },
+ }
+ if got := p.rechunkEmailAttachments(ctx, image, 0); len(got) != 0 {
+ t.Fatalf("image attachment should be skipped, got %d items", len(got))
+ }
+
+ // A .txt attachment re-parses into a searchable text item.
+ txt := map[string]any{
+ "attachments": []map[string]any{
+ {"filename": "note.txt", "payload": "hello attachment"},
+ },
+ }
+ got := p.rechunkEmailAttachments(ctx, txt, 0)
+ if len(got) != 1 {
+ t.Fatalf("txt attachment should yield 1 item, got %d", len(got))
+ }
+ if got[0]["doc_type_kwd"] != "text" {
+ t.Errorf("item doc_type_kwd = %v, want text", got[0]["doc_type_kwd"])
+ }
+ if text, ok := got[0]["text"].(string); !ok || !strings.Contains(text, "hello attachment") {
+ t.Errorf("item text = %v, want it to contain %q", got[0]["text"], "hello attachment")
+ }
+}
diff --git a/internal/service/document/document_artifact.go b/internal/service/document/document_artifact.go
index 4144a6ecb6..5d64eec8c0 100644
--- a/internal/service/document/document_artifact.go
+++ b/internal/service/document/document_artifact.go
@@ -314,11 +314,21 @@ func (s *DocumentService) GetDocumentPreview(ctx context.Context, userID, docID
}
// extractDOCPreviewText extracts plain text from a legacy .doc
-// document via the office_oxide-backed DOCParser.
+// document via the office_oxide-backed DOCParser. The parser only emits JSON
+// payloads, so the preview text is flattened from the JSON items.
func extractDOCPreviewText(ctx context.Context, filename string, data []byte) (string, error) {
res := parser.NewDOCParser().ParseWithResult(ctx, filename, data)
if res.Err != nil {
return "", res.Err
}
- return res.Text, nil
+ if len(res.JSON) == 0 {
+ return "", fmt.Errorf("doc preview: no content extracted")
+ }
+ parts := make([]string, 0, len(res.JSON))
+ for _, item := range res.JSON {
+ if text, ok := item.ToMap()["text"].(string); ok {
+ parts = append(parts, text)
+ }
+ }
+ return strings.Join(parts, "\n"), nil
}
diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.go
index 7db25ec4fc..9bdf9ac38f 100644
--- a/internal/service/file/file_content.go
+++ b/internal/service/file/file_content.go
@@ -242,8 +242,6 @@ func parseAgentUploadContent(ctx context.Context, filename string, data []byte,
// text field and serializing items without one as a final fallback.
func parseResultText(res parser.ParseResult) (string, error) {
switch strings.ToLower(strings.TrimSpace(res.OutputFormat)) {
- case "text":
- return res.Text, nil
case "markdown":
return res.Markdown, nil
case "json":
@@ -270,10 +268,8 @@ func parseResultText(res parser.ParseResult) (string, error) {
// Some legacy parsers mark the result as JSON while only populating a
// rendered companion field. Preserve that content instead of returning
// an empty sys.files value.
- for _, fallback := range []string{res.Markdown, res.Text} {
- if fallback != "" {
- return fallback, nil
- }
+ if res.Markdown != "" {
+ return res.Markdown, nil
}
return "", nil
default: