RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): prefer structured IR over flatter views for .doc (#19147)
xugangqiang4168b3f9df26
diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.goindex 0f0f3cf5ec..d4df6fa880 100644--- a/internal/parser/parser/doc_parser.go+++ b/internal/parser/parser/doc_parser.go@@ -40,11 +40,12 @@ func (p *DOCParser) String() string {// office_oxide, which supports legacy .doc. We prefer the structured IR// (flattened to plain text) and fall back to ToMarkdown, then to PlainText.//-// Note: office_oxide's .doc reader only surfaces paragraph and (heuristically-// detected) heading lines — it does NOT extract table or list structure from-// legacy .doc (the table stream's PAP/TAP/LSTF are ignored; see-// office_oxide issue #115). So this path normalizes paragraph/heading line-// breaks but does not recover tables/lists. OutputFormat stays "text" to keep+// office_oxide recovers table structure from legacy .doc through the IR+// (office_oxide/go v0.1.9, #116); list structure depends on whether the .doc+// reader surfaces list elements. Paragraph and heading (heuristically+// detected) lines are always present. When the IR carries table/list+// structure it is preferred so that content unique to the IR view is never+// shadowed by a longer but flatter view. OutputFormat stays "text" to keep// the downstream contract unchanged.func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {doc, err := officeOxide.OpenFromBytes(data, "doc")@@ -68,33 +69,62 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [// extractDocText returns the best-effort plain text for a legacy .doc// document. office_oxide exposes three text views:// - ToIRJSON: a structured DocumentIR, flattened to plain text via-// flattenDocIR. For .doc this only carries paragraph and (heuristic)-// heading lines — no table/list structure (office_oxide issue #115).+// flattenDocIR. Recovers table structure (office_oxide/go v0.1.9, #116)+// and, when the reader surfaces them, list structure.// - ToMarkdown: paragraphs separated by blank lines.// - PlainText: the raw concatenated text.//-// We return the longest non-empty view so a sparser view can never shadow a-// more complete one (e.g. the IR flatten can be shorter than PlainText for-// some documents). A failure at every stage degrades to the PlainText error,-// preserving the original "no text at all" failure semantics.+// The IR is preferred when it carries table/list structure (that content is+// unique to the IR view and would be silently dropped under a pure length+// comparison — a prose-heavy document can make PlainText longer than the IR+// even though the IR is more complete). Otherwise the longest non-empty view+// is chosen so a sparser view never shadows a more complete one. A failure at+// every stage degrades to the PlainText error, preserving the original "no+// text at all" failure semantics.func extractDocText(doc *officeOxide.Document) (string, error) {- var best string- if irJSON, err := doc.ToIRJSON(); err == nil {- if t := flattenDocIR(irJSON); len(strings.TrimSpace(t)) > len(strings.TrimSpace(best)) {- best = t- }+ var irJSON, irText, mdText, plainText string+ if j, err := doc.ToIRJSON(); err == nil {+ irJSON = j+ irText = flattenDocIR(j)}if md, err := doc.ToMarkdown(); err == nil {- if len(strings.TrimSpace(md)) > len(strings.TrimSpace(best)) {- best = md- }+ mdText = md}if plain, err := doc.PlainText(); err == nil {- if len(strings.TrimSpace(plain)) > len(strings.TrimSpace(best)) {- best = plain- }- } else if best == "" {+ plainText = plain+ } else if strings.TrimSpace(irText) == "" && strings.TrimSpace(mdText) == "" {+ // Every view failed (or produced nothing): keep the original+ // "no text at all" failure semantics.return "", err}- return best, nil+ return selectDocTextView(irJSON, irText, mdText, plainText), nil+}++// selectDocTextView chooses the best plain-text rendering from office_oxide's+// three views. irJSON is the raw IR (used only to detect structured elements);+// irText is its flattened form. When the IR recovers table or list structure+// it is preferred, because that content is unique to the IR view. Otherwise the+// longest non-empty view wins, so a sparser view never shadows a more complete+// one.+func selectDocTextView(irJSON, irText, mdText, plainText string) string {+ if strings.TrimSpace(irText) != "" && irHasStructuredContent(irJSON) {+ return irText+ }+ best := irText+ if len(strings.TrimSpace(mdText)) > len(strings.TrimSpace(best)) {+ best = mdText+ }+ if len(strings.TrimSpace(plainText)) > len(strings.TrimSpace(best)) {+ best = plainText+ }+ return best+}++// irHasStructuredContent reports whether the office_oxide IR JSON carries+// table or list elements. Those structures are only surfaced by the IR view;+// ToMarkdown and PlainText flatten them away. Detection keys on the canonical+// `"type":"table"` / `"type":"list"` field emitted by office_oxide.+func irHasStructuredContent(irJSON string) bool {+ return strings.Contains(irJSON, `"type":"table"`) ||+ strings.Contains(irJSON, `"type":"list"`)}diff --git a/internal/parser/parser/doc_text_select_test.go b/internal/parser/parser/doc_text_select_test.gonew file mode 100644index 0000000000..b5faa19412--- /dev/null+++ b/internal/parser/parser/doc_text_select_test.go@@ -0,0 +1,59 @@+//go:build cgo++package parser++import "testing"++// TestSelectDocTextViewPrefersStructuredIR locks the fix for the fragile+// "longest view" heuristic in extractDocText: when the office_oxide IR carries+// table (or list) structure, that content is unique to the IR view and must+// win even if PlainText/ToMarkdown produce a longer (but flatter) string.+// Before the fix, a prose-heavy .doc could make PlainText longer than the IR+// and silently drop the recovered table.+func TestSelectDocTextViewPrefersStructuredIR(t *testing.T) {+ // IR JSON that carries a table element (the structure only the IR view+ // surfaces). irText is its flattened form with the " | " separator.+ irJSON := `{"sections":[{"elements":[{"type":"table","rows":[]}]}]}`+ irText := "Name | Value\nAlice | 1"++ // PlainText is deliberately longer than the IR flatten — the old+ // length-only heuristic would have shadowed the table here.+ plainText := "This is a long block of prose that is clearly longer " ++ "than the few table cells above, so a naive longest-view selection " ++ "would pick this string and drop the recovered table structure."++ got := selectDocTextView(irJSON, irText, "", plainText)+ if got != irText {+ t.Errorf("structured IR not preferred:\n got=%q\nwant=%q", got, irText)+ }+ if !contains(got, " | ") {+ t.Errorf("table separator lost in selected view: %q", got)+ }+}++// TestSelectDocTextViewFallsBackToLongest verifies that, absent any structured+// element, the selection still prefers the longest non-empty view (the+// original behavior, preserved so a sparser view never shadows a more+// complete one).+func TestSelectDocTextViewFallsBackToLongest(t *testing.T) {+ // No table/list in the IR.+ irJSON := `{"sections":[{"elements":[{"type":"paragraph"}]}]}`++ irText := "short paragraph"+ mdText := "a medium length markdown rendering of the document body"+ plainText := "the longest plain text among the three available views here"++ got := selectDocTextView(irJSON, irText, mdText, plainText)+ if got != plainText {+ t.Errorf("longest view not selected:\n got=%q\nwant=%q", got, plainText)+ }+}++func contains(s, sub string) bool {+ for i := 0; i+len(sub) <= len(s); i++ {+ if s[i:i+len(sub)] == sub {+ return true+ }+ }+ return false+}