Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): prefer structured IR over flatter views for .doc (#19147)

xugangqiang4168b3f9df26
diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.go
index 0f0f3cf5ec..d4df6fa880 100644
--- a/internal/parser/parser/doc_parser.go
+++ b/internal/parser/parser/doc_parser.go
@@ -40,11 +40,12 @@ func (p *DOCParser) String() string {
// office_oxide, which supports legacy .doc. We prefer the structured IR
// (flattened to plain text) and fall back to ToMarkdown, then to PlainText.
//
-// Note: office_oxide's .doc reader only surfaces paragraph and (heuristically
-// detected) heading lines — it does NOT extract table or list structure from
-// legacy .doc (the table stream's PAP/TAP/LSTF are ignored; see
-// office_oxide issue #115). So this path normalizes paragraph/heading line
-// breaks but does not recover tables/lists. OutputFormat stays "text" to keep
+// office_oxide recovers table structure from legacy .doc through the IR
+// (office_oxide/go v0.1.9, #116); list structure depends on whether the .doc
+// reader surfaces list elements. Paragraph and heading (heuristically
+// detected) lines are always present. When the IR carries table/list
+// structure it is preferred so that content unique to the IR view is never
+// shadowed by a longer but flatter view. OutputFormat stays "text" to keep
// the downstream contract unchanged.
func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
doc, err := officeOxide.OpenFromBytes(data, "doc")
@@ -68,33 +69,62 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
// extractDocText returns the best-effort plain text for a legacy .doc
// document. office_oxide exposes three text views:
// - ToIRJSON: a structured DocumentIR, flattened to plain text via
-// flattenDocIR. For .doc this only carries paragraph and (heuristic)
-// heading lines — no table/list structure (office_oxide issue #115).
+// flattenDocIR. Recovers table structure (office_oxide/go v0.1.9, #116)
+// and, when the reader surfaces them, list structure.
// - ToMarkdown: paragraphs separated by blank lines.
// - PlainText: the raw concatenated text.
//
-// We return the longest non-empty view so a sparser view can never shadow a
-// more complete one (e.g. the IR flatten can be shorter than PlainText for
-// some documents). A failure at every stage degrades to the PlainText error,
-// preserving the original "no text at all" failure semantics.
+// The IR is preferred when it carries table/list structure (that content is
+// unique to the IR view and would be silently dropped under a pure length
+// comparison — a prose-heavy document can make PlainText longer than the IR
+// even though the IR is more complete). Otherwise the longest non-empty view
+// is chosen so a sparser view never shadows a more complete one. A failure at
+// every stage degrades to the PlainText error, preserving the original "no
+// text at all" failure semantics.
func extractDocText(doc *officeOxide.Document) (string, error) {
- var best string
- if irJSON, err := doc.ToIRJSON(); err == nil {
- if t := flattenDocIR(irJSON); len(strings.TrimSpace(t)) > len(strings.TrimSpace(best)) {
- best = t
- }
+ var irJSON, irText, mdText, plainText string
+ if j, err := doc.ToIRJSON(); err == nil {
+ irJSON = j
+ irText = flattenDocIR(j)
}
if md, err := doc.ToMarkdown(); err == nil {
- if len(strings.TrimSpace(md)) > len(strings.TrimSpace(best)) {
- best = md
- }
+ mdText = md
}
if plain, err := doc.PlainText(); err == nil {
- if len(strings.TrimSpace(plain)) > len(strings.TrimSpace(best)) {
- best = plain
- }
- } else if best == "" {
+ plainText = plain
+ } else if strings.TrimSpace(irText) == "" && strings.TrimSpace(mdText) == "" {
+ // Every view failed (or produced nothing): keep the original
+ // "no text at all" failure semantics.
return "", err
}
- return best, nil
+ return selectDocTextView(irJSON, irText, mdText, plainText), nil
+}
+
+// selectDocTextView chooses the best plain-text rendering from office_oxide's
+// three views. irJSON is the raw IR (used only to detect structured elements);
+// irText is its flattened form. When the IR recovers table or list structure
+// it is preferred, because that content is unique to the IR view. Otherwise the
+// longest non-empty view wins, so a sparser view never shadows a more complete
+// one.
+func selectDocTextView(irJSON, irText, mdText, plainText string) string {
+ if strings.TrimSpace(irText) != "" && irHasStructuredContent(irJSON) {
+ return irText
+ }
+ best := irText
+ if len(strings.TrimSpace(mdText)) > len(strings.TrimSpace(best)) {
+ best = mdText
+ }
+ if len(strings.TrimSpace(plainText)) > len(strings.TrimSpace(best)) {
+ best = plainText
+ }
+ return best
+}
+
+// irHasStructuredContent reports whether the office_oxide IR JSON carries
+// table or list elements. Those structures are only surfaced by the IR view;
+// ToMarkdown and PlainText flatten them away. Detection keys on the canonical
+// `"type":"table"` / `"type":"list"` field emitted by office_oxide.
+func irHasStructuredContent(irJSON string) bool {
+ return strings.Contains(irJSON, `"type":"table"`) ||
+ strings.Contains(irJSON, `"type":"list"`)
}
diff --git a/internal/parser/parser/doc_text_select_test.go b/internal/parser/parser/doc_text_select_test.go
new file mode 100644
index 0000000000..b5faa19412
--- /dev/null
+++ b/internal/parser/parser/doc_text_select_test.go
@@ -0,0 +1,59 @@
+//go:build cgo
+
+package parser
+
+import "testing"
+
+// TestSelectDocTextViewPrefersStructuredIR locks the fix for the fragile
+// "longest view" heuristic in extractDocText: when the office_oxide IR carries
+// table (or list) structure, that content is unique to the IR view and must
+// win even if PlainText/ToMarkdown produce a longer (but flatter) string.
+// Before the fix, a prose-heavy .doc could make PlainText longer than the IR
+// and silently drop the recovered table.
+func TestSelectDocTextViewPrefersStructuredIR(t *testing.T) {
+ // IR JSON that carries a table element (the structure only the IR view
+ // surfaces). irText is its flattened form with the " | " separator.
+ irJSON := `{"sections":[{"elements":[{"type":"table","rows":[]}]}]}`
+ irText := "Name | Value\nAlice | 1"
+
+ // PlainText is deliberately longer than the IR flatten — the old
+ // length-only heuristic would have shadowed the table here.
+ plainText := "This is a long block of prose that is clearly longer " +
+ "than the few table cells above, so a naive longest-view selection " +
+ "would pick this string and drop the recovered table structure."
+
+ got := selectDocTextView(irJSON, irText, "", plainText)
+ if got != irText {
+ t.Errorf("structured IR not preferred:\n got=%q\nwant=%q", got, irText)
+ }
+ if !contains(got, " | ") {
+ t.Errorf("table separator lost in selected view: %q", got)
+ }
+}
+
+// TestSelectDocTextViewFallsBackToLongest verifies that, absent any structured
+// element, the selection still prefers the longest non-empty view (the
+// original behavior, preserved so a sparser view never shadows a more
+// complete one).
+func TestSelectDocTextViewFallsBackToLongest(t *testing.T) {
+ // No table/list in the IR.
+ irJSON := `{"sections":[{"elements":[{"type":"paragraph"}]}]}`
+
+ irText := "short paragraph"
+ mdText := "a medium length markdown rendering of the document body"
+ plainText := "the longest plain text among the three available views here"
+
+ got := selectDocTextView(irJSON, irText, mdText, plainText)
+ if got != plainText {
+ t.Errorf("longest view not selected:\n got=%q\nwant=%q", got, plainText)
+ }
+}
+
+func contains(s, sub string) bool {
+ for i := 0; i+len(sub) <= len(s); i++ {
+ if s[i:i+len(sub)] == sub {
+ return true
+ }
+ }
+ return false
+}