Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix(parser): raise legacy .doc parsing fidelity via structured IR (#18192)

xugangqiang2132798f153d
diff --git a/internal/parser/parser/doc_flatten.go b/internal/parser/parser/doc_flatten.go
new file mode 100644
index 0000000000..c5091a4d17
--- /dev/null
+++ b/internal/parser/parser/doc_flatten.go
@@ -0,0 +1,96 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Package parser: this file flattens office_oxide's format-independent IR
+// JSON into plain text for legacy .doc files. It is intentionally cgo-free
+// (like docx_ir.go) so it can be unit-tested without the native library; the
+// cgo boundary lives entirely in doc_parser.go (officeOxide.OpenFromBytes).
+//
+// The IR schema produced by Document.ToIRJSON is shared across office formats
+// by office_oxide, so the docx_ir.go model and helpers (joinDOCXIRRuns,
+// joinCellText, extractTextFromListItem, extractTextFromBlockElements) apply
+// here unchanged. Flattening — rather than returning ToMarkdown directly —
+// keeps the .doc output in the same plain-text field the downstream already
+// consumes, while still recovering paragraph/heading/list/table content that
+// PlainText alone drops.
+
+package parser
+
+import (
+ "encoding/json"
+ "strings"
+)
+
+// flattenDocIR converts an office_oxide IR JSON string (from Document.ToIRJSON)
+// into plain text that preserves paragraph, heading, list and table content.
+// Tables are flattened with cells separated by " | " and rows by newlines, so
+// their text is not silently dropped — this is the text-fidelity gain over the
+// previous PlainText-only path and mirrors what Python's Tika text extraction
+// recovers for legacy .doc files.
+//
+// Returns "" if irJSON cannot be parsed or yields no text; the caller then
+// falls back to ToMarkdown / PlainText.
+func flattenDocIR(irJSON string) string {
+ var ir docxIRDocument
+ if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {
+ return ""
+ }
+ var parts []string
+ for _, sec := range ir.Sections {
+ if t := strings.TrimSpace(sec.Title); t != "" {
+ parts = append(parts, t)
+ }
+ for _, el := range sec.Elements {
+ if t := docElementText(el); t != "" {
+ parts = append(parts, t)
+ }
+ }
+ }
+ return strings.Join(parts, "\n")
+}
+
+// docElementText returns the plain-text rendering of a single IR element for
+// the .doc flatten path. It mirrors docxElementText but separates table cells
+// with " | " (docxElementText concatenates them) so flattened table text
+// stays readable. Returns "" for image and unknown types.
+func docElementText(el docxIRElement) string {
+ switch el.Type {
+ case "paragraph", "heading":
+ return joinDOCXIRRuns(el.contentRuns())
+ case "table":
+ var rows []string
+ for _, row := range el.Rows {
+ var cells []string
+ for _, cell := range row.Cells {
+ cells = append(cells, joinCellText(cell))
+ }
+ rows = append(rows, strings.Join(cells, " | "))
+ }
+ return strings.Join(rows, "\n")
+ case "list":
+ var lines []string
+ for _, item := range el.Items {
+ if t := extractTextFromListItem(item); t != "" {
+ lines = append(lines, t)
+ }
+ }
+ return strings.Join(lines, "\n")
+ case "text_box":
+ return extractTextFromBlockElements(el.contentBlocks())
+ default:
+ return ""
+ }
+}
diff --git a/internal/parser/parser/doc_flatten_test.go b/internal/parser/parser/doc_flatten_test.go
new file mode 100644
index 0000000000..9d22c2790b
--- /dev/null
+++ b/internal/parser/parser/doc_flatten_test.go
@@ -0,0 +1,93 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package parser
+
+import (
+ "strings"
+ "testing"
+)
+
+// sampleDocIR is a hand-built office_oxide IR JSON covering the element types
+// a legacy .doc typically contains: a heading, paragraphs, a list, and a
+// table. It exercises flattenDocIR without needing a real binary .doc fixture
+// (the cgo integration test owns that). The schema mirrors docx_ir.go and
+// office_oxide's format-independent IR.
+const sampleDocIR = `{
+ "sections": [
+ {
+ "title": "Quarterly Report",
+ "elements": [
+ {"type": "heading", "level": 1, "content": [{"type": "text", "text": "Summary"}]},
+ {"type": "paragraph", "content": [{"type": "text", "text": "Revenue grew this quarter."}]},
+ {"type": "paragraph", "content": [{"type": "text", "text": "Costs were controlled."}]},
+ {"type": "list", "ordered": false, "items": [
+ {"content": [{"type": "paragraph", "content": [{"type": "text", "text": "Hired two engineers"}]}]},
+ {"content": [{"type": "paragraph", "content": [{"type": "text", "text": "Shipped the parser"}]}]}
+ ]},
+ {"type": "table", "rows": [
+ {"cells": [{"content": [{"type": "paragraph", "content": [{"type": "text", "text": "Region"}]}]}, {"content": [{"type": "paragraph", "content": [{"type": "text", "text": "Sales"}]}]}]},
+ {"cells": [{"content": [{"type": "paragraph", "content": [{"type": "text", "text": "East"}]}]}, {"content": [{"type": "paragraph", "content": [{"type": "text", "text": "120"}]}]}]},
+ {"cells": [{"content": [{"type": "paragraph", "content": [{"type": "text", "text": "West"}]}]}, {"content": [{"type": "paragraph", "content": [{"type": "text", "text": "95"}]}]}]}
+ ]}
+ ]
+ }
+ ]
+}`
+
+func TestFlattenDocIR_PreservesStructure(t *testing.T) {
+ got := flattenDocIR(sampleDocIR)
+ if strings.TrimSpace(got) == "" {
+ t.Fatalf("flattenDocIR returned empty text")
+ }
+
+ // Heading / section title and paragraph text must survive.
+ for _, want := range []string{
+ "Quarterly Report",
+ "Summary",
+ "Revenue grew this quarter.",
+ "Costs were controlled.",
+ } {
+ if !strings.Contains(got, want) {
+ t.Errorf("flattened text missing %q\n got=%q", want, got)
+ }
+ }
+
+ // List items must survive (not dropped).
+ for _, want := range []string{"Hired two engineers", "Shipped the parser"} {
+ if !strings.Contains(got, want) {
+ t.Errorf("flattened text missing list item %q\n got=%q", want, got)
+ }
+ }
+
+ // Table cells must survive, separated by " | " within a row and by
+ // newlines across rows — this is the fidelity gain over PlainText.
+ for _, want := range []string{
+ "Region | Sales",
+ "East | 120",
+ "West | 95",
+ } {
+ if !strings.Contains(got, want) {
+ t.Errorf("flattened text missing table cell group %q\n got=%q", want, got)
+ }
+ }
+}
+
+func TestFlattenDocIR_InvalidJSONReturnsEmpty(t *testing.T) {
+ if got := flattenDocIR("not json"); got != "" {
+ t.Errorf("flattenDocIR(invalid) = %q, want empty", got)
+ }
+}
diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.go
index 2aabc61b6a..0f0f3cf5ec 100644
--- a/internal/parser/parser/doc_parser.go
+++ b/internal/parser/parser/doc_parser.go
@@ -21,6 +21,7 @@ package parser
import (
"context"
"fmt"
+ "strings"
officeOxide "github.com/yfedoseev/office_oxide/go"
)
@@ -35,11 +36,16 @@ func (p *DOCParser) String() string {
return "DOCParser"
}
-// ParseWithResult captures the office_oxide PlainText output for
-// the DOC family. Python parser.py routes .doc through tika;
-// the Go side uses office_oxide which supports DOC via PlainText.
-// OutputFormat="text" — the python side falls back to text for
-// legacy DOC files since structured extraction is unreliable.
+// ParseWithResult extracts text for the DOC family. The Go side uses
+// office_oxide, which supports legacy .doc. We prefer the structured IR
+// (flattened to plain text) and fall back to ToMarkdown, then to PlainText.
+//
+// Note: office_oxide's .doc reader only surfaces paragraph and (heuristically
+// detected) heading lines — it does NOT extract table or list structure from
+// legacy .doc (the table stream's PAP/TAP/LSTF are ignored; see
+// office_oxide issue #115). So this path normalizes paragraph/heading line
+// breaks but does not recover tables/lists. OutputFormat stays "text" to keep
+// the downstream contract unchanged.
func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
doc, err := officeOxide.OpenFromBytes(data, "doc")
if err != nil {
@@ -47,9 +53,9 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
}
defer doc.Close()
- text, err := doc.PlainText()
+ text, err := extractDocText(doc)
if err != nil {
- return ParseResult{Err: fmt.Errorf("doc plain-text: %w", err)}
+ return ParseResult{Err: fmt.Errorf("doc extract: %w", err)}
}
return ParseResult{
@@ -58,3 +64,37 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
Text: text,
}
}
+
+// extractDocText returns the best-effort plain text for a legacy .doc
+// document. office_oxide exposes three text views:
+// - ToIRJSON: a structured DocumentIR, flattened to plain text via
+// flattenDocIR. For .doc this only carries paragraph and (heuristic)
+// heading lines — no table/list structure (office_oxide issue #115).
+// - ToMarkdown: paragraphs separated by blank lines.
+// - PlainText: the raw concatenated text.
+//
+// We return the longest non-empty view so a sparser view can never shadow a
+// more complete one (e.g. the IR flatten can be shorter than PlainText for
+// some documents). A failure at every stage degrades to the PlainText error,
+// preserving the original "no text at all" failure semantics.
+func extractDocText(doc *officeOxide.Document) (string, error) {
+ var best string
+ if irJSON, err := doc.ToIRJSON(); err == nil {
+ if t := flattenDocIR(irJSON); len(strings.TrimSpace(t)) > len(strings.TrimSpace(best)) {
+ best = t
+ }
+ }
+ if md, err := doc.ToMarkdown(); err == nil {
+ if len(strings.TrimSpace(md)) > len(strings.TrimSpace(best)) {
+ best = md
+ }
+ }
+ if plain, err := doc.PlainText(); err == nil {
+ if len(strings.TrimSpace(plain)) > len(strings.TrimSpace(best)) {
+ best = plain
+ }
+ } else if best == "" {
+ return "", err
+ }
+ return best, nil
+}
diff --git a/internal/parser/parser/doc_parser_doc_table_test.go b/internal/parser/parser/doc_parser_doc_table_test.go
new file mode 100644
index 0000000000..4cd7e3dc3b
--- /dev/null
+++ b/internal/parser/parser/doc_parser_doc_table_test.go
@@ -0,0 +1,41 @@
+//go:build cgo
+
+package parser
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// TestDOCParser_RecoversTableFromLegacyDoc verifies that the legacy .doc path
+// recovers table structure through office_oxide's structured IR. The fixture
+// (testdata/table.doc) is a real Word binary containing a multi-column table.
+//
+// The " | " separator is emitted ONLY by flattenDocIR when office_oxide's
+// ToIRJSON returns a table element (cells joined with " | " per row). Before
+// office_oxide/go v0.1.9 (#116) the .doc reader ignored the table stream, so
+// no table element was produced and the separator never appeared — this test
+// is therefore a direct regression gate for the v0.1.9 dependency bump.
+func TestDOCParser_RecoversTableFromLegacyDoc(t *testing.T) {
+ data, err := os.ReadFile(filepath.Join("testdata", "table.doc"))
+ if err != nil {
+ t.Fatalf("read fixture: %v", err)
+ }
+
+ res := NewDOCParser().ParseWithResult(context.Background(), "table.doc", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult error: %v", res.Err)
+ }
+ if strings.TrimSpace(res.Text) == "" {
+ t.Fatalf("parsed text is empty")
+ }
+
+ // Table cells must be separated by " | " — the signature of IR-driven
+ // table recovery. PlainText / ToMarkdown never emit this separator.
+ if !strings.Contains(res.Text, " | ") {
+ t.Errorf("table structure not recovered from legacy .doc\n got=%q", res.Text)
+ }
+}
diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.go
new file mode 100644
index 0000000000..65c6056c01
--- /dev/null
+++ b/internal/parser/parser/doc_parser_synth_test.go
@@ -0,0 +1,160 @@
+//go:build cgo
+
+package parser
+
+import (
+ "bytes"
+ "context"
+ "encoding/binary"
+ "testing"
+ "unicode/utf16"
+
+ officeOxide "github.com/yfedoseev/office_oxide/go"
+)
+
+// buildSyntheticDoc constructs a minimal but valid legacy Word binary (.doc)
+// entirely in memory — an OLE2/CFB container with a WordDocument stream
+// (FIB + piece-table-referenced text) and a 0Table stream (CLX). No external
+// tool or committed fixture is required, so the .doc parser can be exercised
+// end-to-end against dynamically generated input.
+//
+// Layout (sector size = 512):
+//
+// sector 0: FAT
+// sector 1: directory (Root Entry, WordDocument, 0Table)
+// sector 2: WordDocument stream (FIB + CP1252 text piece)
+// sector 3: 0Table stream (CLX)
+//
+// The structure follows office_oxide's reader expectations
+// (src/cfb/*, src/doc/fib.rs, src/doc/piece_table.rs).
+func buildSyntheticDoc(t *testing.T, text string) []byte {
+ t.Helper()
+ const sector = 512
+
+ // ── CLX (piece table) in the 0Table stream ────────────────────────────
+ // 1 piece covering all text, compressed (CP1252): fc bit 30 set,
+ // real byte offset = (fc & ~0x40000000) / 2.
+ const textOffset = 426 // safely past the FIB reads (max offset 0x1AA=426)
+ fc := uint32(0x40000000) | uint32(textOffset*2)
+
+ clx := make([]byte, 21)
+ clx[0] = 0x02 // Pcdt
+ binary.LittleEndian.PutUint32(clx[1:], 16) // PlcPcd size
+ binary.LittleEndian.PutUint32(clx[5:], 0) // CP[0] = 0
+ binary.LittleEndian.PutUint32(clx[9:], uint32(len(text))) // CP[1] = text_len
+ binary.LittleEndian.PutUint16(clx[13:], 0) // PCD u16 unused
+ binary.LittleEndian.PutUint32(clx[15:], fc) // PCD fc
+ binary.LittleEndian.PutUint16(clx[19:], 0) // PCD prm
+
+ // ── WordDocument stream: FIB + text ───────────────────────────────────
+ wordDoc := make([]byte, sector)
+ binary.LittleEndian.PutUint16(wordDoc[0:], 0xA5EC) // wIdent (Word 97+)
+ binary.LittleEndian.PutUint16(wordDoc[2:], 0x00C1) // nFib
+ binary.LittleEndian.PutUint16(wordDoc[0x0A:], 0x0000) // flags: bit9=0 → 0Table
+ binary.LittleEndian.PutUint32(wordDoc[0x4C:], uint32(len(text))) // ccpText
+ binary.LittleEndian.PutUint32(wordDoc[0x01A2:], 0) // fcClx (offset 0 in 0Table)
+ binary.LittleEndian.PutUint32(wordDoc[0x01A6:], uint32(len(clx))) // lcbClx
+ copy(wordDoc[textOffset:], []byte(text))
+
+ // ── CFB header ────────────────────────────────────────────────────────
+ header := make([]byte, sector)
+ copy(header[0:8], []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1})
+ binary.LittleEndian.PutUint16(header[0x18:], 0x003E) // minor version
+ binary.LittleEndian.PutUint16(header[0x1A:], 3) // major version (v3)
+ binary.LittleEndian.PutUint16(header[0x1C:], 0xFFFE) // byte order
+ binary.LittleEndian.PutUint16(header[0x1E:], 9) // sector shift (512)
+ binary.LittleEndian.PutUint16(header[0x20:], 6) // mini sector shift (64)
+ binary.LittleEndian.PutUint32(header[0x2C:], 1) // FAT sector count
+ binary.LittleEndian.PutUint32(header[0x30:], 1) // first directory sector
+ binary.LittleEndian.PutUint32(header[0x38:], 4096) // mini-stream cutoff
+ binary.LittleEndian.PutUint32(header[0x3C:], 0xFFFFFFFE) // first mini-FAT = END
+ binary.LittleEndian.PutUint32(header[0x44:], 0xFFFFFFFE) // first DIFAT = END
+ for i := 0; i < 109; i++ {
+ off := 0x4C + i*4
+ if i == 0 {
+ binary.LittleEndian.PutUint32(header[off:], 0) // FAT lives in sector 0
+ } else {
+ binary.LittleEndian.PutUint32(header[off:], 0xFFFFFFFE) // ignored
+ }
+ }
+
+ // ── FAT sector ────────────────────────────────────────────────────────
+ fat := bytes.Repeat([]byte{0xFF}, sector) // all FREE (0xFFFFFFFF)
+ binary.LittleEndian.PutUint32(fat[0*4:], 0xFFFFFFFD) // sector 0 = FAT itself
+ binary.LittleEndian.PutUint32(fat[1*4:], 0xFFFFFFFE) // directory = END
+ binary.LittleEndian.PutUint32(fat[2*4:], 0xFFFFFFFE) // WordDocument = END
+ binary.LittleEndian.PutUint32(fat[3*4:], 0xFFFFFFFE) // 0Table = END
+
+ // ── Directory sector: 4 entries ───────────────────────────────────────
+ dir := make([]byte, sector)
+ copy(dir[0*128:], dirEntry("Root Entry", 5, 0xFFFFFFFF, 0))
+ copy(dir[1*128:], dirEntry("WordDocument", 2, 2, uint64(len(wordDoc))))
+ copy(dir[2*128:], dirEntry("0Table", 2, 3, uint64(len(clx))))
+ copy(dir[3*128:], dirEntry("", 0, 0xFFFFFFFF, 0))
+
+ // ── Assemble ──────────────────────────────────────────────────────────
+ tableSector := make([]byte, sector)
+ copy(tableSector, clx)
+
+ out := make([]byte, 5*sector)
+ copy(out[0*sector:], header)
+ copy(out[1*sector:], fat)
+ copy(out[2*sector:], dir)
+ copy(out[3*sector:], wordDoc)
+ copy(out[4*sector:], tableSector)
+ return out
+}
+
+func dirEntry(name string, entryType byte, startSector uint32, streamSize uint64) []byte {
+ e := make([]byte, 128)
+ if name != "" {
+ utf16Name := utf16.Encode([]rune(name))
+ var nameBytes []byte
+ for _, r := range utf16Name {
+ nameBytes = append(nameBytes, byte(r&0xFF), byte(r>>8))
+ }
+ nameBytes = append(nameBytes, 0, 0) // UTF-16LE null terminator
+ copy(e[0:], nameBytes)
+ binary.LittleEndian.PutUint16(e[0x40:], uint16(len(nameBytes)))
+ }
+ e[0x42] = entryType
+ e[0x43] = 1 // color (black)
+ binary.LittleEndian.PutUint32(e[0x44:], 0xFFFFFFFF)
+ binary.LittleEndian.PutUint32(e[0x48:], 0xFFFFFFFF)
+ binary.LittleEndian.PutUint32(e[0x4C:], 0xFFFFFFFF)
+ binary.LittleEndian.PutUint32(e[0x74:], startSector)
+ binary.LittleEndian.PutUint32(e[0x78:], uint32(streamSize)) // v3: low 32 bits
+ return e
+}
+
+// TestDOCParser_SyntheticDocRoundTrip verifies the legacy DOC path end-to-end
+// against a .doc that is generated at runtime (no committed binary, no external
+// converter). It exercises office_oxide.OpenFromBytes(data, "doc") and the
+// extractDocText fallback chain (IR → Markdown → PlainText) inside
+// ParseWithResult.
+func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {
+ const body = "SYNTHETIC DOC\rFirst para alpha.\rSecond para beta."
+ data := buildSyntheticDoc(t, body)
+
+ doc, err := officeOxide.OpenFromBytes(data, "doc")
+ if err != nil {
+ t.Fatalf("OpenFromBytes(doc) failed: %v", err)
+ }
+ defer doc.Close()
+
+ pr := NewDOCParser().ParseWithResult(context.Background(), "synthetic.doc", data)
+ if pr.Err != nil {
+ t.Fatalf("ParseWithResult error: %v", pr.Err)
+ }
+ if pr.OutputFormat != "text" {
+ t.Fatalf("OutputFormat = %q, want %q", pr.OutputFormat, "text")
+ }
+ if pr.Text == "" {
+ t.Fatal("ParseWithResult returned empty text — synthetic .doc was not parsed")
+ }
+ for _, want := range []string{"SYNTHETIC DOC", "First para alpha.", "Second para beta."} {
+ if !bytes.Contains([]byte(pr.Text), []byte(want)) {
+ t.Errorf("parsed text missing %q; got:\n%s", want, pr.Text)
+ }
+ }
+}
diff --git a/internal/parser/parser/testdata/table.doc b/internal/parser/parser/testdata/table.doc
new file mode 100644
index 0000000000..c6ab0d79db
Binary files /dev/null and b/internal/parser/parser/testdata/table.doc differ