RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix(parser): raise legacy .doc parsing fidelity via structured IR (#18192)
xugangqiang2132798f153d
diff --git a/internal/parser/parser/doc_flatten.go b/internal/parser/parser/doc_flatten.gonew file mode 100644index 0000000000..c5091a4d17--- /dev/null+++ b/internal/parser/parser/doc_flatten.go@@ -0,0 +1,96 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// Package parser: this file flattens office_oxide's format-independent IR+// JSON into plain text for legacy .doc files. It is intentionally cgo-free+// (like docx_ir.go) so it can be unit-tested without the native library; the+// cgo boundary lives entirely in doc_parser.go (officeOxide.OpenFromBytes).+//+// The IR schema produced by Document.ToIRJSON is shared across office formats+// by office_oxide, so the docx_ir.go model and helpers (joinDOCXIRRuns,+// joinCellText, extractTextFromListItem, extractTextFromBlockElements) apply+// here unchanged. Flattening — rather than returning ToMarkdown directly —+// keeps the .doc output in the same plain-text field the downstream already+// consumes, while still recovering paragraph/heading/list/table content that+// PlainText alone drops.++package parser++import (+ "encoding/json"+ "strings"+)++// flattenDocIR converts an office_oxide IR JSON string (from Document.ToIRJSON)+// into plain text that preserves paragraph, heading, list and table content.+// Tables are flattened with cells separated by " | " and rows by newlines, so+// their text is not silently dropped — this is the text-fidelity gain over the+// previous PlainText-only path and mirrors what Python's Tika text extraction+// recovers for legacy .doc files.+//+// Returns "" if irJSON cannot be parsed or yields no text; the caller then+// falls back to ToMarkdown / PlainText.+func flattenDocIR(irJSON string) string {+ var ir docxIRDocument+ if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {+ return ""+ }+ var parts []string+ for _, sec := range ir.Sections {+ if t := strings.TrimSpace(sec.Title); t != "" {+ parts = append(parts, t)+ }+ for _, el := range sec.Elements {+ if t := docElementText(el); t != "" {+ parts = append(parts, t)+ }+ }+ }+ return strings.Join(parts, "\n")+}++// docElementText returns the plain-text rendering of a single IR element for+// the .doc flatten path. It mirrors docxElementText but separates table cells+// with " | " (docxElementText concatenates them) so flattened table text+// stays readable. Returns "" for image and unknown types.+func docElementText(el docxIRElement) string {+ switch el.Type {+ case "paragraph", "heading":+ return joinDOCXIRRuns(el.contentRuns())+ case "table":+ var rows []string+ for _, row := range el.Rows {+ var cells []string+ for _, cell := range row.Cells {+ cells = append(cells, joinCellText(cell))+ }+ rows = append(rows, strings.Join(cells, " | "))+ }+ return strings.Join(rows, "\n")+ case "list":+ var lines []string+ for _, item := range el.Items {+ if t := extractTextFromListItem(item); t != "" {+ lines = append(lines, t)+ }+ }+ return strings.Join(lines, "\n")+ case "text_box":+ return extractTextFromBlockElements(el.contentBlocks())+ default:+ return ""+ }+}diff --git a/internal/parser/parser/doc_flatten_test.go b/internal/parser/parser/doc_flatten_test.gonew file mode 100644index 0000000000..9d22c2790b--- /dev/null+++ b/internal/parser/parser/doc_flatten_test.go@@ -0,0 +1,93 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package parser++import (+ "strings"+ "testing"+)++// sampleDocIR is a hand-built office_oxide IR JSON covering the element types+// a legacy .doc typically contains: a heading, paragraphs, a list, and a+// table. It exercises flattenDocIR without needing a real binary .doc fixture+// (the cgo integration test owns that). The schema mirrors docx_ir.go and+// office_oxide's format-independent IR.+const sampleDocIR = `{+ "sections": [+ {+ "title": "Quarterly Report",+ "elements": [+ {"type": "heading", "level": 1, "content": [{"type": "text", "text": "Summary"}]},+ {"type": "paragraph", "content": [{"type": "text", "text": "Revenue grew this quarter."}]},+ {"type": "paragraph", "content": [{"type": "text", "text": "Costs were controlled."}]},+ {"type": "list", "ordered": false, "items": [+ {"content": [{"type": "paragraph", "content": [{"type": "text", "text": "Hired two engineers"}]}]},+ {"content": [{"type": "paragraph", "content": [{"type": "text", "text": "Shipped the parser"}]}]}+ ]},+ {"type": "table", "rows": [+ {"cells": [{"content": [{"type": "paragraph", "content": [{"type": "text", "text": "Region"}]}]}, {"content": [{"type": "paragraph", "content": [{"type": "text", "text": "Sales"}]}]}]},+ {"cells": [{"content": [{"type": "paragraph", "content": [{"type": "text", "text": "East"}]}]}, {"content": [{"type": "paragraph", "content": [{"type": "text", "text": "120"}]}]}]},+ {"cells": [{"content": [{"type": "paragraph", "content": [{"type": "text", "text": "West"}]}]}, {"content": [{"type": "paragraph", "content": [{"type": "text", "text": "95"}]}]}]}+ ]}+ ]+ }+ ]+}`++func TestFlattenDocIR_PreservesStructure(t *testing.T) {+ got := flattenDocIR(sampleDocIR)+ if strings.TrimSpace(got) == "" {+ t.Fatalf("flattenDocIR returned empty text")+ }++ // Heading / section title and paragraph text must survive.+ for _, want := range []string{+ "Quarterly Report",+ "Summary",+ "Revenue grew this quarter.",+ "Costs were controlled.",+ } {+ if !strings.Contains(got, want) {+ t.Errorf("flattened text missing %q\n got=%q", want, got)+ }+ }++ // List items must survive (not dropped).+ for _, want := range []string{"Hired two engineers", "Shipped the parser"} {+ if !strings.Contains(got, want) {+ t.Errorf("flattened text missing list item %q\n got=%q", want, got)+ }+ }++ // Table cells must survive, separated by " | " within a row and by+ // newlines across rows — this is the fidelity gain over PlainText.+ for _, want := range []string{+ "Region | Sales",+ "East | 120",+ "West | 95",+ } {+ if !strings.Contains(got, want) {+ t.Errorf("flattened text missing table cell group %q\n got=%q", want, got)+ }+ }+}++func TestFlattenDocIR_InvalidJSONReturnsEmpty(t *testing.T) {+ if got := flattenDocIR("not json"); got != "" {+ t.Errorf("flattenDocIR(invalid) = %q, want empty", got)+ }+}diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.goindex 2aabc61b6a..0f0f3cf5ec 100644--- a/internal/parser/parser/doc_parser.go+++ b/internal/parser/parser/doc_parser.go@@ -21,6 +21,7 @@ package parserimport ("context""fmt"+ "strings"officeOxide "github.com/yfedoseev/office_oxide/go")@@ -35,11 +36,16 @@ func (p *DOCParser) String() string {return "DOCParser"}-// ParseWithResult captures the office_oxide PlainText output for-// the DOC family. Python parser.py routes .doc through tika;-// the Go side uses office_oxide which supports DOC via PlainText.-// OutputFormat="text" — the python side falls back to text for-// legacy DOC files since structured extraction is unreliable.+// ParseWithResult extracts text for the DOC family. The Go side uses+// office_oxide, which supports legacy .doc. We prefer the structured IR+// (flattened to plain text) and fall back to ToMarkdown, then to PlainText.+//+// Note: office_oxide's .doc reader only surfaces paragraph and (heuristically+// detected) heading lines — it does NOT extract table or list structure from+// legacy .doc (the table stream's PAP/TAP/LSTF are ignored; see+// office_oxide issue #115). So this path normalizes paragraph/heading line+// breaks but does not recover tables/lists. OutputFormat stays "text" to keep+// the downstream contract unchanged.func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {doc, err := officeOxide.OpenFromBytes(data, "doc")if err != nil {@@ -47,9 +53,9 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [}defer doc.Close()- text, err := doc.PlainText()+ text, err := extractDocText(doc)if err != nil {- return ParseResult{Err: fmt.Errorf("doc plain-text: %w", err)}+ return ParseResult{Err: fmt.Errorf("doc extract: %w", err)}}return ParseResult{@@ -58,3 +64,37 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [Text: text,}}++// extractDocText returns the best-effort plain text for a legacy .doc+// document. office_oxide exposes three text views:+// - ToIRJSON: a structured DocumentIR, flattened to plain text via+// flattenDocIR. For .doc this only carries paragraph and (heuristic)+// heading lines — no table/list structure (office_oxide issue #115).+// - ToMarkdown: paragraphs separated by blank lines.+// - PlainText: the raw concatenated text.+//+// We return the longest non-empty view so a sparser view can never shadow a+// more complete one (e.g. the IR flatten can be shorter than PlainText for+// some documents). A failure at every stage degrades to the PlainText error,+// preserving the original "no text at all" failure semantics.+func extractDocText(doc *officeOxide.Document) (string, error) {+ var best string+ if irJSON, err := doc.ToIRJSON(); err == nil {+ if t := flattenDocIR(irJSON); len(strings.TrimSpace(t)) > len(strings.TrimSpace(best)) {+ best = t+ }+ }+ if md, err := doc.ToMarkdown(); err == nil {+ if len(strings.TrimSpace(md)) > len(strings.TrimSpace(best)) {+ best = md+ }+ }+ if plain, err := doc.PlainText(); err == nil {+ if len(strings.TrimSpace(plain)) > len(strings.TrimSpace(best)) {+ best = plain+ }+ } else if best == "" {+ return "", err+ }+ return best, nil+}diff --git a/internal/parser/parser/doc_parser_doc_table_test.go b/internal/parser/parser/doc_parser_doc_table_test.gonew file mode 100644index 0000000000..4cd7e3dc3b--- /dev/null+++ b/internal/parser/parser/doc_parser_doc_table_test.go@@ -0,0 +1,41 @@+//go:build cgo++package parser++import (+ "context"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++// TestDOCParser_RecoversTableFromLegacyDoc verifies that the legacy .doc path+// recovers table structure through office_oxide's structured IR. The fixture+// (testdata/table.doc) is a real Word binary containing a multi-column table.+//+// The " | " separator is emitted ONLY by flattenDocIR when office_oxide's+// ToIRJSON returns a table element (cells joined with " | " per row). Before+// office_oxide/go v0.1.9 (#116) the .doc reader ignored the table stream, so+// no table element was produced and the separator never appeared — this test+// is therefore a direct regression gate for the v0.1.9 dependency bump.+func TestDOCParser_RecoversTableFromLegacyDoc(t *testing.T) {+ data, err := os.ReadFile(filepath.Join("testdata", "table.doc"))+ if err != nil {+ t.Fatalf("read fixture: %v", err)+ }++ res := NewDOCParser().ParseWithResult(context.Background(), "table.doc", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult error: %v", res.Err)+ }+ if strings.TrimSpace(res.Text) == "" {+ t.Fatalf("parsed text is empty")+ }++ // Table cells must be separated by " | " — the signature of IR-driven+ // table recovery. PlainText / ToMarkdown never emit this separator.+ if !strings.Contains(res.Text, " | ") {+ t.Errorf("table structure not recovered from legacy .doc\n got=%q", res.Text)+ }+}diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.gonew file mode 100644index 0000000000..65c6056c01--- /dev/null+++ b/internal/parser/parser/doc_parser_synth_test.go@@ -0,0 +1,160 @@+//go:build cgo++package parser++import (+ "bytes"+ "context"+ "encoding/binary"+ "testing"+ "unicode/utf16"++ officeOxide "github.com/yfedoseev/office_oxide/go"+)++// buildSyntheticDoc constructs a minimal but valid legacy Word binary (.doc)+// entirely in memory — an OLE2/CFB container with a WordDocument stream+// (FIB + piece-table-referenced text) and a 0Table stream (CLX). No external+// tool or committed fixture is required, so the .doc parser can be exercised+// end-to-end against dynamically generated input.+//+// Layout (sector size = 512):+//+// sector 0: FAT+// sector 1: directory (Root Entry, WordDocument, 0Table)+// sector 2: WordDocument stream (FIB + CP1252 text piece)+// sector 3: 0Table stream (CLX)+//+// The structure follows office_oxide's reader expectations+// (src/cfb/*, src/doc/fib.rs, src/doc/piece_table.rs).+func buildSyntheticDoc(t *testing.T, text string) []byte {+ t.Helper()+ const sector = 512++ // ── CLX (piece table) in the 0Table stream ────────────────────────────+ // 1 piece covering all text, compressed (CP1252): fc bit 30 set,+ // real byte offset = (fc & ~0x40000000) / 2.+ const textOffset = 426 // safely past the FIB reads (max offset 0x1AA=426)+ fc := uint32(0x40000000) | uint32(textOffset*2)++ clx := make([]byte, 21)+ clx[0] = 0x02 // Pcdt+ binary.LittleEndian.PutUint32(clx[1:], 16) // PlcPcd size+ binary.LittleEndian.PutUint32(clx[5:], 0) // CP[0] = 0+ binary.LittleEndian.PutUint32(clx[9:], uint32(len(text))) // CP[1] = text_len+ binary.LittleEndian.PutUint16(clx[13:], 0) // PCD u16 unused+ binary.LittleEndian.PutUint32(clx[15:], fc) // PCD fc+ binary.LittleEndian.PutUint16(clx[19:], 0) // PCD prm++ // ── WordDocument stream: FIB + text ───────────────────────────────────+ wordDoc := make([]byte, sector)+ binary.LittleEndian.PutUint16(wordDoc[0:], 0xA5EC) // wIdent (Word 97+)+ binary.LittleEndian.PutUint16(wordDoc[2:], 0x00C1) // nFib+ binary.LittleEndian.PutUint16(wordDoc[0x0A:], 0x0000) // flags: bit9=0 → 0Table+ binary.LittleEndian.PutUint32(wordDoc[0x4C:], uint32(len(text))) // ccpText+ binary.LittleEndian.PutUint32(wordDoc[0x01A2:], 0) // fcClx (offset 0 in 0Table)+ binary.LittleEndian.PutUint32(wordDoc[0x01A6:], uint32(len(clx))) // lcbClx+ copy(wordDoc[textOffset:], []byte(text))++ // ── CFB header ────────────────────────────────────────────────────────+ header := make([]byte, sector)+ copy(header[0:8], []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1})+ binary.LittleEndian.PutUint16(header[0x18:], 0x003E) // minor version+ binary.LittleEndian.PutUint16(header[0x1A:], 3) // major version (v3)+ binary.LittleEndian.PutUint16(header[0x1C:], 0xFFFE) // byte order+ binary.LittleEndian.PutUint16(header[0x1E:], 9) // sector shift (512)+ binary.LittleEndian.PutUint16(header[0x20:], 6) // mini sector shift (64)+ binary.LittleEndian.PutUint32(header[0x2C:], 1) // FAT sector count+ binary.LittleEndian.PutUint32(header[0x30:], 1) // first directory sector+ binary.LittleEndian.PutUint32(header[0x38:], 4096) // mini-stream cutoff+ binary.LittleEndian.PutUint32(header[0x3C:], 0xFFFFFFFE) // first mini-FAT = END+ binary.LittleEndian.PutUint32(header[0x44:], 0xFFFFFFFE) // first DIFAT = END+ for i := 0; i < 109; i++ {+ off := 0x4C + i*4+ if i == 0 {+ binary.LittleEndian.PutUint32(header[off:], 0) // FAT lives in sector 0+ } else {+ binary.LittleEndian.PutUint32(header[off:], 0xFFFFFFFE) // ignored+ }+ }++ // ── FAT sector ────────────────────────────────────────────────────────+ fat := bytes.Repeat([]byte{0xFF}, sector) // all FREE (0xFFFFFFFF)+ binary.LittleEndian.PutUint32(fat[0*4:], 0xFFFFFFFD) // sector 0 = FAT itself+ binary.LittleEndian.PutUint32(fat[1*4:], 0xFFFFFFFE) // directory = END+ binary.LittleEndian.PutUint32(fat[2*4:], 0xFFFFFFFE) // WordDocument = END+ binary.LittleEndian.PutUint32(fat[3*4:], 0xFFFFFFFE) // 0Table = END++ // ── Directory sector: 4 entries ───────────────────────────────────────+ dir := make([]byte, sector)+ copy(dir[0*128:], dirEntry("Root Entry", 5, 0xFFFFFFFF, 0))+ copy(dir[1*128:], dirEntry("WordDocument", 2, 2, uint64(len(wordDoc))))+ copy(dir[2*128:], dirEntry("0Table", 2, 3, uint64(len(clx))))+ copy(dir[3*128:], dirEntry("", 0, 0xFFFFFFFF, 0))++ // ── Assemble ──────────────────────────────────────────────────────────+ tableSector := make([]byte, sector)+ copy(tableSector, clx)++ out := make([]byte, 5*sector)+ copy(out[0*sector:], header)+ copy(out[1*sector:], fat)+ copy(out[2*sector:], dir)+ copy(out[3*sector:], wordDoc)+ copy(out[4*sector:], tableSector)+ return out+}++func dirEntry(name string, entryType byte, startSector uint32, streamSize uint64) []byte {+ e := make([]byte, 128)+ if name != "" {+ utf16Name := utf16.Encode([]rune(name))+ var nameBytes []byte+ for _, r := range utf16Name {+ nameBytes = append(nameBytes, byte(r&0xFF), byte(r>>8))+ }+ nameBytes = append(nameBytes, 0, 0) // UTF-16LE null terminator+ copy(e[0:], nameBytes)+ binary.LittleEndian.PutUint16(e[0x40:], uint16(len(nameBytes)))+ }+ e[0x42] = entryType+ e[0x43] = 1 // color (black)+ binary.LittleEndian.PutUint32(e[0x44:], 0xFFFFFFFF)+ binary.LittleEndian.PutUint32(e[0x48:], 0xFFFFFFFF)+ binary.LittleEndian.PutUint32(e[0x4C:], 0xFFFFFFFF)+ binary.LittleEndian.PutUint32(e[0x74:], startSector)+ binary.LittleEndian.PutUint32(e[0x78:], uint32(streamSize)) // v3: low 32 bits+ return e+}++// TestDOCParser_SyntheticDocRoundTrip verifies the legacy DOC path end-to-end+// against a .doc that is generated at runtime (no committed binary, no external+// converter). It exercises office_oxide.OpenFromBytes(data, "doc") and the+// extractDocText fallback chain (IR → Markdown → PlainText) inside+// ParseWithResult.+func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {+ const body = "SYNTHETIC DOC\rFirst para alpha.\rSecond para beta."+ data := buildSyntheticDoc(t, body)++ doc, err := officeOxide.OpenFromBytes(data, "doc")+ if err != nil {+ t.Fatalf("OpenFromBytes(doc) failed: %v", err)+ }+ defer doc.Close()++ pr := NewDOCParser().ParseWithResult(context.Background(), "synthetic.doc", data)+ if pr.Err != nil {+ t.Fatalf("ParseWithResult error: %v", pr.Err)+ }+ if pr.OutputFormat != "text" {+ t.Fatalf("OutputFormat = %q, want %q", pr.OutputFormat, "text")+ }+ if pr.Text == "" {+ t.Fatal("ParseWithResult returned empty text — synthetic .doc was not parsed")+ }+ for _, want := range []string{"SYNTHETIC DOC", "First para alpha.", "Second para beta."} {+ if !bytes.Contains([]byte(pr.Text), []byte(want)) {+ t.Errorf("parsed text missing %q; got:\n%s", want, pr.Text)+ }+ }+}diff --git a/internal/parser/parser/testdata/table.doc b/internal/parser/parser/testdata/table.docnew file mode 100644index 0000000000..c6ab0d79dbBinary files /dev/null and b/internal/parser/parser/testdata/table.doc differ