Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): emit DOC tables as structured items instead of plain text (#21201)

xugangqiang404950b41527
diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.go
index 252a883975..6c4a81d785 100644
--- a/internal/parser/parser/doc_parser.go
+++ b/internal/parser/parser/doc_parser.go
@@ -90,9 +90,32 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
return res
}
- res.OutputFormat = "json"
- res.JSON = []DocItem{NewTextJSONItem(text)}
- return res
+ // JSON path: prefer the structured IR, which office_oxide recovers table
+ // (and list) structure for. Emitting structured sections lets the chunker
+ // render tables as HTML (#20918) instead of losing them to plain text — the
+ // legacy path flattened the whole document (tables included) into a single
+ // text item. Fall back to the flattened text only when the IR yields no
+ // usable sections, so a document whose IR is empty/unusable still keeps its
+ // content (PlainText / ToMarkdown) rather than silently producing 0 chunks.
+ if strings.TrimSpace(irJSON) != "" {
+ budget := newEmbeddedMediaBudget()
+ if sections := buildDOCXJSONSections(irJSON, budget); len(sections) > 0 {
+ return ParseResult{
+ OutputFormat: "json",
+ File: map[string]any{"name": filename, "format": "doc"},
+ JSON: MapsToItems(sections),
+ Warnings: budget.warnings(),
+ }
+ }
+ }
+ if strings.TrimSpace(text) == "" {
+ return ParseResult{Err: fmt.Errorf("doc extract: no text content")}
+ }
+ return ParseResult{
+ OutputFormat: "json",
+ File: map[string]any{"name": filename, "format": "doc"},
+ JSON: []DocItem{NewTextJSONItem(text)},
+ }
}
// extractDocText returns the best-effort plain text for a legacy .doc
diff --git a/internal/parser/parser/doc_parser_doc_table_test.go b/internal/parser/parser/doc_parser_doc_table_test.go
index 4cd7e3dc3b..773bba7e69 100644
--- a/internal/parser/parser/doc_parser_doc_table_test.go
+++ b/internal/parser/parser/doc_parser_doc_table_test.go
@@ -8,18 +8,23 @@ import (
"path/filepath"
"strings"
"testing"
+
+ "ragflow/internal/entity"
)
-// TestDOCParser_RecoversTableFromLegacyDoc verifies that the legacy .doc path
-// recovers table structure through office_oxide's structured IR. The fixture
-// (testdata/table.doc) is a real Word binary containing a multi-column table.
+// TestDOCParser_TableEmittedAsStructuredItem verifies that the legacy .doc
+// path emits a structured doc_type_kwd:"table" item carrying a typed
+// *entity.TableData, instead of flattening the whole document (tables
+// included) into a single plain-text item with " | " separators.
//
-// The " | " separator is emitted ONLY by flattenDocIR when office_oxide's
-// ToIRJSON returns a table element (cells joined with " | " per row). Before
-// office_oxide/go v0.1.9 (#116) the .doc reader ignored the table stream, so
-// no table element was produced and the separator never appeared — this test
-// is therefore a direct regression gate for the v0.1.9 dependency bump.
-func TestDOCParser_RecoversTableFromLegacyDoc(t *testing.T) {
+// Before this change the DOC parser flattened every document through
+// flattenDocIR, so tables appeared as pipe-delimited plain text in chunks and
+// the chunker's HTML-table rendering (#20918) had no structured table to work
+// with. office_oxide/go v0.1.9 (#116) recovers the table stream as an IR
+// "type":"table" element; buildDOCXJSONSections projects that into the typed
+// table contract the chunker reads. The fixture (testdata/table.doc) is a real
+// Word binary containing a multi-column table.
+func TestDOCParser_TableEmittedAsStructuredItem(t *testing.T) {
data, err := os.ReadFile(filepath.Join("testdata", "table.doc"))
if err != nil {
t.Fatalf("read fixture: %v", err)
@@ -29,13 +34,79 @@ func TestDOCParser_RecoversTableFromLegacyDoc(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult error: %v", res.Err)
}
- if strings.TrimSpace(res.Text) == "" {
- t.Fatalf("parsed text is empty")
+ if res.OutputFormat != "json" {
+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
+ }
+ if len(res.JSON) == 0 {
+ t.Fatalf("JSON result is empty: %#v", res)
+ }
+
+ var tableItem map[string]any
+ for _, it := range res.JSON {
+ m := it.ToMap()
+ if dt, _ := m[DocTypeKey].(string); dt == DocTypeTable {
+ tableItem = m
+ break
+ }
+ }
+ if tableItem == nil {
+ t.Fatalf("no doc_type_kwd:table item in JSON result; items=%#v", res.JSON)
+ }
+
+ // The table must be emitted as a structured TableData — the contract the
+ // chunker's HTML-table rendering (#20918) reads — not as pipe-delimited
+ // plain text. projectPDFTable moves the HTML into the typed `table` field
+ // and deletes the redundant `text`, which is the intended shape.
+ td, ok := tableItem["table"].(*entity.TableData)
+ if !ok || td == nil {
+ t.Fatalf("table item has no structured TableData; item=%#v", tableItem)
+ }
+ cells := td.CellTexts()
+ if len(cells) == 0 {
+ t.Fatalf("table TableData has no rows; item=%#v", tableItem)
+ }
+ var joined strings.Builder
+ for _, row := range cells {
+ for _, c := range row {
+ joined.WriteString(c)
+ joined.WriteString(" ")
+ }
+ }
+ if strings.TrimSpace(joined.String()) == "" {
+ t.Errorf("table TableData has no cell text; cells=%#v", cells)
+ }
+
+ // The legacy pipe-delimited text form must not survive anywhere in the
+ // structured JSON: every item's `text` must be free of " | ".
+ for _, it := range res.JSON {
+ m := it.ToMap()
+ if txt, _ := m["text"].(string); strings.Contains(txt, " | ") {
+ t.Errorf("item still carries pipe-separated text: %q", txt)
+ }
+ }
+}
+
+// TestDOCParser_NoTableDocStaysTextItem pins the fallback for a .doc whose IR
+// yields no structured sections: the parser must still emit the flattened text
+// as a single text item (never silently drop content to 0 chunks). The fixture
+// is the same table.doc driven through the text output format, which bypasses
+// the IR sections path and must keep the legacy behavior intact.
+func TestDOCParser_TextOutputFormatStaysText(t *testing.T) {
+ data, err := os.ReadFile(filepath.Join("testdata", "table.doc"))
+ if err != nil {
+ t.Fatalf("read fixture: %v", err)
}
- // Table cells must be separated by " | " — the signature of IR-driven
- // table recovery. PlainText / ToMarkdown never emit this separator.
- if !strings.Contains(res.Text, " | ") {
- t.Errorf("table structure not recovered from legacy .doc\n got=%q", res.Text)
+ p := NewDOCParser()
+ p.OutputFormat = "text"
+ res := p.ParseWithResult(context.Background(), "table.doc", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult error: %v", res.Err)
+ }
+ if res.OutputFormat != "text" {
+ t.Fatalf("OutputFormat = %q, want text", res.OutputFormat)
+ }
+ if strings.TrimSpace(res.Text) == "" {
+ t.Fatalf("text output is empty: %#v", res)
}
}
diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.go
index d48e7b3f9b..c3f0981013 100644
--- a/internal/parser/parser/doc_parser_synth_test.go
+++ b/internal/parser/parser/doc_parser_synth_test.go
@@ -6,95 +6,76 @@ import (
"bytes"
"context"
"encoding/binary"
+ "strings"
"testing"
"unicode/utf16"
+ "ragflow/internal/entity"
+
officeOxide "github.com/yfedoseev/office_oxide/go"
)
// buildSyntheticDoc constructs a minimal but valid legacy Word binary (.doc)
-// entirely in memory — an OLE2/CFB container with a WordDocument stream
-// (FIB + piece-table-referenced text) and a 0Table stream (CLX). No external
-// tool or committed fixture is required, so the .doc parser can be exercised
-// end-to-end against dynamically generated input.
-//
-// Layout (sector size = 512):
-//
-// sector 0: FAT
-// sector 1: directory (Root Entry, WordDocument, 0Table)
-// sector 2: WordDocument stream (FIB + CP1252 text piece)
-// sector 3: 0Table stream (CLX)
-//
-// The structure follows office_oxide's reader expectations
-// (src/cfb/*, src/doc/fib.rs, src/doc/piece_table.rs).
+// entirely in memory.
func buildSyntheticDoc(t *testing.T, text string) []byte {
t.Helper()
const sector = 512
-
- // ── CLX (piece table) in the 0Table stream ────────────────────────────
- // 1 piece covering all text, compressed (CP1252): fc bit 30 set,
- // real byte offset = (fc & ~0x40000000) / 2.
- const textOffset = 426 // safely past the FIB reads (max offset 0x1AA=426)
+ const textOffset = 426
fc := uint32(0x40000000) | uint32(textOffset*2)
clx := make([]byte, 21)
- clx[0] = 0x02 // Pcdt
- binary.LittleEndian.PutUint32(clx[1:], 16) // PlcPcd size
- binary.LittleEndian.PutUint32(clx[5:], 0) // CP[0] = 0
- binary.LittleEndian.PutUint32(clx[9:], uint32(len(text))) // CP[1] = text_len
- binary.LittleEndian.PutUint16(clx[13:], 0) // PCD u16 unused
- binary.LittleEndian.PutUint32(clx[15:], fc) // PCD fc
- binary.LittleEndian.PutUint16(clx[19:], 0) // PCD prm
-
- // ── WordDocument stream: FIB + text ───────────────────────────────────
+ clx[0] = 0x02
+ binary.LittleEndian.PutUint32(clx[1:], 16)
+ binary.LittleEndian.PutUint32(clx[5:], 0)
+ binary.LittleEndian.PutUint32(clx[9:], uint32(len(text)))
+ binary.LittleEndian.PutUint16(clx[13:], 0)
+ binary.LittleEndian.PutUint32(clx[15:], fc)
+ binary.LittleEndian.PutUint16(clx[19:], 0)
+
wordDoc := make([]byte, sector)
- binary.LittleEndian.PutUint16(wordDoc[0:], 0xA5EC) // wIdent (Word 97+)
- binary.LittleEndian.PutUint16(wordDoc[2:], 0x00C1) // nFib
- binary.LittleEndian.PutUint16(wordDoc[0x0A:], 0x0000) // flags: bit9=0 → 0Table
- binary.LittleEndian.PutUint32(wordDoc[0x4C:], uint32(len(text))) // ccpText
- binary.LittleEndian.PutUint32(wordDoc[0x01A2:], 0) // fcClx (offset 0 in 0Table)
- binary.LittleEndian.PutUint32(wordDoc[0x01A6:], uint32(len(clx))) // lcbClx
+ binary.LittleEndian.PutUint16(wordDoc[0:], 0xA5EC)
+ binary.LittleEndian.PutUint16(wordDoc[2:], 0x00C1)
+ binary.LittleEndian.PutUint16(wordDoc[0x0A:], 0x0000)
+ binary.LittleEndian.PutUint32(wordDoc[0x4C:], uint32(len(text)))
+ binary.LittleEndian.PutUint32(wordDoc[0x01A2:], 0)
+ binary.LittleEndian.PutUint32(wordDoc[0x01A6:], uint32(len(clx)))
copy(wordDoc[textOffset:], []byte(text))
- // ── CFB header ────────────────────────────────────────────────────────
header := make([]byte, sector)
copy(header[0:8], []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1})
- binary.LittleEndian.PutUint16(header[0x18:], 0x003E) // minor version
- binary.LittleEndian.PutUint16(header[0x1A:], 3) // major version (v3)
- binary.LittleEndian.PutUint16(header[0x1C:], 0xFFFE) // byte order
- binary.LittleEndian.PutUint16(header[0x1E:], 9) // sector shift (512)
- binary.LittleEndian.PutUint16(header[0x20:], 6) // mini sector shift (64)
- binary.LittleEndian.PutUint32(header[0x2C:], 1) // FAT sector count
- binary.LittleEndian.PutUint32(header[0x30:], 1) // first directory sector
- binary.LittleEndian.PutUint32(header[0x38:], 4096) // mini-stream cutoff
- binary.LittleEndian.PutUint32(header[0x3C:], 0xFFFFFFFE) // first mini-FAT = END
- binary.LittleEndian.PutUint32(header[0x44:], 0xFFFFFFFE) // first DIFAT = END
+ binary.LittleEndian.PutUint16(header[0x18:], 0x003E)
+ binary.LittleEndian.PutUint16(header[0x1A:], 3)
+ binary.LittleEndian.PutUint16(header[0x1C:], 0xFFFE)
+ binary.LittleEndian.PutUint16(header[0x1E:], 9)
+ binary.LittleEndian.PutUint16(header[0x20:], 6)
+ binary.LittleEndian.PutUint32(header[0x2C:], 1)
+ binary.LittleEndian.PutUint32(header[0x30:], 1)
+ binary.LittleEndian.PutUint32(header[0x38:], 4096)
+ binary.LittleEndian.PutUint32(header[0x3C:], 0xFFFFFFFE)
+ binary.LittleEndian.PutUint32(header[0x44:], 0xFFFFFFFE)
for i := 0; i < 109; i++ {
off := 0x4C + i*4
if i == 0 {
- binary.LittleEndian.PutUint32(header[off:], 0) // FAT lives in sector 0
+ binary.LittleEndian.PutUint32(header[off:], 0)
} else {
- binary.LittleEndian.PutUint32(header[off:], 0xFFFFFFFE) // ignored
+ binary.LittleEndian.PutUint32(header[off:], 0xFFFFFFFE)
}
}
- // ── FAT sector ────────────────────────────────────────────────────────
- fat := bytes.Repeat([]byte{0xFF}, sector) // all FREE (0xFFFFFFFF)
- binary.LittleEndian.PutUint32(fat[0*4:], 0xFFFFFFFD) // sector 0 = FAT itself
- binary.LittleEndian.PutUint32(fat[1*4:], 0xFFFFFFFE) // directory = END
- binary.LittleEndian.PutUint32(fat[2*4:], 0xFFFFFFFE) // WordDocument = END
- binary.LittleEndian.PutUint32(fat[3*4:], 0xFFFFFFFE) // 0Table = END
+ fat := bytes.Repeat([]byte{0xFF}, sector)
+ binary.LittleEndian.PutUint32(fat[0*4:], 0xFFFFFFFD)
+ binary.LittleEndian.PutUint32(fat[1*4:], 0xFFFFFFFE)
+ binary.LittleEndian.PutUint32(fat[2*4:], 0xFFFFFFFE)
+ binary.LittleEndian.PutUint32(fat[3*4:], 0xFFFFFFFE)
- // ── Directory sector: 4 entries ───────────────────────────────────────
dir := make([]byte, sector)
copy(dir[0*128:], dirEntry("Root Entry", 5, 0xFFFFFFFF, 0))
copy(dir[1*128:], dirEntry("WordDocument", 2, 2, uint64(len(wordDoc))))
copy(dir[2*128:], dirEntry("0Table", 2, 3, uint64(len(clx))))
copy(dir[3*128:], dirEntry("", 0, 0xFFFFFFFF, 0))
- binary.LittleEndian.PutUint32(dir[0*128+0x4C:], 1) // root child: WordDocument
- binary.LittleEndian.PutUint32(dir[1*128+0x44:], 2) // left sibling: 0Table
+ binary.LittleEndian.PutUint32(dir[0*128+0x4C:], 1)
+ binary.LittleEndian.PutUint32(dir[1*128+0x44:], 2)
- // ── Assemble ──────────────────────────────────────────────────────────
tableSector := make([]byte, sector)
copy(tableSector, clx)
@@ -115,25 +96,24 @@ func dirEntry(name string, entryType byte, startSector uint32, streamSize uint64
for _, r := range utf16Name {
nameBytes = append(nameBytes, byte(r&0xFF), byte(r>>8))
}
- nameBytes = append(nameBytes, 0, 0) // UTF-16LE null terminator
+ nameBytes = append(nameBytes, 0, 0)
copy(e[0:], nameBytes)
binary.LittleEndian.PutUint16(e[0x40:], uint16(len(nameBytes)))
}
e[0x42] = entryType
- e[0x43] = 1 // color (black)
+ e[0x43] = 1
binary.LittleEndian.PutUint32(e[0x44:], 0xFFFFFFFF)
binary.LittleEndian.PutUint32(e[0x48:], 0xFFFFFFFF)
binary.LittleEndian.PutUint32(e[0x4C:], 0xFFFFFFFF)
binary.LittleEndian.PutUint32(e[0x74:], startSector)
- binary.LittleEndian.PutUint32(e[0x78:], uint32(streamSize)) // v3: low 32 bits
+ binary.LittleEndian.PutUint32(e[0x78:], uint32(streamSize))
return e
}
// TestDOCParser_SyntheticDocRoundTrip verifies the legacy DOC path end-to-end
-// against a .doc that is generated at runtime (no committed binary, no external
-// converter). It exercises office_oxide.OpenFromBytes(data, "doc") and the
-// extractDocText fallback chain (IR → Markdown → PlainText) inside
-// ParseWithResult.
+// against a .doc generated at runtime. The JSON path emits structured sections
+// (one per paragraph/table/list), so this test asserts the recovered text
+// fragments appear across the JSON items.
func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {
const body = "SYNTHETIC DOC\rFirst para alpha.\rSecond para beta."
data := buildSyntheticDoc(t, body)
@@ -151,22 +131,33 @@ func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {
if pr.OutputFormat != "json" {
t.Fatalf("OutputFormat = %q, want %q", pr.OutputFormat, "json")
}
- if len(pr.JSON) != 1 {
- t.Fatalf("ParseWithResult returned %d JSON items, want 1", len(pr.JSON))
+ if len(pr.JSON) == 0 {
+ t.Fatalf("ParseWithResult returned 0 JSON items, want at least 1")
}
- if pr.Text == "" {
- t.Fatal("ParseWithResult returned empty text — synthetic .doc was not parsed")
+
+ var b strings.Builder
+ for _, it := range pr.JSON {
+ m := it.ToMap()
+ if txt, _ := m["text"].(string); txt != "" {
+ b.WriteString(txt)
+ b.WriteString("\n")
+ }
+ if td, _ := m["table"].(*entity.TableData); td != nil {
+ for _, row := range td.CellTexts() {
+ for _, c := range row {
+ b.WriteString(c)
+ b.WriteString(" ")
+ }
+ }
+ }
}
+ got := b.String()
for _, want := range []string{"SYNTHETIC DOC", "First para alpha.", "Second para beta."} {
- if !bytes.Contains([]byte(pr.Text), []byte(want)) {
- t.Errorf("parsed text missing %q; got:\n%s", want, pr.Text)
+ if !strings.Contains(got, want) {
+ t.Errorf("parsed JSON missing %q; got:\n%s", want, got)
}
}
- if got, _ := pr.JSON[0].ToMap()["text"].(string); got != pr.Text {
- t.Errorf("JSON text = %q, want complete parser text %q", got, pr.Text)
- }
- // Also verify that explicit output_format="text" in setup is respected
pText := NewDOCParser()
pText.ConfigureFromSetup(map[string]any{"output_format": "text"})
prText := pText.ParseWithResult(context.Background(), "synthetic.doc", data)