RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): emit DOC tables as structured items instead of plain text (#21201)
xugangqiang404950b41527
diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.goindex 252a883975..6c4a81d785 100644--- a/internal/parser/parser/doc_parser.go+++ b/internal/parser/parser/doc_parser.go@@ -90,9 +90,32 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [return res}- res.OutputFormat = "json"- res.JSON = []DocItem{NewTextJSONItem(text)}- return res+ // JSON path: prefer the structured IR, which office_oxide recovers table+ // (and list) structure for. Emitting structured sections lets the chunker+ // render tables as HTML (#20918) instead of losing them to plain text — the+ // legacy path flattened the whole document (tables included) into a single+ // text item. Fall back to the flattened text only when the IR yields no+ // usable sections, so a document whose IR is empty/unusable still keeps its+ // content (PlainText / ToMarkdown) rather than silently producing 0 chunks.+ if strings.TrimSpace(irJSON) != "" {+ budget := newEmbeddedMediaBudget()+ if sections := buildDOCXJSONSections(irJSON, budget); len(sections) > 0 {+ return ParseResult{+ OutputFormat: "json",+ File: map[string]any{"name": filename, "format": "doc"},+ JSON: MapsToItems(sections),+ Warnings: budget.warnings(),+ }+ }+ }+ if strings.TrimSpace(text) == "" {+ return ParseResult{Err: fmt.Errorf("doc extract: no text content")}+ }+ return ParseResult{+ OutputFormat: "json",+ File: map[string]any{"name": filename, "format": "doc"},+ JSON: []DocItem{NewTextJSONItem(text)},+ }}// extractDocText returns the best-effort plain text for a legacy .docdiff --git a/internal/parser/parser/doc_parser_doc_table_test.go b/internal/parser/parser/doc_parser_doc_table_test.goindex 4cd7e3dc3b..773bba7e69 100644--- a/internal/parser/parser/doc_parser_doc_table_test.go+++ b/internal/parser/parser/doc_parser_doc_table_test.go@@ -8,18 +8,23 @@ import ("path/filepath""strings""testing"++ "ragflow/internal/entity")-// TestDOCParser_RecoversTableFromLegacyDoc verifies that the legacy .doc path-// recovers table structure through office_oxide's structured IR. The fixture-// (testdata/table.doc) is a real Word binary containing a multi-column table.+// TestDOCParser_TableEmittedAsStructuredItem verifies that the legacy .doc+// path emits a structured doc_type_kwd:"table" item carrying a typed+// *entity.TableData, instead of flattening the whole document (tables+// included) into a single plain-text item with " | " separators.//-// The " | " separator is emitted ONLY by flattenDocIR when office_oxide's-// ToIRJSON returns a table element (cells joined with " | " per row). Before-// office_oxide/go v0.1.9 (#116) the .doc reader ignored the table stream, so-// no table element was produced and the separator never appeared — this test-// is therefore a direct regression gate for the v0.1.9 dependency bump.-func TestDOCParser_RecoversTableFromLegacyDoc(t *testing.T) {+// Before this change the DOC parser flattened every document through+// flattenDocIR, so tables appeared as pipe-delimited plain text in chunks and+// the chunker's HTML-table rendering (#20918) had no structured table to work+// with. office_oxide/go v0.1.9 (#116) recovers the table stream as an IR+// "type":"table" element; buildDOCXJSONSections projects that into the typed+// table contract the chunker reads. The fixture (testdata/table.doc) is a real+// Word binary containing a multi-column table.+func TestDOCParser_TableEmittedAsStructuredItem(t *testing.T) {data, err := os.ReadFile(filepath.Join("testdata", "table.doc"))if err != nil {t.Fatalf("read fixture: %v", err)@@ -29,13 +34,79 @@ func TestDOCParser_RecoversTableFromLegacyDoc(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult error: %v", res.Err)}- if strings.TrimSpace(res.Text) == "" {- t.Fatalf("parsed text is empty")+ if res.OutputFormat != "json" {+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)+ }+ if len(res.JSON) == 0 {+ t.Fatalf("JSON result is empty: %#v", res)+ }++ var tableItem map[string]any+ for _, it := range res.JSON {+ m := it.ToMap()+ if dt, _ := m[DocTypeKey].(string); dt == DocTypeTable {+ tableItem = m+ break+ }+ }+ if tableItem == nil {+ t.Fatalf("no doc_type_kwd:table item in JSON result; items=%#v", res.JSON)+ }++ // The table must be emitted as a structured TableData — the contract the+ // chunker's HTML-table rendering (#20918) reads — not as pipe-delimited+ // plain text. projectPDFTable moves the HTML into the typed `table` field+ // and deletes the redundant `text`, which is the intended shape.+ td, ok := tableItem["table"].(*entity.TableData)+ if !ok || td == nil {+ t.Fatalf("table item has no structured TableData; item=%#v", tableItem)+ }+ cells := td.CellTexts()+ if len(cells) == 0 {+ t.Fatalf("table TableData has no rows; item=%#v", tableItem)+ }+ var joined strings.Builder+ for _, row := range cells {+ for _, c := range row {+ joined.WriteString(c)+ joined.WriteString(" ")+ }+ }+ if strings.TrimSpace(joined.String()) == "" {+ t.Errorf("table TableData has no cell text; cells=%#v", cells)+ }++ // The legacy pipe-delimited text form must not survive anywhere in the+ // structured JSON: every item's `text` must be free of " | ".+ for _, it := range res.JSON {+ m := it.ToMap()+ if txt, _ := m["text"].(string); strings.Contains(txt, " | ") {+ t.Errorf("item still carries pipe-separated text: %q", txt)+ }+ }+}++// TestDOCParser_NoTableDocStaysTextItem pins the fallback for a .doc whose IR+// yields no structured sections: the parser must still emit the flattened text+// as a single text item (never silently drop content to 0 chunks). The fixture+// is the same table.doc driven through the text output format, which bypasses+// the IR sections path and must keep the legacy behavior intact.+func TestDOCParser_TextOutputFormatStaysText(t *testing.T) {+ data, err := os.ReadFile(filepath.Join("testdata", "table.doc"))+ if err != nil {+ t.Fatalf("read fixture: %v", err)}- // Table cells must be separated by " | " — the signature of IR-driven- // table recovery. PlainText / ToMarkdown never emit this separator.- if !strings.Contains(res.Text, " | ") {- t.Errorf("table structure not recovered from legacy .doc\n got=%q", res.Text)+ p := NewDOCParser()+ p.OutputFormat = "text"+ res := p.ParseWithResult(context.Background(), "table.doc", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult error: %v", res.Err)+ }+ if res.OutputFormat != "text" {+ t.Fatalf("OutputFormat = %q, want text", res.OutputFormat)+ }+ if strings.TrimSpace(res.Text) == "" {+ t.Fatalf("text output is empty: %#v", res)}}diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.goindex d48e7b3f9b..c3f0981013 100644--- a/internal/parser/parser/doc_parser_synth_test.go+++ b/internal/parser/parser/doc_parser_synth_test.go@@ -6,95 +6,76 @@ import ("bytes""context""encoding/binary"+ "strings""testing""unicode/utf16"+ "ragflow/internal/entity"+officeOxide "github.com/yfedoseev/office_oxide/go")// buildSyntheticDoc constructs a minimal but valid legacy Word binary (.doc)-// entirely in memory — an OLE2/CFB container with a WordDocument stream-// (FIB + piece-table-referenced text) and a 0Table stream (CLX). No external-// tool or committed fixture is required, so the .doc parser can be exercised-// end-to-end against dynamically generated input.-//-// Layout (sector size = 512):-//-// sector 0: FAT-// sector 1: directory (Root Entry, WordDocument, 0Table)-// sector 2: WordDocument stream (FIB + CP1252 text piece)-// sector 3: 0Table stream (CLX)-//-// The structure follows office_oxide's reader expectations-// (src/cfb/*, src/doc/fib.rs, src/doc/piece_table.rs).+// entirely in memory.func buildSyntheticDoc(t *testing.T, text string) []byte {t.Helper()const sector = 512-- // ── CLX (piece table) in the 0Table stream ────────────────────────────- // 1 piece covering all text, compressed (CP1252): fc bit 30 set,- // real byte offset = (fc & ~0x40000000) / 2.- const textOffset = 426 // safely past the FIB reads (max offset 0x1AA=426)+ const textOffset = 426fc := uint32(0x40000000) | uint32(textOffset*2)clx := make([]byte, 21)- clx[0] = 0x02 // Pcdt- binary.LittleEndian.PutUint32(clx[1:], 16) // PlcPcd size- binary.LittleEndian.PutUint32(clx[5:], 0) // CP[0] = 0- binary.LittleEndian.PutUint32(clx[9:], uint32(len(text))) // CP[1] = text_len- binary.LittleEndian.PutUint16(clx[13:], 0) // PCD u16 unused- binary.LittleEndian.PutUint32(clx[15:], fc) // PCD fc- binary.LittleEndian.PutUint16(clx[19:], 0) // PCD prm-- // ── WordDocument stream: FIB + text ───────────────────────────────────+ clx[0] = 0x02+ binary.LittleEndian.PutUint32(clx[1:], 16)+ binary.LittleEndian.PutUint32(clx[5:], 0)+ binary.LittleEndian.PutUint32(clx[9:], uint32(len(text)))+ binary.LittleEndian.PutUint16(clx[13:], 0)+ binary.LittleEndian.PutUint32(clx[15:], fc)+ binary.LittleEndian.PutUint16(clx[19:], 0)+wordDoc := make([]byte, sector)- binary.LittleEndian.PutUint16(wordDoc[0:], 0xA5EC) // wIdent (Word 97+)- binary.LittleEndian.PutUint16(wordDoc[2:], 0x00C1) // nFib- binary.LittleEndian.PutUint16(wordDoc[0x0A:], 0x0000) // flags: bit9=0 → 0Table- binary.LittleEndian.PutUint32(wordDoc[0x4C:], uint32(len(text))) // ccpText- binary.LittleEndian.PutUint32(wordDoc[0x01A2:], 0) // fcClx (offset 0 in 0Table)- binary.LittleEndian.PutUint32(wordDoc[0x01A6:], uint32(len(clx))) // lcbClx+ binary.LittleEndian.PutUint16(wordDoc[0:], 0xA5EC)+ binary.LittleEndian.PutUint16(wordDoc[2:], 0x00C1)+ binary.LittleEndian.PutUint16(wordDoc[0x0A:], 0x0000)+ binary.LittleEndian.PutUint32(wordDoc[0x4C:], uint32(len(text)))+ binary.LittleEndian.PutUint32(wordDoc[0x01A2:], 0)+ binary.LittleEndian.PutUint32(wordDoc[0x01A6:], uint32(len(clx)))copy(wordDoc[textOffset:], []byte(text))- // ── CFB header ────────────────────────────────────────────────────────header := make([]byte, sector)copy(header[0:8], []byte{0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1})- binary.LittleEndian.PutUint16(header[0x18:], 0x003E) // minor version- binary.LittleEndian.PutUint16(header[0x1A:], 3) // major version (v3)- binary.LittleEndian.PutUint16(header[0x1C:], 0xFFFE) // byte order- binary.LittleEndian.PutUint16(header[0x1E:], 9) // sector shift (512)- binary.LittleEndian.PutUint16(header[0x20:], 6) // mini sector shift (64)- binary.LittleEndian.PutUint32(header[0x2C:], 1) // FAT sector count- binary.LittleEndian.PutUint32(header[0x30:], 1) // first directory sector- binary.LittleEndian.PutUint32(header[0x38:], 4096) // mini-stream cutoff- binary.LittleEndian.PutUint32(header[0x3C:], 0xFFFFFFFE) // first mini-FAT = END- binary.LittleEndian.PutUint32(header[0x44:], 0xFFFFFFFE) // first DIFAT = END+ binary.LittleEndian.PutUint16(header[0x18:], 0x003E)+ binary.LittleEndian.PutUint16(header[0x1A:], 3)+ binary.LittleEndian.PutUint16(header[0x1C:], 0xFFFE)+ binary.LittleEndian.PutUint16(header[0x1E:], 9)+ binary.LittleEndian.PutUint16(header[0x20:], 6)+ binary.LittleEndian.PutUint32(header[0x2C:], 1)+ binary.LittleEndian.PutUint32(header[0x30:], 1)+ binary.LittleEndian.PutUint32(header[0x38:], 4096)+ binary.LittleEndian.PutUint32(header[0x3C:], 0xFFFFFFFE)+ binary.LittleEndian.PutUint32(header[0x44:], 0xFFFFFFFE)for i := 0; i < 109; i++ {off := 0x4C + i*4if i == 0 {- binary.LittleEndian.PutUint32(header[off:], 0) // FAT lives in sector 0+ binary.LittleEndian.PutUint32(header[off:], 0)} else {- binary.LittleEndian.PutUint32(header[off:], 0xFFFFFFFE) // ignored+ binary.LittleEndian.PutUint32(header[off:], 0xFFFFFFFE)}}- // ── FAT sector ────────────────────────────────────────────────────────- fat := bytes.Repeat([]byte{0xFF}, sector) // all FREE (0xFFFFFFFF)- binary.LittleEndian.PutUint32(fat[0*4:], 0xFFFFFFFD) // sector 0 = FAT itself- binary.LittleEndian.PutUint32(fat[1*4:], 0xFFFFFFFE) // directory = END- binary.LittleEndian.PutUint32(fat[2*4:], 0xFFFFFFFE) // WordDocument = END- binary.LittleEndian.PutUint32(fat[3*4:], 0xFFFFFFFE) // 0Table = END+ fat := bytes.Repeat([]byte{0xFF}, sector)+ binary.LittleEndian.PutUint32(fat[0*4:], 0xFFFFFFFD)+ binary.LittleEndian.PutUint32(fat[1*4:], 0xFFFFFFFE)+ binary.LittleEndian.PutUint32(fat[2*4:], 0xFFFFFFFE)+ binary.LittleEndian.PutUint32(fat[3*4:], 0xFFFFFFFE)- // ── Directory sector: 4 entries ───────────────────────────────────────dir := make([]byte, sector)copy(dir[0*128:], dirEntry("Root Entry", 5, 0xFFFFFFFF, 0))copy(dir[1*128:], dirEntry("WordDocument", 2, 2, uint64(len(wordDoc))))copy(dir[2*128:], dirEntry("0Table", 2, 3, uint64(len(clx))))copy(dir[3*128:], dirEntry("", 0, 0xFFFFFFFF, 0))- binary.LittleEndian.PutUint32(dir[0*128+0x4C:], 1) // root child: WordDocument- binary.LittleEndian.PutUint32(dir[1*128+0x44:], 2) // left sibling: 0Table+ binary.LittleEndian.PutUint32(dir[0*128+0x4C:], 1)+ binary.LittleEndian.PutUint32(dir[1*128+0x44:], 2)- // ── Assemble ──────────────────────────────────────────────────────────tableSector := make([]byte, sector)copy(tableSector, clx)@@ -115,25 +96,24 @@ func dirEntry(name string, entryType byte, startSector uint32, streamSize uint64for _, r := range utf16Name {nameBytes = append(nameBytes, byte(r&0xFF), byte(r>>8))}- nameBytes = append(nameBytes, 0, 0) // UTF-16LE null terminator+ nameBytes = append(nameBytes, 0, 0)copy(e[0:], nameBytes)binary.LittleEndian.PutUint16(e[0x40:], uint16(len(nameBytes)))}e[0x42] = entryType- e[0x43] = 1 // color (black)+ e[0x43] = 1binary.LittleEndian.PutUint32(e[0x44:], 0xFFFFFFFF)binary.LittleEndian.PutUint32(e[0x48:], 0xFFFFFFFF)binary.LittleEndian.PutUint32(e[0x4C:], 0xFFFFFFFF)binary.LittleEndian.PutUint32(e[0x74:], startSector)- binary.LittleEndian.PutUint32(e[0x78:], uint32(streamSize)) // v3: low 32 bits+ binary.LittleEndian.PutUint32(e[0x78:], uint32(streamSize))return e}// TestDOCParser_SyntheticDocRoundTrip verifies the legacy DOC path end-to-end-// against a .doc that is generated at runtime (no committed binary, no external-// converter). It exercises office_oxide.OpenFromBytes(data, "doc") and the-// extractDocText fallback chain (IR → Markdown → PlainText) inside-// ParseWithResult.+// against a .doc generated at runtime. The JSON path emits structured sections+// (one per paragraph/table/list), so this test asserts the recovered text+// fragments appear across the JSON items.func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {const body = "SYNTHETIC DOC\rFirst para alpha.\rSecond para beta."data := buildSyntheticDoc(t, body)@@ -151,22 +131,33 @@ func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {if pr.OutputFormat != "json" {t.Fatalf("OutputFormat = %q, want %q", pr.OutputFormat, "json")}- if len(pr.JSON) != 1 {- t.Fatalf("ParseWithResult returned %d JSON items, want 1", len(pr.JSON))+ if len(pr.JSON) == 0 {+ t.Fatalf("ParseWithResult returned 0 JSON items, want at least 1")}- if pr.Text == "" {- t.Fatal("ParseWithResult returned empty text — synthetic .doc was not parsed")++ var b strings.Builder+ for _, it := range pr.JSON {+ m := it.ToMap()+ if txt, _ := m["text"].(string); txt != "" {+ b.WriteString(txt)+ b.WriteString("\n")+ }+ if td, _ := m["table"].(*entity.TableData); td != nil {+ for _, row := range td.CellTexts() {+ for _, c := range row {+ b.WriteString(c)+ b.WriteString(" ")+ }+ }+ }}+ got := b.String()for _, want := range []string{"SYNTHETIC DOC", "First para alpha.", "Second para beta."} {- if !bytes.Contains([]byte(pr.Text), []byte(want)) {- t.Errorf("parsed text missing %q; got:\n%s", want, pr.Text)+ if !strings.Contains(got, want) {+ t.Errorf("parsed JSON missing %q; got:\n%s", want, got)}}- if got, _ := pr.JSON[0].ToMap()["text"].(string); got != pr.Text {- t.Errorf("JSON text = %q, want complete parser text %q", got, pr.Text)- }- // Also verify that explicit output_format="text" in setup is respectedpText := NewDOCParser()pText.ConfigureFromSetup(map[string]any{"output_format": "text"})prText := pText.ParseWithResult(context.Background(), "synthetic.doc", data)