RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): merge .doc header-row table fragments split by office_oxide (#21222)
xugangqiang3a4cef0b779a
diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.goindex f47832d11a..77ab144e32 100644--- a/internal/parser/parser/docx_ir.go+++ b/internal/parser/parser/docx_ir.go@@ -334,6 +334,16 @@ func docxElementText(el docxIRElement, cellSep string) string {}}+// docxIRTableCols returns the number of cells in a table's first row, used+// to decide whether two adjacent table elements are fragments of one logical+// table. It returns 0 for an empty table.+func docxIRTableCols(el docxIRElement) int {+ if len(el.Rows) == 0 {+ return 0+ }+ return len(el.Rows[0].Cells)+}+// docxHeading is one entry of the heading outline stack.type docxHeading struct {level int@@ -414,8 +424,60 @@ func buildDOCXJSONSectionsCore(irJSON string, budget *embeddedMediaBudget, tablevar sections []map[string]anytableSequence := 0var headings []docxHeading- for _, sec := range ir.Sections {- for _, el := range sec.Elements {+ for si := range ir.Sections {+ sec := &ir.Sections[si]+ for ei := 0; ei < len(sec.Elements); ei++ {+ el := sec.Elements[ei]+ // TEMPORARY: this merge is a stopgap. Revisit it once office_oxide+ // populates `TableRow.is_header` for .doc tables (upstream issue+ // yfedoseev/office_oxide#456) — at that point prefer rejoining via+ // the real `is_header` flag and drop the structural proxy below.+ //+ // office_oxide splits a .doc table whose first row is a styled+ // header/title row into a 1-row "header" table immediately+ // followed by the body table (the legacy binary .doc format+ // stores such a table as two separate Table objects with a blank+ // paragraph between them; walk_paragraphs flushes a new Element::Table+ // at every non-fInTable paragraph, and emit_prose drops the blank+ // one, so the two tables end up adjacent in the IR). Rejoin a+ // 1-row table fragment with the adjacent following table when their+ // first rows carry the same column count, so one logical table is+ // emitted as a single item instead of a header chunk + body chunk+ // (each would otherwise carry the same caption).+ //+ // Why a structural proxy instead of the row's `is_header` flag:+ // office_oxide's IR DOES carry `TableRow.is_header`, but the .doc+ // converter (convert_doc.rs) never populates it — every .doc row+ // arrives with is_header=false, verified on the real document that+ // triggered this bug. Only the .docx converter sets it, and .docx+ // never splits. So `is_header` is a dead signal for .doc and cannot+ // be used to confirm the leading fragment is a header or that the+ // following table lacks one. The 1-row gate is the strongest+ // available proxy: a 1-row table adjacent to a same-width table is+ // overwhelmingly a header fragment, because a genuinely standalone+ // 1-row table immediately followed by another table (with only a+ // blank paragraph between) is rare.+ //+ // Regression guards baked into the merge condition:+ // - leading fragment is exactly 1 row: two genuinely separate+ // MULTI-row tables (same width, blank paragraph between) are+ // never merged.+ // - same first-row column count: differently-shaped tables stay+ // separate.+ // - adjacency (no element between): any heading/paragraph/image+ // in between keeps the two tables distinct.+ if el.Type == "table" && len(el.Rows) == 1 && ei+1 < len(sec.Elements) {+ if nxt := sec.Elements[ei+1]; nxt.Type == "table" &&+ docxIRTableCols(el) > 0 && docxIRTableCols(el) == docxIRTableCols(nxt) {+ merged := nxt+ rows := make([]docxIRRow, 0, len(nxt.Rows)+len(el.Rows))+ rows = append(rows, el.Rows...)+ rows = append(rows, nxt.Rows...)+ merged.Rows = rows+ el = merged+ ei++ // fold the following table into this one; skip it+ }+ }switch el.Type {case "paragraph", "heading":if el.Type == "heading" {diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.goindex 6d29a1b530..23037c6807 100644--- a/internal/parser/parser/docx_ir_test.go+++ b/internal/parser/parser/docx_ir_test.go@@ -55,6 +55,123 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) {}}+// TestBuildDOCXJSONSections_MergesSplitDocTable pins the fix for the+// office_oxide .doc artifact where a table whose first row is a styled+// header is emitted as a 1-row "header" table immediately followed by the+// body table. The two fragments share the same column count and sit under+// the same heading, so they must be rejoined into a single table item+// (otherwise the chunker emits a header chunk + body chunk, each carrying+// the same caption).+func TestBuildDOCXJSONSections_MergesSplitDocTable(t *testing.T) {+ const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}`+ // header: 1 row x 3 cols; body: 2 rows x 3 cols — same width.+ irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[+ {"type":"heading","level":1,"content":[{"type":"text","text":"H"}]},+ {"type":"table","rows":[{"cells":[%s,%s,%s]}]},+ {"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}+ ]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell)++ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ var tables []map[string]any+ for _, it := range got {+ if dt, _ := it["doc_type_kwd"].(string); dt == "table" {+ tables = append(tables, it)+ }+ }+ if len(tables) != 1 {+ t.Fatalf("expected 1 merged table item, got %d: %#v", len(tables), got)+ }+ td, ok := tables[0]["table"].(*entity.TableData)+ if !ok || td == nil {+ t.Fatalf("merged table has no TableData: %#v", tables[0])+ }+ if len(td.Grid) != 3 {+ t.Errorf("merged table should have 3 rows (1 header + 2 body), got %d: %#v", len(td.Grid), td.Grid)+ }+ if td.Caption != "Table Location: H" {+ t.Errorf("merged table caption = %q, want %q", td.Caption, "Table Location: H")+ }+}++// TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate pins that+// two adjacent tables of different column counts are NOT merged: they are+// genuinely separate tables, not fragments of one.+func TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate(t *testing.T) {+ const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}`+ // a 1-row 3-col table followed by a 2-row 2-col table: different widths.+ irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[+ {"type":"table","rows":[{"cells":[%s,%s,%s]}]},+ {"type":"table","rows":[{"cells":[%s,%s]},{"cells":[%s,%s]}]}+ ]}]}`, cell, cell, cell, cell, cell, cell, cell)+ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ var n int+ for _, it := range got {+ if dt, _ := it["doc_type_kwd"].(string); dt == "table" {+ n+++ }+ }+ if n != 2 {+ t.Fatalf("expected 2 separate tables (different column counts), got %d: %#v", n, got)+ }+}++// TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate is the key+// regression guard for the 1-row merge gate: two genuinely separate multi-row+// tables of the SAME width, stacked with only a blank paragraph between them+// (so they are adjacent in the IR), must NOT be merged. Only a 1-row header+// fragment is folded into the following table.+func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T) {+ const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}`+ // table A: 2 rows x 3 cols; table B: 2 rows x 3 cols — same width, both+ // multi-row, no element between. These are two distinct tables.+ irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[+ {"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]},+ {"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}+ ]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell, cell)+ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ var tables []map[string]any+ for _, it := range got {+ if dt, _ := it["doc_type_kwd"].(string); dt == "table" {+ tables = append(tables, it)+ }+ }+ if len(tables) != 2 {+ t.Fatalf("expected 2 separate multi-row tables, got %d: %#v", len(tables), got)+ }+ for i, tb := range tables {+ td, ok := tb["table"].(*entity.TableData)+ if !ok || td == nil {+ t.Fatalf("table %d has no TableData: %#v", i, tb)+ }+ if len(td.Grid) != 2 {+ t.Errorf("table %d should keep its 2 rows, got %d: %#v", i, len(td.Grid), td.Grid)+ }+ }+}++// TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit pins+// that a 1-row header fragment is only merged when the body is the VERY NEXT+// element. A paragraph (even blank) between them keeps them as distinct+// tables, so the gate never reaches across intervening content.+func TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit(t *testing.T) {+ const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}`+ irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[+ {"type":"table","rows":[{"cells":[%s,%s,%s]}]},+ {"type":"paragraph","content":[{"type":"text","text":"note"}]},+ {"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}+ ]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell)+ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ var n int+ for _, it := range got {+ if dt, _ := it["doc_type_kwd"].(string); dt == "table" {+ n+++ }+ }+ if n != 2 {+ t.Fatalf("expected 2 separate tables (paragraph between), got %d: %#v", n, got)+ }+}+func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {irJSON := `{"sections":[{"elements":[{"type":"paragraph","content":[