RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): preserve Office table cell spans (#22012)
jay77721afd9a7f1dab5
diff --git a/internal/ingestion/component/chunker/general_table_policy_test.go b/internal/ingestion/component/chunker/general_table_policy_test.gonew file mode 100644index 0000000000..42e0ded5f3--- /dev/null+++ b/internal/ingestion/component/chunker/general_table_policy_test.go@@ -0,0 +1,67 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "context"+ "strings"+ "testing"+)++// TestGeneralChunkerKeepsLargeTablesWhole pins the table policy shared by the+// PDF and DOCX paths: a table is an atomic media unit, so chunk_token_size+// never splits it. Text is still merged and split by the configured budget.+func TestGeneralChunkerKeepsLargeTablesWhole(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})+ if err != nil {+ t.Fatal(err)+ }++ table := "<table><tr><th>header</th></tr>" ++ strings.Repeat("<tr><td>cell value</td></tr>", 400) ++ "</table>"+ if tokens := tokenizeStr(table); tokens <= 512 {+ t.Fatalf("test table has %d tokens, want more than the 512 budget", tokens)+ }++ for _, fileType := range []string{"docx", "pdf"} {+ t.Run(fileType, func(t *testing.T) {+ out, err := component.Invoke(context.Background(), nil, map[string]any{+ "name": "large." + fileType,+ "file_type": fileType,+ "output_format": "json",+ "json": []map[string]any{{+ "text": table,+ "doc_type_kwd": "table",+ }},+ })+ if err != nil {+ t.Fatal(err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 1 {+ t.Fatalf("chunks = %d, want 1: a table is not split by the token budget", len(chunks))+ }+ if got := chunks[0]["doc_type_kwd"]; got != "table" {+ t.Fatalf("doc_type_kwd = %v, want table", got)+ }+ if got, _ := chunks[0]["text"].(string); got != table {+ t.Fatalf("table text changed: got %d bytes, want %d", len(got), len(table))+ }+ })+ }+}diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.goindex f7bf8113fd..f6781e9c28 100644--- a/internal/parser/parser/docx_ir.go+++ b/internal/parser/parser/docx_ir.go@@ -99,11 +99,14 @@ type docxIRRun struct {}type docxIRRow struct {- Cells []docxIRCell `json:"cells"`+ IsHeader bool `json:"is_header"` // repeated table header row+ Cells []docxIRCell `json:"cells"`}type docxIRCell struct {Content []docxIRElement `json:"content"` // block-level content (typically paragraphs; may also hold list/table)+ ColSpan int `json:"col_span"`+ RowSpan int `json:"row_span"`}// joinDOCXIRRuns concatenates inline runs into plain text. Hard line@@ -180,16 +183,26 @@ func joinCellText(cell docxIRCell) string {}// officeIRTableData projects native Office rows directly onto the table contract.+// The row/column spans office_oxide recovers for merged cells are preserved so+// BuildTableData can rebuild the same grid; header flags mark repeated header+// rows, falling back to BuildTableData's default when the format does not carry+// them.func officeIRTableData(el docxIRElement, caption string) *entity.TableData {rows := make([][]tableutil.CellInput, 0, len(el.Rows))+ headerFlags := make([]bool, 0, len(el.Rows))for _, row := range el.Rows {cells := make([]tableutil.CellInput, 0, len(row.Cells))for _, cell := range row.Cells {- cells = append(cells, tableutil.CellInput{Text: joinCellText(cell)})+ cells = append(cells, tableutil.CellInput{+ Text: joinCellText(cell),+ ColSpan: cell.ColSpan,+ RowSpan: cell.RowSpan,+ })}rows = append(rows, cells)+ headerFlags = append(headerFlags, row.IsHeader)}- return tableutil.BuildTableData(rows, nil, strings.TrimSpace(caption))+ return tableutil.BuildTableData(rows, headerFlags, strings.TrimSpace(caption))}type docxTableImage struct {@@ -302,16 +315,6 @@ func docxElementText(el docxIRElement, cellSep string) string {}}-// docxIRTableCols returns the number of cells in a table's first row, used-// to decide whether two adjacent table elements are fragments of one logical-// table. It returns 0 for an empty table.-func docxIRTableCols(el docxIRElement) int {- if len(el.Rows) == 0 {- return 0- }- return len(el.Rows[0].Cells)-}-// docxHeading is one entry of the heading outline stack.type docxHeading struct {level intdiff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.goindex 1d1576b0a6..692edd47cd 100644--- a/internal/parser/parser/docx_ir_test.go+++ b/internal/parser/parser/docx_ir_test.go@@ -54,12 +54,126 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) {}}-// TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate pins that-// two adjacent tables of different column counts are NOT merged: they are-// genuinely separate tables, not fragments of one.+// TestBuildDOCXJSONSections_PreservesMergedHeaderSpans pins that office_oxide's+// row/column spans and repeated-header flags survive the IR -> TableData+// projection. The header is two rows: row 0 carries the colspan/rowspan+// geometry, row 1 fills the seven columns the vertical merges leave open, and+// the body has ten columns.+func TestBuildDOCXJSONSections_PreservesMergedHeaderSpans(t *testing.T) {+ cell := func(text string, colSpan, rowSpan int) docxIRCell {+ return docxIRCell{+ Content: []docxIRElement{para(text)},+ ColSpan: colSpan,+ RowSpan: rowSpan,+ }+ }+ row := func(header bool, cells ...docxIRCell) docxIRRow {+ return docxIRRow{IsHeader: header, Cells: cells}+ }+ ir := docxIRDocument{Sections: []docxIRSection{{+ Elements: []docxIRElement{+ {Type: "heading", Level: 1, Content: rawJSON([]docxIRRun{{Type: "text", Text: "H"}})},+ {Type: "table", Rows: []docxIRRow{+ row(true,+ cell("单位名称", 1, 2),+ cell("集团公司采用", 3, 1),+ cell("中国中铁采用", 2, 1),+ cell("国资委采用", 2, 1),+ cell("得分", 1, 2),+ cell("排名", 1, 2),+ ),+ row(true,+ cell("政务", 1, 1),+ cell("政工", 1, 1),+ cell("得分", 1, 1),+ cell("数量", 1, 1),+ cell("得分", 1, 1),+ cell("数量", 1, 1),+ cell("得分", 1, 1),+ ),+ row(false,+ cell("西安公司", 1, 1),+ cell("15(1)", 1, 1),+ cell("12", 1, 1),+ cell("265", 1, 1),+ cell("4(3)", 1, 1),+ cell("50", 1, 1),+ cell("2(2)", 1, 1),+ cell("40", 1, 1),+ cell("355", 1, 1),+ cell("1", 1, 1),+ ),+ }},+ },+ }}}+ encoded, err := json.Marshal(ir)+ if err != nil {+ t.Fatal(err)+ }++ got := buildDOCXJSONSections(string(encoded), nil)+ var tables []map[string]any+ for _, it := range got {+ if dt, _ := it["doc_type_kwd"].(string); dt == "table" {+ tables = append(tables, it)+ }+ }+ if len(tables) != 1 {+ t.Fatalf("expected 1 table item, got %d: %#v", len(tables), got)+ }+ td, ok := tables[0]["table"].(*entity.TableData)+ if !ok || td == nil {+ t.Fatalf("table has no TableData: %#v", tables[0])+ }+ if td.HeaderRows != 2 {+ t.Fatalf("HeaderRows = %d, want 2", td.HeaderRows)+ }+ if td.Caption != "Table Location: H" {+ t.Errorf("caption = %q, want %q", td.Caption, "Table Location: H")+ }+ if len(td.Grid) != 3 {+ t.Fatalf("grid rows = %d, want 3", len(td.Grid))+ }+ for i, row := range td.Grid {+ if len(row) != 10 {+ t.Fatalf("grid row %d width = %d, want 10: %#v", i, len(row), row)+ }+ }+ if got := td.Grid[0][0]; got.Text != "单位名称" || got.RowSpan != 2 {+ t.Errorf("header cell 0 = %#v, want 单位名称 rowspan 2", got)+ }+ if got := td.Grid[0][1]; got.Text != "集团公司采用" || got.ColSpan != 3 {+ t.Errorf("header cell 1 = %#v, want 集团公司采用 colspan 3", got)+ }+ if got := td.Grid[0][4]; got.Text != "中国中铁采用" || got.ColSpan != 2 {+ t.Errorf("header cell at col 4 = %#v, want 中国中铁采用 colspan 2", got)+ }+ if got := td.Grid[0][6]; got.Text != "国资委采用" || got.ColSpan != 2 {+ t.Errorf("header cell at col 6 = %#v, want 国资委采用 colspan 2", got)+ }+ if got := td.Grid[0][8]; got.Text != "得分" || got.RowSpan != 2 {+ t.Errorf("header cell at col 8 = %#v, want 得分 rowspan 2", got)+ }+ if got := td.Grid[0][9]; got.Text != "排名" || got.RowSpan != 2 {+ t.Errorf("header cell at col 9 = %#v, want 排名 rowspan 2", got)+ }+ for _, col := range []int{2, 3, 5, 7} {+ if got := td.Grid[0][col]; !got.Covered {+ t.Errorf("row 0 col %d = %#v, want covered by a colspan", col, got)+ }+ }+ if got := td.Grid[1][0]; !got.Covered {+ t.Errorf("row 1 col 0 = %#v, want covered by the rowspan", got)+ }+ if got := td.Grid[1][1]; got.Text != "政务" {+ t.Errorf("row 1 col 1 = %#v, want 政务", got)+ }+}++// TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate pins that two+// adjacent tables of different column counts remain separate.func TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate(t *testing.T) {const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}`- // a 1-row 3-col table followed by a 2-row 2-col table: different widths.irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[{"type":"table","rows":[{"cells":[%s,%s,%s]}]},{"type":"table","rows":[{"cells":[%s,%s]},{"cells":[%s,%s]}]}@@ -76,15 +190,11 @@ func TestBuildDOCXJSONSections_DifferentColumnTablesStaySeparate(t *testing.T) {}}-// TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate is the key-// regression guard for the 1-row merge gate: two genuinely separate multi-row-// tables of the SAME width, stacked with only a blank paragraph between them-// (so they are adjacent in the IR), must NOT be merged. Only a 1-row header-// fragment is folded into the following table.-func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T) {+// TestBuildDOCXJSONSections_AdjacentTablesStaySeparate pins that adjacent+// tables remain separate items. The parser no longer merges table elements;+// office_oxide now emits one element per logical table.+func TestBuildDOCXJSONSections_AdjacentTablesStaySeparate(t *testing.T) {const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}`- // table A: 2 rows x 3 cols; table B: 2 rows x 3 cols — same width, both- // multi-row, no element between. These are two distinct tables.irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]},{"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}@@ -97,7 +207,7 @@ func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T)}}if len(tables) != 2 {- t.Fatalf("expected 2 separate multi-row tables, got %d: %#v", len(tables), got)+ t.Fatalf("expected 2 separate tables, got %d: %#v", len(tables), got)}for i, tb := range tables {td, ok := tb["table"].(*entity.TableData)@@ -110,29 +220,6 @@ func TestBuildDOCXJSONSections_AdjacentMultiRowTablesStaySeparate(t *testing.T)}}-// TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit pins-// that a 1-row header fragment is only merged when the body is the VERY NEXT-// element. A paragraph (even blank) between them keeps them as distinct-// tables, so the gate never reaches across intervening content.-func TestBuildDOCXJSONSections_HeaderFragmentWithParagraphBetweenStaysSplit(t *testing.T) {- const cell = `{"content":[{"type":"paragraph","content":[{"type":"text","text":"x"}]}]}`- irJSON := fmt.Sprintf(`{"sections":[{"title":"","elements":[- {"type":"table","rows":[{"cells":[%s,%s,%s]}]},- {"type":"paragraph","content":[{"type":"text","text":"note"}]},- {"type":"table","rows":[{"cells":[%s,%s,%s]},{"cells":[%s,%s,%s]}]}- ]}]}`, cell, cell, cell, cell, cell, cell, cell, cell, cell)- got := buildDOCXJSONSections(irJSON, nil)- var n int- for _, it := range got {- if dt, _ := it["doc_type_kwd"].(string); dt == "table" {- n++- }- }- if n != 2 {- t.Fatalf("expected 2 separate tables (paragraph between), got %d: %#v", n, got)- }-}-func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {imageData := officeTestPNG(t, 1)irJSON := `{"sections":[{"elements":[diff --git a/internal/parser/parser/table_structure_test.go b/internal/parser/parser/table_structure_test.goindex bc95630196..77dd4e4150 100644--- a/internal/parser/parser/table_structure_test.go+++ b/internal/parser/parser/table_structure_test.go@@ -252,13 +252,25 @@ func TestPPTXTableOnlySlideIsNotEmpty(t *testing.T) {}}-func TestOfficeTableRetainsDefaultHeaderCount(t *testing.T) {+func TestOfficeTableUsesRepeatedHeaderFlags(t *testing.T) {ir := `{"sections":[{"elements":[{"type":"table","rows":[{"is_header":true,"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H1"}]}]}]},{"is_header":true,"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H2"}]}]}]}]}]}]}`items := buildDOCXJSONSections(ir, nil)if len(items) != 1 {t.Fatalf("items=%#v", items)}td := items[0]["table"].(*entity.TableData)+ if td.HeaderRows != 2 || len(td.Grid) != 2 {+ t.Fatalf("table=%+v, want two rows and two repeated header rows", td)+ }+}++func TestOfficeTableDefaultsToOneHeaderWithoutFlags(t *testing.T) {+ ir := `{"sections":[{"elements":[{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H1"}]}]}]},{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"H2"}]}]}]}]}]}]}`+ items := buildDOCXJSONSections(ir, nil)+ if len(items) != 1 {+ t.Fatalf("items=%#v", items)+ }+ td := items[0]["table"].(*entity.TableData)if td.HeaderRows != 1 || len(td.Grid) != 2 {t.Fatalf("table=%+v, want two rows and one default header", td)}