Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): stop charging the replicated table header against the split budget (#20845)

jay77721a2a629e9728c
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index 4dcb080274..5c61decece 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -964,7 +964,7 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream
chunks := make([]schema.ChunkDoc, 0, len(units))
for _, unit := range units {
if itemDocType(unit) == "table" && tableDataOf(unit) != nil {
- chunks = append(chunks, c.splitSpreadsheetTable(unit)...)
+ chunks = append(chunks, c.splitSpreadsheetTable(unit, upstream.DocID)...)
continue
}
chunks = append(chunks, cloneChunkDoc(unit))
@@ -984,15 +984,19 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream
// whole; a split whose matrix cannot be aligned to the markup rows drops
// positions rather than pointing every sub-table row at the full matrix (R1).
//
+// The budget covers data rows only — the replicated header is fixed
+// boilerplate and does not consume the per-part allowance (see
+// splitLargeTable). docID attributes the oversized-header warning.
+//
// The legacy path merged adjacent rows by token budget into multi-row chunks
// and deferred the header; a segment now passes through whole unless the
// budget forces a cut, and the header repeats in every part.
-func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []schema.ChunkDoc {
+func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc, docID string) []schema.ChunkDoc {
td := tableDataOf(unit)
if td == nil {
return []schema.ChunkDoc{cloneChunkDoc(unit)}
}
- parts, ranges, headerRows := splitLargeTable(td, c.param.ChunkTokenSize, tokenizeStr)
+ parts, ranges, headerRows := splitLargeTable(td, c.param.ChunkTokenSize, tokenizeStr, docID)
if ranges == nil {
// The whole table fits one chunk: render it as HTML on Text so the
// persisted content and the id (hash of Text) stay byte-consistent and
diff --git a/internal/ingestion/component/chunker/table_html_coverage_test.go b/internal/ingestion/component/chunker/table_html_coverage_test.go
index 93fef6dc1d..be101367c0 100644
--- a/internal/ingestion/component/chunker/table_html_coverage_test.go
+++ b/internal/ingestion/component/chunker/table_html_coverage_test.go
@@ -131,17 +131,21 @@ func TestSplitLargeTableBudgetsOnHTML(t *testing.T) {
whole := tokenizeStr(tableutil.RenderTableHTML(td))
budget := whole / 2
- parts, ranges, headerRows := splitLargeTable(td, budget, tokenizeStr)
+ parts, ranges, headerRows := splitLargeTable(td, budget, tokenizeStr, "")
if len(parts) < 2 {
t.Fatalf("table did not split under the HTML budget: whole=%d budget=%d parts=%d", whole, budget, len(parts))
}
if headerRows != 1 {
t.Fatalf("headerRows = %d, want 1", headerRows)
}
+ // The split budget covers data rows only: every part replicates the header,
+ // so a part may exceed the budget by up to the header's own HTML cost —
+ // but never beyond that (which is what clean-projection counting would do).
+ headerTokens := tokenizeStr(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:1], HeaderRows: 1}))
covered := 0
for i, p := range parts {
- if n := tokenizeStr(tableutil.RenderTableHTML(&p)); n > budget {
- t.Fatalf("part %d HTML tokens %d exceed budget %d (budget must cover the stored HTML)", i, n, budget)
+ if n := tokenizeStr(tableutil.RenderTableHTML(&p)); n > budget+headerTokens {
+ t.Fatalf("part %d HTML tokens %d exceed budget %d + header %d (budget must cover the stored HTML)", i, n, budget, headerTokens)
}
covered += ranges[i][1] - ranges[i][0]
}
@@ -168,7 +172,7 @@ func TestSplitLargeTableHTMLBudgetKeepsSingleOverBudgetRowWhole(t *testing.T) {
t.Fatalf("test setup: budget %d should be below the huge row HTML %d", budget, hugeRow)
}
- parts, _, _ := splitLargeTable(td, budget, tokenizeStr)
+ parts, _, _ := splitLargeTable(td, budget, tokenizeStr, "")
if len(parts) < 2 {
t.Fatalf("expected the huge row to isolate into its own part, got %d parts", len(parts))
}
@@ -211,7 +215,7 @@ func TestSplitLargeTableHTMLBudgetKeepsRowspanGroup(t *testing.T) {
Grid: [][]entity.TableCell{td.Grid[0], td.Grid[1]},
HeaderRows: 1,
}))
- parts, _, _ := splitLargeTable(td, budget, tokenizeStr)
+ parts, _, _ := splitLargeTable(td, budget, tokenizeStr, "")
if len(parts) < 2 {
t.Fatalf("test setup: expected a split, got %d parts", len(parts))
}
diff --git a/internal/ingestion/component/chunker/table_split.go b/internal/ingestion/component/chunker/table_split.go
index ce3c0efa6d..6779392b5c 100644
--- a/internal/ingestion/component/chunker/table_split.go
+++ b/internal/ingestion/component/chunker/table_split.go
@@ -17,6 +17,9 @@
package chunker
import (
+ "go.uber.org/zap"
+
+ "ragflow/internal/common"
"ragflow/internal/entity"
"ragflow/internal/parser/tableutil"
)
@@ -34,6 +37,15 @@ import (
// it never claims rows the part does not contain. Every part therefore
// satisfies the same grid contract as a freshly parsed table.
//
+// The budget covers DATA rows only. Header rows and the caption are replicated
+// into every part — each part has to stand alone — but they are fixed
+// boilerplate that does not grow with content, so charging them per part would
+// shrink every part's data allowance and, on a wide-header table, fragment the
+// data into one-row chunks. A part may therefore exceed maxTokens by up to the
+// size of the replicated header. When that header alone meets or exceeds the
+// whole budget the table width and the configured chunk size disagree, which is
+// reported through a warning rather than silently absorbed.
+//
// Alongside the parts it returns dataRanges and headerRows: for part i, the
// half-open range [start, end) of DATA rows (rows below the header) the part
// contains, and the number of leading header rows every part replicates. A
@@ -41,7 +53,8 @@ import (
// matrix[:headerRows] + matrix[headerRows+start : headerRows+end] to stay
// aligned with the sub-table. headerRows and ranges are 0/nil when the table
// fits in a single chunk (the caller then emits the whole table unchanged).
-func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(string) int) (parts []entity.TableData, dataRanges [][2]int, headerRows int) {
+// docID is only used to attribute the oversized-header warning.
+func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(string) int, docID string) (parts []entity.TableData, dataRanges [][2]int, headerRows int) {
if td == nil || len(td.Grid) == 0 {
return nil, nil, 0
}
@@ -60,10 +73,19 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strin
// counted on the HTML rendering — the text actually stored on the chunk —
// so a rendered part never exceeds the budget. The <table> wrapper + caption
// appear once per part; emptyWrapper isolates them so each data row below is
- // charged only its own <tr> contribution.
+ // charged only its own <tr> contribution. The base count no longer feeds
+ // the splitting budget; it is kept only to detect a header that cannot fit
+ // a part on its own.
base := &entity.TableData{Grid: headerGrid, Caption: td.Caption}
baseTokens := countTokens(tableutil.RenderTableHTML(base))
emptyWrapper := countTokens(tableutil.RenderTableHTML(&entity.TableData{Caption: td.Caption}))
+ if baseTokens >= maxTokens {
+ common.Warn("chunker: table header alone exceeds chunk budget",
+ zap.String("doc_id", docID),
+ zap.Int("header_tokens", baseTokens),
+ zap.Int("chunk_token_size", maxTokens),
+ zap.Int("header_rows", headerRows))
+ }
// spanThrough[i] is the last data-row index any rowspan originating at or
// before row i still covers (-1 while no span is active). Header-origin
@@ -97,6 +119,9 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strin
}
var start int
+ // currentTokens counts the data rows batched into the current part. It
+ // starts at zero on purpose: the replicated header is not charged, see the
+ // doc comment above.
var currentTokens int
flush := func(end int) {
if end <= start {
@@ -111,10 +136,9 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strin
parts = append(parts, part)
dataRanges = append(dataRanges, [2]int{start, end})
start = end
- currentTokens = baseTokens
+ currentTokens = 0
}
- currentTokens = baseTokens
for i, row := range dataRows {
if i < start {
continue // already belongs to a flushed part
diff --git a/internal/ingestion/component/chunker/table_split_test.go b/internal/ingestion/component/chunker/table_split_test.go
index 4f54154e78..9cc22e117c 100644
--- a/internal/ingestion/component/chunker/table_split_test.go
+++ b/internal/ingestion/component/chunker/table_split_test.go
@@ -16,8 +16,13 @@ package chunker
import (
"reflect"
+ "strings"
"testing"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zaptest/observer"
+
+ "ragflow/internal/common"
"ragflow/internal/entity"
"ragflow/internal/parser/tableutil"
)
@@ -35,7 +40,7 @@ func makeTable(dataRows int) *entity.TableData {
func TestSplitLargeTableFitsWhole(t *testing.T) {
td := makeTable(3)
- parts, ranges, headerRows := splitLargeTable(td, 1_000_000, charTokens)
+ parts, ranges, headerRows := splitLargeTable(td, 1_000_000, charTokens, "")
if ranges != nil || len(parts) != 0 {
t.Fatalf("small table should not split: parts=%v ranges=%v", parts, ranges)
}
@@ -46,8 +51,9 @@ func TestSplitLargeTableFitsWhole(t *testing.T) {
func TestSplitLargeTableRowGranular(t *testing.T) {
td := makeTable(20)
- // Budget bounded so a part holds ~3 data rows.
- parts, ranges, headerRows := splitLargeTable(td, 22, charTokens)
+ // Budget bounded so a part holds a couple of data rows (the replicated
+ // header is not charged against it; rows cost ~36 tokens as HTML here).
+ parts, ranges, headerRows := splitLargeTable(td, 80, charTokens, "")
if headerRows != 1 {
t.Fatalf("headerRows = %d, want 1", headerRows)
}
@@ -99,7 +105,7 @@ func TestSplitLargeTableSingleRowOverBudgetKeepsRow(t *testing.T) {
Grid: [][]entity.TableCell{{{Text: "H"}}, {{Text: "x" + string(make([]byte, 200))}}},
HeaderRows: 1,
}
- parts, ranges, _ := splitLargeTable(td, 5, charTokens)
+ parts, ranges, _ := splitLargeTable(td, 5, charTokens, "")
if ranges != nil || len(parts) != 0 {
t.Fatalf("over-budget single row must emit whole (ranges nil): parts=%v ranges=%v", parts, ranges)
}
@@ -107,7 +113,7 @@ func TestSplitLargeTableSingleRowOverBudgetKeepsRow(t *testing.T) {
func TestSplitLargeTableHeaderOnly(t *testing.T) {
td := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}}, HeaderRows: 1}
- parts, ranges, _ := splitLargeTable(td, 10, charTokens)
+ parts, ranges, _ := splitLargeTable(td, 10, charTokens, "")
if ranges != nil || len(parts) != 0 {
t.Fatalf("header-only table should not split")
}
@@ -169,11 +175,13 @@ func TestSplitLargeTableHeaderRowSpanClippedPerPart(t *testing.T) {
if err != nil || td == nil {
t.Fatalf("parse: %v", err)
}
- // Budget = the HTML cost of the header + the two rows A spans (y, z), so
+ // Budget = the HTML cost of the two rows A spans (y, z), so
// the first part is exactly those rows and keeps A's RowSpan 3; later parts
// carry none of them, so A degrades to a plain header cell (RowSpan 1).
- budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:3], HeaderRows: 1}))
- parts, ranges, headerRows := splitLargeTable(td, budget, charTokens)
+ // The header cost is subtracted: the split budget covers data rows only.
+ headerCost := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:1], HeaderRows: 1}))
+ budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:3], HeaderRows: 1})) - headerCost
+ parts, ranges, headerRows := splitLargeTable(td, budget, charTokens, "")
if headerRows != 1 || len(parts) < 2 {
t.Fatalf("parts=%d ranges=%v headerRows=%d", len(parts), ranges, headerRows)
}
@@ -197,11 +205,12 @@ func TestSplitLargeTableHeaderRowSpanClippedPerPart(t *testing.T) {
t.Fatalf("parts cover %d data rows, want 5", covered)
}
- // Budget = the HTML cost of the header + only the first spanned row (y), so
+ // Budget = the HTML cost of only the first spanned row (y), so
// the first part carries exactly one of A's spanned rows and A keeps
// RowSpan 2 (header + that one row), not the full 3.
- budget = charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1}))
- parts, ranges, _ = splitLargeTable(td, budget, charTokens)
+ // The header cost is subtracted: the split budget covers data rows only.
+ budget = charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1})) - headerCost
+ parts, ranges, _ = splitLargeTable(td, budget, charTokens, "")
if len(parts) < 2 || ranges[0] != [2]int{0, 1} {
t.Fatalf("parts=%d ranges=%v, want first part [0 1]", len(parts), ranges)
}
@@ -233,11 +242,13 @@ func TestSplitLargeTableNeverSplitsInsideRowSpan(t *testing.T) {
if err != nil || td == nil {
t.Fatalf("parse: %v", err)
}
- // Budget = the HTML cost of the header + B's own row only. The budget alone
+ // Budget = the HTML cost of B's own row only. The budget alone
// would cut right after data row 0, inside B's span (B covers data rows
// 0..2); span-safe cut must move the boundary to after row 2, i.e. to 3.
- budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1}))
- parts, ranges, headerRows := splitLargeTable(td, budget, charTokens)
+ // The header cost is subtracted: the split budget covers data rows only.
+ headerCost := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:1], HeaderRows: 1}))
+ budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1})) - headerCost
+ parts, ranges, headerRows := splitLargeTable(td, budget, charTokens, "")
if headerRows != 1 {
t.Fatalf("headerRows=%d want 1", headerRows)
}
@@ -269,3 +280,101 @@ func TestSplitLargeTableNeverSplitsInsideRowSpan(t *testing.T) {
t.Fatalf("merged value B appears %d times across parts, want 1", seen)
}
}
+
+// A header wider than the whole budget must not eat the data allowance. While
+// the header was charged per part, every data row alone blew the budget and the
+// table collapsed to one row per part (20 parts here); the budget is now spent
+// on data rows alone, so a part packs as many rows as fit.
+func TestSplitLargeTableHeaderDoesNotConsumeDataBudget(t *testing.T) {
+ header := strings.Repeat("x", 100) // wide header: 136 tokens as HTML
+ grid := [][]entity.TableCell{{{Text: header}}}
+ for i := 0; i < 20; i++ {
+ grid = append(grid, []entity.TableCell{{Text: strings.Repeat("y", 10)}}) // 29 tokens each as HTML
+ }
+ td := &entity.TableData{Grid: grid, HeaderRows: 1}
+
+ // 150-token budget: data alone packs 5 rows per part -> 4 parts.
+ // While the 136-token header was charged per part, every data row alone
+ // blew the budget and the table collapsed to 20 one-row parts.
+ parts, ranges, headerRows := splitLargeTable(td, 150, charTokens, "")
+ if headerRows != 1 {
+ t.Fatalf("headerRows = %d, want 1", headerRows)
+ }
+ if len(parts) <= 1 {
+ t.Fatalf("expected the table to split, got %d parts", len(parts))
+ }
+ if len(parts) > 5 {
+ t.Fatalf("header consumed the data budget: got %d parts, want <= 5", len(parts))
+ }
+ if len(parts) != len(ranges) {
+ t.Fatalf("parts(%d) != ranges(%d)", len(parts), len(ranges))
+ }
+ seen := 0
+ for i := range parts {
+ part := &parts[i]
+ assertPartGridContract(t, part, i)
+ if part.Grid[0][0].Text != header {
+ t.Fatalf("part %d missing the replicated header: %v", i, part.Grid[0])
+ }
+ if len(part.Grid) < 2 {
+ t.Fatalf("part %d holds no data rows: %v", i, part.CellTexts())
+ }
+ r := ranges[i]
+ if r[0] != seen {
+ t.Fatalf("part %d range start %d != running %d", i, r[0], seen)
+ }
+ seen = r[1]
+ }
+ if seen != 20 {
+ t.Fatalf("data rows covered = %d, want 20", seen)
+ }
+}
+
+// A header that alone meets or exceeds the budget means the table width and
+// ChunkTokenSize disagree; that must be surfaced as a warning, not absorbed.
+func TestSplitLargeTableWarnsWhenHeaderExceedsBudget(t *testing.T) {
+ const msg = "chunker: table header alone exceeds chunk budget"
+
+ newObservedLogs := func(t *testing.T) *observer.ObservedLogs {
+ t.Helper()
+ core, logs := observer.New(zap.WarnLevel)
+ originalLogger := common.Logger
+ common.Logger = zap.New(core)
+ t.Cleanup(func() { common.Logger = originalLogger })
+ return logs
+ }
+
+ t.Run("header at or over budget warns", func(t *testing.T) {
+ logs := newObservedLogs(t)
+ td := &entity.TableData{
+ Grid: [][]entity.TableCell{{{Text: strings.Repeat("x", 100)}}, {{Text: "cell"}}},
+ HeaderRows: 1,
+ }
+ splitLargeTable(td, 50, charTokens, "doc-123")
+
+ entries := logs.FilterMessage(msg).All()
+ if len(entries) != 1 {
+ t.Fatalf("want exactly 1 warning, got %d (logs=%v)", len(entries), logs.All())
+ }
+ fields := entries[0].ContextMap()
+ if fields["doc_id"] != "doc-123" {
+ t.Fatalf("doc_id = %v, want doc-123", fields["doc_id"])
+ }
+ if fields["chunk_token_size"] != int64(50) {
+ t.Fatalf("chunk_token_size = %v, want 50", fields["chunk_token_size"])
+ }
+ if got, ok := fields["header_tokens"].(int64); !ok || got < 50 {
+ t.Fatalf("header_tokens = %v, want >= 50", fields["header_tokens"])
+ }
+ })
+
+ t.Run("header under budget stays quiet", func(t *testing.T) {
+ logs := newObservedLogs(t)
+ // makeTable's header costs 52 tokens as HTML; budget 100 stays above it.
+ splitLargeTable(makeTable(3), 100, charTokens, "doc-123")
+
+ if n := logs.FilterMessage(msg).Len(); n != 0 {
+ t.Fatalf("want no warning, got %d (logs=%v)", n, logs.All())
+ }
+ })
+}