RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): stop charging the replicated table header against the split budget (#20845)
jay77721a2a629e9728c
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex 4dcb080274..5c61decece 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -964,7 +964,7 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstreamchunks := make([]schema.ChunkDoc, 0, len(units))for _, unit := range units {if itemDocType(unit) == "table" && tableDataOf(unit) != nil {- chunks = append(chunks, c.splitSpreadsheetTable(unit)...)+ chunks = append(chunks, c.splitSpreadsheetTable(unit, upstream.DocID)...)continue}chunks = append(chunks, cloneChunkDoc(unit))@@ -984,15 +984,19 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream// whole; a split whose matrix cannot be aligned to the markup rows drops// positions rather than pointing every sub-table row at the full matrix (R1).//+// The budget covers data rows only — the replicated header is fixed+// boilerplate and does not consume the per-part allowance (see+// splitLargeTable). docID attributes the oversized-header warning.+//// The legacy path merged adjacent rows by token budget into multi-row chunks// and deferred the header; a segment now passes through whole unless the// budget forces a cut, and the header repeats in every part.-func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc) []schema.ChunkDoc {+func (c *GeneralChunkerComponent) splitSpreadsheetTable(unit schema.ChunkDoc, docID string) []schema.ChunkDoc {td := tableDataOf(unit)if td == nil {return []schema.ChunkDoc{cloneChunkDoc(unit)}}- parts, ranges, headerRows := splitLargeTable(td, c.param.ChunkTokenSize, tokenizeStr)+ parts, ranges, headerRows := splitLargeTable(td, c.param.ChunkTokenSize, tokenizeStr, docID)if ranges == nil {// The whole table fits one chunk: render it as HTML on Text so the// persisted content and the id (hash of Text) stay byte-consistent anddiff --git a/internal/ingestion/component/chunker/table_html_coverage_test.go b/internal/ingestion/component/chunker/table_html_coverage_test.goindex 93fef6dc1d..be101367c0 100644--- a/internal/ingestion/component/chunker/table_html_coverage_test.go+++ b/internal/ingestion/component/chunker/table_html_coverage_test.go@@ -131,17 +131,21 @@ func TestSplitLargeTableBudgetsOnHTML(t *testing.T) {whole := tokenizeStr(tableutil.RenderTableHTML(td))budget := whole / 2- parts, ranges, headerRows := splitLargeTable(td, budget, tokenizeStr)+ parts, ranges, headerRows := splitLargeTable(td, budget, tokenizeStr, "")if len(parts) < 2 {t.Fatalf("table did not split under the HTML budget: whole=%d budget=%d parts=%d", whole, budget, len(parts))}if headerRows != 1 {t.Fatalf("headerRows = %d, want 1", headerRows)}+ // The split budget covers data rows only: every part replicates the header,+ // so a part may exceed the budget by up to the header's own HTML cost —+ // but never beyond that (which is what clean-projection counting would do).+ headerTokens := tokenizeStr(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:1], HeaderRows: 1}))covered := 0for i, p := range parts {- if n := tokenizeStr(tableutil.RenderTableHTML(&p)); n > budget {- t.Fatalf("part %d HTML tokens %d exceed budget %d (budget must cover the stored HTML)", i, n, budget)+ if n := tokenizeStr(tableutil.RenderTableHTML(&p)); n > budget+headerTokens {+ t.Fatalf("part %d HTML tokens %d exceed budget %d + header %d (budget must cover the stored HTML)", i, n, budget, headerTokens)}covered += ranges[i][1] - ranges[i][0]}@@ -168,7 +172,7 @@ func TestSplitLargeTableHTMLBudgetKeepsSingleOverBudgetRowWhole(t *testing.T) {t.Fatalf("test setup: budget %d should be below the huge row HTML %d", budget, hugeRow)}- parts, _, _ := splitLargeTable(td, budget, tokenizeStr)+ parts, _, _ := splitLargeTable(td, budget, tokenizeStr, "")if len(parts) < 2 {t.Fatalf("expected the huge row to isolate into its own part, got %d parts", len(parts))}@@ -211,7 +215,7 @@ func TestSplitLargeTableHTMLBudgetKeepsRowspanGroup(t *testing.T) {Grid: [][]entity.TableCell{td.Grid[0], td.Grid[1]},HeaderRows: 1,}))- parts, _, _ := splitLargeTable(td, budget, tokenizeStr)+ parts, _, _ := splitLargeTable(td, budget, tokenizeStr, "")if len(parts) < 2 {t.Fatalf("test setup: expected a split, got %d parts", len(parts))}diff --git a/internal/ingestion/component/chunker/table_split.go b/internal/ingestion/component/chunker/table_split.goindex ce3c0efa6d..6779392b5c 100644--- a/internal/ingestion/component/chunker/table_split.go+++ b/internal/ingestion/component/chunker/table_split.go@@ -17,6 +17,9 @@package chunkerimport (+ "go.uber.org/zap"++ "ragflow/internal/common""ragflow/internal/entity""ragflow/internal/parser/tableutil")@@ -34,6 +37,15 @@ import (// it never claims rows the part does not contain. Every part therefore// satisfies the same grid contract as a freshly parsed table.//+// The budget covers DATA rows only. Header rows and the caption are replicated+// into every part — each part has to stand alone — but they are fixed+// boilerplate that does not grow with content, so charging them per part would+// shrink every part's data allowance and, on a wide-header table, fragment the+// data into one-row chunks. A part may therefore exceed maxTokens by up to the+// size of the replicated header. When that header alone meets or exceeds the+// whole budget the table width and the configured chunk size disagree, which is+// reported through a warning rather than silently absorbed.+//// Alongside the parts it returns dataRanges and headerRows: for part i, the// half-open range [start, end) of DATA rows (rows below the header) the part// contains, and the number of leading header rows every part replicates. A@@ -41,7 +53,8 @@ import (// matrix[:headerRows] + matrix[headerRows+start : headerRows+end] to stay// aligned with the sub-table. headerRows and ranges are 0/nil when the table// fits in a single chunk (the caller then emits the whole table unchanged).-func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(string) int) (parts []entity.TableData, dataRanges [][2]int, headerRows int) {+// docID is only used to attribute the oversized-header warning.+func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(string) int, docID string) (parts []entity.TableData, dataRanges [][2]int, headerRows int) {if td == nil || len(td.Grid) == 0 {return nil, nil, 0}@@ -60,10 +73,19 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strin// counted on the HTML rendering — the text actually stored on the chunk —// so a rendered part never exceeds the budget. The <table> wrapper + caption// appear once per part; emptyWrapper isolates them so each data row below is- // charged only its own <tr> contribution.+ // charged only its own <tr> contribution. The base count no longer feeds+ // the splitting budget; it is kept only to detect a header that cannot fit+ // a part on its own.base := &entity.TableData{Grid: headerGrid, Caption: td.Caption}baseTokens := countTokens(tableutil.RenderTableHTML(base))emptyWrapper := countTokens(tableutil.RenderTableHTML(&entity.TableData{Caption: td.Caption}))+ if baseTokens >= maxTokens {+ common.Warn("chunker: table header alone exceeds chunk budget",+ zap.String("doc_id", docID),+ zap.Int("header_tokens", baseTokens),+ zap.Int("chunk_token_size", maxTokens),+ zap.Int("header_rows", headerRows))+ }// spanThrough[i] is the last data-row index any rowspan originating at or// before row i still covers (-1 while no span is active). Header-origin@@ -97,6 +119,9 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strin}var start int+ // currentTokens counts the data rows batched into the current part. It+ // starts at zero on purpose: the replicated header is not charged, see the+ // doc comment above.var currentTokens intflush := func(end int) {if end <= start {@@ -111,10 +136,9 @@ func splitLargeTable(td *entity.TableData, maxTokens int, countTokens func(strinparts = append(parts, part)dataRanges = append(dataRanges, [2]int{start, end})start = end- currentTokens = baseTokens+ currentTokens = 0}- currentTokens = baseTokensfor i, row := range dataRows {if i < start {continue // already belongs to a flushed partdiff --git a/internal/ingestion/component/chunker/table_split_test.go b/internal/ingestion/component/chunker/table_split_test.goindex 4f54154e78..9cc22e117c 100644--- a/internal/ingestion/component/chunker/table_split_test.go+++ b/internal/ingestion/component/chunker/table_split_test.go@@ -16,8 +16,13 @@ package chunkerimport ("reflect"+ "strings""testing"+ "go.uber.org/zap"+ "go.uber.org/zap/zaptest/observer"++ "ragflow/internal/common""ragflow/internal/entity""ragflow/internal/parser/tableutil")@@ -35,7 +40,7 @@ func makeTable(dataRows int) *entity.TableData {func TestSplitLargeTableFitsWhole(t *testing.T) {td := makeTable(3)- parts, ranges, headerRows := splitLargeTable(td, 1_000_000, charTokens)+ parts, ranges, headerRows := splitLargeTable(td, 1_000_000, charTokens, "")if ranges != nil || len(parts) != 0 {t.Fatalf("small table should not split: parts=%v ranges=%v", parts, ranges)}@@ -46,8 +51,9 @@ func TestSplitLargeTableFitsWhole(t *testing.T) {func TestSplitLargeTableRowGranular(t *testing.T) {td := makeTable(20)- // Budget bounded so a part holds ~3 data rows.- parts, ranges, headerRows := splitLargeTable(td, 22, charTokens)+ // Budget bounded so a part holds a couple of data rows (the replicated+ // header is not charged against it; rows cost ~36 tokens as HTML here).+ parts, ranges, headerRows := splitLargeTable(td, 80, charTokens, "")if headerRows != 1 {t.Fatalf("headerRows = %d, want 1", headerRows)}@@ -99,7 +105,7 @@ func TestSplitLargeTableSingleRowOverBudgetKeepsRow(t *testing.T) {Grid: [][]entity.TableCell{{{Text: "H"}}, {{Text: "x" + string(make([]byte, 200))}}},HeaderRows: 1,}- parts, ranges, _ := splitLargeTable(td, 5, charTokens)+ parts, ranges, _ := splitLargeTable(td, 5, charTokens, "")if ranges != nil || len(parts) != 0 {t.Fatalf("over-budget single row must emit whole (ranges nil): parts=%v ranges=%v", parts, ranges)}@@ -107,7 +113,7 @@ func TestSplitLargeTableSingleRowOverBudgetKeepsRow(t *testing.T) {func TestSplitLargeTableHeaderOnly(t *testing.T) {td := &entity.TableData{Grid: [][]entity.TableCell{{{Text: "Name"}, {Text: "Age"}}}, HeaderRows: 1}- parts, ranges, _ := splitLargeTable(td, 10, charTokens)+ parts, ranges, _ := splitLargeTable(td, 10, charTokens, "")if ranges != nil || len(parts) != 0 {t.Fatalf("header-only table should not split")}@@ -169,11 +175,13 @@ func TestSplitLargeTableHeaderRowSpanClippedPerPart(t *testing.T) {if err != nil || td == nil {t.Fatalf("parse: %v", err)}- // Budget = the HTML cost of the header + the two rows A spans (y, z), so+ // Budget = the HTML cost of the two rows A spans (y, z), so// the first part is exactly those rows and keeps A's RowSpan 3; later parts// carry none of them, so A degrades to a plain header cell (RowSpan 1).- budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:3], HeaderRows: 1}))- parts, ranges, headerRows := splitLargeTable(td, budget, charTokens)+ // The header cost is subtracted: the split budget covers data rows only.+ headerCost := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:1], HeaderRows: 1}))+ budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:3], HeaderRows: 1})) - headerCost+ parts, ranges, headerRows := splitLargeTable(td, budget, charTokens, "")if headerRows != 1 || len(parts) < 2 {t.Fatalf("parts=%d ranges=%v headerRows=%d", len(parts), ranges, headerRows)}@@ -197,11 +205,12 @@ func TestSplitLargeTableHeaderRowSpanClippedPerPart(t *testing.T) {t.Fatalf("parts cover %d data rows, want 5", covered)}- // Budget = the HTML cost of the header + only the first spanned row (y), so+ // Budget = the HTML cost of only the first spanned row (y), so// the first part carries exactly one of A's spanned rows and A keeps// RowSpan 2 (header + that one row), not the full 3.- budget = charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1}))- parts, ranges, _ = splitLargeTable(td, budget, charTokens)+ // The header cost is subtracted: the split budget covers data rows only.+ budget = charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1})) - headerCost+ parts, ranges, _ = splitLargeTable(td, budget, charTokens, "")if len(parts) < 2 || ranges[0] != [2]int{0, 1} {t.Fatalf("parts=%d ranges=%v, want first part [0 1]", len(parts), ranges)}@@ -233,11 +242,13 @@ func TestSplitLargeTableNeverSplitsInsideRowSpan(t *testing.T) {if err != nil || td == nil {t.Fatalf("parse: %v", err)}- // Budget = the HTML cost of the header + B's own row only. The budget alone+ // Budget = the HTML cost of B's own row only. The budget alone// would cut right after data row 0, inside B's span (B covers data rows// 0..2); span-safe cut must move the boundary to after row 2, i.e. to 3.- budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1}))- parts, ranges, headerRows := splitLargeTable(td, budget, charTokens)+ // The header cost is subtracted: the split budget covers data rows only.+ headerCost := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:1], HeaderRows: 1}))+ budget := charTokens(tableutil.RenderTableHTML(&entity.TableData{Grid: td.Grid[:2], HeaderRows: 1})) - headerCost+ parts, ranges, headerRows := splitLargeTable(td, budget, charTokens, "")if headerRows != 1 {t.Fatalf("headerRows=%d want 1", headerRows)}@@ -269,3 +280,101 @@ func TestSplitLargeTableNeverSplitsInsideRowSpan(t *testing.T) {t.Fatalf("merged value B appears %d times across parts, want 1", seen)}}++// A header wider than the whole budget must not eat the data allowance. While+// the header was charged per part, every data row alone blew the budget and the+// table collapsed to one row per part (20 parts here); the budget is now spent+// on data rows alone, so a part packs as many rows as fit.+func TestSplitLargeTableHeaderDoesNotConsumeDataBudget(t *testing.T) {+ header := strings.Repeat("x", 100) // wide header: 136 tokens as HTML+ grid := [][]entity.TableCell{{{Text: header}}}+ for i := 0; i < 20; i++ {+ grid = append(grid, []entity.TableCell{{Text: strings.Repeat("y", 10)}}) // 29 tokens each as HTML+ }+ td := &entity.TableData{Grid: grid, HeaderRows: 1}++ // 150-token budget: data alone packs 5 rows per part -> 4 parts.+ // While the 136-token header was charged per part, every data row alone+ // blew the budget and the table collapsed to 20 one-row parts.+ parts, ranges, headerRows := splitLargeTable(td, 150, charTokens, "")+ if headerRows != 1 {+ t.Fatalf("headerRows = %d, want 1", headerRows)+ }+ if len(parts) <= 1 {+ t.Fatalf("expected the table to split, got %d parts", len(parts))+ }+ if len(parts) > 5 {+ t.Fatalf("header consumed the data budget: got %d parts, want <= 5", len(parts))+ }+ if len(parts) != len(ranges) {+ t.Fatalf("parts(%d) != ranges(%d)", len(parts), len(ranges))+ }+ seen := 0+ for i := range parts {+ part := &parts[i]+ assertPartGridContract(t, part, i)+ if part.Grid[0][0].Text != header {+ t.Fatalf("part %d missing the replicated header: %v", i, part.Grid[0])+ }+ if len(part.Grid) < 2 {+ t.Fatalf("part %d holds no data rows: %v", i, part.CellTexts())+ }+ r := ranges[i]+ if r[0] != seen {+ t.Fatalf("part %d range start %d != running %d", i, r[0], seen)+ }+ seen = r[1]+ }+ if seen != 20 {+ t.Fatalf("data rows covered = %d, want 20", seen)+ }+}++// A header that alone meets or exceeds the budget means the table width and+// ChunkTokenSize disagree; that must be surfaced as a warning, not absorbed.+func TestSplitLargeTableWarnsWhenHeaderExceedsBudget(t *testing.T) {+ const msg = "chunker: table header alone exceeds chunk budget"++ newObservedLogs := func(t *testing.T) *observer.ObservedLogs {+ t.Helper()+ core, logs := observer.New(zap.WarnLevel)+ originalLogger := common.Logger+ common.Logger = zap.New(core)+ t.Cleanup(func() { common.Logger = originalLogger })+ return logs+ }++ t.Run("header at or over budget warns", func(t *testing.T) {+ logs := newObservedLogs(t)+ td := &entity.TableData{+ Grid: [][]entity.TableCell{{{Text: strings.Repeat("x", 100)}}, {{Text: "cell"}}},+ HeaderRows: 1,+ }+ splitLargeTable(td, 50, charTokens, "doc-123")++ entries := logs.FilterMessage(msg).All()+ if len(entries) != 1 {+ t.Fatalf("want exactly 1 warning, got %d (logs=%v)", len(entries), logs.All())+ }+ fields := entries[0].ContextMap()+ if fields["doc_id"] != "doc-123" {+ t.Fatalf("doc_id = %v, want doc-123", fields["doc_id"])+ }+ if fields["chunk_token_size"] != int64(50) {+ t.Fatalf("chunk_token_size = %v, want 50", fields["chunk_token_size"])+ }+ if got, ok := fields["header_tokens"].(int64); !ok || got < 50 {+ t.Fatalf("header_tokens = %v, want >= 50", fields["header_tokens"])+ }+ })++ t.Run("header under budget stays quiet", func(t *testing.T) {+ logs := newObservedLogs(t)+ // makeTable's header costs 52 tokens as HTML; budget 100 stays above it.+ splitLargeTable(makeTable(3), 100, charTokens, "doc-123")++ if n := logs.FilterMessage(msg).Len(); n != 0 {+ t.Fatalf("want no warning, got %d (logs=%v)", n, logs.All())+ }+ })+}