Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): preserve spreadsheet identity in rebuilt chunks (#20825)

jay77721ccb78ef37dca
diff --git a/internal/ingestion/component/chunker/one.go b/internal/ingestion/component/chunker/one.go
index b0dcc25cc8..9f21d806e2 100644
--- a/internal/ingestion/component/chunker/one.go
+++ b/internal/ingestion/component/chunker/one.go
@@ -153,6 +153,8 @@ func emitOneFromItems(items, chunks []schema.ChunkDoc) map[string]any {
Image: it.Image,
Positions: it.Positions,
PDFPositions: it.PDFPositions,
+ Sheet: it.Sheet,
+ SheetIndex: it.SheetIndex,
}
return chunkOutputs([]schema.ChunkDoc{out})
}
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go
index 5ef0f1a953..1fc8c30b8e 100644
--- a/internal/ingestion/component/chunker/qa.go
+++ b/internal/ingestion/component/chunker/qa.go
@@ -147,6 +147,8 @@ func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (m
DocType: "text",
ContentLtks: contentLTKS,
ContentSmLtks: contentSMLTKS,
+ Sheet: pair.Sheet,
+ SheetIndex: pair.SheetIndex,
}
//
// index), image id + coordinates carried from the source item.
@@ -187,6 +189,8 @@ type qaPair struct {
Image string
PDFPositions json.RawMessage
Positions json.RawMessage
+ Sheet string
+ SheetIndex *int
}
// rmQAPrefixRe mirrors Python qa.py:241 `[\t:: ]+` — one-or-more separator
@@ -431,7 +435,7 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {
rows = td.Rows
}
if len(rows) > 0 {
- tmp = qaPairsFromRows(rows, strictCSV, item.Positions, item.SheetIndex != nil)
+ tmp = qaPairsFromRows(rows, strictCSV, item.Positions, item.SheetIndex != nil || item.Sheet != "")
} else {
tmp = extractQAText(txt)
}
@@ -440,6 +444,8 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {
// pair its own tuple (R1: a row chunk must not carry the whole
// table's matrix in positions[0]).
for _, p := range tmp {
+ p.Sheet = item.Sheet
+ p.SheetIndex = item.SheetIndex
p.Image = item.Image
p.PDFPositions = item.PDFPositions
if len(p.Positions) == 0 {
diff --git a/internal/ingestion/component/chunker/sheet_identity_test.go b/internal/ingestion/component/chunker/sheet_identity_test.go
new file mode 100644
index 0000000000..ef60f8aabc
--- /dev/null
+++ b/internal/ingestion/component/chunker/sheet_identity_test.go
@@ -0,0 +1,188 @@
+package chunker
+
+import (
+ "encoding/json"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/xuri/excelize/v2"
+ "ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/task/indexdoc"
+ "ragflow/internal/parser/parser"
+)
+
+func TestSpreadsheetChunkersPreserveSheetIdentityThroughIndexing(t *testing.T) {
+ book := excelize.NewFile()
+ defer book.Close()
+ if _, err := book.NewSheet("Orders"); err != nil {
+ t.Fatal(err)
+ }
+ for _, cell := range []struct{ sheet, axis, value string }{
+ {"Sheet1", "A1", "question"}, {"Sheet1", "B1", "answer"}, {"Sheet1", "A42", "first question"}, {"Sheet1", "B42", "first answer"},
+ {"Orders", "A1", "question"}, {"Orders", "B1", "answer"}, {"Orders", "A7", "second question"}, {"Orders", "B7", "second answer"},
+ } {
+ if err := book.SetCellValue(cell.sheet, cell.axis, cell.value); err != nil {
+ t.Fatal(err)
+ }
+ }
+ data, err := book.WriteToBuffer()
+ if err != nil {
+ t.Fatal(err)
+ }
+ xlsx, err := parser.NewXLSXParser("")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, source := range []struct {
+ name string
+ result parser.ParseResult
+ }{
+ {"rows.csv", parser.NewCSVParser().ParseWithResult(t.Context(), "rows.csv", []byte("question,answer\nhello,world\n"))},
+ {"book.xlsx", xlsx.ParseWithResult(t.Context(), "book.xlsx", data.Bytes())},
+ } {
+ if source.result.Err != nil {
+ t.Fatal(source.result.Err)
+ }
+ for _, tc := range []struct {
+ name string
+ factory func(map[string]any) (runtime.Component, error)
+ }{{"QA", NewQAChunker}, {"Token", NewTokenChunker}, {"One", NewOneChunker}} {
+ t.Run(source.name+"/"+tc.name, func(t *testing.T) {
+ batches := [][]map[string]any{source.result.JSON}
+ // One preserves positions only for a single source item.
+ if tc.name == "One" {
+ batches = nil
+ for _, item := range source.result.JSON {
+ batches = append(batches, []map[string]any{item})
+ }
+ }
+ for _, items := range batches {
+ comp, err := tc.factory(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := comp.Invoke(t.Context(), nil, map[string]any{"name": source.name, "output_format": source.result.OutputFormat, "json": items})
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks := indexdoc.NormalizeChunks(out)
+ if len(chunks) == 0 {
+ t.Fatal("no chunks")
+ }
+ var want [][]int
+ for _, item := range items {
+ raw, err := json.Marshal(item["positions"])
+ if err != nil {
+ t.Fatal(err)
+ }
+ var ranges [][]int
+ if err = json.Unmarshal(raw, &ranges); err != nil {
+ t.Fatal(err)
+ }
+ want = append(want, ranges...)
+ }
+ var got [][]int
+ for _, chunk := range chunks {
+ var sourceItem map[string]any
+ for _, item := range items {
+ if chunk["sheet_index"] == float64(item["sheet_index"].(int)) {
+ sourceItem = item
+ break
+ }
+ }
+ if sourceItem == nil || chunk["sheet"] != sourceItem["sheet"] {
+ t.Fatalf("sheet identity lost: chunk=%v", chunk)
+ }
+ beforeTop := chunk["top_int"]
+ if _, err = indexdoc.ProcessChunksForPipeline([]map[string]any{chunk}, "doc", source.name, time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := chunk["page_num_int"]; ok {
+ t.Fatalf("spreadsheet interpreted as PDF: %v", chunk)
+ }
+ positions, ok := chunk["position_int"].([][]int)
+ if !ok {
+ t.Fatalf("missing spreadsheet positions: %v", chunk)
+ }
+ got = append(got, positions...)
+ if tc.name == "QA" {
+ if !reflect.DeepEqual(chunk["top_int"], beforeTop) || !reflect.DeepEqual(beforeTop, []any{float64(positions[0][1] - 1)}) {
+ t.Fatalf("QA source row overwritten: %v", chunk)
+ }
+ } else if _, ok := chunk["top_int"]; ok {
+ t.Fatalf("spreadsheet column became PDF top: %v", chunk)
+ }
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("selections=%v, want %v", got, want)
+ }
+ }
+ })
+ }
+ }
+}
+
+func TestQAChunkerSheetNameOnlyPreservesSourceRows(t *testing.T) {
+ book := excelize.NewFile()
+ defer book.Close()
+ for cell, value := range map[string]string{"A1": "question", "B1": "answer", "A42": "source question", "B42": "source answer"} {
+ if err := book.SetCellValue("Sheet1", cell, value); err != nil {
+ t.Fatal(err)
+ }
+ }
+ data, err := book.WriteToBuffer()
+ if err != nil {
+ t.Fatal(err)
+ }
+ xlsx, err := parser.NewXLSXParser("")
+ if err != nil {
+ t.Fatal(err)
+ }
+ parsed := xlsx.ParseWithResult(t.Context(), "rows.xlsx", data.Bytes())
+ if parsed.Err != nil {
+ t.Fatal(parsed.Err)
+ }
+ if len(parsed.JSON) != 1 {
+ t.Fatalf("parser items=%d, want 1", len(parsed.JSON))
+ }
+ // Keep real parser selections, but exercise the supported name-only identity.
+ delete(parsed.JSON[0], "sheet_index")
+ comp, err := NewQAChunker(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := comp.Invoke(t.Context(), nil, map[string]any{"name": "rows.xlsx", "output_format": parsed.OutputFormat, "json": parsed.JSON})
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks := indexdoc.NormalizeChunks(out)
+ if len(chunks) != 2 {
+ t.Fatalf("chunks=%d, want 2", len(chunks))
+ }
+ for i, row := range []int{1, 42} {
+ chunk := chunks[i]
+ if chunk["sheet"] != "Sheet1" {
+ t.Fatalf("sheet=%v, want Sheet1", chunk["sheet"])
+ }
+ if _, ok := chunk["sheet_index"]; ok {
+ t.Fatal("name-only identity acquired sheet_index")
+ }
+ wantTop := []any{float64(row - 1)}
+ if !reflect.DeepEqual(chunk["top_int"], wantTop) {
+ t.Errorf("chunk %d row=%v, want %v", i, chunk["top_int"], wantTop)
+ }
+ if want := [][]float64{{1, float64(row), float64(row), 1, 2}}; !reflect.DeepEqual(chunk["positions"], want) {
+ t.Errorf("chunk %d selection=%v, want %v", i, chunk["positions"], want)
+ }
+ if _, err = indexdoc.ProcessChunksForPipeline([]map[string]any{chunk}, "doc", "rows.xlsx", time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(chunk["top_int"], wantTop) || !reflect.DeepEqual(chunk["position_int"], [][]int{{1, row, row, 1, 2}}) {
+ t.Errorf("chunk %d indexed row/selection=%v", i, chunk)
+ }
+ if _, ok := chunk["page_num_int"]; ok {
+ t.Fatal("sheet interpreted as PDF")
+ }
+ }
+}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 89d68eaf67..f710e1f177 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -820,6 +820,8 @@ func buildChunkDoc(it schema.ChunkDoc, ckType, text, ctxAbove, ctxBelow string)
Positions: it.Positions,
Image: it.Image,
PageNumber: it.PageNumber,
+ Sheet: it.Sheet,
+ SheetIndex: it.SheetIndex,
Extra: cloneExtra(it.Extra),
}
if ctxAbove != "" {
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index b7cb1cce7a..27e21e1238 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -718,6 +718,9 @@ func TestTokenChunker_SpreadsheetTablePreservesSelectionRange(t *testing.T) {
if got := chunks[0]["positions"]; !reflect.DeepEqual(got, wantPositions) {
t.Errorf("positions = %#v, want unchanged %#v", got, wantPositions)
}
+ if got := chunks[0]["sheet"]; got != "Sheet1" {
+ t.Errorf("sheet = %v, want Sheet1", got)
+ }
}
// TestTokenChunker_NewRejectsBadParam enforces the param validation