RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): preserve spreadsheet identity in rebuilt chunks (#20825)
jay77721ccb78ef37dca
diff --git a/internal/ingestion/component/chunker/one.go b/internal/ingestion/component/chunker/one.goindex b0dcc25cc8..9f21d806e2 100644--- a/internal/ingestion/component/chunker/one.go+++ b/internal/ingestion/component/chunker/one.go@@ -153,6 +153,8 @@ func emitOneFromItems(items, chunks []schema.ChunkDoc) map[string]any {Image: it.Image,Positions: it.Positions,PDFPositions: it.PDFPositions,+ Sheet: it.Sheet,+ SheetIndex: it.SheetIndex,}return chunkOutputs([]schema.ChunkDoc{out})}diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.goindex 5ef0f1a953..1fc8c30b8e 100644--- a/internal/ingestion/component/chunker/qa.go+++ b/internal/ingestion/component/chunker/qa.go@@ -147,6 +147,8 @@ func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (mDocType: "text",ContentLtks: contentLTKS,ContentSmLtks: contentSMLTKS,+ Sheet: pair.Sheet,+ SheetIndex: pair.SheetIndex,}//// index), image id + coordinates carried from the source item.@@ -187,6 +189,8 @@ type qaPair struct {Image stringPDFPositions json.RawMessagePositions json.RawMessage+ Sheet string+ SheetIndex *int}// rmQAPrefixRe mirrors Python qa.py:241 `[\t:: ]+` — one-or-more separator@@ -431,7 +435,7 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {rows = td.Rows}if len(rows) > 0 {- tmp = qaPairsFromRows(rows, strictCSV, item.Positions, item.SheetIndex != nil)+ tmp = qaPairsFromRows(rows, strictCSV, item.Positions, item.SheetIndex != nil || item.Sheet != "")} else {tmp = extractQAText(txt)}@@ -440,6 +444,8 @@ func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {// pair its own tuple (R1: a row chunk must not carry the whole// table's matrix in positions[0]).for _, p := range tmp {+ p.Sheet = item.Sheet+ p.SheetIndex = item.SheetIndexp.Image = item.Imagep.PDFPositions = item.PDFPositionsif len(p.Positions) == 0 {diff --git a/internal/ingestion/component/chunker/sheet_identity_test.go b/internal/ingestion/component/chunker/sheet_identity_test.gonew file mode 100644index 0000000000..ef60f8aabc--- /dev/null+++ b/internal/ingestion/component/chunker/sheet_identity_test.go@@ -0,0 +1,188 @@+package chunker++import (+ "encoding/json"+ "reflect"+ "testing"+ "time"++ "github.com/xuri/excelize/v2"+ "ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/task/indexdoc"+ "ragflow/internal/parser/parser"+)++func TestSpreadsheetChunkersPreserveSheetIdentityThroughIndexing(t *testing.T) {+ book := excelize.NewFile()+ defer book.Close()+ if _, err := book.NewSheet("Orders"); err != nil {+ t.Fatal(err)+ }+ for _, cell := range []struct{ sheet, axis, value string }{+ {"Sheet1", "A1", "question"}, {"Sheet1", "B1", "answer"}, {"Sheet1", "A42", "first question"}, {"Sheet1", "B42", "first answer"},+ {"Orders", "A1", "question"}, {"Orders", "B1", "answer"}, {"Orders", "A7", "second question"}, {"Orders", "B7", "second answer"},+ } {+ if err := book.SetCellValue(cell.sheet, cell.axis, cell.value); err != nil {+ t.Fatal(err)+ }+ }+ data, err := book.WriteToBuffer()+ if err != nil {+ t.Fatal(err)+ }+ xlsx, err := parser.NewXLSXParser("")+ if err != nil {+ t.Fatal(err)+ }+ for _, source := range []struct {+ name string+ result parser.ParseResult+ }{+ {"rows.csv", parser.NewCSVParser().ParseWithResult(t.Context(), "rows.csv", []byte("question,answer\nhello,world\n"))},+ {"book.xlsx", xlsx.ParseWithResult(t.Context(), "book.xlsx", data.Bytes())},+ } {+ if source.result.Err != nil {+ t.Fatal(source.result.Err)+ }+ for _, tc := range []struct {+ name string+ factory func(map[string]any) (runtime.Component, error)+ }{{"QA", NewQAChunker}, {"Token", NewTokenChunker}, {"One", NewOneChunker}} {+ t.Run(source.name+"/"+tc.name, func(t *testing.T) {+ batches := [][]map[string]any{source.result.JSON}+ // One preserves positions only for a single source item.+ if tc.name == "One" {+ batches = nil+ for _, item := range source.result.JSON {+ batches = append(batches, []map[string]any{item})+ }+ }+ for _, items := range batches {+ comp, err := tc.factory(nil)+ if err != nil {+ t.Fatal(err)+ }+ out, err := comp.Invoke(t.Context(), nil, map[string]any{"name": source.name, "output_format": source.result.OutputFormat, "json": items})+ if err != nil {+ t.Fatal(err)+ }+ chunks := indexdoc.NormalizeChunks(out)+ if len(chunks) == 0 {+ t.Fatal("no chunks")+ }+ var want [][]int+ for _, item := range items {+ raw, err := json.Marshal(item["positions"])+ if err != nil {+ t.Fatal(err)+ }+ var ranges [][]int+ if err = json.Unmarshal(raw, &ranges); err != nil {+ t.Fatal(err)+ }+ want = append(want, ranges...)+ }+ var got [][]int+ for _, chunk := range chunks {+ var sourceItem map[string]any+ for _, item := range items {+ if chunk["sheet_index"] == float64(item["sheet_index"].(int)) {+ sourceItem = item+ break+ }+ }+ if sourceItem == nil || chunk["sheet"] != sourceItem["sheet"] {+ t.Fatalf("sheet identity lost: chunk=%v", chunk)+ }+ beforeTop := chunk["top_int"]+ if _, err = indexdoc.ProcessChunksForPipeline([]map[string]any{chunk}, "doc", source.name, time.Now()); err != nil {+ t.Fatal(err)+ }+ if _, ok := chunk["page_num_int"]; ok {+ t.Fatalf("spreadsheet interpreted as PDF: %v", chunk)+ }+ positions, ok := chunk["position_int"].([][]int)+ if !ok {+ t.Fatalf("missing spreadsheet positions: %v", chunk)+ }+ got = append(got, positions...)+ if tc.name == "QA" {+ if !reflect.DeepEqual(chunk["top_int"], beforeTop) || !reflect.DeepEqual(beforeTop, []any{float64(positions[0][1] - 1)}) {+ t.Fatalf("QA source row overwritten: %v", chunk)+ }+ } else if _, ok := chunk["top_int"]; ok {+ t.Fatalf("spreadsheet column became PDF top: %v", chunk)+ }+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("selections=%v, want %v", got, want)+ }+ }+ })+ }+ }+}++func TestQAChunkerSheetNameOnlyPreservesSourceRows(t *testing.T) {+ book := excelize.NewFile()+ defer book.Close()+ for cell, value := range map[string]string{"A1": "question", "B1": "answer", "A42": "source question", "B42": "source answer"} {+ if err := book.SetCellValue("Sheet1", cell, value); err != nil {+ t.Fatal(err)+ }+ }+ data, err := book.WriteToBuffer()+ if err != nil {+ t.Fatal(err)+ }+ xlsx, err := parser.NewXLSXParser("")+ if err != nil {+ t.Fatal(err)+ }+ parsed := xlsx.ParseWithResult(t.Context(), "rows.xlsx", data.Bytes())+ if parsed.Err != nil {+ t.Fatal(parsed.Err)+ }+ if len(parsed.JSON) != 1 {+ t.Fatalf("parser items=%d, want 1", len(parsed.JSON))+ }+ // Keep real parser selections, but exercise the supported name-only identity.+ delete(parsed.JSON[0], "sheet_index")+ comp, err := NewQAChunker(nil)+ if err != nil {+ t.Fatal(err)+ }+ out, err := comp.Invoke(t.Context(), nil, map[string]any{"name": "rows.xlsx", "output_format": parsed.OutputFormat, "json": parsed.JSON})+ if err != nil {+ t.Fatal(err)+ }+ chunks := indexdoc.NormalizeChunks(out)+ if len(chunks) != 2 {+ t.Fatalf("chunks=%d, want 2", len(chunks))+ }+ for i, row := range []int{1, 42} {+ chunk := chunks[i]+ if chunk["sheet"] != "Sheet1" {+ t.Fatalf("sheet=%v, want Sheet1", chunk["sheet"])+ }+ if _, ok := chunk["sheet_index"]; ok {+ t.Fatal("name-only identity acquired sheet_index")+ }+ wantTop := []any{float64(row - 1)}+ if !reflect.DeepEqual(chunk["top_int"], wantTop) {+ t.Errorf("chunk %d row=%v, want %v", i, chunk["top_int"], wantTop)+ }+ if want := [][]float64{{1, float64(row), float64(row), 1, 2}}; !reflect.DeepEqual(chunk["positions"], want) {+ t.Errorf("chunk %d selection=%v, want %v", i, chunk["positions"], want)+ }+ if _, err = indexdoc.ProcessChunksForPipeline([]map[string]any{chunk}, "doc", "rows.xlsx", time.Now()); err != nil {+ t.Fatal(err)+ }+ if !reflect.DeepEqual(chunk["top_int"], wantTop) || !reflect.DeepEqual(chunk["position_int"], [][]int{{1, row, row, 1, 2}}) {+ t.Errorf("chunk %d indexed row/selection=%v", i, chunk)+ }+ if _, ok := chunk["page_num_int"]; ok {+ t.Fatal("sheet interpreted as PDF")+ }+ }+}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 89d68eaf67..f710e1f177 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -820,6 +820,8 @@ func buildChunkDoc(it schema.ChunkDoc, ckType, text, ctxAbove, ctxBelow string)Positions: it.Positions,Image: it.Image,PageNumber: it.PageNumber,+ Sheet: it.Sheet,+ SheetIndex: it.SheetIndex,Extra: cloneExtra(it.Extra),}if ctxAbove != "" {diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex b7cb1cce7a..27e21e1238 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -718,6 +718,9 @@ func TestTokenChunker_SpreadsheetTablePreservesSelectionRange(t *testing.T) {if got := chunks[0]["positions"]; !reflect.DeepEqual(got, wantPositions) {t.Errorf("positions = %#v, want unchanged %#v", got, wantPositions)}+ if got := chunks[0]["sheet"]; got != "Sheet1" {+ t.Errorf("sheet = %v, want Sheet1", got)+ }}// TestTokenChunker_NewRejectsBadParam enforces the param validation