Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(infinity): repair chunk ingestion and dataset save (#19773)

qinling0210850132e6cbd2
diff --git a/conf/infinity_mapping.json b/conf/infinity_mapping.json
index 95de7aeac4..15ba437143 100644
--- a/conf/infinity_mapping.json
+++ b/conf/infinity_mapping.json
@@ -77,7 +77,6 @@
"claims": {"type": "json", "default": "[]", "enabled": false},
"page_ids": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},
"page_id": {"type": "varchar", "default": ""},
- "deleted_doc_id": {"type": "varchar", "default": ""},
"source_chunk_hashes": {"type": "json", "default": "{}", "enabled": false},
"map_checksum": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index 5c4dabc928..4d56e17012 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -2371,6 +2371,107 @@ func getChunkScore(chunk map[string]interface{}) float64 {
return 0.0
}
+// numericValue reports whether v is a number the hex encoder accepts.
+func numericValue(v interface{}) (interface{}, bool) {
+ switch n := v.(type) {
+ case int, int64, float64:
+ return n, true
+ }
+ return nil, false
+}
+
+// numericRow appends one flat row of numbers: a JSON-decoded []interface{}, or
+// the Go-typed []int / []int64 / []float64 a chunk built in process carries.
+func numericRow(row interface{}, out *[]interface{}) bool {
+ if vals, ok := row.([]interface{}); ok {
+ for _, item := range vals {
+ n, isNum := numericValue(item)
+ if !isNum {
+ return false
+ }
+ *out = append(*out, n)
+ }
+ return true
+ }
+ switch vals := row.(type) {
+ case []int:
+ for _, n := range vals {
+ *out = append(*out, n)
+ }
+ case []int64:
+ for _, n := range vals {
+ *out = append(*out, n)
+ }
+ case []float64:
+ for _, n := range vals {
+ *out = append(*out, n)
+ }
+ default:
+ return false
+ }
+ return true
+}
+
+// numericSlice flattens a numeric slice — or a slice of numeric slices — into the
+// flat []interface{} the hex encoder takes. Two shapes reach the engine: values
+// decoded from JSON arrive as []interface{} (numbers as float64), while a chunk
+// built IN PROCESS carries Go's typed slices (the ingestion pipeline's
+// AddPositions emits []int / [][]int). Both must be encoded, because these
+// columns are VARCHAR holding the hex form: handing Infinity the typed slice
+// makes it try to store an int64 tensor and fail with
+// "Not support to convert Tensor(int64,5) to Varchar" (InfinityException 3049).
+// A scalar or a non-numeric value reports ok=false, so the caller keeps its
+// pass-through instead of hex-encoding something it cannot parse back.
+func numericSlice(v interface{}) ([]interface{}, bool) {
+ out := make([]interface{}, 0, 5)
+ switch vals := v.(type) {
+ case []interface{}:
+ // JSON shape: plain numbers, and/or position rows that are slices too.
+ for _, item := range vals {
+ if n, ok := numericValue(item); ok {
+ out = append(out, n)
+ continue
+ }
+ if !numericRow(item, &out) {
+ return nil, false
+ }
+ }
+ case []int:
+ for _, n := range vals {
+ out = append(out, n)
+ }
+ case []int64:
+ for _, n := range vals {
+ out = append(out, n)
+ }
+ case []float64:
+ for _, n := range vals {
+ out = append(out, n)
+ }
+ case [][]int:
+ for _, row := range vals {
+ if !numericRow(row, &out) {
+ return nil, false
+ }
+ }
+ case [][]int64:
+ for _, row := range vals {
+ if !numericRow(row, &out) {
+ return nil, false
+ }
+ }
+ case [][]float64:
+ for _, row := range vals {
+ if !numericRow(row, &out) {
+ return nil, false
+ }
+ }
+ default:
+ return nil, false
+ }
+ return out, true
+}
+
// transformChunkFields converts chunk field names to Infinity format.
// Converts internal field names (like docnm_kwd) to Infinity column names (docnm).
// Also handles:
@@ -2396,23 +2497,23 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]inter
d["docnm"] = utility.ConvertToString(v)
}
case "important_kwd":
- if list, ok := v.([]interface{}); ok {
- emptyCount := 0
- tokens := make([]string, 0)
- for _, item := range list {
- if str, ok := item.(string); ok {
- if str == "" {
- emptyCount++
- } else {
- tokens = append(tokens, str)
- }
- }
+ // Python: list2str(tokens, ",") plus the count of the empty entries
+ // (infinity_conn.py:528-535). The extractor and the tokenizer hand
+ // over a Go-NATIVE []string, which the old []interface{}-only branch
+ // fed to ConvertToString — writing "[a b]" into important_keywords
+ // and leaving important_kwd_empty_count unset.
+ parts := utility.ConvertToStringSlice(v)
+ tokens := make([]string, 0, len(parts))
+ emptyCount := 0
+ for _, str := range parts {
+ if str == "" {
+ emptyCount++
+ continue
}
- d["important_keywords"] = strings.Join(tokens, ",")
- d["important_kwd_empty_count"] = emptyCount
- } else {
- d["important_keywords"] = utility.ConvertToString(v)
+ tokens = append(tokens, str)
}
+ d["important_keywords"] = strings.Join(tokens, ",")
+ d["important_kwd_empty_count"] = emptyCount
case "important_tks":
if _, exists := chunk["important_kwd"]; !exists {
d["important_keywords"] = v
@@ -2465,15 +2566,15 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]inter
// 3. Otherwise assign v directly
d["kb_id"] = v
}
- case "position_int":
- if list, ok := v.([]interface{}); ok {
- d["position_int"] = utility.ConvertPositionIntArrayToHex(list)
- } else {
- d["position_int"] = v
- }
- case "page_num_int", "top_int":
- if list, ok := v.([]interface{}); ok {
- d[k] = utility.ConvertIntArrayToHex(list)
+ case "position_int", "page_num_int", "top_int":
+ // Python flattens the position rows and hex-encodes every number
+ // (infinity_conn.py: `[num for row in v for num in row]`, "%08x").
+ // The input may be Go-NATIVE: the ingestion pipeline's AddPositions
+ // emits []int / [][]int, and those used to miss the
+ // []interface{}-only branch and reach Infinity as a raw tensor
+ // ("Not support to convert Tensor(int64,5) to Varchar", 3049).
+ if nums, ok := numericSlice(v); ok {
+ d[k] = utility.ConvertIntArrayToHex(nums)
} else {
d[k] = v
}
diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.go
index 60af458523..5cab6154a3 100644
--- a/internal/engine/infinity/chunk_transform_test.go
+++ b/internal/engine/infinity/chunk_transform_test.go
@@ -78,3 +78,64 @@ func TestTransformChunkFields_IngestionShape(t *testing.T) {
}
}
}
+
+// TestTransformChunkFieldsHexEncodesNativeIntSlices pins the Go-native slice
+// shapes the ingestion pipeline produces: AddPositions emits []int / [][]int
+// (internal/ingestion/task/indexdoc/position.go), and the Infinity columns are
+// VARCHAR holding the hex form. Left unconverted, Infinity rejects the insert
+// with "Not support to convert Tensor(int64,5) to Varchar" (3049).
+func TestTransformChunkFieldsHexEncodesNativeIntSlices(t *testing.T) {
+ chunk := map[string]interface{}{
+ "position_int": [][]int{{1, 10, 20, 30, 40}},
+ "page_num_int": []int{1},
+ "top_int": []int{30},
+ }
+
+ got := transformChunkFields(chunk, nil)
+
+ // Python: "_".join(f"{num:08x}" for num in flattened positions).
+ if want := "00000001_0000000a_00000014_0000001e_00000028"; got["position_int"] != want {
+ t.Errorf("position_int = %v, want %v", got["position_int"], want)
+ }
+ if want := "00000001"; got["page_num_int"] != want {
+ t.Errorf("page_num_int = %v, want %v", got["page_num_int"], want)
+ }
+ if want := "0000001e"; got["top_int"] != want {
+ t.Errorf("top_int = %v, want %v", got["top_int"], want)
+ }
+}
+
+// TestTransformChunkFieldsHexEncodesJSONPositionShape keeps the decoded-JSON
+// shape working: numbers arrive as float64 inside []interface{} rows.
+func TestTransformChunkFieldsHexEncodesJSONPositionShape(t *testing.T) {
+ chunk := map[string]interface{}{
+ "position_int": []interface{}{[]interface{}{float64(1), float64(10), float64(20), float64(30), float64(40)}},
+ "page_num_int": []interface{}{float64(1)},
+ }
+
+ got := transformChunkFields(chunk, nil)
+
+ if want := "00000001_0000000a_00000014_0000001e_00000028"; got["position_int"] != want {
+ t.Errorf("position_int = %v, want %v", got["position_int"], want)
+ }
+ if want := "00000001"; got["page_num_int"] != want {
+ t.Errorf("page_num_int = %v, want %v", got["page_num_int"], want)
+ }
+}
+
+// TestTransformChunkFieldsJoinsNativeKeywordSlice pins the other Go-native slice
+// the ingestion hands over: important_kwd is a plain []string (SplitKeywords /
+// the extractor / the tokenizer), and Python joins it with "," while counting
+// the empty entries (infinity_conn.py:528-535).
+func TestTransformChunkFieldsJoinsNativeKeywordSlice(t *testing.T) {
+ got := transformChunkFields(map[string]interface{}{
+ "important_kwd": []string{"alpha", "", "beta"},
+ }, nil)
+
+ if want := "alpha,beta"; got["important_keywords"] != want {
+ t.Errorf("important_keywords = %v, want %v", got["important_keywords"], want)
+ }
+ if got["important_kwd_empty_count"] != 1 {
+ t.Errorf("important_kwd_empty_count = %v, want 1", got["important_kwd_empty_count"])
+ }
+}
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index 1638718900..f7ba149f1e 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -866,9 +866,12 @@ func TestGeneralChunkerMediaContextReachesChunkIDAndIndexContent(t *testing.T) {
}
const wantText = "before<table><tr><td>A</td></tr></table>after"
+ // The table chunk is selected by the STORED type column: ck_type is chunker
+ // bookkeeping and the index boundary strips it (it has no chunk-store
+ // column, and a strict engine rejects the whole insert over one).
var table map[string]any
for _, ck := range chunks {
- if ck["ck_type"] == "table" {
+ if ck["doc_type_kwd"] == "table" {
table = ck
break
}
diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.go
index 99b78ac8e7..453172dfbd 100644
--- a/internal/ingestion/task/indexdoc/process.go
+++ b/internal/ingestion/task/indexdoc/process.go
@@ -94,6 +94,7 @@ func ProcessChunksForPipeline(
}
cleanupConsumedChunkFields(ck)
+ stripPipelineOnlyFields(ck)
metadata = mergeChunkMetadata(metadata, ck)
RenameTextToContentWithWeight(ck)
processChunkPositions(ck)
@@ -139,6 +140,33 @@ func cleanupConsumedChunkFields(ck map[string]any) {
delete(ck, "summary")
}
+// pipelineOnlyFields are the parser/chunker BOOKKEEPING keys: each one is
+// consumed inside the pipeline (ck_type drives chunk merging and the image-crop
+// decision, tk_nums carries the chunker's token count into the Tokenizer,
+// layout*/image/context_* describe the media block, and page_number/table_id/
+// sheet/headers/cells describe the table or spreadsheet block) and NONE of them
+// is a chunk-store column.
+//
+// The Python index doc carries none of them — its chunk builder emits only the
+// persist fields — but Go's chunker hands them on, and the write boundary is
+// strict about unknown columns: Infinity rejects the whole insert with
+// "Column ck_type not found in table" (InfinityException 3013). Elasticsearch
+// merely swallowed them, because a dynamic mapping accepts any field.
+var pipelineOnlyFields = []string{
+ "ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image",
+ "context_above", "context_below", "page_number",
+ "table_id", "sheet", "sheet_index", "headers", "cells",
+ "row_start", "row_end", "col_start", "col_end",
+}
+
+// stripPipelineOnlyFields drops those bookkeeping keys at the index boundary,
+// leaving the chunk with the persist schema only.
+func stripPipelineOnlyFields(ck map[string]any) {
+ for _, key := range pipelineOnlyFields {
+ delete(ck, key)
+ }
+}
+
func mergeChunkMetadata(metadata map[string]any, ck map[string]any) map[string]any {
metaVal, exists := ck["metadata"]
if !exists {
diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.go
index 15584ccb47..4fe492a740 100644
--- a/internal/ingestion/task/indexdoc/process_test.go
+++ b/internal/ingestion/task/indexdoc/process_test.go
@@ -463,3 +463,37 @@ func TestCleanupConsumedChunkFields_ImportantKwdDropsEmptyParts(t *testing.T) {
t.Fatalf("executor important_kwd = %v, want %v (empty parts dropped)", kwd, want)
}
}
+
+// TestProcessChunksForPipeline_StripsPipelineOnlyFields pins the index boundary
+// against the parser/chunker bookkeeping keys: none of them is a chunk-store
+// column, and a strict engine rejects the whole insert over one of them
+// ("Column ck_type not found in table", InfinityException 3013). ES only
+// swallowed them because its mapping is dynamic.
+func TestProcessChunksForPipeline_StripsPipelineOnlyFields(t *testing.T) {
+ ck := map[string]any{
+ "text": "hello",
+ // Every bookkeeping key the chunkers/parsers can leave on a chunk.
+ "ck_type": "text", "tk_nums": 3, "layout": "text", "layout_type": "text",
+ "layoutno": "0", "image": "data:image/png;base64,AAAA",
+ "context_above": "above", "context_below": "below", "page_number": 2,
+ "table_id": "t1", "sheet": "s1", "sheet_index": 0,
+ "headers": []string{"h"}, "cells": []string{"c"},
+ "row_start": 0, "row_end": 1, "col_start": 0, "col_end": 1,
+ }
+
+ if _, err := ProcessChunksForPipeline([]map[string]any{ck}, "doc-1", "Doc", time.Now()); err != nil {
+ t.Fatalf("ProcessChunksForPipeline: %v", err)
+ }
+
+ for _, key := range pipelineOnlyFields {
+ if _, exists := ck[key]; exists {
+ t.Errorf("%q must be stripped before persist (no chunk column; a strict engine rejects the insert)", key)
+ }
+ }
+ if ck["content_with_weight"] != "hello" {
+ t.Errorf("content_with_weight = %v, want the chunk text (the strip must not touch persist fields)", ck["content_with_weight"])
+ }
+ if ck["doc_id"] != "doc-1" {
+ t.Errorf("doc_id = %v, want doc-1 (the strip must not touch persist fields)", ck["doc_id"])
+ }
+}
diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.go
index 9c9a4efd71..2db7d151e6 100644
--- a/internal/service/dataset/update.go
+++ b/internal/service/dataset/update.go
@@ -160,9 +160,6 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
if d.docEngine == nil {
return nil, common.CodeServerError, errors.New("document engine is not initialized")
}
- if !d.docEngine.SupportsPageRank() {
- return nil, common.CodeDataError, errors.New("'pagerank' can only be set when doc_engine is elasticsearch")
- }
}
requestedAnyUpdate := len(simpleUpdates) > 0 || connectorsProvided || parserIDProvided ||
@@ -253,15 +250,17 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
updates["parser_config"] = preserveDatasetParserConfigMetadata(parserConfig, lockedKB.ParserConfig, req.ParserConfig)
}
}
- if pagerankRequested {
+ if pagerankRequested && requestedPagerank != lockedKB.Pagerank {
+ if !d.docEngine.SupportsPageRank() {
+ txCode = common.CodeDataError
+ return errors.New("'pagerank' can only be set when doc_engine is elasticsearch")
+ }
pagerankUpdate = &datasetPagerankUpdate{
value: requestedPagerank,
index: fmt.Sprintf("ragflow_%s", lockedKB.TenantID),
datasetID: lockedKB.ID,
}
- if requestedPagerank != lockedKB.Pagerank {
- updates["pagerank"] = requestedPagerank
- }
+ updates["pagerank"] = requestedPagerank
}
if parserIDProvided && parserID != lockedKB.ParserID {
if _, ok := updates["parser_config"]; !ok {
diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.go
index 1dcf09425d..ca59493cd8 100644
--- a/internal/service/dataset/update_test.go
+++ b/internal/service/dataset/update_test.go
@@ -1250,3 +1250,42 @@ func TestUpdateDataset_SwitchCanvasToBuiltinValidatesAgainstBuiltin(t *testing.T
t.Fatalf("expected success code, got %d", code)
}
}
+
+// TestUpdateDatasetPagerankUnchangedIsANoOpOnInfinity pins Python's guard: the
+// engine capability is consulted ONLY when the requested pagerank differs from
+// the stored one (`if "pagerank" in req and req["pagerank"] != kb.pagerank`,
+// dataset_api_service.py:392). A settings form re-sending the value the dataset
+// already has must not fail on a non-ES engine; a real change still must
+// (Python's own tests assert that message:
+// test/testcases/restful_api/test_datasets.py:620).
+func TestUpdateDatasetPagerankUnchangedIsANoOpOnInfinity(t *testing.T) {
+ db := setupDatasetUpdateTestDB(t)
+ pushServiceDB(t, db)
+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")
+ // fakeChatDocEngine.SupportsPageRank() == false — the Infinity case.
+ svc := testDatasetUpdateService(t)
+ svc.docEngine = fakeChatDocEngine{}
+
+ unchanged := int64(0) // insertDatasetUpdateKB stores pagerank 0
+ if _, code, err := svc.UpdateDataset(t.Context(), "kb-1", "tenant-1",
+ service.UpdateDatasetRequest{Pagerank: &unchanged}); err != nil || code != common.CodeSuccess {
+ t.Fatalf("unchanged pagerank: code=%d err=%v, want success (a no-op, as in Python)", code, err)
+ }
+
+ changed := int64(7)
+ _, code, err := svc.UpdateDataset(t.Context(), "kb-1", "tenant-1",
+ service.UpdateDatasetRequest{Pagerank: &changed})
+ if err == nil || !strings.Contains(err.Error(), "can only be set when doc_engine is elasticsearch") {
+ t.Fatalf("changed pagerank: code=%d err=%v, want the doc_engine message", code, err)
+ }
+ if code != common.CodeDataError {
+ t.Fatalf("changed pagerank: code=%d, want CodeDataError", code)
+ }
+ persisted, getErr := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
+ if getErr != nil {
+ t.Fatalf("get kb: %v", getErr)
+ }
+ if persisted.Pagerank != 0 {
+ t.Fatalf("pagerank = %d, want the rejected update rolled back", persisted.Pagerank)
+ }
+}