RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(infinity): repair chunk ingestion and dataset save (#19773)
qinling0210850132e6cbd2
diff --git a/conf/infinity_mapping.json b/conf/infinity_mapping.jsonindex 95de7aeac4..15ba437143 100644--- a/conf/infinity_mapping.json+++ b/conf/infinity_mapping.json@@ -77,7 +77,6 @@"claims": {"type": "json", "default": "[]", "enabled": false},"page_ids": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},"page_id": {"type": "varchar", "default": ""},- "deleted_doc_id": {"type": "varchar", "default": ""},"source_chunk_hashes": {"type": "json", "default": "{}", "enabled": false},"map_checksum": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex 5c4dabc928..4d56e17012 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -2371,6 +2371,107 @@ func getChunkScore(chunk map[string]interface{}) float64 {return 0.0}+// numericValue reports whether v is a number the hex encoder accepts.+func numericValue(v interface{}) (interface{}, bool) {+ switch n := v.(type) {+ case int, int64, float64:+ return n, true+ }+ return nil, false+}++// numericRow appends one flat row of numbers: a JSON-decoded []interface{}, or+// the Go-typed []int / []int64 / []float64 a chunk built in process carries.+func numericRow(row interface{}, out *[]interface{}) bool {+ if vals, ok := row.([]interface{}); ok {+ for _, item := range vals {+ n, isNum := numericValue(item)+ if !isNum {+ return false+ }+ *out = append(*out, n)+ }+ return true+ }+ switch vals := row.(type) {+ case []int:+ for _, n := range vals {+ *out = append(*out, n)+ }+ case []int64:+ for _, n := range vals {+ *out = append(*out, n)+ }+ case []float64:+ for _, n := range vals {+ *out = append(*out, n)+ }+ default:+ return false+ }+ return true+}++// numericSlice flattens a numeric slice — or a slice of numeric slices — into the+// flat []interface{} the hex encoder takes. Two shapes reach the engine: values+// decoded from JSON arrive as []interface{} (numbers as float64), while a chunk+// built IN PROCESS carries Go's typed slices (the ingestion pipeline's+// AddPositions emits []int / [][]int). Both must be encoded, because these+// columns are VARCHAR holding the hex form: handing Infinity the typed slice+// makes it try to store an int64 tensor and fail with+// "Not support to convert Tensor(int64,5) to Varchar" (InfinityException 3049).+// A scalar or a non-numeric value reports ok=false, so the caller keeps its+// pass-through instead of hex-encoding something it cannot parse back.+func numericSlice(v interface{}) ([]interface{}, bool) {+ out := make([]interface{}, 0, 5)+ switch vals := v.(type) {+ case []interface{}:+ // JSON shape: plain numbers, and/or position rows that are slices too.+ for _, item := range vals {+ if n, ok := numericValue(item); ok {+ out = append(out, n)+ continue+ }+ if !numericRow(item, &out) {+ return nil, false+ }+ }+ case []int:+ for _, n := range vals {+ out = append(out, n)+ }+ case []int64:+ for _, n := range vals {+ out = append(out, n)+ }+ case []float64:+ for _, n := range vals {+ out = append(out, n)+ }+ case [][]int:+ for _, row := range vals {+ if !numericRow(row, &out) {+ return nil, false+ }+ }+ case [][]int64:+ for _, row := range vals {+ if !numericRow(row, &out) {+ return nil, false+ }+ }+ case [][]float64:+ for _, row := range vals {+ if !numericRow(row, &out) {+ return nil, false+ }+ }+ default:+ return nil, false+ }+ return out, true+}+// transformChunkFields converts chunk field names to Infinity format.// Converts internal field names (like docnm_kwd) to Infinity column names (docnm).// Also handles:@@ -2396,23 +2497,23 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]interd["docnm"] = utility.ConvertToString(v)}case "important_kwd":- if list, ok := v.([]interface{}); ok {- emptyCount := 0- tokens := make([]string, 0)- for _, item := range list {- if str, ok := item.(string); ok {- if str == "" {- emptyCount++- } else {- tokens = append(tokens, str)- }- }+ // Python: list2str(tokens, ",") plus the count of the empty entries+ // (infinity_conn.py:528-535). The extractor and the tokenizer hand+ // over a Go-NATIVE []string, which the old []interface{}-only branch+ // fed to ConvertToString — writing "[a b]" into important_keywords+ // and leaving important_kwd_empty_count unset.+ parts := utility.ConvertToStringSlice(v)+ tokens := make([]string, 0, len(parts))+ emptyCount := 0+ for _, str := range parts {+ if str == "" {+ emptyCount+++ continue}- d["important_keywords"] = strings.Join(tokens, ",")- d["important_kwd_empty_count"] = emptyCount- } else {- d["important_keywords"] = utility.ConvertToString(v)+ tokens = append(tokens, str)}+ d["important_keywords"] = strings.Join(tokens, ",")+ d["important_kwd_empty_count"] = emptyCountcase "important_tks":if _, exists := chunk["important_kwd"]; !exists {d["important_keywords"] = v@@ -2465,15 +2566,15 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]inter// 3. Otherwise assign v directlyd["kb_id"] = v}- case "position_int":- if list, ok := v.([]interface{}); ok {- d["position_int"] = utility.ConvertPositionIntArrayToHex(list)- } else {- d["position_int"] = v- }- case "page_num_int", "top_int":- if list, ok := v.([]interface{}); ok {- d[k] = utility.ConvertIntArrayToHex(list)+ case "position_int", "page_num_int", "top_int":+ // Python flattens the position rows and hex-encodes every number+ // (infinity_conn.py: `[num for row in v for num in row]`, "%08x").+ // The input may be Go-NATIVE: the ingestion pipeline's AddPositions+ // emits []int / [][]int, and those used to miss the+ // []interface{}-only branch and reach Infinity as a raw tensor+ // ("Not support to convert Tensor(int64,5) to Varchar", 3049).+ if nums, ok := numericSlice(v); ok {+ d[k] = utility.ConvertIntArrayToHex(nums)} else {d[k] = v}diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.goindex 60af458523..5cab6154a3 100644--- a/internal/engine/infinity/chunk_transform_test.go+++ b/internal/engine/infinity/chunk_transform_test.go@@ -78,3 +78,64 @@ func TestTransformChunkFields_IngestionShape(t *testing.T) {}}}++// TestTransformChunkFieldsHexEncodesNativeIntSlices pins the Go-native slice+// shapes the ingestion pipeline produces: AddPositions emits []int / [][]int+// (internal/ingestion/task/indexdoc/position.go), and the Infinity columns are+// VARCHAR holding the hex form. Left unconverted, Infinity rejects the insert+// with "Not support to convert Tensor(int64,5) to Varchar" (3049).+func TestTransformChunkFieldsHexEncodesNativeIntSlices(t *testing.T) {+ chunk := map[string]interface{}{+ "position_int": [][]int{{1, 10, 20, 30, 40}},+ "page_num_int": []int{1},+ "top_int": []int{30},+ }++ got := transformChunkFields(chunk, nil)++ // Python: "_".join(f"{num:08x}" for num in flattened positions).+ if want := "00000001_0000000a_00000014_0000001e_00000028"; got["position_int"] != want {+ t.Errorf("position_int = %v, want %v", got["position_int"], want)+ }+ if want := "00000001"; got["page_num_int"] != want {+ t.Errorf("page_num_int = %v, want %v", got["page_num_int"], want)+ }+ if want := "0000001e"; got["top_int"] != want {+ t.Errorf("top_int = %v, want %v", got["top_int"], want)+ }+}++// TestTransformChunkFieldsHexEncodesJSONPositionShape keeps the decoded-JSON+// shape working: numbers arrive as float64 inside []interface{} rows.+func TestTransformChunkFieldsHexEncodesJSONPositionShape(t *testing.T) {+ chunk := map[string]interface{}{+ "position_int": []interface{}{[]interface{}{float64(1), float64(10), float64(20), float64(30), float64(40)}},+ "page_num_int": []interface{}{float64(1)},+ }++ got := transformChunkFields(chunk, nil)++ if want := "00000001_0000000a_00000014_0000001e_00000028"; got["position_int"] != want {+ t.Errorf("position_int = %v, want %v", got["position_int"], want)+ }+ if want := "00000001"; got["page_num_int"] != want {+ t.Errorf("page_num_int = %v, want %v", got["page_num_int"], want)+ }+}++// TestTransformChunkFieldsJoinsNativeKeywordSlice pins the other Go-native slice+// the ingestion hands over: important_kwd is a plain []string (SplitKeywords /+// the extractor / the tokenizer), and Python joins it with "," while counting+// the empty entries (infinity_conn.py:528-535).+func TestTransformChunkFieldsJoinsNativeKeywordSlice(t *testing.T) {+ got := transformChunkFields(map[string]interface{}{+ "important_kwd": []string{"alpha", "", "beta"},+ }, nil)++ if want := "alpha,beta"; got["important_keywords"] != want {+ t.Errorf("important_keywords = %v, want %v", got["important_keywords"], want)+ }+ if got["important_kwd_empty_count"] != 1 {+ t.Errorf("important_kwd_empty_count = %v, want 1", got["important_kwd_empty_count"])+ }+}diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex 1638718900..f7ba149f1e 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -866,9 +866,12 @@ func TestGeneralChunkerMediaContextReachesChunkIDAndIndexContent(t *testing.T) {}const wantText = "before<table><tr><td>A</td></tr></table>after"+ // The table chunk is selected by the STORED type column: ck_type is chunker+ // bookkeeping and the index boundary strips it (it has no chunk-store+ // column, and a strict engine rejects the whole insert over one).var table map[string]anyfor _, ck := range chunks {- if ck["ck_type"] == "table" {+ if ck["doc_type_kwd"] == "table" {table = ckbreak}diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.goindex 99b78ac8e7..453172dfbd 100644--- a/internal/ingestion/task/indexdoc/process.go+++ b/internal/ingestion/task/indexdoc/process.go@@ -94,6 +94,7 @@ func ProcessChunksForPipeline(}cleanupConsumedChunkFields(ck)+ stripPipelineOnlyFields(ck)metadata = mergeChunkMetadata(metadata, ck)RenameTextToContentWithWeight(ck)processChunkPositions(ck)@@ -139,6 +140,33 @@ func cleanupConsumedChunkFields(ck map[string]any) {delete(ck, "summary")}+// pipelineOnlyFields are the parser/chunker BOOKKEEPING keys: each one is+// consumed inside the pipeline (ck_type drives chunk merging and the image-crop+// decision, tk_nums carries the chunker's token count into the Tokenizer,+// layout*/image/context_* describe the media block, and page_number/table_id/+// sheet/headers/cells describe the table or spreadsheet block) and NONE of them+// is a chunk-store column.+//+// The Python index doc carries none of them — its chunk builder emits only the+// persist fields — but Go's chunker hands them on, and the write boundary is+// strict about unknown columns: Infinity rejects the whole insert with+// "Column ck_type not found in table" (InfinityException 3013). Elasticsearch+// merely swallowed them, because a dynamic mapping accepts any field.+var pipelineOnlyFields = []string{+ "ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image",+ "context_above", "context_below", "page_number",+ "table_id", "sheet", "sheet_index", "headers", "cells",+ "row_start", "row_end", "col_start", "col_end",+}++// stripPipelineOnlyFields drops those bookkeeping keys at the index boundary,+// leaving the chunk with the persist schema only.+func stripPipelineOnlyFields(ck map[string]any) {+ for _, key := range pipelineOnlyFields {+ delete(ck, key)+ }+}+func mergeChunkMetadata(metadata map[string]any, ck map[string]any) map[string]any {metaVal, exists := ck["metadata"]if !exists {diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.goindex 15584ccb47..4fe492a740 100644--- a/internal/ingestion/task/indexdoc/process_test.go+++ b/internal/ingestion/task/indexdoc/process_test.go@@ -463,3 +463,37 @@ func TestCleanupConsumedChunkFields_ImportantKwdDropsEmptyParts(t *testing.T) {t.Fatalf("executor important_kwd = %v, want %v (empty parts dropped)", kwd, want)}}++// TestProcessChunksForPipeline_StripsPipelineOnlyFields pins the index boundary+// against the parser/chunker bookkeeping keys: none of them is a chunk-store+// column, and a strict engine rejects the whole insert over one of them+// ("Column ck_type not found in table", InfinityException 3013). ES only+// swallowed them because its mapping is dynamic.+func TestProcessChunksForPipeline_StripsPipelineOnlyFields(t *testing.T) {+ ck := map[string]any{+ "text": "hello",+ // Every bookkeeping key the chunkers/parsers can leave on a chunk.+ "ck_type": "text", "tk_nums": 3, "layout": "text", "layout_type": "text",+ "layoutno": "0", "image": "data:image/png;base64,AAAA",+ "context_above": "above", "context_below": "below", "page_number": 2,+ "table_id": "t1", "sheet": "s1", "sheet_index": 0,+ "headers": []string{"h"}, "cells": []string{"c"},+ "row_start": 0, "row_end": 1, "col_start": 0, "col_end": 1,+ }++ if _, err := ProcessChunksForPipeline([]map[string]any{ck}, "doc-1", "Doc", time.Now()); err != nil {+ t.Fatalf("ProcessChunksForPipeline: %v", err)+ }++ for _, key := range pipelineOnlyFields {+ if _, exists := ck[key]; exists {+ t.Errorf("%q must be stripped before persist (no chunk column; a strict engine rejects the insert)", key)+ }+ }+ if ck["content_with_weight"] != "hello" {+ t.Errorf("content_with_weight = %v, want the chunk text (the strip must not touch persist fields)", ck["content_with_weight"])+ }+ if ck["doc_id"] != "doc-1" {+ t.Errorf("doc_id = %v, want doc-1 (the strip must not touch persist fields)", ck["doc_id"])+ }+}diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.goindex 9c9a4efd71..2db7d151e6 100644--- a/internal/service/dataset/update.go+++ b/internal/service/dataset/update.go@@ -160,9 +160,6 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantIDif d.docEngine == nil {return nil, common.CodeServerError, errors.New("document engine is not initialized")}- if !d.docEngine.SupportsPageRank() {- return nil, common.CodeDataError, errors.New("'pagerank' can only be set when doc_engine is elasticsearch")- }}requestedAnyUpdate := len(simpleUpdates) > 0 || connectorsProvided || parserIDProvided ||@@ -253,15 +250,17 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantIDupdates["parser_config"] = preserveDatasetParserConfigMetadata(parserConfig, lockedKB.ParserConfig, req.ParserConfig)}}- if pagerankRequested {+ if pagerankRequested && requestedPagerank != lockedKB.Pagerank {+ if !d.docEngine.SupportsPageRank() {+ txCode = common.CodeDataError+ return errors.New("'pagerank' can only be set when doc_engine is elasticsearch")+ }pagerankUpdate = &datasetPagerankUpdate{value: requestedPagerank,index: fmt.Sprintf("ragflow_%s", lockedKB.TenantID),datasetID: lockedKB.ID,}- if requestedPagerank != lockedKB.Pagerank {- updates["pagerank"] = requestedPagerank- }+ updates["pagerank"] = requestedPagerank}if parserIDProvided && parserID != lockedKB.ParserID {if _, ok := updates["parser_config"]; !ok {diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.goindex 1dcf09425d..ca59493cd8 100644--- a/internal/service/dataset/update_test.go+++ b/internal/service/dataset/update_test.go@@ -1250,3 +1250,42 @@ func TestUpdateDataset_SwitchCanvasToBuiltinValidatesAgainstBuiltin(t *testing.Tt.Fatalf("expected success code, got %d", code)}}++// TestUpdateDatasetPagerankUnchangedIsANoOpOnInfinity pins Python's guard: the+// engine capability is consulted ONLY when the requested pagerank differs from+// the stored one (`if "pagerank" in req and req["pagerank"] != kb.pagerank`,+// dataset_api_service.py:392). A settings form re-sending the value the dataset+// already has must not fail on a non-ES engine; a real change still must+// (Python's own tests assert that message:+// test/testcases/restful_api/test_datasets.py:620).+func TestUpdateDatasetPagerankUnchangedIsANoOpOnInfinity(t *testing.T) {+ db := setupDatasetUpdateTestDB(t)+ pushServiceDB(t, db)+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")+ // fakeChatDocEngine.SupportsPageRank() == false — the Infinity case.+ svc := testDatasetUpdateService(t)+ svc.docEngine = fakeChatDocEngine{}++ unchanged := int64(0) // insertDatasetUpdateKB stores pagerank 0+ if _, code, err := svc.UpdateDataset(t.Context(), "kb-1", "tenant-1",+ service.UpdateDatasetRequest{Pagerank: &unchanged}); err != nil || code != common.CodeSuccess {+ t.Fatalf("unchanged pagerank: code=%d err=%v, want success (a no-op, as in Python)", code, err)+ }++ changed := int64(7)+ _, code, err := svc.UpdateDataset(t.Context(), "kb-1", "tenant-1",+ service.UpdateDatasetRequest{Pagerank: &changed})+ if err == nil || !strings.Contains(err.Error(), "can only be set when doc_engine is elasticsearch") {+ t.Fatalf("changed pagerank: code=%d err=%v, want the doc_engine message", code, err)+ }+ if code != common.CodeDataError {+ t.Fatalf("changed pagerank: code=%d, want CodeDataError", code)+ }+ persisted, getErr := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")+ if getErr != nil {+ t.Fatalf("get kb: %v", getErr)+ }+ if persisted.Pagerank != 0 {+ t.Fatalf("pagerank = %d, want the rejected update rolled back", persisted.Pagerank)+ }+}