RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(infinity): drop row keys that are not table columns (#20513)
Yash Raj Pandey65bfcf443e75
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex 94088db6a0..0eb27bbfd8 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -21,6 +21,7 @@ import ("encoding/json""fmt""hash/fnv"+ "maps""os""ragflow/internal/common""ragflow/internal/engine/types"@@ -371,11 +372,13 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface// ShowColumns returns a result set where Data contains arrays of column valuesre := regexp.MustCompile(`Embedding\([a-z]+,(\d+)\)`)hasChunkData := false+ columns := make(map[string]bool)if nameArr, ok := result.Data["name"]; ok {if typeArr, ok := result.Data["type"]; ok {for i := 0; i < len(nameArr); i++ {colName, _ := nameArr[i].(string)colType, _ := typeArr[i].(string)+ columns[colName] = trueif colName == "chunk_data" {hasChunkData = true}@@ -406,6 +409,9 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface// so the producer value (if any) is intentionally overridden here.insertChunks[i]["kb_id"] = datasetID}+ if err := e.dropUndeclaredFields(tableName, insertChunks, columns); err != nil {+ return nil, err+ }// Delete existing rows with matching IDsif len(insertChunks) > 0 {@@ -446,6 +452,67 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interfacereturn []string{}, nil}+// vectorColumnPattern matches the q_<dim>_vec columns the engine adds per+// embedding size; no mapping file declares them.+var vectorColumnPattern = regexp.MustCompile(`^q_\d+_vec$`)++// dropUndeclaredFields removes each row key that is not a column of the table+// and that no mapping file declares, such as parser bookkeeping keys+// (slide_number, __outline__). Infinity fails the whole batch on one unknown+// column. A declared key that the table lacks is kept, so a stale table still+// fails the insert instead of losing that value.+func (e *Engine) dropUndeclaredFields(tableName string, rows []map[string]interface{}, columns map[string]bool) error {+ var declared map[string]bool+ dropped := make(map[string]bool)+ for _, row := range rows {+ for key := range row {+ if columns[key] || key == "chunk_data" || vectorColumnPattern.MatchString(key) {+ continue+ }+ if declared == nil {+ var err error+ if declared, err = declaredChunkColumns(e.mappingFileName); err != nil {+ return err+ }+ }+ if !declared[key] {+ delete(row, key)+ dropped[key] = true+ }+ }+ }+ if len(dropped) > 0 {+ common.Warn("Dropping fields that are not Infinity columns",+ zap.String("tableName", tableName),+ zap.Strings("fields", slices.Sorted(maps.Keys(dropped))))+ }+ return nil+}++// declaredChunkColumns returns the columns of the chunk mapping and of the+// memory message mapping, because memory messages go through InsertChunks too.+func declaredChunkColumns(mappingFileName string) (map[string]bool, error) {+ declared := make(map[string]bool)+ for _, name := range []string{mappingFileName, "message_infinity_mapping.json"} {+ path, err := utility.FindConfFileInProject(name)+ if err != nil {+ return nil, err+ }+ data, err := os.ReadFile(*path)+ if err != nil {+ return nil, fmt.Errorf("failed to read mapping file %s: %w", *path, err)+ }+ var schema map[string]json.RawMessage+ if err := json.Unmarshal(data, &schema); err != nil {+ return nil, fmt.Errorf("failed to parse mapping file %s: %w", *path, err)+ }+ for column := range schema {+ declared[column] = true+ }+ }+ return declared, nil+}+// UpdateChunks updates chunks in a dataset// Table name format: {baseName}_{datasetID}func (e *Engine) UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error {diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.goindex 5b2d13f54a..67a96f76ef 100644--- a/internal/engine/infinity/chunk_transform_test.go+++ b/internal/engine/infinity/chunk_transform_test.go@@ -835,3 +835,87 @@ func TestTagKwdRoundTripsAsList(t *testing.T) {})}}++// TestDropUndeclaredFieldsKeepsColumnsAndDeclaredKeys covers #20500: parser+// keys that are not columns (slide_number, __outline__) failed the whole batch.+func TestDropUndeclaredFieldsKeepsColumnsAndDeclaredKeys(t *testing.T) {+ outline := []map[string]interface{}{{"title": "Intro", "depth": 1}}+ vector := []float64{0.1, 0.2}+ rows := []map[string]interface{}{+ transformChunkFields(map[string]interface{}{+ "id": "c1", "content_with_weight": "slide text", "docnm_kwd": "deck.pptx",+ "slide_number": 2, "q_1024_vec": vector, "legacy_col": "kept",+ }, nil),+ transformChunkFields(map[string]interface{}{+ "id": "c2", "content_with_weight": "pdf text", "__outline__": outline,+ "important_kwd": []string{"a"}, "memory_id": "m1", "chunk_data": map[string]interface{}{"k": "v"},+ }, nil),+ }+ // The table lacks q_1024_vec, important_keywords, memory_id and chunk_data,+ // but the engine or a mapping declares each one. They must reach the insert.+ // legacy_col is a table column that no mapping declares.+ columns := map[string]bool{"id": true, "content": true, "docnm": true, "legacy_col": true}++ if err := (&Engine{mappingFileName: "infinity_mapping.json"}).dropUndeclaredFields("ragflow_t_kb", rows, columns); err != nil {+ t.Fatalf("dropUndeclaredFields: %v", err)+ }++ want := []map[string]interface{}{+ {"id": "c1", "content": "slide text", "docnm": "deck.pptx", "q_1024_vec": vector, "legacy_col": "kept"},+ {+ "id": "c2", "content": "pdf text", "important_keywords": "a", "important_kwd_empty_count": 0,+ "memory_id": "m1", "chunk_data": `{"k":"v"}`,+ },+ }+ if !reflect.DeepEqual(rows, want) {+ t.Fatalf("rows = %#v\nwant %#v", rows, want)+ }+}++func TestDropUndeclaredFieldsKeepsEveryMappingColumn(t *testing.T) {+ path, err := utility.FindConfFileInProject("message_infinity_mapping.json")+ if err != nil {+ t.Fatalf("locate message_infinity_mapping.json: %v", err)+ }+ data, err := os.ReadFile(*path)+ if err != nil {+ t.Fatalf("read %s: %v", *path, err)+ }+ var messageColumns map[string]json.RawMessage+ if err := json.Unmarshal(data, &messageColumns); err != nil {+ t.Fatalf("parse %s: %v", *path, err)+ }++ row := map[string]interface{}{}+ for column := range chunkMappingColumns(t) {+ row[column] = "v"+ }+ for column := range messageColumns {+ row[column] = "v"+ }+ before := len(row)++ if err := (&Engine{mappingFileName: "infinity_mapping.json"}).dropUndeclaredFields("ragflow_t_kb", []map[string]interface{}{row}, map[string]bool{}); err != nil {+ t.Fatalf("dropUndeclaredFields: %v", err)+ }+ if len(row) != before {+ t.Fatalf("dropped %d mapping columns from a table with no columns", before-len(row))+ }+}++func TestDropUndeclaredFieldsFailsWhenMappingIsMissing(t *testing.T) {+ engine := &Engine{mappingFileName: "no_such_mapping.json"}++ allColumns := []map[string]interface{}{{"id": "c1"}}+ if err := engine.dropUndeclaredFields("ragflow_t_kb", allColumns, map[string]bool{"id": true}); err != nil {+ t.Fatalf("a row with only table columns must not read the mapping: %v", err)+ }++ unknown := []map[string]interface{}{{"id": "c1", "slide_number": 2}}+ if err := engine.dropUndeclaredFields("ragflow_t_kb", unknown, map[string]bool{"id": true}); err == nil {+ t.Fatal("expected an error when the mapping file cannot be read")+ }+ if _, ok := unknown[0]["slide_number"]; !ok {+ t.Fatal("a failed mapping read must leave the row unchanged")+ }+}diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.goindex 5e80a9c937..f3b06ddbbd 100644--- a/internal/ingestion/task/indexdoc/process.go+++ b/internal/ingestion/task/indexdoc/process.go@@ -152,9 +152,8 @@ func cleanupConsumedChunkFields(ck map[string]any) {//// The Python index doc carries none of them — its chunk builder emits only the// persist fields — but Go's chunker hands them on, and the write boundary is-// strict about unknown columns: Infinity rejects the whole insert with-// "Column ck_type not found in table" (InfinityException 3013). Elasticsearch-// merely swallowed them, because a dynamic mapping accepts any field.+// where they stop: Elasticsearch and Vastbase would add a field for each key,+// and Infinity would drop each key at insert with a warning.var pipelineOnlyFields = []string{"ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image","context_above", "context_below", "page_number",