Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(infinity): drop row keys that are not table columns (#20513)

Yash Raj Pandey65bfcf443e75
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index 94088db6a0..0eb27bbfd8 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -21,6 +21,7 @@ import (
"encoding/json"
"fmt"
"hash/fnv"
+ "maps"
"os"
"ragflow/internal/common"
"ragflow/internal/engine/types"
@@ -371,11 +372,13 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
// ShowColumns returns a result set where Data contains arrays of column values
re := regexp.MustCompile(`Embedding\([a-z]+,(\d+)\)`)
hasChunkData := false
+ columns := make(map[string]bool)
if nameArr, ok := result.Data["name"]; ok {
if typeArr, ok := result.Data["type"]; ok {
for i := 0; i < len(nameArr); i++ {
colName, _ := nameArr[i].(string)
colType, _ := typeArr[i].(string)
+ columns[colName] = true
if colName == "chunk_data" {
hasChunkData = true
}
@@ -406,6 +409,9 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
// so the producer value (if any) is intentionally overridden here.
insertChunks[i]["kb_id"] = datasetID
}
+ if err := e.dropUndeclaredFields(tableName, insertChunks, columns); err != nil {
+ return nil, err
+ }
// Delete existing rows with matching IDs
if len(insertChunks) > 0 {
@@ -446,6 +452,67 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
return []string{}, nil
}
+// vectorColumnPattern matches the q_<dim>_vec columns the engine adds per
+// embedding size; no mapping file declares them.
+var vectorColumnPattern = regexp.MustCompile(`^q_\d+_vec$`)
+
+// dropUndeclaredFields removes each row key that is not a column of the table
+// and that no mapping file declares, such as parser bookkeeping keys
+// (slide_number, __outline__). Infinity fails the whole batch on one unknown
+// column. A declared key that the table lacks is kept, so a stale table still
+// fails the insert instead of losing that value.
+func (e *Engine) dropUndeclaredFields(tableName string, rows []map[string]interface{}, columns map[string]bool) error {
+ var declared map[string]bool
+ dropped := make(map[string]bool)
+ for _, row := range rows {
+ for key := range row {
+ if columns[key] || key == "chunk_data" || vectorColumnPattern.MatchString(key) {
+ continue
+ }
+ if declared == nil {
+ var err error
+ if declared, err = declaredChunkColumns(e.mappingFileName); err != nil {
+ return err
+ }
+ }
+ if !declared[key] {
+ delete(row, key)
+ dropped[key] = true
+ }
+ }
+ }
+ if len(dropped) > 0 {
+ common.Warn("Dropping fields that are not Infinity columns",
+ zap.String("tableName", tableName),
+ zap.Strings("fields", slices.Sorted(maps.Keys(dropped))))
+ }
+ return nil
+}
+
+// declaredChunkColumns returns the columns of the chunk mapping and of the
+// memory message mapping, because memory messages go through InsertChunks too.
+func declaredChunkColumns(mappingFileName string) (map[string]bool, error) {
+ declared := make(map[string]bool)
+ for _, name := range []string{mappingFileName, "message_infinity_mapping.json"} {
+ path, err := utility.FindConfFileInProject(name)
+ if err != nil {
+ return nil, err
+ }
+ data, err := os.ReadFile(*path)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read mapping file %s: %w", *path, err)
+ }
+ var schema map[string]json.RawMessage
+ if err := json.Unmarshal(data, &schema); err != nil {
+ return nil, fmt.Errorf("failed to parse mapping file %s: %w", *path, err)
+ }
+ for column := range schema {
+ declared[column] = true
+ }
+ }
+ return declared, nil
+}
+
// UpdateChunks updates chunks in a dataset
// Table name format: {baseName}_{datasetID}
func (e *Engine) UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error {
diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.go
index 5b2d13f54a..67a96f76ef 100644
--- a/internal/engine/infinity/chunk_transform_test.go
+++ b/internal/engine/infinity/chunk_transform_test.go
@@ -835,3 +835,87 @@ func TestTagKwdRoundTripsAsList(t *testing.T) {
})
}
}
+
+// TestDropUndeclaredFieldsKeepsColumnsAndDeclaredKeys covers #20500: parser
+// keys that are not columns (slide_number, __outline__) failed the whole batch.
+func TestDropUndeclaredFieldsKeepsColumnsAndDeclaredKeys(t *testing.T) {
+ outline := []map[string]interface{}{{"title": "Intro", "depth": 1}}
+ vector := []float64{0.1, 0.2}
+ rows := []map[string]interface{}{
+ transformChunkFields(map[string]interface{}{
+ "id": "c1", "content_with_weight": "slide text", "docnm_kwd": "deck.pptx",
+ "slide_number": 2, "q_1024_vec": vector, "legacy_col": "kept",
+ }, nil),
+ transformChunkFields(map[string]interface{}{
+ "id": "c2", "content_with_weight": "pdf text", "__outline__": outline,
+ "important_kwd": []string{"a"}, "memory_id": "m1", "chunk_data": map[string]interface{}{"k": "v"},
+ }, nil),
+ }
+ // The table lacks q_1024_vec, important_keywords, memory_id and chunk_data,
+ // but the engine or a mapping declares each one. They must reach the insert.
+ // legacy_col is a table column that no mapping declares.
+ columns := map[string]bool{"id": true, "content": true, "docnm": true, "legacy_col": true}
+
+ if err := (&Engine{mappingFileName: "infinity_mapping.json"}).dropUndeclaredFields("ragflow_t_kb", rows, columns); err != nil {
+ t.Fatalf("dropUndeclaredFields: %v", err)
+ }
+
+ want := []map[string]interface{}{
+ {"id": "c1", "content": "slide text", "docnm": "deck.pptx", "q_1024_vec": vector, "legacy_col": "kept"},
+ {
+ "id": "c2", "content": "pdf text", "important_keywords": "a", "important_kwd_empty_count": 0,
+ "memory_id": "m1", "chunk_data": `{"k":"v"}`,
+ },
+ }
+ if !reflect.DeepEqual(rows, want) {
+ t.Fatalf("rows = %#v\nwant %#v", rows, want)
+ }
+}
+
+func TestDropUndeclaredFieldsKeepsEveryMappingColumn(t *testing.T) {
+ path, err := utility.FindConfFileInProject("message_infinity_mapping.json")
+ if err != nil {
+ t.Fatalf("locate message_infinity_mapping.json: %v", err)
+ }
+ data, err := os.ReadFile(*path)
+ if err != nil {
+ t.Fatalf("read %s: %v", *path, err)
+ }
+ var messageColumns map[string]json.RawMessage
+ if err := json.Unmarshal(data, &messageColumns); err != nil {
+ t.Fatalf("parse %s: %v", *path, err)
+ }
+
+ row := map[string]interface{}{}
+ for column := range chunkMappingColumns(t) {
+ row[column] = "v"
+ }
+ for column := range messageColumns {
+ row[column] = "v"
+ }
+ before := len(row)
+
+ if err := (&Engine{mappingFileName: "infinity_mapping.json"}).dropUndeclaredFields("ragflow_t_kb", []map[string]interface{}{row}, map[string]bool{}); err != nil {
+ t.Fatalf("dropUndeclaredFields: %v", err)
+ }
+ if len(row) != before {
+ t.Fatalf("dropped %d mapping columns from a table with no columns", before-len(row))
+ }
+}
+
+func TestDropUndeclaredFieldsFailsWhenMappingIsMissing(t *testing.T) {
+ engine := &Engine{mappingFileName: "no_such_mapping.json"}
+
+ allColumns := []map[string]interface{}{{"id": "c1"}}
+ if err := engine.dropUndeclaredFields("ragflow_t_kb", allColumns, map[string]bool{"id": true}); err != nil {
+ t.Fatalf("a row with only table columns must not read the mapping: %v", err)
+ }
+
+ unknown := []map[string]interface{}{{"id": "c1", "slide_number": 2}}
+ if err := engine.dropUndeclaredFields("ragflow_t_kb", unknown, map[string]bool{"id": true}); err == nil {
+ t.Fatal("expected an error when the mapping file cannot be read")
+ }
+ if _, ok := unknown[0]["slide_number"]; !ok {
+ t.Fatal("a failed mapping read must leave the row unchanged")
+ }
+}
diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.go
index 5e80a9c937..f3b06ddbbd 100644
--- a/internal/ingestion/task/indexdoc/process.go
+++ b/internal/ingestion/task/indexdoc/process.go
@@ -152,9 +152,8 @@ func cleanupConsumedChunkFields(ck map[string]any) {
//
// The Python index doc carries none of them — its chunk builder emits only the
// persist fields — but Go's chunker hands them on, and the write boundary is
-// strict about unknown columns: Infinity rejects the whole insert with
-// "Column ck_type not found in table" (InfinityException 3013). Elasticsearch
-// merely swallowed them, because a dynamic mapping accepts any field.
+// where they stop: Elasticsearch and Vastbase would add a field for each key,
+// and Infinity would drop each key at insert with a warning.
var pipelineOnlyFields = []string{
"ck_type", "tk_nums", "layout", "layout_type", "layoutno", "image",
"context_above", "context_below", "page_number",