Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(infinity): decode tag_feas JSON text on chunk detail reads (#21779)

qinling0210d459d51912ad
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index 619fc1fe39..0ed34b4ca3 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -1928,6 +1928,7 @@ func (e *Engine) GetChunk(ctx context.Context, tableName, chunkID string, datase
common.Debug("infinity get chunk", zap.String("chunkID", chunkID), zap.Any("tables", tableNames))
decodeJSONFields(chunk)
+ decodeVarcharJSONFields(chunk)
// Memory rows come back under the message mapping's storage columns, which
// the chunk field mappings below would mangle.
@@ -3408,6 +3409,31 @@ func decodeJSONFields(chunk map[string]interface{}) {
}
}
+// decodeVarcharJSONFields parses the JSON text transformChunkFields writes into
+// fieldVarcharJSON columns back into a map; empty or nil becomes {}.
+func decodeVarcharJSONFields(chunk map[string]interface{}) {
+ for fieldName, value := range chunk {
+ if !fieldVarcharJSON(fieldName) {
+ continue
+ }
+ text, ok := value.(string)
+ if !ok {
+ if value == nil {
+ chunk[fieldName] = map[string]interface{}{}
+ }
+ continue
+ }
+ if text == "" {
+ chunk[fieldName] = map[string]interface{}{}
+ continue
+ }
+ var decoded interface{}
+ if err := json.Unmarshal([]byte(text), &decoded); err == nil {
+ chunk[fieldName] = decoded
+ }
+ }
+}
+
// splitLengthPrefixedSegments splits Infinity's `[int32 little-endian
// length][payload]` framing into its payloads, each the JSON text of one value.
// parseLengthPrefixedJSON (metadata.go) unpacks the same framing for the
@@ -3837,7 +3863,7 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]inter
default:
if fieldJSON(k) {
d[k] = encodeJSONField(k, v)
- } else if strings.HasSuffix(k, "_feas") {
+ } else if fieldVarcharJSON(k) {
jsonBytes, _ := json.Marshal(v)
d[k] = string(jsonBytes)
} else if fieldKeyword(k) {
diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.go
index 7a689d62ee..6cfacdb00d 100644
--- a/internal/engine/infinity/chunk_transform_test.go
+++ b/internal/engine/infinity/chunk_transform_test.go
@@ -300,6 +300,22 @@ func TestFieldJSONMatchesJSONMappingColumns(t *testing.T) {
}
}
+// TestFieldVarcharJSONMatchesVarcharColumns keeps fieldVarcharJSON aligned with
+// the schema: a column it decodes must exist as a varchar column, and it must
+// not claim a json-typed column fieldJSON already owns.
+func TestFieldVarcharJSONMatchesVarcharColumns(t *testing.T) {
+ for _, column := range []string{"tag_feas"} {
+ if !fieldVarcharJSON(column) {
+ t.Errorf("fieldVarcharJSON(%q) = false, want true", column)
+ }
+ }
+ for column, columnType := range mappingColumnTypes(t) {
+ if fieldVarcharJSON(column) && columnType != "varchar" {
+ t.Errorf("fieldVarcharJSON(%q) = true, but the mapping declares type %q", column, columnType)
+ }
+ }
+}
+
// chunkMappingColumns reads the column set the chunk table is created from.
func chunkMappingColumns(t *testing.T) map[string]bool {
columns := mappingColumnTypes(t)
@@ -505,6 +521,60 @@ func TestJSONColumnsMatchCaseInsensitively(t *testing.T) {
}
}
+// TestDecodeVarcharJSONFieldsRoundTripsTagFeatures pins that the read-side
+// decode of tag_feas yields the map the write path marshaled.
+func TestDecodeVarcharJSONFieldsRoundTripsTagFeatures(t *testing.T) {
+ tags := map[string]int{"物流查询": 7, "售后服务": 8}
+ row := transformChunkFields(map[string]interface{}{"tag_feas": tags}, nil)
+ if _, ok := row["tag_feas"].(string); !ok {
+ t.Fatalf("write path should store tag_feas as JSON text, got %#v", row["tag_feas"])
+ }
+
+ decodeVarcharJSONFields(row)
+ got, ok := row["tag_feas"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("decodeVarcharJSONFields left tag_feas as %T, want map", row["tag_feas"])
+ }
+ if want := float64(7); got["物流查询"] != want {
+ t.Errorf("tag_feas[物流查询] = %#v, want %#v", got["物流查询"], want)
+ }
+ if want := float64(8); got["售后服务"] != want {
+ t.Errorf("tag_feas[售后服务] = %#v, want %#v", got["售后服务"], want)
+ }
+}
+
+// TestDecodeVarcharJSONFieldsEmptyAndPreDecoded covers empty/nil text (→ {}),
+// an already decoded value, and that pagerank_fea is left untouched.
+func TestDecodeVarcharJSONFieldsEmptyAndPreDecoded(t *testing.T) {
+ row := map[string]interface{}{"tag_feas": ""}
+ decodeVarcharJSONFields(row)
+ if want := map[string]interface{}{}; !reflect.DeepEqual(row["tag_feas"], want) {
+ t.Errorf("empty tag_feas = %#v, want %#v", row["tag_feas"], want)
+ }
+ row = map[string]interface{}{"tag_feas": nil}
+ decodeVarcharJSONFields(row)
+ if want := map[string]interface{}{}; !reflect.DeepEqual(row["tag_feas"], want) {
+ t.Errorf("nil tag_feas = %#v, want %#v", row["tag_feas"], want)
+ }
+
+ decoded := map[string]interface{}{"tag": float64(9)}
+ row = map[string]interface{}{"tag_feas": decoded}
+ decodeVarcharJSONFields(row)
+ if !reflect.DeepEqual(row["tag_feas"], decoded) {
+ t.Errorf("pre-decoded tag_feas = %#v, want %#v", row["tag_feas"], decoded)
+ }
+
+ // pagerank_fea ends in "_fea", not "_feas": it must stay untouched.
+ ranked := map[string]interface{}{"pagerank_fea": int64(7), "tag_feas": `{"t":1}`}
+ decodeVarcharJSONFields(ranked)
+ if want := int64(7); ranked["pagerank_fea"] != want {
+ t.Errorf("pagerank_fea = %#v, want %#v", ranked["pagerank_fea"], want)
+ }
+ if _, ok := ranked["tag_feas"].(map[string]interface{}); !ok {
+ t.Errorf("tag_feas = %#v, want a decoded map", ranked["tag_feas"])
+ }
+}
+
// TestTransformChunkFieldsKeepsTitleColumn pins that title_kwd survives as its
// own column. The compiled wiki/nav rows carry their label only in title_kwd and
// nav keys every cluster update on it (title_kwd = <cluster name>); folding the
diff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.go
index 376f9d6916..344c002e51 100644
--- a/internal/engine/infinity/common.go
+++ b/internal/engine/infinity/common.go
@@ -229,6 +229,18 @@ func fieldJSONList(fieldName string) bool {
}
}
+// fieldVarcharJSON reports varchar columns that hold JSON text: the write path
+// json-encodes their values and readers must parse them back. They are not
+// json-typed columns, so fieldJSON does not cover them.
+func fieldVarcharJSON(fieldName string) bool {
+ switch strings.ToLower(fieldName) {
+ case "tag_feas":
+ return true
+ default:
+ return false
+ }
+}
+
// joinBalanced renders a boolean chain as a balanced tree instead of the flat,
// left-deep chain strings.Join produces. Infinity's filter string is parsed by
// the SDK into a Thrift expression tree (infinity-go-sdk expression_parser.go)