RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(infinity): decode tag_feas JSON text on chunk detail reads (#21779)
qinling0210d459d51912ad
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex 619fc1fe39..0ed34b4ca3 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -1928,6 +1928,7 @@ func (e *Engine) GetChunk(ctx context.Context, tableName, chunkID string, datasecommon.Debug("infinity get chunk", zap.String("chunkID", chunkID), zap.Any("tables", tableNames))decodeJSONFields(chunk)+ decodeVarcharJSONFields(chunk)// Memory rows come back under the message mapping's storage columns, which// the chunk field mappings below would mangle.@@ -3408,6 +3409,31 @@ func decodeJSONFields(chunk map[string]interface{}) {}}+// decodeVarcharJSONFields parses the JSON text transformChunkFields writes into+// fieldVarcharJSON columns back into a map; empty or nil becomes {}.+func decodeVarcharJSONFields(chunk map[string]interface{}) {+ for fieldName, value := range chunk {+ if !fieldVarcharJSON(fieldName) {+ continue+ }+ text, ok := value.(string)+ if !ok {+ if value == nil {+ chunk[fieldName] = map[string]interface{}{}+ }+ continue+ }+ if text == "" {+ chunk[fieldName] = map[string]interface{}{}+ continue+ }+ var decoded interface{}+ if err := json.Unmarshal([]byte(text), &decoded); err == nil {+ chunk[fieldName] = decoded+ }+ }+}+// splitLengthPrefixedSegments splits Infinity's `[int32 little-endian// length][payload]` framing into its payloads, each the JSON text of one value.// parseLengthPrefixedJSON (metadata.go) unpacks the same framing for the@@ -3837,7 +3863,7 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]interdefault:if fieldJSON(k) {d[k] = encodeJSONField(k, v)- } else if strings.HasSuffix(k, "_feas") {+ } else if fieldVarcharJSON(k) {jsonBytes, _ := json.Marshal(v)d[k] = string(jsonBytes)} else if fieldKeyword(k) {diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.goindex 7a689d62ee..6cfacdb00d 100644--- a/internal/engine/infinity/chunk_transform_test.go+++ b/internal/engine/infinity/chunk_transform_test.go@@ -300,6 +300,22 @@ func TestFieldJSONMatchesJSONMappingColumns(t *testing.T) {}}+// TestFieldVarcharJSONMatchesVarcharColumns keeps fieldVarcharJSON aligned with+// the schema: a column it decodes must exist as a varchar column, and it must+// not claim a json-typed column fieldJSON already owns.+func TestFieldVarcharJSONMatchesVarcharColumns(t *testing.T) {+ for _, column := range []string{"tag_feas"} {+ if !fieldVarcharJSON(column) {+ t.Errorf("fieldVarcharJSON(%q) = false, want true", column)+ }+ }+ for column, columnType := range mappingColumnTypes(t) {+ if fieldVarcharJSON(column) && columnType != "varchar" {+ t.Errorf("fieldVarcharJSON(%q) = true, but the mapping declares type %q", column, columnType)+ }+ }+}+// chunkMappingColumns reads the column set the chunk table is created from.func chunkMappingColumns(t *testing.T) map[string]bool {columns := mappingColumnTypes(t)@@ -505,6 +521,60 @@ func TestJSONColumnsMatchCaseInsensitively(t *testing.T) {}}+// TestDecodeVarcharJSONFieldsRoundTripsTagFeatures pins that the read-side+// decode of tag_feas yields the map the write path marshaled.+func TestDecodeVarcharJSONFieldsRoundTripsTagFeatures(t *testing.T) {+ tags := map[string]int{"物流查询": 7, "售后服务": 8}+ row := transformChunkFields(map[string]interface{}{"tag_feas": tags}, nil)+ if _, ok := row["tag_feas"].(string); !ok {+ t.Fatalf("write path should store tag_feas as JSON text, got %#v", row["tag_feas"])+ }++ decodeVarcharJSONFields(row)+ got, ok := row["tag_feas"].(map[string]interface{})+ if !ok {+ t.Fatalf("decodeVarcharJSONFields left tag_feas as %T, want map", row["tag_feas"])+ }+ if want := float64(7); got["物流查询"] != want {+ t.Errorf("tag_feas[物流查询] = %#v, want %#v", got["物流查询"], want)+ }+ if want := float64(8); got["售后服务"] != want {+ t.Errorf("tag_feas[售后服务] = %#v, want %#v", got["售后服务"], want)+ }+}++// TestDecodeVarcharJSONFieldsEmptyAndPreDecoded covers empty/nil text (→ {}),+// an already decoded value, and that pagerank_fea is left untouched.+func TestDecodeVarcharJSONFieldsEmptyAndPreDecoded(t *testing.T) {+ row := map[string]interface{}{"tag_feas": ""}+ decodeVarcharJSONFields(row)+ if want := map[string]interface{}{}; !reflect.DeepEqual(row["tag_feas"], want) {+ t.Errorf("empty tag_feas = %#v, want %#v", row["tag_feas"], want)+ }+ row = map[string]interface{}{"tag_feas": nil}+ decodeVarcharJSONFields(row)+ if want := map[string]interface{}{}; !reflect.DeepEqual(row["tag_feas"], want) {+ t.Errorf("nil tag_feas = %#v, want %#v", row["tag_feas"], want)+ }++ decoded := map[string]interface{}{"tag": float64(9)}+ row = map[string]interface{}{"tag_feas": decoded}+ decodeVarcharJSONFields(row)+ if !reflect.DeepEqual(row["tag_feas"], decoded) {+ t.Errorf("pre-decoded tag_feas = %#v, want %#v", row["tag_feas"], decoded)+ }++ // pagerank_fea ends in "_fea", not "_feas": it must stay untouched.+ ranked := map[string]interface{}{"pagerank_fea": int64(7), "tag_feas": `{"t":1}`}+ decodeVarcharJSONFields(ranked)+ if want := int64(7); ranked["pagerank_fea"] != want {+ t.Errorf("pagerank_fea = %#v, want %#v", ranked["pagerank_fea"], want)+ }+ if _, ok := ranked["tag_feas"].(map[string]interface{}); !ok {+ t.Errorf("tag_feas = %#v, want a decoded map", ranked["tag_feas"])+ }+}+// TestTransformChunkFieldsKeepsTitleColumn pins that title_kwd survives as its// own column. The compiled wiki/nav rows carry their label only in title_kwd and// nav keys every cluster update on it (title_kwd = <cluster name>); folding thediff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.goindex 376f9d6916..344c002e51 100644--- a/internal/engine/infinity/common.go+++ b/internal/engine/infinity/common.go@@ -229,6 +229,18 @@ func fieldJSONList(fieldName string) bool {}}+// fieldVarcharJSON reports varchar columns that hold JSON text: the write path+// json-encodes their values and readers must parse them back. They are not+// json-typed columns, so fieldJSON does not cover them.+func fieldVarcharJSON(fieldName string) bool {+ switch strings.ToLower(fieldName) {+ case "tag_feas":+ return true+ default:+ return false+ }+}+// joinBalanced renders a boolean chain as a balanced tree instead of the flat,// left-deep chain strings.Join produces. Infinity's filter string is parsed by// the SDK into a Thrift expression tree (infinity-go-sdk expression_parser.go)