RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunk): persist tag_kwd on Go UpdateChunk (#20138) (#20536)
Xavierandob33b458b6113
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex 83f9035d09..94088db6a0 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -1514,9 +1514,30 @@ func (e *Engine) GetChunk(ctx context.Context, tableName, chunkID string, datasechunk["position_int"] = []interface{}{}}+ if val, ok := chunk["tag_kwd"].(string); ok {+ chunk["tag_kwd"], _ = splitKeywordList("tag_kwd", val)+ }+return chunk, nil}+// splitKeywordList reads a "###"-joined keyword column back as a list.+// transformChunkFields always joins tag_kwd, so it reads back as a list even+// with one tag or none; any other keyword column is split only when it holds+// several values, since most of them are scalar.+func splitKeywordList(field, value string) ([]interface{}, bool) {+ if field != "tag_kwd" && !strings.Contains(value, "###") {+ return nil, false+ }+ list := []interface{}{}+ for part := range strings.SplitSeq(value, "###") {+ if part != "" {+ list = append(list, part)+ }+ }+ return list, true+}+// applyFieldMappings applies field mappings to chunks (side-effect only).// Used by Search() to mutate chunks with derived fields before returning.func applyFieldMappings(chunks []map[string]interface{}) {@@ -1605,16 +1626,10 @@ func applyFieldMappings(chunks []map[string]interface{}) {if !ok || val == nil || val == "" {chunk[colName] = []interface{}{}} else if !kwdNoSplit[colName] {- // Split by "###" for _kwd fields- if strVal, ok := val.(string); ok && strings.Contains(strVal, "###") {- parts := strings.Split(strVal, "###")- var filtered []interface{}- for _, p := range parts {- if p != "" {- filtered = append(filtered, p)- }+ if strVal, ok := val.(string); ok {+ if list, split := splitKeywordList(colName, strVal); split {+ chunk[colName] = list}- chunk[colName] = filtered}}}@@ -1825,15 +1840,10 @@ func (e *Engine) GetFields(chunks []map[string]interface{}, fields []string) mapneedsSplit = true}if needsSplit {- if strVal, ok := val.(string); ok && strings.Contains(strVal, "###") {- parts := strings.Split(strVal, "###")- var filtered []interface{}- for _, p := range parts {- if p != "" {- filtered = append(filtered, p)- }+ if strVal, ok := val.(string); ok {+ if list, split := splitKeywordList(fieldLower, strVal); split {+ chunk[field] = list}- chunk[field] = filtered}continue}diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.goindex 4febde65ab..5b2d13f54a 100644--- a/internal/engine/infinity/chunk_transform_test.go+++ b/internal/engine/infinity/chunk_transform_test.go@@ -790,3 +790,48 @@ func TestRealignJSONColumnCells(t *testing.T) {t.Errorf("non-JSON segments must be left untouched, got %#v", notJSON[0])}}++// TestTagKwdRoundTripsAsList pins that tag_kwd, which the write path always+// stores "###"-joined, reads back as a list on every read path even when it+// holds a single tag or none, while scalar keyword columns stay strings.+func TestTagKwdRoundTripsAsList(t *testing.T) {+ cases := []struct {+ name string+ tags []string+ want []interface{}+ }{+ {"single tag", []string{"alpha"}, []interface{}{"alpha"}},+ {"several tags", []string{"alpha", "beta"}, []interface{}{"alpha", "beta"}},+ {"no tags", []string{}, []interface{}{}},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ stored := transformChunkFields(map[string]interface{}{"tag_kwd": tc.tags, "doc_type_kwd": "pdf"}, nil)++ searched := []map[string]interface{}{{"tag_kwd": stored["tag_kwd"], "doc_type_kwd": stored["doc_type_kwd"]}}+ applyFieldMappings(searched)+ if !reflect.DeepEqual(searched[0]["tag_kwd"], tc.want) {+ t.Fatalf("search tag_kwd = %#v, want %#v", searched[0]["tag_kwd"], tc.want)+ }++ fields := (&Engine{}).GetFields(+ []map[string]interface{}{{"id": "c1", "tag_kwd": stored["tag_kwd"], "doc_type_kwd": stored["doc_type_kwd"]}},+ []string{"tag_kwd", "doc_type_kwd"},+ )+ if !reflect.DeepEqual(fields["c1"]["tag_kwd"], tc.want) {+ t.Fatalf("get fields tag_kwd = %#v, want %#v", fields["c1"]["tag_kwd"], tc.want)+ }+ if fields["c1"]["doc_type_kwd"] != "pdf" {+ t.Fatalf("get fields doc_type_kwd = %#v, want scalar \"pdf\"", fields["c1"]["doc_type_kwd"])+ }++ tagKwd, _ := stored["tag_kwd"].(string)+ if got, ok := splitKeywordList("tag_kwd", tagKwd); !ok || !reflect.DeepEqual(got, tc.want) {+ t.Fatalf("get chunk tag_kwd = %#v (split %v), want %#v", got, ok, tc.want)+ }+ if _, ok := splitKeywordList("doc_type_kwd", "pdf"); ok {+ t.Fatal("a single-valued scalar keyword column must stay a string")+ }+ })+ }+}diff --git a/internal/handler/chunk.go b/internal/handler/chunk.goindex f63f41d9cc..f2e46c0dfb 100644--- a/internal/handler/chunk.go+++ b/internal/handler/chunk.go@@ -20,6 +20,7 @@ import ("encoding/json""errors""fmt"+ "io""net/http""ragflow/internal/common""strconv"@@ -535,11 +536,19 @@ func (h *ChunkHandler) UpdateChunk(c *gin.Context) {}// Validate allowed update fields and get IDs from body+ body, err := io.ReadAll(c.Request.Body)var rawBody map[string]interface{}- if err := json.NewDecoder(c.Request.Body).Decode(&rawBody); err != nil {+ if err == nil {+ err = json.Unmarshal(body, &rawBody)+ }+ if err != nil {common.ResponseWithHttpCodeData(c, http.StatusBadRequest, common.CodeArgumentError, nil, "invalid JSON body: "+err.Error())return}+ // The same body, undecoded, for the list validator shared with AddChunk.+ // It parsed above, so it cannot fail here.+ var rawFields map[string]json.RawMessage+ _ = json.Unmarshal(body, &rawFields)// Get required ID fieldsdatasetID := strings.TrimSpace(c.Param("dataset_id"))@@ -570,17 +579,12 @@ func (h *ChunkHandler) UpdateChunk(c *gin.Context) {"positions": true,"image_base64": true,"image_update_mode": true,- // Accepted but never persisted: the UI always sends tag_kwd, so- // dropping it here would 400 every chunk edit. Go's tagger writes- // tag_feas; tag_kwd belongs to a Python tag dataset- // (rag/app/tag.py::beAdoc), which Go never builds. Left out of the- // error text below so it does not read as updatable.- "tag_kwd": true,- "tag_feas": true,+ "tag_kwd": true,+ "tag_feas": true,}for field := range rawBody {if field != "dataset_id" && field != "document_id" && field != "chunk_id" && !allowedFields[field] {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Update field '"+field+"' is not supported. Updatable fields: content, important_keywords, questions, available, positions, image_base64, image_update_mode, tag_feas")+ common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Update field '"+field+"' is not supported. Updatable fields: content, important_keywords, questions, available, positions, image_base64, image_update_mode, tag_kwd, tag_feas")return}}@@ -612,6 +616,12 @@ func (h *ChunkHandler) UpdateChunk(c *gin.Context) {if positions, ok := rawBody["positions"].([]interface{}); ok {req.Positions = positions}+ tagKwd, err := addChunkStringListField(rawFields, "tag_kwd", "`tag_kwd` is required to be a list", "`tag_kwd` must be a list of strings")+ if err != nil {+ common.ResponseWithHttpCodeData(c, http.StatusBadRequest, common.CodeDataError, nil, err.Error())+ return+ }+ req.TagKwd = tagKwdreq.TagFeas = rawBody["tag_feas"]imageBase64, err := optionalBodyString(rawBody, "image_base64", "`image_base64` must be a non-empty string")if err != nil {diff --git a/internal/handler/chunk_test.go b/internal/handler/chunk_test.goindex 9e285a2f6d..eb008babd9 100644--- a/internal/handler/chunk_test.go+++ b/internal/handler/chunk_test.go@@ -383,6 +383,89 @@ func TestChunkHandlerUpdateChunkRejectsNonStringImageFields(t *testing.T) {}}+func TestChunkHandlerUpdateChunkForwardsTagKwd(t *testing.T) {+ tests := []struct {+ name string+ body string+ want []string+ }{+ {"tags", `{"tag_kwd":["alpha","beta"]}`, []string{"alpha", "beta"}},+ {"empty list clears tags", `{"tag_kwd":[]}`, []string{}},+ {"absent", `{"content":"updated"}`, nil},+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ mock := &mockChunkSvc{}+ r, h := setupChunkHandlerWithUser("user-1", mock)+ r.PATCH("/api/v1/datasets/:dataset_id/documents/:document_id/chunks/:chunk_id", h.UpdateChunk)++ called := false+ mock.updateChunkFn = func(ctx context.Context, req *service.UpdateChunkRequest, userID string) error {+ called = true+ if !reflect.DeepEqual(req.TagKwd, tc.want) {+ t.Fatalf("tag_kwd = %#v, want %#v", req.TagKwd, tc.want)+ }+ return nil+ }++ req := httptest.NewRequest(http.MethodPatch, "/api/v1/datasets/kb-1/documents/doc-1/chunks/chunk-1", strings.NewReader(tc.body))+ req.Header.Set("Content-Type", "application/json")+ w := httptest.NewRecorder()+ r.ServeHTTP(w, req)++ if w.Code != http.StatusOK {+ t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())+ }+ if !called {+ t.Fatal("service was not called")+ }+ })+ }+}++func TestChunkHandlerUpdateChunkRejectsInvalidTagKwd(t *testing.T) {+ cases := []struct {+ name string+ body string+ want string+ }{+ {"string", `{"tag_kwd":"alpha"}`, "`tag_kwd` is required to be a list"},+ {"object", `{"tag_kwd":{}}`, "`tag_kwd` is required to be a list"},+ {"non-string item", `{"tag_kwd":["alpha",1]}`, "`tag_kwd` must be a list of strings"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ mock := &mockChunkSvc{}+ r, h := setupChunkHandlerWithUser("user-1", mock)+ r.PATCH("/api/v1/datasets/:dataset_id/documents/:document_id/chunks/:chunk_id", h.UpdateChunk)++ mock.updateChunkFn = func(context.Context, *service.UpdateChunkRequest, string) error {+ t.Fatal("service must not be called for a malformed tag_kwd")+ return nil+ }++ req := httptest.NewRequest(http.MethodPatch, "/api/v1/datasets/kb-1/documents/doc-1/chunks/chunk-1", strings.NewReader(tc.body))+ req.Header.Set("Content-Type", "application/json")+ w := httptest.NewRecorder()+ r.ServeHTTP(w, req)++ if w.Code != http.StatusBadRequest {+ t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())+ }+ var resp map[string]interface{}+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {+ t.Fatalf("decode response: %v", err)+ }+ if resp["message"] != tc.want {+ t.Fatalf("message = %v, want %q", resp["message"], tc.want)+ }+ if resp["code"] != float64(common.CodeDataError) {+ t.Fatalf("code = %v, want %d", resp["code"], common.CodeDataError)+ }+ })+ }+}+func TestChunkHandlerUpdateChunkStillRejectsUnknownFields(t *testing.T) {mock := &mockChunkSvc{}r, h := setupChunkHandlerWithUser("user-1", mock)@@ -406,7 +489,7 @@ func TestChunkHandlerUpdateChunkStillRejectsUnknownFields(t *testing.T) {t.Fatalf("decode response: %v", err)}message, _ := resp["message"].(string)- for _, field := range []string{"image_base64", "image_update_mode"} {+ for _, field := range []string{"image_base64", "image_update_mode", "tag_kwd"} {if !strings.Contains(message, field) {t.Fatalf("message = %q, want it to list %s as updatable", message, field)}diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex bf73d41da2..56b5e58508 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -1191,6 +1191,10 @@ func (s *ChunkService) UpdateChunk(ctx context.Context, req *service.UpdateChunkd["position_int"] = req.Positions}+ if req.TagKwd != nil {+ d["tag_kwd"] = req.TagKwd+ }+// Tag featuresif req.TagFeas != nil {tagFeas, err := validateTagFeatures(req.TagFeas)diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.goindex 79a10feb6e..a4fddd8055 100644--- a/internal/service/chunk/chunk_test.go+++ b/internal/service/chunk/chunk_test.go@@ -753,6 +753,61 @@ func TestUpdateChunkUpdatesSameDocumentWithDocumentCondition(t *testing.T) {}}+func TestUpdateChunkStoresTagKwdOnlyWhenSent(t *testing.T) {+ tests := []struct {+ name string+ tagKwd []string+ want []string+ present bool+ }{+ {name: "tags sent", tagKwd: []string{"alpha", "beta"}, want: []string{"alpha", "beta"}, present: true},+ {name: "empty list clears tags", tagKwd: []string{}, want: []string{}, present: true},+ {name: "absent leaves tags untouched", tagKwd: nil, present: false},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ db := setupChunkTestDB(t)+ pushChunkTestDB(t, db)++ insertChunkTestUserTenant(t, "user-1", "tenant-1")+ insertChunkTestKB(t, "kb-1", "tenant-1")+ insertChunkTestDoc(t, "doc-a", "kb-1")++ engine := &updateChunkTestEngine{+ existingChunk: map[string]interface{}{+ "doc_id": "doc-a",+ "content_with_weight": "existing content",+ },+ }+ svc := &ChunkService{+ docEngine: engine,+ kbDAO: dao.NewKnowledgebaseDAO(),+ userTenantDAO: dao.NewUserTenantDAO(),+ }++ err := svc.UpdateChunk(t.Context(), &service.UpdateChunkRequest{+ DatasetID: "kb-1",+ DocumentID: "doc-a",+ ChunkID: "chunk-1",+ TagKwd: tt.tagKwd,+ }, "user-1")+ if err != nil {+ t.Fatalf("UpdateChunk() error = %v", err)+ }+ if len(engine.updateCalls) != 1 {+ t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))+ }+ got, ok := engine.updateCalls[0].newValue["tag_kwd"]+ if ok != tt.present {+ t.Fatalf("tag_kwd present = %v, want %v (newValue=%#v)", ok, tt.present, engine.updateCalls[0].newValue)+ }+ if tt.present && !reflect.DeepEqual(got, tt.want) {+ t.Fatalf("tag_kwd = %#v, want %#v", got, tt.want)+ }+ })+ }+}+func TestUpdateChunkStoresImageAndFlagsImageChunk(t *testing.T) {cases := []struct {name stringdiff --git a/internal/service/chunk_types.go b/internal/service/chunk_types.goindex 0e9164dd4c..8dee3a72a6 100644--- a/internal/service/chunk_types.go+++ b/internal/service/chunk_types.go@@ -151,6 +151,7 @@ type UpdateChunkRequest struct {Questions []string `json:"questions,omitempty"`Available *bool `json:"available,omitempty"`Positions []interface{} `json:"positions,omitempty"`+ TagKwd []string `json:"tag_kwd,omitempty"`TagFeas interface{} `json:"tag_feas,omitempty"`// ImageBase64 and ImageUpdateMode replace or drop the chunk's stored image.// ImageUpdateMode is one of append (default), replace, remove.