Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunk): persist tag_kwd on Go UpdateChunk (#20138) (#20536)

Xavierandob33b458b6113
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index 83f9035d09..94088db6a0 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -1514,9 +1514,30 @@ func (e *Engine) GetChunk(ctx context.Context, tableName, chunkID string, datase
chunk["position_int"] = []interface{}{}
}
+ if val, ok := chunk["tag_kwd"].(string); ok {
+ chunk["tag_kwd"], _ = splitKeywordList("tag_kwd", val)
+ }
+
return chunk, nil
}
+// splitKeywordList reads a "###"-joined keyword column back as a list.
+// transformChunkFields always joins tag_kwd, so it reads back as a list even
+// with one tag or none; any other keyword column is split only when it holds
+// several values, since most of them are scalar.
+func splitKeywordList(field, value string) ([]interface{}, bool) {
+ if field != "tag_kwd" && !strings.Contains(value, "###") {
+ return nil, false
+ }
+ list := []interface{}{}
+ for part := range strings.SplitSeq(value, "###") {
+ if part != "" {
+ list = append(list, part)
+ }
+ }
+ return list, true
+}
+
// applyFieldMappings applies field mappings to chunks (side-effect only).
// Used by Search() to mutate chunks with derived fields before returning.
func applyFieldMappings(chunks []map[string]interface{}) {
@@ -1605,16 +1626,10 @@ func applyFieldMappings(chunks []map[string]interface{}) {
if !ok || val == nil || val == "" {
chunk[colName] = []interface{}{}
} else if !kwdNoSplit[colName] {
- // Split by "###" for _kwd fields
- if strVal, ok := val.(string); ok && strings.Contains(strVal, "###") {
- parts := strings.Split(strVal, "###")
- var filtered []interface{}
- for _, p := range parts {
- if p != "" {
- filtered = append(filtered, p)
- }
+ if strVal, ok := val.(string); ok {
+ if list, split := splitKeywordList(colName, strVal); split {
+ chunk[colName] = list
}
- chunk[colName] = filtered
}
}
}
@@ -1825,15 +1840,10 @@ func (e *Engine) GetFields(chunks []map[string]interface{}, fields []string) map
needsSplit = true
}
if needsSplit {
- if strVal, ok := val.(string); ok && strings.Contains(strVal, "###") {
- parts := strings.Split(strVal, "###")
- var filtered []interface{}
- for _, p := range parts {
- if p != "" {
- filtered = append(filtered, p)
- }
+ if strVal, ok := val.(string); ok {
+ if list, split := splitKeywordList(fieldLower, strVal); split {
+ chunk[field] = list
}
- chunk[field] = filtered
}
continue
}
diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.go
index 4febde65ab..5b2d13f54a 100644
--- a/internal/engine/infinity/chunk_transform_test.go
+++ b/internal/engine/infinity/chunk_transform_test.go
@@ -790,3 +790,48 @@ func TestRealignJSONColumnCells(t *testing.T) {
t.Errorf("non-JSON segments must be left untouched, got %#v", notJSON[0])
}
}
+
+// TestTagKwdRoundTripsAsList pins that tag_kwd, which the write path always
+// stores "###"-joined, reads back as a list on every read path even when it
+// holds a single tag or none, while scalar keyword columns stay strings.
+func TestTagKwdRoundTripsAsList(t *testing.T) {
+ cases := []struct {
+ name string
+ tags []string
+ want []interface{}
+ }{
+ {"single tag", []string{"alpha"}, []interface{}{"alpha"}},
+ {"several tags", []string{"alpha", "beta"}, []interface{}{"alpha", "beta"}},
+ {"no tags", []string{}, []interface{}{}},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ stored := transformChunkFields(map[string]interface{}{"tag_kwd": tc.tags, "doc_type_kwd": "pdf"}, nil)
+
+ searched := []map[string]interface{}{{"tag_kwd": stored["tag_kwd"], "doc_type_kwd": stored["doc_type_kwd"]}}
+ applyFieldMappings(searched)
+ if !reflect.DeepEqual(searched[0]["tag_kwd"], tc.want) {
+ t.Fatalf("search tag_kwd = %#v, want %#v", searched[0]["tag_kwd"], tc.want)
+ }
+
+ fields := (&Engine{}).GetFields(
+ []map[string]interface{}{{"id": "c1", "tag_kwd": stored["tag_kwd"], "doc_type_kwd": stored["doc_type_kwd"]}},
+ []string{"tag_kwd", "doc_type_kwd"},
+ )
+ if !reflect.DeepEqual(fields["c1"]["tag_kwd"], tc.want) {
+ t.Fatalf("get fields tag_kwd = %#v, want %#v", fields["c1"]["tag_kwd"], tc.want)
+ }
+ if fields["c1"]["doc_type_kwd"] != "pdf" {
+ t.Fatalf("get fields doc_type_kwd = %#v, want scalar \"pdf\"", fields["c1"]["doc_type_kwd"])
+ }
+
+ tagKwd, _ := stored["tag_kwd"].(string)
+ if got, ok := splitKeywordList("tag_kwd", tagKwd); !ok || !reflect.DeepEqual(got, tc.want) {
+ t.Fatalf("get chunk tag_kwd = %#v (split %v), want %#v", got, ok, tc.want)
+ }
+ if _, ok := splitKeywordList("doc_type_kwd", "pdf"); ok {
+ t.Fatal("a single-valued scalar keyword column must stay a string")
+ }
+ })
+ }
+}
diff --git a/internal/handler/chunk.go b/internal/handler/chunk.go
index f63f41d9cc..f2e46c0dfb 100644
--- a/internal/handler/chunk.go
+++ b/internal/handler/chunk.go
@@ -20,6 +20,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "io"
"net/http"
"ragflow/internal/common"
"strconv"
@@ -535,11 +536,19 @@ func (h *ChunkHandler) UpdateChunk(c *gin.Context) {
}
// Validate allowed update fields and get IDs from body
+ body, err := io.ReadAll(c.Request.Body)
var rawBody map[string]interface{}
- if err := json.NewDecoder(c.Request.Body).Decode(&rawBody); err != nil {
+ if err == nil {
+ err = json.Unmarshal(body, &rawBody)
+ }
+ if err != nil {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, common.CodeArgumentError, nil, "invalid JSON body: "+err.Error())
return
}
+ // The same body, undecoded, for the list validator shared with AddChunk.
+ // It parsed above, so it cannot fail here.
+ var rawFields map[string]json.RawMessage
+ _ = json.Unmarshal(body, &rawFields)
// Get required ID fields
datasetID := strings.TrimSpace(c.Param("dataset_id"))
@@ -570,17 +579,12 @@ func (h *ChunkHandler) UpdateChunk(c *gin.Context) {
"positions": true,
"image_base64": true,
"image_update_mode": true,
- // Accepted but never persisted: the UI always sends tag_kwd, so
- // dropping it here would 400 every chunk edit. Go's tagger writes
- // tag_feas; tag_kwd belongs to a Python tag dataset
- // (rag/app/tag.py::beAdoc), which Go never builds. Left out of the
- // error text below so it does not read as updatable.
- "tag_kwd": true,
- "tag_feas": true,
+ "tag_kwd": true,
+ "tag_feas": true,
}
for field := range rawBody {
if field != "dataset_id" && field != "document_id" && field != "chunk_id" && !allowedFields[field] {
- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Update field '"+field+"' is not supported. Updatable fields: content, important_keywords, questions, available, positions, image_base64, image_update_mode, tag_feas")
+ common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Update field '"+field+"' is not supported. Updatable fields: content, important_keywords, questions, available, positions, image_base64, image_update_mode, tag_kwd, tag_feas")
return
}
}
@@ -612,6 +616,12 @@ func (h *ChunkHandler) UpdateChunk(c *gin.Context) {
if positions, ok := rawBody["positions"].([]interface{}); ok {
req.Positions = positions
}
+ tagKwd, err := addChunkStringListField(rawFields, "tag_kwd", "`tag_kwd` is required to be a list", "`tag_kwd` must be a list of strings")
+ if err != nil {
+ common.ResponseWithHttpCodeData(c, http.StatusBadRequest, common.CodeDataError, nil, err.Error())
+ return
+ }
+ req.TagKwd = tagKwd
req.TagFeas = rawBody["tag_feas"]
imageBase64, err := optionalBodyString(rawBody, "image_base64", "`image_base64` must be a non-empty string")
if err != nil {
diff --git a/internal/handler/chunk_test.go b/internal/handler/chunk_test.go
index 9e285a2f6d..eb008babd9 100644
--- a/internal/handler/chunk_test.go
+++ b/internal/handler/chunk_test.go
@@ -383,6 +383,89 @@ func TestChunkHandlerUpdateChunkRejectsNonStringImageFields(t *testing.T) {
}
}
+func TestChunkHandlerUpdateChunkForwardsTagKwd(t *testing.T) {
+ tests := []struct {
+ name string
+ body string
+ want []string
+ }{
+ {"tags", `{"tag_kwd":["alpha","beta"]}`, []string{"alpha", "beta"}},
+ {"empty list clears tags", `{"tag_kwd":[]}`, []string{}},
+ {"absent", `{"content":"updated"}`, nil},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ mock := &mockChunkSvc{}
+ r, h := setupChunkHandlerWithUser("user-1", mock)
+ r.PATCH("/api/v1/datasets/:dataset_id/documents/:document_id/chunks/:chunk_id", h.UpdateChunk)
+
+ called := false
+ mock.updateChunkFn = func(ctx context.Context, req *service.UpdateChunkRequest, userID string) error {
+ called = true
+ if !reflect.DeepEqual(req.TagKwd, tc.want) {
+ t.Fatalf("tag_kwd = %#v, want %#v", req.TagKwd, tc.want)
+ }
+ return nil
+ }
+
+ req := httptest.NewRequest(http.MethodPatch, "/api/v1/datasets/kb-1/documents/doc-1/chunks/chunk-1", strings.NewReader(tc.body))
+ req.Header.Set("Content-Type", "application/json")
+ w := httptest.NewRecorder()
+ r.ServeHTTP(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
+ }
+ if !called {
+ t.Fatal("service was not called")
+ }
+ })
+ }
+}
+
+func TestChunkHandlerUpdateChunkRejectsInvalidTagKwd(t *testing.T) {
+ cases := []struct {
+ name string
+ body string
+ want string
+ }{
+ {"string", `{"tag_kwd":"alpha"}`, "`tag_kwd` is required to be a list"},
+ {"object", `{"tag_kwd":{}}`, "`tag_kwd` is required to be a list"},
+ {"non-string item", `{"tag_kwd":["alpha",1]}`, "`tag_kwd` must be a list of strings"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ mock := &mockChunkSvc{}
+ r, h := setupChunkHandlerWithUser("user-1", mock)
+ r.PATCH("/api/v1/datasets/:dataset_id/documents/:document_id/chunks/:chunk_id", h.UpdateChunk)
+
+ mock.updateChunkFn = func(context.Context, *service.UpdateChunkRequest, string) error {
+ t.Fatal("service must not be called for a malformed tag_kwd")
+ return nil
+ }
+
+ req := httptest.NewRequest(http.MethodPatch, "/api/v1/datasets/kb-1/documents/doc-1/chunks/chunk-1", strings.NewReader(tc.body))
+ req.Header.Set("Content-Type", "application/json")
+ w := httptest.NewRecorder()
+ r.ServeHTTP(w, req)
+
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
+ }
+ var resp map[string]interface{}
+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+ if resp["message"] != tc.want {
+ t.Fatalf("message = %v, want %q", resp["message"], tc.want)
+ }
+ if resp["code"] != float64(common.CodeDataError) {
+ t.Fatalf("code = %v, want %d", resp["code"], common.CodeDataError)
+ }
+ })
+ }
+}
+
func TestChunkHandlerUpdateChunkStillRejectsUnknownFields(t *testing.T) {
mock := &mockChunkSvc{}
r, h := setupChunkHandlerWithUser("user-1", mock)
@@ -406,7 +489,7 @@ func TestChunkHandlerUpdateChunkStillRejectsUnknownFields(t *testing.T) {
t.Fatalf("decode response: %v", err)
}
message, _ := resp["message"].(string)
- for _, field := range []string{"image_base64", "image_update_mode"} {
+ for _, field := range []string{"image_base64", "image_update_mode", "tag_kwd"} {
if !strings.Contains(message, field) {
t.Fatalf("message = %q, want it to list %s as updatable", message, field)
}
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index bf73d41da2..56b5e58508 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -1191,6 +1191,10 @@ func (s *ChunkService) UpdateChunk(ctx context.Context, req *service.UpdateChunk
d["position_int"] = req.Positions
}
+ if req.TagKwd != nil {
+ d["tag_kwd"] = req.TagKwd
+ }
+
// Tag features
if req.TagFeas != nil {
tagFeas, err := validateTagFeatures(req.TagFeas)
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index 79a10feb6e..a4fddd8055 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -753,6 +753,61 @@ func TestUpdateChunkUpdatesSameDocumentWithDocumentCondition(t *testing.T) {
}
}
+func TestUpdateChunkStoresTagKwdOnlyWhenSent(t *testing.T) {
+ tests := []struct {
+ name string
+ tagKwd []string
+ want []string
+ present bool
+ }{
+ {name: "tags sent", tagKwd: []string{"alpha", "beta"}, want: []string{"alpha", "beta"}, present: true},
+ {name: "empty list clears tags", tagKwd: []string{}, want: []string{}, present: true},
+ {name: "absent leaves tags untouched", tagKwd: nil, present: false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ db := setupChunkTestDB(t)
+ pushChunkTestDB(t, db)
+
+ insertChunkTestUserTenant(t, "user-1", "tenant-1")
+ insertChunkTestKB(t, "kb-1", "tenant-1")
+ insertChunkTestDoc(t, "doc-a", "kb-1")
+
+ engine := &updateChunkTestEngine{
+ existingChunk: map[string]interface{}{
+ "doc_id": "doc-a",
+ "content_with_weight": "existing content",
+ },
+ }
+ svc := &ChunkService{
+ docEngine: engine,
+ kbDAO: dao.NewKnowledgebaseDAO(),
+ userTenantDAO: dao.NewUserTenantDAO(),
+ }
+
+ err := svc.UpdateChunk(t.Context(), &service.UpdateChunkRequest{
+ DatasetID: "kb-1",
+ DocumentID: "doc-a",
+ ChunkID: "chunk-1",
+ TagKwd: tt.tagKwd,
+ }, "user-1")
+ if err != nil {
+ t.Fatalf("UpdateChunk() error = %v", err)
+ }
+ if len(engine.updateCalls) != 1 {
+ t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))
+ }
+ got, ok := engine.updateCalls[0].newValue["tag_kwd"]
+ if ok != tt.present {
+ t.Fatalf("tag_kwd present = %v, want %v (newValue=%#v)", ok, tt.present, engine.updateCalls[0].newValue)
+ }
+ if tt.present && !reflect.DeepEqual(got, tt.want) {
+ t.Fatalf("tag_kwd = %#v, want %#v", got, tt.want)
+ }
+ })
+ }
+}
+
func TestUpdateChunkStoresImageAndFlagsImageChunk(t *testing.T) {
cases := []struct {
name string
diff --git a/internal/service/chunk_types.go b/internal/service/chunk_types.go
index 0e9164dd4c..8dee3a72a6 100644
--- a/internal/service/chunk_types.go
+++ b/internal/service/chunk_types.go
@@ -151,6 +151,7 @@ type UpdateChunkRequest struct {
Questions []string `json:"questions,omitempty"`
Available *bool `json:"available,omitempty"`
Positions []interface{} `json:"positions,omitempty"`
+ TagKwd []string `json:"tag_kwd,omitempty"`
TagFeas interface{} `json:"tag_feas,omitempty"`
// ImageBase64 and ImageUpdateMode replace or drop the chunk's stored image.
// ImageUpdateMode is one of append (default), replace, remove.