Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(api): persist tag_kwd on Go AddChunk (#20138) (#20146)

Harsh Kashyapd513bc2afa89
diff --git a/internal/handler/chunk.go b/internal/handler/chunk.go
index d18fd9510e..f63f41d9cc 100644
--- a/internal/handler/chunk.go
+++ b/internal/handler/chunk.go
@@ -725,6 +725,9 @@ func addChunkStringListField(rawBody map[string]json.RawMessage, field, listMess
if !ok {
return nil, nil
}
+ if string(raw) == "null" {
+ return nil, errors.New(listMessage)
+ }
var values []interface{}
if err := json.Unmarshal(raw, &values); err != nil {
return nil, errors.New(listMessage)
@@ -779,6 +782,11 @@ func (h *ChunkHandler) AddChunk(c *gin.Context) {
common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
return
}
+ tagKwd, err := addChunkStringListField(rawBody, "tag_kwd", "`tag_kwd` is required to be a list", "`tag_kwd` must be a list of strings")
+ if err != nil {
+ common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
+ return
+ }
imageBase64, err := addChunkStringPtrField(rawBody, "image_base64")
if err != nil {
common.ResponseWithCodeData(c, common.CodeArgumentError, nil, err.Error())
@@ -798,6 +806,7 @@ func (h *ChunkHandler) AddChunk(c *gin.Context) {
Content: content,
ImportantKeywords: importantKeywords,
Questions: questions,
+ TagKwd: tagKwd,
TagFeas: tagFeas,
ImageBase64: imageBase64,
}
diff --git a/internal/handler/chunk_test.go b/internal/handler/chunk_test.go
index 6367cb5654..9e285a2f6d 100644
--- a/internal/handler/chunk_test.go
+++ b/internal/handler/chunk_test.go
@@ -856,6 +856,21 @@ func TestChunkHandlerAddChunkValidatesListFields(t *testing.T) {
body: `{"content":"chunk body","important_keywords":{}}`,
wantMsg: "`important_keywords` is required to be a list",
},
+ {
+ name: "tag_kwd null",
+ body: `{"content":"chunk body","tag_kwd":null}`,
+ wantMsg: "`tag_kwd` is required to be a list",
+ },
+ {
+ name: "tag_kwd type",
+ body: `{"content":"chunk body","tag_kwd":{}}`,
+ wantMsg: "`tag_kwd` is required to be a list",
+ },
+ {
+ name: "tag_kwd element type",
+ body: `{"content":"chunk body","tag_kwd":["ok",42]}`,
+ wantMsg: "`tag_kwd` must be a list of strings",
+ },
}
for _, tt := range tests {
@@ -890,6 +905,75 @@ func TestChunkHandlerAddChunkValidatesListFields(t *testing.T) {
}
}
+// TestChunkHandlerAddChunkForwardsTagKwd pins the Go/Python parity contract
+// that callers can attach a tag list at write time and the field survives on
+// the service-side AddChunkRequest. Issue #20138 documented that the Go path
+// was silently dropping the field while the Python chunk API accepted it.
+func TestChunkHandlerAddChunkForwardsTagKwd(t *testing.T) {
+ cases := []struct {
+ name string
+ body string
+ wantTag []string
+ wantNil bool
+ }{
+ {
+ name: "tag_kwd omitted",
+ body: `{"content":"chunk body"}`,
+ wantNil: true,
+ },
+ {
+ name: "tag_kwd explicit empty list",
+ body: `{"content":"chunk body","tag_kwd":[]}`,
+ wantTag: []string{},
+ },
+ {
+ name: "tag_kwd with values",
+ body: `{"content":"chunk body","tag_kwd":["alpha","beta"]}`,
+ wantTag: []string{"alpha", "beta"},
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ var got *service.AddChunkRequest
+ mock := &mockChunkSvc{
+ addChunkFn: func(ctx context.Context, req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error) {
+ got = req
+ return &service.AddChunkResponse{Chunk: map[string]interface{}{"id": "chunk-1"}}, nil
+ },
+ }
+ gin.SetMode(gin.TestMode)
+ r := gin.New()
+ r.Use(func(c *gin.Context) {
+ c.Set("user", &entity.User{ID: "user1"})
+ })
+ h := &ChunkHandler{chunkService: mock}
+ r.POST("/api/v1/datasets/:dataset_id/documents/:document_id/chunks", h.AddChunk)
+
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("POST", "/api/v1/datasets/kb1/documents/doc1/chunks", strings.NewReader(tc.body))
+ req.Header.Set("Content-Type", "application/json")
+ r.ServeHTTP(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String())
+ }
+ if got == nil {
+ t.Fatal("service was not called")
+ }
+ if tc.wantNil {
+ if got.TagKwd != nil {
+ t.Fatalf("TagKwd = %v, want nil", got.TagKwd)
+ }
+ return
+ }
+ if !reflect.DeepEqual(got.TagKwd, tc.wantTag) {
+ t.Fatalf("TagKwd = %v, want %v", got.TagKwd, tc.wantTag)
+ }
+ })
+ }
+}
+
func TestChunkHandlerAddChunkHidesServerErrorDetails(t *testing.T) {
mock := &mockChunkSvc{
addChunkFn: func(ctx context.Context, req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error) {
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index 44837510e4..bf73d41da2 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -1418,6 +1418,13 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkReques
"docnm_kwd": docName,
"doc_id": req.DocumentID,
}
+ if req.TagKwd != nil {
+ // Mirror Python chunk_api.py: `tag_kwd` is only persisted when the
+ // caller actually supplied it. Omitting the field when absent avoids
+ // a Go/Python behavioural divergence on chunks that were not tagged
+ // at write time. See issue #20138.
+ chunkData["tag_kwd"] = req.TagKwd
+ }
if tagFeas != nil {
chunkData["tag_feas"] = tagFeas
}
@@ -1485,6 +1492,9 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkReques
"create_timestamp": chunkData["create_timestamp_flt"],
"create_time": chunkData["create_time"],
}
+ if tagKwd, ok := chunkData["tag_kwd"]; ok {
+ renamedChunk["tag_kwd"] = tagKwd
+ }
if imgID, ok := chunkData["img_id"]; ok {
renamedChunk["image_id"] = imgID
}
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index 9119de75bc..79a10feb6e 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -1092,6 +1092,122 @@ func TestAddChunkSuccess(t *testing.T) {
}
}
+// TestAddChunkPersistsTagKwdOnlyWhenProvided pins the Go/Python parity
+// contract that mirrors api/apps/restful_apis/chunk_api.py: the `tag_kwd`
+// field is only persisted on the inserted chunk when the caller actually
+// supplied it, and is echoed back in the response when present. Issue
+// #20138 documented that the Go path was silently dropping the field.
+func TestAddChunkPersistsTagKwdOnlyWhenProvided(t *testing.T) {
+ ctx := t.Context()
+ db := setupChunkTestDB(t)
+ pushChunkTestDB(t, db)
+ userID, datasetID, documentID := "user-1", "kb-1", "doc-1"
+ insertChunkTestKB(t, datasetID, userID)
+ insertChunkTestDoc(t, documentID, datasetID)
+
+ makeService := func() (*ChunkService, *addChunkTestEngine) {
+ engine := &addChunkTestEngine{}
+ svc := &ChunkService{
+ docEngine: engine,
+ kbDAO: dao.NewKnowledgebaseDAO(),
+ documentDAO: dao.NewDocumentDAO(),
+ accessibleFunc: func(datasetIDArg, userIDArg string) bool {
+ return datasetIDArg == datasetID && userIDArg == userID
+ },
+ getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
+ return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1"}, nil
+ },
+ getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
+ driver := &stubEmbeddingDriver{
+ embeddings: []models.EmbeddingData{
+ {Embedding: []float64{1, 2}},
+ {Embedding: []float64{3, 4}},
+ },
+ }
+ modelName := "embed-1"
+ return models.NewEmbeddingModel(driver, &modelName, &models.APIConfig{}, 0), nil
+ },
+ incrementChunkStatsFunc: func(_, _ string, _, _ int64, _ float64) error { return nil },
+ tokenizeFunc: func(text string) (string, error) { return text, nil },
+ fineGrainedTokenizeFunc: func(text string) (string, error) { return text + "_fg", nil },
+ numTokensFunc: func(text string) int { return len(text) },
+ }
+ return svc, engine
+ }
+
+ t.Run("omitted by caller", func(t *testing.T) {
+ svc, engine := makeService()
+ resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
+ DatasetID: datasetID,
+ DocumentID: documentID,
+ Content: "chunk body",
+ }, userID)
+ if err != nil {
+ t.Fatalf("AddChunk() error = %v", err)
+ }
+ if len(engine.insertedChunks) != 1 {
+ t.Fatalf("inserted chunks = %d, want 1", len(engine.insertedChunks))
+ }
+ if _, ok := engine.insertedChunks[0]["tag_kwd"]; ok {
+ t.Fatalf("expected tag_kwd to be absent when caller omits it; got %#v", engine.insertedChunks[0]["tag_kwd"])
+ }
+ if _, ok := resp.Chunk["tag_kwd"]; ok {
+ t.Fatalf("response should not include tag_kwd when caller omits it; got %#v", resp.Chunk["tag_kwd"])
+ }
+ })
+
+ t.Run("explicit empty list persists empty slice", func(t *testing.T) {
+ svc, engine := makeService()
+ resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
+ DatasetID: datasetID,
+ DocumentID: documentID,
+ Content: "chunk body",
+ TagKwd: []string{},
+ }, userID)
+ if err != nil {
+ t.Fatalf("AddChunk() error = %v", err)
+ }
+ got, ok := engine.insertedChunks[0]["tag_kwd"].([]string)
+ if !ok {
+ t.Fatalf("expected []string in tag_kwd, got %T", engine.insertedChunks[0]["tag_kwd"])
+ }
+ if len(got) != 0 {
+ t.Fatalf("expected empty tag_kwd, got %v", got)
+ }
+ if echoed, _ := resp.Chunk["tag_kwd"].([]string); len(echoed) != 0 {
+ t.Fatalf("response tag_kwd = %v, want empty", resp.Chunk["tag_kwd"])
+ }
+ })
+
+ t.Run("values are persisted and echoed", func(t *testing.T) {
+ svc, engine := makeService()
+ want := []string{"alpha", "beta"}
+ resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
+ DatasetID: datasetID,
+ DocumentID: documentID,
+ Content: "chunk body",
+ TagKwd: want,
+ }, userID)
+ if err != nil {
+ t.Fatalf("AddChunk() error = %v", err)
+ }
+ got, ok := engine.insertedChunks[0]["tag_kwd"].([]string)
+ if !ok {
+ t.Fatalf("expected []string in tag_kwd, got %T", engine.insertedChunks[0]["tag_kwd"])
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("inserted tag_kwd = %v, want %v", got, want)
+ }
+ echoed, ok := resp.Chunk["tag_kwd"].([]string)
+ if !ok {
+ t.Fatalf("response tag_kwd type = %T, want []string", resp.Chunk["tag_kwd"])
+ }
+ if !reflect.DeepEqual(echoed, want) {
+ t.Fatalf("response tag_kwd = %v, want %v", echoed, want)
+ }
+ })
+}
+
func TestAddChunkValidationErrors(t *testing.T) {
ctx := t.Context()
db := setupChunkTestDB(t)
diff --git a/internal/service/chunk_types.go b/internal/service/chunk_types.go
index 3c252ca5e2..0e9164dd4c 100644
--- a/internal/service/chunk_types.go
+++ b/internal/service/chunk_types.go
@@ -76,6 +76,7 @@ type AddChunkRequest struct {
Content string `json:"content"`
ImportantKeywords []string `json:"important_keywords,omitempty"`
Questions []string `json:"questions,omitempty"`
+ TagKwd []string `json:"tag_kwd,omitempty"`
TagFeas interface{} `json:"tag_feas,omitempty"`
ImageBase64 *string `json:"image_base64,omitempty"`
}