Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: scope chunk deletion to route dataset and add Go API coverage (#21000)

buua4360c7adf2a8e75
diff --git a/internal/handler/chunk.go b/internal/handler/chunk.go
index 546f464eb5..f30832a683 100644
--- a/internal/handler/chunk.go
+++ b/internal/handler/chunk.go
@@ -569,10 +569,10 @@ func (h *ChunkHandler) RemoveChunks(c *gin.Context) {
return
}
- // Get document_id from URL path
+ datasetID := c.Param("dataset_id")
docID := c.Param("document_id")
- if docID == "" {
- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "document_id is required")
+ if datasetID == "" || docID == "" {
+ common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "dataset_id and document_id are required")
return
}
@@ -582,13 +582,9 @@ func (h *ChunkHandler) RemoveChunks(c *gin.Context) {
return
}
+ req.DatasetID = datasetID
req.DocID = docID
- if req.DocID == "" {
- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "doc_id is required")
- return
- }
-
ctx := c.Request.Context()
deletedCount, err := h.chunkService.RemoveChunks(ctx, &req, user.ID)
if err != nil {
diff --git a/internal/handler/chunk_test.go b/internal/handler/chunk_test.go
index b9791ec5aa..e312fc9795 100644
--- a/internal/handler/chunk_test.go
+++ b/internal/handler/chunk_test.go
@@ -12,6 +12,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/entity"
+ "ragflow/internal/permission"
"ragflow/internal/service"
"github.com/gin-gonic/gin"
@@ -25,6 +26,7 @@ type mockChunkSvc struct {
switchChunksFn func(ctx context.Context, userID, datasetID, documentID string, availableInt int, chunkIDs []string) error
updateChunkFn func(ctx context.Context, req *service.UpdateChunkRequest, userID string) error
stopParsingFn func(ctx context.Context, userID, datasetID string, req service.StopParsingRequest) (*service.StopParsingResponse, common.ErrorCode, error)
+ removeChunksFn func(context.Context, *service.RemoveChunksRequest, string) (int64, error)
}
type codedTestError struct {
@@ -61,9 +63,57 @@ func (m *mockChunkSvc) UpdateChunk(ctx context.Context, req *service.UpdateChunk
}
panic("not implemented")
}
-func (m *mockChunkSvc) RemoveChunks(context.Context, *service.RemoveChunksRequest, string) (int64, error) {
+func (m *mockChunkSvc) RemoveChunks(ctx context.Context, req *service.RemoveChunksRequest, userID string) (int64, error) {
+ if m.removeChunksFn != nil {
+ return m.removeChunksFn(ctx, req, userID)
+ }
panic("not implemented")
}
+
+func TestChunkRemoveUsesRouteIDs(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ err error
+ code int
+ }{
+ {name: "success", code: 0},
+ {name: "mismatched dataset", err: permission.ErrResourceNotFound, code: 404},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ called := false
+ mock := &mockChunkSvc{removeChunksFn: func(_ context.Context, req *service.RemoveChunksRequest, userID string) (int64, error) {
+ called = true
+ if req.DatasetID != "kb-1" || req.DocID != "doc-1" || userID != "user-1" {
+ t.Fatalf("unexpected delete target: %#v, user %q", req, userID)
+ }
+ if !reflect.DeepEqual(req.ChunkIDs, []string{"chunk-1"}) {
+ t.Fatalf("unexpected chunk IDs: %v", req.ChunkIDs)
+ }
+ return 1, tc.err
+ }}
+ r, h := setupChunkHandlerWithUser("user-1", mock)
+ r.DELETE("/api/v1/datasets/:dataset_id/documents/:document_id/chunks", h.RemoveChunks)
+ request := httptest.NewRequest(http.MethodDelete, "/api/v1/datasets/kb-1/documents/doc-1/chunks",
+ strings.NewReader(`{"dataset_id":"body-kb","doc_id":"body-doc","chunk_ids":["chunk-1"]}`))
+ request.Header.Set("Content-Type", "application/json")
+ response := httptest.NewRecorder()
+ r.ServeHTTP(response, request)
+ if !called {
+ t.Fatal("delete service was not called")
+ }
+ var payload map[string]interface{}
+ if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
+ t.Fatal(err)
+ }
+ if response.Code != http.StatusOK || payload["code"] != float64(tc.code) {
+ t.Fatalf("unexpected response: HTTP %d, %s", response.Code, response.Body.String())
+ }
+ if tc.err != nil && (payload["message"] != "Resource not found" || payload["data"] != nil) {
+ t.Fatalf("unexpected resource error: %s", response.Body.String())
+ }
+ })
+ }
+}
func (m *mockChunkSvc) StopParsing(ctx context.Context, userID, datasetID string, req service.StopParsingRequest) (*service.StopParsingResponse, common.ErrorCode, error) {
if m.stopParsingFn != nil {
return m.stopParsingFn(ctx, userID, datasetID, req)
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index c0b45fb063..232c90d62d 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -1262,16 +1262,19 @@ func (s *ChunkService) RemoveChunks(ctx context.Context, req *service.RemoveChun
return 0, fmt.Errorf("doc engine not initialized")
}
- if req.DocID == "" {
- return 0, fmt.Errorf("doc_id is required")
+ if req.DatasetID == "" || req.DocID == "" {
+ return 0, fmt.Errorf("dataset_id and doc_id are required")
}
- // Verify document exists and belongs to a dataset (do this first to get doc.KbID)
+ // Bind deletion to the dataset and document named in the route.
docDAO := dao.NewDocumentDAO()
doc, err := docDAO.GetByID(ctx, dao.DB, req.DocID)
if err != nil || doc == nil {
return 0, fmt.Errorf("document not found")
}
+ if doc.KbID != req.DatasetID {
+ return 0, permission.ErrResourceNotFound
+ }
kb, err := s.getAccessibleKnowledgebase(ctx, doc.KbID, userID, permission.OperationUpdate)
if err != nil {
diff --git a/internal/service/chunk/chunk_permissions_test.go b/internal/service/chunk/chunk_permissions_test.go
index 3588f91063..da42a48e4b 100644
--- a/internal/service/chunk/chunk_permissions_test.go
+++ b/internal/service/chunk/chunk_permissions_test.go
@@ -85,7 +85,7 @@ func TestChunkManagementDatasetPermissions(t *testing.T) {
}},
{"remove", func(s *ChunkService) error {
s.docEngine = &parseTestDocEngine{}
- _, err := s.RemoveChunks(ctx, &service.RemoveChunksRequest{DocID: "doc-1", DeleteAll: true}, tt.userID)
+ _, err := s.RemoveChunks(ctx, &service.RemoveChunksRequest{DatasetID: "kb-1", DocID: "doc-1", DeleteAll: true}, tt.userID)
return err
}},
} {
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index c19eed1212..cfcf963224 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -5,6 +5,7 @@ import (
"context"
"encoding/base64"
"errors"
+ "fmt"
"image"
"image/color"
_ "image/jpeg"
@@ -1635,6 +1636,44 @@ func TestChunkImageOperationsShareThePerChunkLock(t *testing.T) {
}
}
+func TestRemoveChunksRejectsInvalidDatasetBeforeMutation(t *testing.T) {
+ for _, datasetID := range []string{"", "kb-2"} {
+ for _, deleteAll := range []bool{false, true} {
+ t.Run(fmt.Sprintf("dataset=%s/delete_all=%t", datasetID, deleteAll), func(t *testing.T) {
+ db := setupChunkTestDB(t)
+ pushChunkTestDB(t, db)
+ insertChunkTestKB(t, "kb-1", "owner")
+ insertChunkTestDoc(t, "doc-1", "kb-1")
+ engine := &parseTestDocEngine{deleteChunksCount: 1}
+ svc := &ChunkService{
+ docEngine: engine,
+ decrementChunkStatsFunc: func(string, string, int64, int64, float64) error {
+ t.Fatal("rejected deletion must not change statistics")
+ return nil
+ },
+ markWikiDirtyFunc: func(string, string, string, []string) {
+ t.Fatal("rejected deletion must not schedule wiki updates")
+ },
+ }
+ req := &service.RemoveChunksRequest{DatasetID: datasetID, DocID: "doc-1", DeleteAll: deleteAll}
+ if !deleteAll {
+ req.ChunkIDs = []string{"chunk-1"}
+ }
+ count, err := svc.RemoveChunks(t.Context(), req, "owner")
+ if err == nil || count != 0 {
+ t.Fatalf("invalid dataset deletion = (%d, %v), want error and zero count", count, err)
+ }
+ if datasetID != "" && !errors.Is(err, permission.ErrResourceNotFound) {
+ t.Fatalf("error = %v, want resource not found", err)
+ }
+ if engine.deleteChunksCalls != 0 {
+ t.Fatalf("rejected deletion reached the engine %d times", engine.deleteChunksCalls)
+ }
+ })
+ }
+ }
+}
+
func TestRemoveChunksDecrementsStatsAfterDelete(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
@@ -1650,8 +1689,9 @@ func TestRemoveChunksDecrementsStatsAfterDelete(t *testing.T) {
}
ctx := t.Context()
deletedCount, err := svc.RemoveChunks(ctx, &service.RemoveChunksRequest{
- DocID: "doc-1",
- ChunkIDs: []string{"chunk-1", "chunk-2", "chunk-3"},
+ DatasetID: "kb-1",
+ DocID: "doc-1",
+ ChunkIDs: []string{"chunk-1", "chunk-2", "chunk-3"},
}, "user-1")
if err != nil {
t.Fatalf("RemoveChunks() error = %v", err)
@@ -1714,6 +1754,7 @@ func TestRemoveChunksSkipsStatsWhenNothingDeleted(t *testing.T) {
}
ctx := t.Context()
deletedCount, err := svc.RemoveChunks(ctx, &service.RemoveChunksRequest{
+ DatasetID: "kb-1",
DocID: "doc-1",
DeleteAll: true,
}, "user-1")
@@ -1747,8 +1788,9 @@ func TestRemoveChunksReturnsStatsError(t *testing.T) {
}
ctx := t.Context()
deletedCount, err := svc.RemoveChunks(ctx, &service.RemoveChunksRequest{
- DocID: "doc-1",
- ChunkIDs: []string{"chunk-1", "chunk-2"},
+ DatasetID: "kb-1",
+ DocID: "doc-1",
+ ChunkIDs: []string{"chunk-1", "chunk-2"},
}, "user-1")
if err == nil || !strings.Contains(err.Error(), "failed to update chunk stats") {
t.Fatalf("expected stats update error, got count=%d err=%v", deletedCount, err)
diff --git a/internal/service/chunk_types.go b/internal/service/chunk_types.go
index 8dee3a72a6..21300e9d9b 100644
--- a/internal/service/chunk_types.go
+++ b/internal/service/chunk_types.go
@@ -161,6 +161,7 @@ type UpdateChunkRequest struct {
// RemoveChunksRequest request for removing chunks
type RemoveChunksRequest struct {
+ DatasetID string `json:"-"`
DocID string `json:"doc_id"`
ChunkIDs []string `json:"chunk_ids,omitempty"`
DeleteAll bool `json:"delete_all,omitempty"`
diff --git a/test/api/chunk_api_test.go b/test/api/chunk_api_test.go
new file mode 100644
index 0000000000..39aa3fafc4
--- /dev/null
+++ b/test/api/chunk_api_test.go
@@ -0,0 +1,776 @@
+//go:build integration
+
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package api
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "mime/multipart"
+ "net/http"
+ "net/url"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+)
+
+type chunkAPIResult struct {
+ status int
+ url string
+ payload map[string]interface{}
+ body string
+ err error
+}
+
+var chunkHTTPClient = &http.Client{Timeout: 60 * time.Second}
+
+func requestChunkAPI(t *testing.T, client *ServerTestConfig, method, path string, body, params map[string]interface{}) chunkAPIResult {
+ t.Helper()
+
+ requestURL, err := chunkRequestURL(client, path, params)
+ if err != nil {
+ return chunkAPIResult{err: err}
+ }
+
+ var requestBody io.Reader
+ payloadText := "<empty>"
+ if body != nil {
+ encoded, err := json.Marshal(body)
+ if err != nil {
+ return chunkAPIResult{url: requestURL, err: err}
+ }
+ payloadText = string(encoded)
+ requestBody = bytes.NewReader(encoded)
+ }
+
+ t.Logf("request: %s %s\npayload: %s", method, requestURL, payloadText)
+ request, err := http.NewRequest(method, requestURL, requestBody)
+ if err != nil {
+ return chunkAPIResult{url: requestURL, err: err}
+ }
+ request.Header = client.buildHeaders(nil)
+
+ response, err := chunkHTTPClient.Do(request)
+ if err != nil {
+ return chunkAPIResult{url: requestURL, err: err}
+ }
+ return readChunkAPIResponse(t, requestURL, response)
+}
+
+func chunkRequestURL(client *ServerTestConfig, path string, params map[string]interface{}) (string, error) {
+ requestURL, err := url.Parse(client.apiBase() + "/" + strings.TrimLeft(path, "/"))
+ if err != nil {
+ return "", err
+ }
+ query := requestURL.Query()
+ for key, value := range params {
+ if value != nil {
+ query.Set(key, fmt.Sprint(value))
+ }
+ }
+ requestURL.RawQuery = query.Encode()
+ return requestURL.String(), nil
+}
+
+func readChunkAPIResponse(t *testing.T, requestURL string, response *http.Response) chunkAPIResult {
+ t.Helper()
+ defer response.Body.Close()
+
+ rawBody, err := io.ReadAll(response.Body)
+ if err != nil {
+ return chunkAPIResult{status: response.StatusCode, url: requestURL, err: err}
+ }
+ body := string(rawBody)
+ t.Logf("response: HTTP %d\nbody: %s", response.StatusCode, body)
+
+ var payload map[string]interface{}
+ if err := json.Unmarshal(rawBody, &payload); err != nil {
+ return chunkAPIResult{
+ status: response.StatusCode,
+ url: requestURL,
+ body: body,
+ err: fmt.Errorf("decode response JSON: %w", err),
+ }
+ }
+ return chunkAPIResult{status: response.StatusCode, url: requestURL, payload: payload, body: body}
+}
+
+func callChunkAPI(t *testing.T, client *ServerTestConfig, method, path string, body, params map[string]interface{}, wantStatus int) map[string]interface{} {
+ t.Helper()
+ result := requestChunkAPI(t, client, method, path, body, params)
+ if result.err != nil {
+ t.Fatalf("%s %s failed: %v; response body: %s", method, result.url, result.err, result.body)
+ }
+ if result.status != wantStatus {
+ t.Fatalf("%s %s status = %d, want %d; response: %s", method, result.url, result.status, wantStatus, result.body)
+ }
+ return result.payload
+}
+
+func requireChunkCode(t *testing.T, payload map[string]interface{}, want int) {
+ t.Helper()
+ code, ok := payload["code"].(float64)
+ if !ok || int(code) != want {
+ t.Fatalf("response code = %v, want %d; payload: %v", payload["code"], want, payload)
+ }
+}
+
+func requireChunkMessage(t *testing.T, payload map[string]interface{}, want string) {
+ t.Helper()
+ message, ok := payload["message"].(string)
+ if !ok || message != want {
+ t.Fatalf("response message = %q, want %q; payload: %v", message, want, payload)
+ }
+}
+
+func chunkData(t *testing.T, payload map[string]interface{}) map[string]interface{} {
+ t.Helper()
+ data, ok := payload["data"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("response data is not an object: %v", payload)
+ }
+ return data
+}
+
+func chunkString(t *testing.T, object map[string]interface{}, key string) string {
+ t.Helper()
+ value, ok := object[key].(string)
+ if !ok {
+ t.Fatalf("%s is not a string: %v", key, object[key])
+ }
+ return value
+}
+
+func chunkPath(datasetID, documentID string) string {
+ return fmt.Sprintf("/datasets/%s/documents/%s/chunks", datasetID, documentID)
+}
+
+func chunkClientWithToken(token string) *ServerTestConfig {
+ client := *TestConfig
+ client.AuthToken = token
+ client.APIKey = ""
+ return &client
+}
+
+func requireChunkAuthError(t *testing.T, client *ServerTestConfig, method, path string, body map[string]interface{}, scenario string) {
+ t.Helper()
+ payload := callChunkAPI(t, client, method, path, body, nil, http.StatusUnauthorized)
+ requireChunkCode(t, payload, http.StatusUnauthorized)
+
+ message := chunkString(t, payload, "message")
+ switch scenario {
+ case "missing token":
+ if message != "Missing Authorization header" {
+ t.Fatalf("missing-token message = %q", message)
+ }
+ case "invalid token":
+ if message != "Invalid access token" && message != "Invalid auth credentials" {
+ t.Fatalf("invalid-token message = %q", message)
+ }
+ default:
+ t.Fatalf("unknown auth scenario %q", scenario)
+ }
+}
+
+func createChunkTestDocument(t *testing.T, filename string) (string, string) {
+ t.Helper()
+ datasetName := fmt.Sprintf("chunk_api_%d", time.Now().UnixNano())
+ datasetPayload := callChunkAPI(t, TestConfig, http.MethodPost, "/datasets",
+ map[string]interface{}{"name": datasetName, "parse_type": 1, "parser_id": "general"}, nil, http.StatusOK)
+ requireChunkCode(t, datasetPayload, 0)
+ datasetID := chunkString(t, chunkData(t, datasetPayload), "id")
+
+ t.Cleanup(func() {
+ deleteDataset := requestChunkAPI(t, TestConfig, http.MethodDelete, "/datasets",
+ map[string]interface{}{"ids": []interface{}{datasetID}}, nil)
+ if deleteDataset.err != nil {
+ t.Errorf("delete test dataset failed: %v", deleteDataset.err)
+ } else if deleteDataset.status != http.StatusOK {
+ t.Errorf("delete test dataset status = %d, want %d; response: %s", deleteDataset.status, http.StatusOK, deleteDataset.body)
+ } else if code, ok := deleteDataset.payload["code"].(float64); !ok || code != 0 {
+ t.Errorf("delete test dataset returned unexpected response: %s", deleteDataset.body)
+ }
+ })
+ return datasetID, uploadChunkTestDocument(t, datasetID, filename)
+}
+
+func uploadChunkTestDocument(t *testing.T, datasetID, filename string) string {
+ t.Helper()
+
+ const fileContents = "ragflow test upload"
+ var uploadBody bytes.Buffer
+ writer := multipart.NewWriter(&uploadBody)
+ file, err := writer.CreateFormFile("file", filename)
+ if err != nil {
+ t.Fatalf("create multipart file field: %v", err)
+ }
+ if _, err := io.WriteString(file, fileContents); err != nil {
+ t.Fatalf("write multipart file: %v", err)
+ }
+ if err := writer.Close(); err != nil {
+ t.Fatalf("close multipart writer: %v", err)
+ }
+
+ requestURL := fmt.Sprintf("%s/datasets/%s/documents", TestConfig.apiBase(), datasetID)
+ t.Logf("request: %s %s\npayload: multipart file %q (%d bytes), content=%q",
+ http.MethodPost, requestURL, filename, len(fileContents), fileContents)
+ request, err := http.NewRequest(http.MethodPost, requestURL, &uploadBody)
+ if err != nil {
+ t.Fatalf("create document upload request: %v", err)
+ }
+ request.Header = TestConfig.buildHeaders(nil)
+ request.Header.Set("Content-Type", writer.FormDataContentType())
+ response, err := chunkHTTPClient.Do(request)
+ if err != nil {
+ t.Fatalf("upload test document: %v", err)
+ }
+ uploadResult := readChunkAPIResponse(t, requestURL, response)
+ if uploadResult.err != nil {
+ t.Fatalf("decode document upload response: %v; body: %s", uploadResult.err, uploadResult.body)
+ }
+ if uploadResult.status != http.StatusOK {
+ t.Fatalf("document upload status = %d, want %d; response: %s", uploadResult.status, http.StatusOK, uploadResult.body)
+ }
+ requireChunkCode(t, uploadResult.payload, 0)
+
+ items, ok := uploadResult.payload["data"].([]interface{})
+ if !ok || len(items) == 0 {
+ t.Fatalf("document upload data is not a non-empty array: %v", uploadResult.payload)
+ }
+ document, ok := items[0].(map[string]interface{})
+ if !ok {
+ t.Fatalf("document upload item is not an object: %v", items[0])
+ }
+ documentID := chunkString(t, document, "id")
+ t.Cleanup(func() {
+ deleted := requestChunkAPI(t, TestConfig, http.MethodDelete,
+ fmt.Sprintf("/datasets/%s/documents", datasetID),
+ map[string]interface{}{"ids": []string{documentID}}, nil)
+ if deleted.err != nil {
+ t.Errorf("delete test document failed: %v", deleted.err)
+ } else if deleted.status != http.StatusOK || deleted.payload["code"] != float64(0) {
+ t.Errorf("delete test document failed: HTTP %d, %s", deleted.status, deleted.body)
+ }
+ })
+ return documentID
+}
+
+func TestChunkAuthRequiresCredentials(t *testing.T) {
+ path := chunkPath(TestConfig.InvalidID, TestConfig.InvalidID)
+ chunkIDPath := path + "/" + TestConfig.InvalidID
+ requests := []struct {
+ name string
+ method string
+ path string
+ body map[string]interface{}
+ }{
+ {name: "add", method: http.MethodPost, path: path, body: map[string]interface{}{"content": "chunk"}},
+ {name: "list", method: http.MethodGet, path: path},
+ {name: "get", method: http.MethodGet, path: chunkIDPath},
+ {name: "update", method: http.MethodPatch, path: chunkIDPath, body: map[string]interface{}{"content": "chunk"}},
+ {name: "delete", method: http.MethodDelete, path: path, body: map[string]interface{}{"chunk_ids": []interface{}{}}},
+ }
+ scenarios := []struct {
+ name string
+ client *ServerTestConfig
+ }{
+ {name: "missing token", client: chunkClientWithToken("")},
+ {name: "invalid token", client: chunkClientWithToken(TestConfig.InvalidAPIKey)},
+ }
+
+ for _, request := range requests {
+ for _, scenario := range scenarios {
+ t.Run(request.name+"/"+scenario.name, func(t *testing.T) {
+ requireChunkAuthError(t, scenario.client, request.method, request.path, request.body, scenario.name)
+ })
+ }
+ }
+}
+
+func TestChunkListRejectsInvalidQueryValues(t *testing.T) {
+ path := chunkPath(TestConfig.InvalidID, TestConfig.InvalidID)
+ cases := []struct {
+ name string
+ key string
+ value string
+ message string
+ }{
+ {name: "page zero", key: "page", value: "0", message: "page must be a positive integer"},
+ {name: "page not numeric", key: "page", value: "first", message: "page must be a positive integer"},
+ {name: "page_size zero", key: "page_size", value: "0", message: "page_size must be a positive integer"},
+ {name: "page_size not numeric", key: "page_size", value: "many", message: "page_size must be a positive integer"},
+ {name: "available invalid", key: "available", value: "sometimes", message: "available must be one of: true, false, 1, 0"},
+ }
+
+ for _, testCase := range cases {
+ t.Run(testCase.name, func(t *testing.T) {
+ payload := callChunkAPI(t, TestConfig, http.MethodGet, path, nil,
+ map[string]interface{}{testCase.key: testCase.value}, http.StatusBadRequest)
+ requireChunkCode(t, payload, 101)
+ requireChunkMessage(t, payload, testCase.message)
+ })
+ }
+}
+
+func TestChunkAddRejectsMalformedPayloadFields(t *testing.T) {
+ path := chunkPath(TestConfig.InvalidID, TestConfig.InvalidID)
+ cases := []struct {
+ name string
+ body map[string]interface{}
+ code int
+ message string
+ }{
+ {
+ name: "content must be a string",
+ body: map[string]interface{}{"content": 42},
+ code: 101,
+ message: "`content` must be a string",
+ },
+ {
+ name: "important_keywords must be a list",
+ body: map[string]interface{}{"content": "chunk", "important_keywords": "keyword"},
+ code: 102,
+ message: "`important_keywords` is required to be a list",
+ },
+ {
+ name: "questions must contain strings",
+ body: map[string]interface{}{"content": "chunk", "questions": []interface{}{42}},
+ code: 102,
+ message: "`questions` must be a list of strings",
+ },
+ {
+ name: "tag_kwd cannot be null",
+ body: map[string]interface{}{"content": "chunk", "tag_kwd": nil},
+ code: 102,
+ message: "`tag_kwd` is required to be a list",
+ },
+ {
+ name: "image_base64 must be a string",
+ body: map[string]interface{}{"content": "chunk", "image_base64": 42},
+ code: 101,
+ message: "`image_base64` must be a string",
+ },
+ }
+
+ for _, testCase := range cases {
+ t.Run(testCase.name, func(t *testing.T) {
+ payload := callChunkAPI(t, TestConfig, http.MethodPost, path, testCase.body, nil, http.StatusOK)
+ requireChunkCode(t, payload, testCase.code)
+ requireChunkMessage(t, payload, testCase.message)
+ })
+ }
+}
+
+func TestChunkUpdateRejectsUnsupportedAndMalformedFields(t *testing.T) {
+ path := chunkPath(TestConfig.InvalidID, TestConfig.InvalidID) + "/" + TestConfig.InvalidID
+ cases := []struct {
+ name string
+ body map[string]interface{}
+ status int
+ code int
+ message string
+ messageHas string
+ }{
+ {
+ name: "unsupported field",
+ body: map[string]interface{}{"unknown_field": true},
+ status: http.StatusBadRequest,
+ code: http.StatusBadRequest,
+ messageHas: "Update field 'unknown_field' is not supported.",
+ },
+ {
+ name: "tag_kwd scalar is rejected",
+ body: map[string]interface{}{"tag_kwd": "tag"},
+ status: http.StatusBadRequest,
+ code: 102,
+ message: "`tag_kwd` is required to be a list",
+ },
+ {
+ name: "tag_kwd elements must be strings",
+ body: map[string]interface{}{"tag_kwd": []interface{}{"tag", 42}},
+ status: http.StatusBadRequest,
+ code: 102,
+ message: "`tag_kwd` must be a list of strings",
+ },
+ {
+ name: "image_base64 must be a string",
+ body: map[string]interface{}{"image_base64": 42},
+ status: http.StatusBadRequest,
+ code: 102,
+ message: "`image_base64` must be a non-empty string",
+ },
+ {
+ name: "image_update_mode must be a string",
+ body: map[string]interface{}{"image_update_mode": false},
+ status: http.StatusBadRequest,
+ code: 102,
+ message: "`image_update_mode` must be a string",
+ },
+ }
+
+ for _, testCase := range cases {
+ t.Run(testCase.name, func(t *testing.T) {
+ payload := callChunkAPI(t, TestConfig, http.MethodPatch, path, testCase.body, nil, testCase.status)
+ requireChunkCode(t, payload, testCase.code)
+ if testCase.messageHas != "" {
+ message := chunkString(t, payload, "message")
+ if !strings.HasPrefix(message, testCase.messageHas) {
+ t.Fatalf("response message = %q, want prefix %q", message, testCase.messageHas)
+ }
+ return
+ }
+ requireChunkMessage(t, payload, testCase.message)
+ })
+ }
+}
+
+func TestChunkAddRejectsMissingOrBlankContent(t *testing.T) {
+ datasetID, documentID := createChunkTestDocument(t, "chunk_missing_content.txt")
+ path := chunkPath(datasetID, documentID)
+ cases := []struct {
+ name string
+ body map[string]interface{}
+ }{
+ {name: "missing content", body: map[string]interface{}{}},
+ {name: "blank content", body: map[string]interface{}{"content": " \n "}},
+ }
+
+ for _, testCase := range cases {
+ t.Run(testCase.name, func(t *testing.T) {
+ payload := callChunkAPI(t, TestConfig, http.MethodPost, path, testCase.body, nil, http.StatusOK)
+ requireChunkCode(t, payload, 102)
+ requireChunkMessage(t, payload, "`content` is required")
+ })
+ }
+}
+
+func TestChunkAddRejectsInvalidDatasetAndDocument(t *testing.T) {
+ datasetID, documentID := createChunkTestDocument(t, "chunk_invalid_targets.txt")
+
+ t.Run("dataset does not exist", func(t *testing.T) {
+ payload := callChunkAPI(t, TestConfig, http.MethodPost,
+ chunkPath(TestConfig.InvalidID, documentID),
+ map[string]interface{}{"content": "chunk"}, nil, http.StatusOK)
+ requireChunkCode(t, payload, 404)
+ requireChunkMessage(t, payload, "Resource not found")
+ })
+
+ t.Run("document does not exist", func(t *testing.T) {
+ payload := callChunkAPI(t, TestConfig, http.MethodPost,
+ chunkPath(datasetID, TestConfig.InvalidID),
+ map[string]interface{}{"content": "chunk"}, nil, http.StatusOK)
+ requireChunkCode(t, payload, 102)
+ requireChunkMessage(t, payload, fmt.Sprintf("you don't own the document %s", TestConfig.InvalidID))
+ })
+
+ t.Run("document belongs to another dataset", func(t *testing.T) {
+ otherDatasetID, _ := createChunkTestDocument(t, "chunk_other_dataset.txt")
+ payload := callChunkAPI(t, TestConfig, http.MethodPost,
+ chunkPath(otherDatasetID, documentID),
+ map[string]interface{}{"content": "chunk"}, nil, http.StatusOK)
+ requireChunkCode(t, payload, 102)
+ requireChunkMessage(t, payload, fmt.Sprintf("you don't own the document %s", documentID))
+ })
+}
+
+func TestChunkLifecycle(t *testing.T) {
+ datasetID, documentID := createChunkTestDocument(t, "chunk_lifecycle.txt")
+ path := chunkPath(datasetID, documentID)
+ var ids []string
+ for _, content := range []string{
+ "Oranges are citrus fruits.", "Bicycles have two wheels.", "Galaxies contain billions of stars.",
+ } {
+ added := callChunkAPI(t, TestConfig, http.MethodPost, path, map[string]interface{}{
+ "content": content, "important_keywords": []string{"lifecycle"},
+ "questions": []string{"What must persist?"}, "tag_kwd": []string{"contract"},
+ }, nil, http.StatusOK)
+ requireChunkCode(t, added, 0)
+ chunk, ok := chunkData(t, added)["chunk"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("created chunk is not an object: %v", added)
+ }
+ id := chunkString(t, chunk, "id")
+ if id == "" || slices.Contains(ids, id) {
+ t.Fatalf("created chunk ID is empty or reused: %q; previous IDs: %v", id, ids)
+ }
+ ids = append(ids, id)
+ get := callChunkAPI(t, TestConfig, http.MethodGet, path+"/"+id, nil, nil, http.StatusOK)
+ requireChunkCode(t, get, 0)
+ stored := chunkData(t, get)
+ for key, want := range map[string]string{
+ "id": id, "doc_id": documentID, "kb_id": datasetID, "content_with_weight": content,
+ } {
+ if got := chunkString(t, stored, key); got != want {
+ t.Fatalf("stored %s = %q, want %q", key, got, want)
+ }
+ }
+ requireChunkStrings(t, stored, "important_kwd", "lifecycle")
+ requireChunkStrings(t, stored, "question_kwd", "What must persist?")
+ requireChunkStrings(t, stored, "tag_kwd", "contract")
+ for key := range stored {
+ if strings.HasSuffix(key, "_vec") || strings.HasSuffix(key, "_tks") || strings.HasSuffix(key, "_ltks") {
+ t.Fatalf("chunk response exposes internal field %q", key)
+ }
+ }
+ }
+ assertChunkIDs(t, listChunkIDs(t, path, nil, 3), ids)
+ var paged []string
+ for page := 1; page <= 2; page++ {
+ pageIDs := listChunkIDs(t, path, map[string]interface{}{"page": page, "page_size": 2}, 3)
+ if want := 3 - page; len(pageIDs) != want {
+ t.Fatalf("page %d contains %d chunks, want %d", page, len(pageIDs), want)
+ }
+ paged = append(paged, pageIDs...)
+ }
+ assertChunkIDs(t, paged, ids)
+ assertChunkIDs(t, listChunkIDs(t, path, map[string]interface{}{"page": 3, "page_size": 2}, 3), nil)
+ assertChunkIDs(t, listChunkIDs(t, path, map[string]interface{}{"chunk_ids": ids[0] + "," + ids[2]}, 2), []string{ids[0], ids[2]})
+ assertChunkIDs(t, listChunkIDs(t, path, map[string]interface{}{"chunk_ids": TestConfig.InvalidID}, 0), nil)
+ assertChunkIDs(t, listChunkIDs(t, path, map[string]interface{}{"keywords": "galaxies"}, 1), ids[2:])
+
+ chunkURL := path + "/" + ids[0]
+ const updated = "The updated content must also persist."
+ patch := callChunkAPI(t, TestConfig, http.MethodPatch, chunkURL,
+ map[string]interface{}{"content": updated, "available": false}, nil, http.StatusOK)
+ requireChunkCode(t, patch, 0)
+ get := callChunkAPI(t, TestConfig, http.MethodGet, chunkURL, nil, nil, http.StatusOK)
+ requireChunkCode(t, get, 0)
+ stored := chunkData(t, get)
+ if got := chunkString(t, stored, "content_with_weight"); got != updated {
+ t.Fatalf("updated content = %q, want %q", got, updated)
+ }
+ if stored["available_int"] != float64(0) {
+ t.Fatalf("updated available_int = %v, want 0", stored["available_int"])
+ }
+ // Omitted lists must survive a partial update; explicit empty lists clear them.
+ requireChunkStrings(t, stored, "important_kwd", "lifecycle")
+ requireChunkStrings(t, stored, "question_kwd", "What must persist?")
+ requireChunkStrings(t, stored, "tag_kwd", "contract")
+ assertChunkIDs(t, listChunkIDs(t, path, nil, 3), ids)
+ assertChunkIDs(t, listChunkIDs(t, path, map[string]interface{}{"available": true}, 2), ids[1:])
+ assertChunkIDs(t, listChunkIDs(t, path, map[string]interface{}{"available": false}, 1), ids[:1])
+ patch = callChunkAPI(t, TestConfig, http.MethodPatch, chunkURL, map[string]interface{}{
+ "important_keywords": []string{}, "questions": []string{}, "tag_kwd": []string{},
+ }, nil, http.StatusOK)
+ requireChunkCode(t, patch, 0)
+ get = callChunkAPI(t, TestConfig, http.MethodGet, chunkURL, nil, nil, http.StatusOK)
+ requireChunkCode(t, get, 0)
+ stored = chunkData(t, get)
+ for _, key := range []string{"important_kwd", "question_kwd", "tag_kwd"} {
+ requireChunkStrings(t, stored, key)
+ }
+ if got := chunkString(t, stored, "content_with_weight"); got != updated {
+ t.Fatalf("metadata update changed content to %q", got)
+ }
+
+ // Keep another chunk alive so an implementation that deletes everything fails.
+ deleted := callChunkAPI(t, TestConfig, http.MethodDelete, path,
+ map[string]interface{}{"chunk_ids": []string{ids[0], ids[1], ids[0]}}, nil, http.StatusOK)
+ requireChunkCode(t, deleted, 0)
+ if deleted["data"] != float64(2) {
+ t.Fatalf("deleted count = %v, want 2 unique chunks", deleted["data"])
+ }
+ assertChunkIDs(t, listChunkIDs(t, path, nil, 1), ids[2:])
+ deleted = callChunkAPI(t, TestConfig, http.MethodDelete, path,
+ map[string]interface{}{"chunk_ids": ids[:2]}, nil, http.StatusOK)
+ requireChunkCode(t, deleted, 0)
+ if deleted["data"] != float64(0) {
+ t.Fatalf("repeated deletion count = %v, want 0", deleted["data"])
+ }
+ deleted = callChunkAPI(t, TestConfig, http.MethodDelete, path,
+ map[string]interface{}{"delete_all": true}, nil, http.StatusOK)
+ requireChunkCode(t, deleted, 0)
+ if deleted["data"] != float64(1) {
+ t.Fatalf("delete_all count = %v, want 1", deleted["data"])
+ }
+ assertChunkIDs(t, listChunkIDs(t, path, nil, 0), nil)
+}
+
+func requireChunkStrings(t *testing.T, object map[string]interface{}, key string, want ...string) {
+ t.Helper()
+ items, ok := object[key].([]interface{})
+ if !ok {
+ t.Fatalf("%s is not an array: %v", key, object[key])
+ }
+ got := make([]string, len(items))
+ for i, item := range items {
+ value, ok := item.(string)
+ if !ok {
+ t.Fatalf("%s[%d] is not a string: %v", key, i, item)
+ }
+ got[i] = value
+ }
+ if !slices.Equal(got, want) {
+ t.Fatalf("%s = %v, want %v", key, got, want)
+ }
+}
+
+func listChunkIDs(t *testing.T, path string, params map[string]interface{}, wantTotal int) []string {
+ t.Helper()
+ listed := callChunkAPI(t, TestConfig, http.MethodGet, path, nil, params, http.StatusOK)
+ requireChunkCode(t, listed, 0)
+ data := chunkData(t, listed)
+ if data["total"] != float64(wantTotal) {
+ t.Fatalf("list total = %v, want %d; response: %v", data["total"], wantTotal, listed)
+ }
+ chunks, ok := data["chunks"].([]interface{})
+ if !ok {
+ t.Fatalf("listed chunks is not an array: %v", data)
+ }
+ ids := make([]string, len(chunks))
+ for i, raw := range chunks {
+ chunk, ok := raw.(map[string]interface{})
+ if !ok {
+ t.Fatalf("listed chunk is not an object: %v", raw)
+ }
+ ids[i] = chunkString(t, chunk, "chunk_id")
+ }
+ return ids
+}
+
+func assertChunkIDs(t *testing.T, got, want []string) {
+ t.Helper()
+ got, want = slices.Clone(got), slices.Clone(want)
+ slices.Sort(got)
+ slices.Sort(want)
+ if !slices.Equal(got, want) {
+ t.Fatalf("chunk IDs = %v, want %v", got, want)
+ }
+}
+
+func TestChunkReadUpdateAndDeleteStayWithinDocument(t *testing.T) {
+ datasetID, documentID := createChunkTestDocument(t, "chunk_isolation_source.txt")
+ otherDocumentID := uploadChunkTestDocument(t, datasetID, "chunk_isolation_other.txt")
+ if otherDocumentID == documentID {
+ t.Fatal("isolation test requires two different documents")
+ }
+ path := chunkPath(datasetID, documentID)
+ otherPath := chunkPath(datasetID, otherDocumentID)
+ const content = "This chunk belongs only to the source document."
+ added := callChunkAPI(t, TestConfig, http.MethodPost, path,
+ map[string]interface{}{"content": content}, nil, http.StatusOK)
+ requireChunkCode(t, added, 0)
+ chunk, ok := chunkData(t, added)["chunk"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("created chunk is not an object: %v", added)
+ }
+ id := chunkString(t, chunk, "id")
+ assertChunkIDs(t, listChunkIDs(t, path, nil, 1), []string{id})
+ assertChunkIDs(t, listChunkIDs(t, otherPath, nil, 0), nil)
+ for _, method := range []string{http.MethodGet, http.MethodPatch} {
+ t.Run(method+" rejects another document's chunk", func(t *testing.T) {
+ var body map[string]interface{}
+ if method == http.MethodPatch {
+ body = map[string]interface{}{"content": "Must not overwrite the source."}
+ }
+ result := requestChunkAPI(t, TestConfig, method, otherPath+"/"+id, body, nil)
+ if result.err != nil {
+ t.Fatal(result.err)
+ }
+ // Check the specific rejection rather than allowing any server error.
+ if code, ok := result.payload["code"].(float64); !ok || code == 0 {
+ t.Fatalf("another document's chunk was accepted: %s", result.body)
+ }
+ requireChunkMessage(t, result.payload, "chunk not found")
+ if result.payload["data"] != nil {
+ t.Fatalf("rejected request exposes chunk data: %s", result.body)
+ }
+ })
+ }
+ deleted := callChunkAPI(t, TestConfig, http.MethodDelete, otherPath,
+ map[string]interface{}{"chunk_ids": []string{id}}, nil, http.StatusOK)
+ requireChunkCode(t, deleted, 0)
+ if deleted["data"] != float64(0) {
+ t.Fatalf("deleted another document's chunk: %v", deleted)
+ }
+ get := callChunkAPI(t, TestConfig, http.MethodGet, path+"/"+id, nil, nil, http.StatusOK)
+ requireChunkCode(t, get, 0)
+ if got := chunkString(t, chunkData(t, get), "content_with_weight"); got != content {
+ t.Fatalf("source chunk changed after rejected mutations: %q", got)
+ }
+ assertChunkIDs(t, listChunkIDs(t, path, nil, 1), []string{id})
+}
+
+func TestChunkDeleteRejectsMismatchedDataset(t *testing.T) {
+ datasetID, documentID := createChunkTestDocument(t, "chunk_delete_dataset_source.txt")
+ otherDatasetID, _ := createChunkTestDocument(t, "chunk_delete_dataset_other.txt")
+ path := chunkPath(datasetID, documentID)
+ added := callChunkAPI(t, TestConfig, http.MethodPost, path,
+ map[string]interface{}{"content": "A mismatched dataset path must not delete this chunk."}, nil, http.StatusOK)
+ requireChunkCode(t, added, 0)
+ chunk, ok := chunkData(t, added)["chunk"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("created chunk is not an object: %v", added)
+ }
+ id := chunkString(t, chunk, "id")
+ assertChunkIDs(t, listChunkIDs(t, path, nil, 1), []string{id})
+ for _, tc := range []struct {
+ name string
+ body map[string]interface{}
+ }{
+ {name: "specific chunks", body: map[string]interface{}{"chunk_ids": []string{id}}},
+ {name: "all chunks", body: map[string]interface{}{"delete_all": true}},
+ {name: "body cannot override route", body: map[string]interface{}{"dataset_id": datasetID, "doc_id": documentID, "chunk_ids": []string{id}}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ payload := callChunkAPI(t, TestConfig, http.MethodDelete, chunkPath(otherDatasetID, documentID), tc.body, nil, http.StatusOK)
+ requireChunkCode(t, payload, 404)
+ requireChunkMessage(t, payload, "Resource not found")
+ if payload["data"] != nil {
+ t.Fatalf("resource error returned data: %v", payload)
+ }
+ assertChunkIDs(t, listChunkIDs(t, path, nil, 1), []string{id})
+ })
+ }
+ // A rejection must leave the source data intact.
+ assertChunkIDs(t, listChunkIDs(t, path, nil, 1), []string{id})
+}
+
+func TestChunkListEmptyDocument(t *testing.T) {
+ datasetID, documentID := createChunkTestDocument(t, "chunk_list_empty.txt")
+ payload := callChunkAPI(t, TestConfig, http.MethodGet, chunkPath(datasetID, documentID), nil, nil, http.StatusOK)
+ requireChunkCode(t, payload, 0)
+
+ data := chunkData(t, payload)
+ chunks, ok := data["chunks"].([]interface{})
+ if !ok {
+ t.Fatalf("response data.chunks is not an array: %v", data["chunks"])
+ }
+ if len(chunks) != 0 {
+ t.Fatalf("empty document returned %d chunks: %v", len(chunks), chunks)
+ }
+ if total, ok := data["total"].(float64); !ok || total != 0 {
+ t.Fatalf("response data.total = %v, want 0", data["total"])
+ }
+ doc, ok := data["doc"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("response data.doc is not an object: %v", data["doc"])
+ }
+ if got := chunkString(t, doc, "id"); got != documentID {
+ t.Fatalf("response document id = %q, want %q", got, documentID)
+ }
+ if got := chunkString(t, doc, "dataset_id"); got != datasetID {
+ t.Fatalf("response document dataset_id = %q, want %q", got, datasetID)
+ }
+ if count, ok := doc["chunk_count"].(float64); !ok || count != 0 {
+ t.Fatalf("response document chunk_count = %v, want 0", doc["chunk_count"])
+ }
+}