Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

test(api): migrate Python dataset API coverage to Go (#21063)

Sbaaoui Idriss012a59e26e80
diff --git a/test/api/chunk_api_test.go b/test/api/chunk_api_test.go
index 39aa3fafc4..93bc6fec6b 100644
--- a/test/api/chunk_api_test.go
+++ b/test/api/chunk_api_test.go
@@ -23,7 +23,6 @@ import (
"encoding/json"
"fmt"
"io"
- "mime/multipart"
"net/http"
"net/url"
"slices"
@@ -215,51 +214,8 @@ func createChunkTestDocument(t *testing.T, filename string) (string, string) {
func uploadChunkTestDocument(t *testing.T, datasetID, filename string) string {
t.Helper()
- const fileContents = "ragflow test upload"
- var uploadBody bytes.Buffer
- writer := multipart.NewWriter(&uploadBody)
- file, err := writer.CreateFormFile("file", filename)
- if err != nil {
- t.Fatalf("create multipart file field: %v", err)
- }
- if _, err := io.WriteString(file, fileContents); err != nil {
- t.Fatalf("write multipart file: %v", err)
- }
- if err := writer.Close(); err != nil {
- t.Fatalf("close multipart writer: %v", err)
- }
-
- requestURL := fmt.Sprintf("%s/datasets/%s/documents", TestConfig.apiBase(), datasetID)
- t.Logf("request: %s %s\npayload: multipart file %q (%d bytes), content=%q",
- http.MethodPost, requestURL, filename, len(fileContents), fileContents)
- request, err := http.NewRequest(http.MethodPost, requestURL, &uploadBody)
- if err != nil {
- t.Fatalf("create document upload request: %v", err)
- }
- request.Header = TestConfig.buildHeaders(nil)
- request.Header.Set("Content-Type", writer.FormDataContentType())
- response, err := chunkHTTPClient.Do(request)
- if err != nil {
- t.Fatalf("upload test document: %v", err)
- }
- uploadResult := readChunkAPIResponse(t, requestURL, response)
- if uploadResult.err != nil {
- t.Fatalf("decode document upload response: %v; body: %s", uploadResult.err, uploadResult.body)
- }
- if uploadResult.status != http.StatusOK {
- t.Fatalf("document upload status = %d, want %d; response: %s", uploadResult.status, http.StatusOK, uploadResult.body)
- }
- requireChunkCode(t, uploadResult.payload, 0)
-
- items, ok := uploadResult.payload["data"].([]interface{})
- if !ok || len(items) == 0 {
- t.Fatalf("document upload data is not a non-empty array: %v", uploadResult.payload)
- }
- document, ok := items[0].(map[string]interface{})
- if !ok {
- t.Fatalf("document upload item is not an object: %v", items[0])
- }
- documentID := chunkString(t, document, "id")
+ items := uploadRESTDocuments(t, TestConfig, datasetID, []documentUpload{{Name: filename, Content: []byte("ragflow test upload")}})
+ documentID := chunkString(t, object(t, items[0]), "id")
t.Cleanup(func() {
deleted := requestChunkAPI(t, TestConfig, http.MethodDelete,
fmt.Sprintf("/datasets/%s/documents", datasetID),
diff --git a/test/api/dataset_api_test.go b/test/api/dataset_api_test.go
index 36c8b39ca6..b4ae611b0b 100644
--- a/test/api/dataset_api_test.go
+++ b/test/api/dataset_api_test.go
@@ -21,561 +21,195 @@ package api
import (
"bytes"
"encoding/base64"
- "encoding/json"
- "fmt"
"image"
"image/color"
- "image/draw"
"image/png"
- "io"
"net/http"
+ "reflect"
"strings"
"testing"
)
-func decodeResponseJSON(resp *http.Response) (map[string]interface{}, error) {
- defer resp.Body.Close()
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, err
- }
- var payload map[string]interface{}
- if err = json.Unmarshal(body, &payload); err != nil {
- return nil, err
- }
- return payload, nil
-}
-
-func requireStatusCode(t *testing.T, resp *http.Response, expected int) map[string]interface{} {
+// createRESTDataset registers ownership before checking the persisted resource.
+// Explicit names are useful for duplicate and boundary tests; all callers own
+// only the returned ID, even when the server deduplicates the requested name.
+func createRESTDataset(t *testing.T, fields apiObject) apiObject {
t.Helper()
- if resp.StatusCode != expected {
- t.Fatalf("expected status code %d, got %d", expected, resp.StatusCode)
+ payload := apiObject{"name": uniqueName("go_dataset"), "parse_type": 1, "parser_id": "general"}
+ for key, value := range fields {
+ payload[key] = value
+ }
+ result := restCall(t, TestConfig, http.MethodPost, "/datasets", payload, nil)
+ if data, ok := result.JSON["data"].(apiObject); ok {
+ if id, ok := data["id"].(string); ok && id != "" {
+ cleanupResource(t, TestConfig, "datasets", id)
+ }
}
- payload, err := decodeResponseJSON(resp)
- if err != nil {
- t.Fatalf("failed to decode response: %v", err)
+ data := object(t, restOK(t, result)["data"])
+ id, ok := data["id"].(string)
+ if !ok || id == "" {
+ t.Fatalf("created dataset has invalid id: %v", data)
+ }
+ persisted := getRESTDataset(t, id)
+ for _, key := range []string{"id", "name", "permission", "parser_id", "embedding_model", "parser_config"} {
+ if !reflect.DeepEqual(data[key], persisted[key]) {
+ t.Fatalf("created %s differs from persisted value: response=%v stored=%v", key, data[key], persisted[key])
+ }
}
- return payload
+ return persisted
}
-func requireCodeZero(t *testing.T, payload map[string]interface{}) {
+func getRESTDataset(t *testing.T, id string) apiObject {
t.Helper()
- code, ok := payload["code"].(float64)
- if !ok || int(code) != 0 {
- t.Fatalf("expected code 0, got %v, payload: %v", payload["code"], payload)
- }
+ return object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id, nil, nil))["data"])
}
-func clearDatasets(t *testing.T) {
+func requireDatasetFields(t *testing.T, id string, expected apiObject) apiObject {
t.Helper()
- resp, err := TestConfig.DeleteJSON("/datasets", map[string]interface{}{"ids": nil, "delete_all": true}, nil)
- if err != nil {
- t.Fatalf("clear datasets request failed: %v", err)
- }
- payload := requireStatusCode(t, resp, http.StatusOK)
- code, _ := payload["code"].(float64)
- if int(code) != 0 && int(code) != 102 {
- t.Fatalf("clear datasets failed: code=%v, payload=%v", payload["code"], payload)
+ actual := getRESTDataset(t, id)
+ for key, want := range expected {
+ if !reflect.DeepEqual(actual[key], want) {
+ t.Errorf("dataset %s field %s=%#v want=%#v", id, key, actual[key], want)
+ }
}
+ return actual
}
-func jsonDeepEqual(a, b interface{}) bool {
- switch av := a.(type) {
- case float64:
- switch bv := b.(type) {
- case float64:
- return av == bv
- case int:
- return av == float64(bv)
- case int64:
- return av == float64(bv)
- default:
- return false
- }
- case int:
- return jsonDeepEqual(float64(av), b)
- case int64:
- return jsonDeepEqual(float64(av), b)
- case string, bool, nil:
- return a == b
- case []interface{}:
- bv, ok := b.([]interface{})
- if !ok || len(av) != len(bv) {
- return false
- }
- for i := range av {
- if !jsonDeepEqual(av[i], bv[i]) {
- return false
- }
- }
- return true
- case map[string]interface{}:
- bv, ok := b.(map[string]interface{})
- if !ok || len(av) != len(bv) {
- return false
- }
- for k, v := range av {
- bvVal, ok := bv[k]
- if !ok || !jsonDeepEqual(v, bvVal) {
- return false
- }
- }
- return true
- default:
- return a == b
- }
+func requireDatasetAbsent(t *testing.T, id string) {
+ t.Helper()
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id, nil, nil), 200, 404)
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, apiObject{"id": id}), 200, 404)
}
func TestDatasetCRUDCycle(t *testing.T) {
- clearDatasets(t)
- // Create
- createResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": "restful_dataset_crud"}, nil)
- if err != nil {
- t.Fatalf("create dataset request failed: %v", err)
- }
- createPayload := requireStatusCode(t, createResp, http.StatusOK)
- requireCodeZero(t, createPayload)
- createData, ok := createPayload["data"].(map[string]interface{})
- if !ok {
- t.Fatalf("create response data is not an object: %v", createPayload)
- }
- datasetID, ok := createData["id"].(string)
- if !ok || datasetID == "" {
- t.Fatalf("create response data.id is not a valid string: %v", createData)
- }
-
- // Get by id
- getResp, err := TestConfig.GetJSON(fmt.Sprintf("/datasets/%s", datasetID), nil, nil)
- if err != nil {
- t.Fatalf("get dataset request failed: %v", err)
- }
- getPayload := requireStatusCode(t, getResp, http.StatusOK)
- requireCodeZero(t, getPayload)
- getData, ok := getPayload["data"].(map[string]interface{})
- if !ok || getData["id"] != datasetID {
- t.Fatalf("get response data.id mismatch: %v", getPayload)
- }
-
- // Update
- updateResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", datasetID), map[string]interface{}{"name": "restful_dataset_crud_updated"}, nil)
- if err != nil {
- t.Fatalf("update dataset request failed: %v", err)
- }
- updatePayload := requireStatusCode(t, updateResp, http.StatusOK)
- requireCodeZero(t, updatePayload)
- updateData, ok := updatePayload["data"].(map[string]interface{})
- if !ok || updateData["name"] != "restful_dataset_crud_updated" {
- t.Fatalf("update response data.name mismatch: %v", updatePayload)
- }
-
- // List with id filter
- listResp, err := TestConfig.GetJSON("/datasets", map[string]interface{}{"id": datasetID}, nil)
- if err != nil {
- t.Fatalf("list dataset request failed: %v", err)
- }
- listPayload := requireStatusCode(t, listResp, http.StatusOK)
- requireCodeZero(t, listPayload)
- listData, ok := listPayload["data"].([]interface{})
- if !ok || len(listData) != 1 {
- t.Fatalf("expected 1 dataset in list, got %v", listPayload)
- }
- firstItem, ok := listData[0].(map[string]interface{})
- if !ok || firstItem["id"] != datasetID {
- t.Fatalf("list response data[0].id mismatch: %v", listPayload)
- }
-
- // Delete
- deleteResp, err := TestConfig.DeleteJSON("/datasets", map[string]interface{}{"ids": []interface{}{datasetID}}, nil)
- if err != nil {
- t.Fatalf("delete dataset request failed: %v", err)
- }
- deletePayload := requireStatusCode(t, deleteResp, http.StatusOK)
- requireCodeZero(t, deletePayload)
-
- // List after delete
- listAfterResp, err := TestConfig.GetJSON("/datasets", nil, nil)
- if err != nil {
- t.Fatalf("list after delete request failed: %v", err)
- }
- listAfterPayload := requireStatusCode(t, listAfterResp, http.StatusOK)
- requireCodeZero(t, listAfterPayload)
- listAfterData, ok := listAfterPayload["data"].([]interface{})
- if !ok {
- t.Fatalf("list after delete data is not an array: %v", listAfterPayload)
- }
- for _, item := range listAfterData {
- dataset, ok := item.(map[string]interface{})
- if !ok {
- continue
- }
- if dataset["id"] == datasetID {
- t.Fatalf("deleted dataset %s still present in list: %v", datasetID, listAfterPayload)
- }
- }
+ dataset := createRESTDataset(t, nil)
+ id := dataset["id"].(string)
+ name := uniqueName("renamed")
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": name}, nil))
+ requireDatasetFields(t, id, apiObject{"name": name})
+ listed := restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, apiObject{"id": id}))
+ items := array(t, listed["data"])
+ if len(items) != 1 || object(t, items[0])["id"] != id || listed["total_datasets"] != float64(1) {
+ t.Fatalf("id-filtered list=%v", listed)
+ }
+ restOK(t, restCall(t, TestConfig, http.MethodDelete, "/datasets", apiObject{"ids": []string{id}}, nil))
+ requireDatasetAbsent(t, id)
}
func TestDatasetUpdateNameAndCaseInsensitiveContract(t *testing.T) {
- clearDatasets(t)
-
- // Create first dataset
- firstResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": "dataset_update_name_source"}, nil)
- if err != nil {
- t.Fatalf("create first dataset request failed: %v", err)
- }
- firstPayload := requireStatusCode(t, firstResp, http.StatusOK)
- requireCodeZero(t, firstPayload)
- firstData, ok := firstPayload["data"].(map[string]interface{})
- if !ok {
- t.Fatalf("first response data is not an object: %v", firstPayload)
- }
- firstDatasetID, ok := firstData["id"].(string)
- if !ok || firstDatasetID == "" {
- t.Fatalf("first dataset id is invalid: %v", firstData)
- }
-
- // Create second dataset
- secondResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": "dataset_update_name_target"}, nil)
- if err != nil {
- t.Fatalf("create second dataset request failed: %v", err)
- }
- secondPayload := requireStatusCode(t, secondResp, http.StatusOK)
- requireCodeZero(t, secondPayload)
- secondData, ok := secondPayload["data"].(map[string]interface{})
- if !ok {
- t.Fatalf("second response data is not an object: %v", secondPayload)
- }
- secondDatasetName, ok := secondData["name"].(string)
- if !ok || secondDatasetName == "" {
- t.Fatalf("second dataset name is invalid: %v", secondData)
- }
-
- // Rename first dataset
- renameResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", firstDatasetID), map[string]interface{}{"name": "dataset_update_name_renamed"}, nil)
- if err != nil {
- t.Fatalf("rename dataset request failed: %v", err)
- }
- renamePayload := requireStatusCode(t, renameResp, http.StatusOK)
- requireCodeZero(t, renamePayload)
- renameData, ok := renamePayload["data"].(map[string]interface{})
- if !ok || renameData["name"] != "dataset_update_name_renamed" {
- t.Fatalf("rename response data.name mismatch: %v", renamePayload)
- }
-
- // List with id filter
- listResp, err := TestConfig.GetJSON("/datasets", map[string]interface{}{"id": firstDatasetID}, nil)
- if err != nil {
- t.Fatalf("list dataset request failed: %v", err)
- }
- listPayload := requireStatusCode(t, listResp, http.StatusOK)
- requireCodeZero(t, listPayload)
- listData, ok := listPayload["data"].([]interface{})
- if !ok || len(listData) != 1 {
- t.Fatalf("expected 1 dataset in list, got %v", listPayload)
- }
- firstItem, ok := listData[0].(map[string]interface{})
- if !ok || firstItem["name"] != "dataset_update_name_renamed" {
- t.Fatalf("list response data[0].name mismatch: %v", listPayload)
- }
-
- // Try duplicate name (case-insensitive)
- duplicateResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", firstDatasetID), map[string]interface{}{"name": strings.ToUpper(secondDatasetName)}, nil)
- if err != nil {
- t.Fatalf("duplicate case rename request failed: %v", err)
- }
- duplicatePayload := requireStatusCode(t, duplicateResp, http.StatusOK)
- duplicateCode, ok := duplicatePayload["code"].(float64)
- if !ok || int(duplicateCode) != 102 {
- t.Fatalf("expected code 102 for duplicate case rename, got %v, payload: %v", duplicatePayload["code"], duplicatePayload)
- }
- duplicateMessage, _ := duplicatePayload["message"].(string)
- if !strings.Contains(duplicateMessage, "already exists") {
- t.Fatalf("expected message to contain 'already exists', got %q, payload: %v", duplicateMessage, duplicatePayload)
- }
+ first := createRESTDataset(t, nil)
+ second := createRESTDataset(t, nil)
+ id := first["id"].(string)
+ name := uniqueName("renamed")
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": name}, nil))
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": strings.ToUpper(name)}, nil))
+ requireDatasetFields(t, id, apiObject{"name": strings.ToUpper(name)})
+ restCode(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": strings.ToUpper(second["name"].(string))}, nil), 200, 102)
+ requireDatasetFields(t, id, apiObject{"name": strings.ToUpper(name)})
}
func encodeAvatar() (string, error) {
- img := image.NewRGBA(image.Rectangle{
- Min: image.Point{X: 0, Y: 0},
- Max: image.Point{X: 100, Y: 100},
- })
- draw.Draw(img, img.Bounds(), &image.Uniform{color.RGBA{R: 0, G: 0, B: 255, A: 255}}, image.Point{}, draw.Src)
+ img := image.NewRGBA(image.Rect(0, 0, 1, 1))
+ img.Set(0, 0, color.RGBA{B: 255, A: 255})
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
return "", err
}
- encoded := base64.StdEncoding.EncodeToString(buf.Bytes())
- return fmt.Sprintf("data:image/png;base64,%s", encoded), nil
+ return "data:image/png;base64," + base64.StdEncoding.EncodeToString(buf.Bytes()), nil
}
func TestDatasetUpdateLanguageConnectorsAvatarAndDescriptionContract(t *testing.T) {
- clearDatasets(t)
-
- createResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": "dataset_update_lang_connectors"}, nil)
- if err != nil {
- t.Fatalf("create dataset request failed: %v", err)
- }
- createPayload := requireStatusCode(t, createResp, http.StatusOK)
- requireCodeZero(t, createPayload)
- createData, ok := createPayload["data"].(map[string]interface{})
- if !ok {
- t.Fatalf("create response data is not an object: %v", createPayload)
- }
- datasetID, ok := createData["id"].(string)
- if !ok || datasetID == "" {
- t.Fatalf("dataset id is invalid: %v", createData)
- }
-
- avatarValue, err := encodeAvatar()
+ dataset := createRESTDataset(t, nil)
+ id := dataset["id"].(string)
+ avatar, err := encodeAvatar()
if err != nil {
- t.Fatalf("failed to encode avatar: %v", err)
- }
-
- updateResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", datasetID), map[string]interface{}{
- "name": "dataset_update_lang_connectors",
- "description": "",
- "parser_id": "naive",
- "parse_type": 1,
- "language": "English",
- "connectors": []interface{}{},
- "avatar": avatarValue,
- }, nil)
- if err != nil {
- t.Fatalf("update dataset request failed: %v", err)
- }
- updatePayload := requireStatusCode(t, updateResp, http.StatusOK)
- requireCodeZero(t, updatePayload)
- updateData, ok := updatePayload["data"].(map[string]interface{})
- if !ok {
- t.Fatalf("update response data is not an object: %v", updatePayload)
- }
- if updateData["language"] != "English" {
- t.Fatalf("language mismatch: expected English, got %v", updatePayload)
- }
- connectors, ok := updateData["connectors"].([]interface{})
- if !ok {
- t.Fatalf("connectors is not an array: %v", updatePayload)
- }
- if len(connectors) != 0 {
- t.Fatalf("connectors should be empty: %v", updatePayload)
- }
- if updateData["avatar"] != avatarValue {
- t.Fatalf("avatar mismatch: %v", updatePayload)
- }
-
- descriptionResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", datasetID), map[string]interface{}{"description": "description"}, nil)
- if err != nil {
- t.Fatalf("update description request failed: %v", err)
- }
- descriptionPayload := requireStatusCode(t, descriptionResp, http.StatusOK)
- requireCodeZero(t, descriptionPayload)
- descriptionData, ok := descriptionPayload["data"].(map[string]interface{})
- if !ok || descriptionData["description"] != "description" {
- t.Fatalf("description mismatch: %v", descriptionPayload)
+ t.Fatal(err)
}
+ fields := apiObject{"language": "English", "connectors": []any{}, "avatar": avatar, "description": "description"}
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, fields, nil))
+ requireDatasetFields(t, id, fields)
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"description": ""}, nil))
+ requireDatasetFields(t, id, apiObject{"description": "", "language": "English", "avatar": avatar, "connectors": []any{}})
}
-func TestDatasetUpdateParserConfigValidMatrixContract(t *testing.T) {
- clearDatasets(t)
- t.Skip("Go validation or response contract does not match the established API contract")
-
- cases := []struct {
- name string
- parserConfig map[string]interface{}
- }{
- {name: "auto_keywords_min", parserConfig: map[string]interface{}{"auto_keywords": 0}},
- {name: "auto_keywords_mid", parserConfig: map[string]interface{}{"auto_keywords": 16}},
- {name: "auto_keywords_max", parserConfig: map[string]interface{}{"auto_keywords": 32}},
- {name: "auto_questions_min", parserConfig: map[string]interface{}{"auto_questions": 0}},
- {name: "auto_questions_mid", parserConfig: map[string]interface{}{"auto_questions": 5}},
- {name: "auto_questions_max", parserConfig: map[string]interface{}{"auto_questions": 10}},
- {name: "chunk_token_num_min", parserConfig: map[string]interface{}{"chunk_token_num": 1}},
- {name: "chunk_token_num_mid", parserConfig: map[string]interface{}{"chunk_token_num": 1024}},
- {name: "chunk_token_num_max", parserConfig: map[string]interface{}{"chunk_token_num": 2048}},
- {name: "delimiter", parserConfig: map[string]interface{}{"delimiter": "\n"}},
- {name: "delimiter_space", parserConfig: map[string]interface{}{"delimiter": " "}},
- {name: "html4excel_true", parserConfig: map[string]interface{}{"html4excel": true}},
- {name: "html4excel_false", parserConfig: map[string]interface{}{"html4excel": false}},
- {name: "layout_recognize_DeepDOC", parserConfig: map[string]interface{}{"layout_recognize": "DeepDOC"}},
- {name: "layout_recognize_navie", parserConfig: map[string]interface{}{"layout_recognize": "Plain Text"}},
- {name: "topn_tags_min", parserConfig: map[string]interface{}{"topn_tags": 1}},
- {name: "topn_tags_mid", parserConfig: map[string]interface{}{"topn_tags": 5}},
- {name: "topn_tags_max", parserConfig: map[string]interface{}{"topn_tags": 10}},
- {name: "tags_tag_file_id", parserConfig: map[string]interface{}{"tags": map[string]interface{}{"tag_file_id": "file-123"}}},
- {name: "filename_embd_weight_min", parserConfig: map[string]interface{}{"filename_embd_weight": 0.1}},
- {name: "filename_embd_weight_mid", parserConfig: map[string]interface{}{"filename_embd_weight": 0.5}},
- {name: "filename_embd_weight_max", parserConfig: map[string]interface{}{"filename_embd_weight": 1.0}},
- {name: "task_page_size_min", parserConfig: map[string]interface{}{"task_page_size": 1}},
- {name: "task_page_size_None", parserConfig: map[string]interface{}{"task_page_size": nil}},
- {name: "pages", parserConfig: map[string]interface{}{"pages": []interface{}{[]interface{}{1, 100}}}},
- {name: "pages_none", parserConfig: map[string]interface{}{"pages": nil}},
- {name: "graphrag_true", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"use_graphrag": true}}},
- {name: "graphrag_false", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"use_graphrag": false}}},
- {name: "graphrag_entity_types", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"entity_types": []interface{}{"age", "sex", "height", "weight"}}}},
- {name: "graphrag_method_general", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"method": "general"}}},
- {name: "graphrag_method_light", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"method": "light"}}},
- {name: "graphrag_community_true", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"community": true}}},
- {name: "graphrag_community_false", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"community": false}}},
- {name: "graphrag_resolution_true", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"resolution": true}}},
- {name: "graphrag_resolution_false", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"resolution": false}}},
- {name: "raptor_true", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"use_raptor": true}}},
- {name: "raptor_false", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"use_raptor": false}}},
- {name: "raptor_prompt", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"prompt": "Who are you?"}}},
- {name: "raptor_max_token_min", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_token": 512}}},
- {name: "raptor_max_token_mid", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_token": 1024}}},
- {name: "raptor_max_token_max", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_token": 2048}}},
- {name: "raptor_clustering_threshold_min", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"clustering_threshold": 0.0}}},
- {name: "raptor_clustering_threshold_mid", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"clustering_threshold": 0.5}}},
- {name: "raptor_clustering_threshold_max", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"clustering_threshold": 1.0}}},
- {name: "raptor_max_cluster_min", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_cluster": 1}}},
- {name: "raptor_max_cluster_mid", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_cluster": 512}}},
- {name: "raptor_max_cluster_max", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_cluster": 1024}}},
- {name: "raptor_random_seed_min", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"random_seed": 0}}},
+func requirePipelineParserConfig(t *testing.T, value any, chunkerPrefix string) apiObject {
+ t.Helper()
+ config := object(t, value)
+ if _, ok := config["File"]; !ok {
+ t.Fatalf("parser config missing File node: %v", config)
+ }
+ parser, chunker := false, false
+ for id, params := range config {
+ object(t, params)
+ parser = parser || strings.HasPrefix(id, "Parser:")
+ chunker = chunker || strings.HasPrefix(id, chunkerPrefix)
+ }
+ if !parser || !chunker {
+ t.Fatalf("parser config missing parser/chunker nodes: %v", config)
+ }
+ for _, key := range []string{"chunk_token_num", "raptor", "graphrag"} {
+ if _, ok := config[key]; ok {
+ t.Fatalf("unexpected flat setting %s: %v", key, config)
+ }
}
+ return config
+}
- for _, tc := range cases {
- tc := tc
+func TestDatasetUpdateParserConfigWithChunkMethodChangeContract(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ provided bool
+ config any
+ }{{"empty", true, apiObject{}}, {"null", true, nil}, {"unset", false, nil}} {
t.Run(tc.name, func(t *testing.T) {
- createResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": fmt.Sprintf("dataset_update_parser_%s", tc.name)}, nil)
- if err != nil {
- t.Fatalf("create dataset request failed: %v", err)
- }
- createPayload := requireStatusCode(t, createResp, http.StatusOK)
- requireCodeZero(t, createPayload)
- createData, ok := createPayload["data"].(map[string]interface{})
- if !ok {
- t.Fatalf("create response data is not an object: %v", createPayload)
- }
- datasetID, ok := createData["id"].(string)
- if !ok || datasetID == "" {
- t.Fatalf("dataset id is invalid: %v", createData)
- }
-
- updateResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", datasetID), map[string]interface{}{"parser_config": tc.parserConfig}, nil)
- if err != nil {
- t.Fatalf("update parser_config request failed: %v", err)
- }
- updatePayload := requireStatusCode(t, updateResp, http.StatusOK)
- requireCodeZero(t, updatePayload)
-
- listResp, err := TestConfig.GetJSON("/datasets", map[string]interface{}{"id": datasetID}, nil)
- if err != nil {
- t.Fatalf("list dataset request failed: %v", err)
- }
- listPayload := requireStatusCode(t, listResp, http.StatusOK)
- requireCodeZero(t, listPayload)
- listData, ok := listPayload["data"].([]interface{})
- if !ok || len(listData) != 1 {
- t.Fatalf("expected 1 dataset in list, got %v", listPayload)
- }
- firstItem, ok := listData[0].(map[string]interface{})
- if !ok {
- t.Fatalf("list data[0] is not an object: %v", listPayload)
- }
- actualParserConfig, ok := firstItem["parser_config"].(map[string]interface{})
- if !ok {
- t.Fatalf("parser_config is not an object: %v", firstItem)
- }
-
- for key, expectedValue := range tc.parserConfig {
- if key == "graphrag" || key == "raptor" {
- if _, exists := actualParserConfig[key]; exists {
- t.Fatalf("expected %s not in parser_config, but got %v", key, actualParserConfig)
- }
- continue
- }
- actualValue, ok := actualParserConfig[key]
- if !ok {
- t.Fatalf("parser_config missing key %s: %v", key, actualParserConfig)
- }
- if !jsonDeepEqual(actualValue, expectedValue) {
- t.Fatalf("parser_config.%s mismatch: expected %v, got %v", key, expectedValue, actualValue)
+ dataset := createRESTDataset(t, nil)
+ id := dataset["id"].(string)
+ fields := apiObject{"parse_type": 1, "parser_id": "qa"}
+ if tc.provided {
+ fields["parser_config"] = tc.config
+ }
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, fields, nil))
+ persisted := requireDatasetFields(t, id, apiObject{"parser_id": "qa"})
+ config := object(t, persisted["parser_config"])
+ if len(config) == 0 {
+ t.Fatal("parser config is empty after method change")
+ }
+ for _, key := range []string{"raptor", "graphrag"} {
+ if _, ok := config[key]; ok {
+ t.Fatalf("unexpected flat key %s", key)
}
}
})
}
}
-func TestDatasetUpdateParserConfigWithChunkMethodChangeContract(t *testing.T) {
- clearDatasets(t)
-
- cases := []struct {
- name string
- updatePayload map[string]interface{}
- }{
- {
- name: "parser_config_empty",
- updatePayload: map[string]interface{}{
- "parser_id": "qa",
- "parse_type": 1,
- "parser_config": map[string]interface{}{},
- },
- },
- {
- name: "parser_config_none",
- updatePayload: map[string]interface{}{
- "parser_id": "qa",
- "parse_type": 1,
- "parser_config": nil,
- },
- },
- {
- name: "parser_config_unset",
- updatePayload: map[string]interface{}{
- "parser_id": "qa",
- "parse_type": 1,
- },
- },
+func TestDatasetComponentScopedParserConfigRoundtrip(t *testing.T) {
+ dataset := createRESTDataset(t, nil)
+ id := dataset["id"].(string)
+ config := requirePipelineParserConfig(t, dataset["parser_config"], "GeneralChunker:")
+ chunkerID := ""
+ for key := range config {
+ if strings.HasPrefix(key, "GeneralChunker:") {
+ chunkerID = key
+ break
+ }
}
-
- for _, tc := range cases {
- tc := tc
- t.Run(tc.name, func(t *testing.T) {
- createResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": fmt.Sprintf("dataset_update_%s", tc.name)}, nil)
- if err != nil {
- t.Fatalf("create dataset request failed: %v", err)
- }
- createPayload := requireStatusCode(t, createResp, http.StatusOK)
- requireCodeZero(t, createPayload)
- createData, ok := createPayload["data"].(map[string]interface{})
- if !ok {
- t.Fatalf("create response data is not an object: %v", createPayload)
- }
- datasetID, ok := createData["id"].(string)
- if !ok || datasetID == "" {
- t.Fatalf("dataset id is invalid: %v", createData)
- }
-
- updateResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", datasetID), tc.updatePayload, nil)
- if err != nil {
- t.Fatalf("update dataset request failed: %v", err)
- }
- updatePayload := requireStatusCode(t, updateResp, http.StatusOK)
- requireCodeZero(t, updatePayload)
-
- listResp, err := TestConfig.GetJSON("/datasets", map[string]interface{}{"id": datasetID}, nil)
- if err != nil {
- t.Fatalf("list dataset request failed: %v", err)
- }
- listPayload := requireStatusCode(t, listResp, http.StatusOK)
- requireCodeZero(t, listPayload)
- listData, ok := listPayload["data"].([]interface{})
- if !ok || len(listData) != 1 {
- t.Fatalf("expected 1 dataset in list, got %v", listPayload)
- }
- firstItem, ok := listData[0].(map[string]interface{})
- if !ok {
- t.Fatalf("list data[0] is not an object: %v", listPayload)
- }
- actualParserConfig, ok := firstItem["parser_config"].(map[string]interface{})
- if !ok || len(actualParserConfig) == 0 {
- t.Fatalf("parser_config should be non-empty map: %v", firstItem)
- }
- if _, exists := actualParserConfig["raptor"]; exists {
- t.Fatalf("raptor should not be in parser_config: %v", actualParserConfig)
- }
- if _, exists := actualParserConfig["graphrag"]; exists {
- t.Fatalf("graphrag should not be in parser_config: %v", actualParserConfig)
+ for _, delimiter := range []string{"|", ";", "\n"} {
+ t.Run(strings.ReplaceAll(delimiter, "\n", "newline"), func(t *testing.T) {
+ fields := apiObject{"parser_config": apiObject{chunkerID: apiObject{"parent_child": apiObject{"use_parent_child": true, "children_delimiter": delimiter}}}}
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, fields, nil))
+ updated := object(t, getRESTDataset(t, id)["parser_config"])
+ chunker := object(t, updated[chunkerID])
+ if object(t, chunker["parent_child"])["use_parent_child"] != true || !reflect.DeepEqual(chunker["children_delimiters"], []any{delimiter}) {
+ t.Fatalf("component settings did not reach chunker: %v", chunker)
+ }
+ for key, params := range config {
+ if key != chunkerID && !reflect.DeepEqual(params, updated[key]) {
+ t.Errorf("unrelated parser node %s changed", key)
+ }
}
})
}
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"parser_config": apiObject{"File": apiObject{"layout_recognize": "DeepDOC"}}}, nil))
+ requirePipelineParserConfig(t, getRESTDataset(t, id)["parser_config"], "GeneralChunker:")
}
diff --git a/test/api/dataset_features_test.go b/test/api/dataset_features_test.go
new file mode 100644
index 0000000000..d58a23626d
--- /dev/null
+++ b/test/api/dataset_features_test.go
@@ -0,0 +1,271 @@
+//go:build integration
+
+package api
+
+import (
+ "context"
+ "net/http"
+ "reflect"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestDatasetMetadataConfigPersistenceAndValidation(t *testing.T) {
+ dataset := createRESTDataset(t, nil)
+ path := "/datasets/" + dataset["id"].(string) + "/metadata/config"
+ empty := apiObject{"enabled": false, "metadata": []any{}, "built_in_metadata": []any{}}
+ read := func() apiObject {
+ return object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, path, nil, nil))["data"])
+ }
+ if got := read(); !reflect.DeepEqual(got, empty) {
+ t.Fatalf("fresh metadata config=%v want=%v", got, empty)
+ }
+ fields := []any{
+ apiObject{"key": "author", "type": "string", "description": "Author name"},
+ apiObject{"key": "tags", "type": "list", "description": "Tag list", "enum": []string{"foo", "bar"}},
+ }
+ body := apiObject{"metadata": fields, "built_in_metadata": []any{apiObject{"key": "size", "type": "number", "description": "File size"}}}
+ expected := apiObject{
+ "enabled": false,
+ "metadata": []any{
+ apiObject{"key": "author", "type": "string", "description": "Author name", "enum": nil},
+ apiObject{"key": "tags", "type": "list", "description": "Tag list", "enum": []any{"foo", "bar"}},
+ },
+ "built_in_metadata": []any{apiObject{"key": "size", "type": "number", "description": "File size", "enum": nil}},
+ }
+ for _, enabled := range []bool{false, true} {
+ if enabled {
+ body["enabled"] = true
+ }
+ expected["enabled"] = enabled
+ result := restOK(t, restCall(t, TestConfig, http.MethodPut, path, body, nil))
+ if !reflect.DeepEqual(result["data"], expected) || !reflect.DeepEqual(read(), expected) {
+ t.Fatalf("metadata config did not persist normalized fields: response=%v stored=%v want=%v", result["data"], read(), expected)
+ }
+ }
+ for _, invalid := range []apiObject{
+ {"metadata": []any{apiObject{"key": " ", "type": "string"}}},
+ {"metadata": []any{apiObject{"key": "author", "type": "unknown"}}},
+ {"metadata": "invalid"},
+ {"enabled": "true"},
+ } {
+ restCode(t, restCall(t, TestConfig, http.MethodPut, path, invalid, nil), 200, 102)
+ if got := read(); !reflect.DeepEqual(got, expected) {
+ t.Fatalf("invalid metadata update changed config: got=%v want=%v", got, expected)
+ }
+ }
+ restOK(t, restCall(t, TestConfig, http.MethodPut, path, apiObject{}, nil))
+ if got := read(); !reflect.DeepEqual(got, empty) {
+ t.Fatalf("empty update did not reset metadata config: %v", got)
+ }
+ for _, method := range []string{http.MethodGet, http.MethodPut} {
+ restCode(t, restCall(t, TestConfig, method, "/datasets/00000000000040008000000000000000/metadata/config", apiObject{}, nil), 200, 404)
+ }
+}
+
+func TestDatasetMetadataSummaryAndFlattenedIdentities(t *testing.T) {
+ first := createRESTDataset(t, nil)["id"].(string)
+ second := createRESTDataset(t, nil)["id"].(string)
+ documents := uploadRESTDocuments(t, TestConfig, first, []documentUpload{
+ {Name: "one.txt", Content: []byte("metadata one")},
+ {Name: "two.txt", Content: []byte("metadata two")},
+ {Name: "three.txt", Content: []byte("metadata three")},
+ })
+ metadata := []apiObject{
+ {"tags": []string{"foo", "bar"}, "author": "alice"},
+ {"tags": []string{"foo"}, "author": "bob"},
+ {"tags": []string{"bar", "baz"}, "author": ""},
+ }
+ var ids []string
+ for i, doc := range documents {
+ id := object(t, doc)["id"].(string)
+ ids = append(ids, id)
+ restOK(t, restCall(t, TestConfig, http.MethodPatch, "/datasets/"+first+"/documents/"+id, apiObject{"meta_fields": metadata[i]}, nil))
+ }
+ foreign := object(t, uploadRESTDocuments(t, TestConfig, second, []documentUpload{{Name: "other.txt", Content: []byte("foreign metadata")}})[0])["id"].(string)
+ restOK(t, restCall(t, TestConfig, http.MethodPatch, "/datasets/"+second+"/documents/"+foreign, apiObject{"meta_fields": apiObject{"tags": []string{"foreign"}}}, nil))
+ // Metadata lives in the document engine. Poll for the precise aggregate,
+ // allowing index refresh latency without accepting an empty result.
+ for _, tc := range []struct {
+ name string
+ query apiObject
+ tags apiObject
+ authors apiObject
+ }{
+ {"all_documents", nil, apiObject{"foo": float64(2), "bar": float64(2), "baz": float64(1)}, apiObject{"alice": float64(1), "bob": float64(1), "": float64(1)}},
+ {"explicit_documents", apiObject{"doc_ids": strings.Join(ids[:2], ",")}, apiObject{"foo": float64(2), "bar": float64(1)}, apiObject{"alice": float64(1), "bob": float64(1)}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second)
+ defer cancel()
+ wantCounts := tc.tags
+ for {
+ data := object(t, restOK(t, restCallContext(t, ctx, TestConfig, http.MethodGet, "/datasets/"+first+"/metadata/summary", nil, tc.query))["data"])
+ summary := object(t, data["summary"])
+ counts := apiObject{}
+ if tags, ok := summary["tags"].(apiObject); ok {
+ for _, value := range array(t, tags["values"]) {
+ pair := array(t, value)
+ if len(pair) != 2 {
+ t.Fatalf("invalid metadata value/count pair: %v", pair)
+ }
+ counts[pair[0].(string)] = pair[1]
+ }
+ }
+ if reflect.DeepEqual(counts, wantCounts) {
+ authors := object(t, summary["author"])
+ got := apiObject{}
+ for _, value := range array(t, authors["values"]) {
+ pair := array(t, value)
+ if len(pair) != 2 {
+ t.Fatalf("invalid author value/count: %v", pair)
+ }
+ got[pair[0].(string)] = pair[1]
+ }
+ if !reflect.DeepEqual(got, tc.authors) {
+ t.Fatalf("author counts differ from stored values: %v", got)
+ }
+ break
+ }
+ select {
+ case <-ctx.Done():
+ t.Fatalf("metadata summary never matched owned documents: got=%v want=%v", counts, wantCounts)
+ case <-time.After(100 * time.Millisecond):
+ }
+ }
+ })
+ }
+ for _, tc := range []struct {
+ datasetIDs string
+ expected map[string][]string
+ }{
+ {first, map[string][]string{"foo": {ids[0], ids[1]}, "bar": {ids[0], ids[2]}, "baz": {ids[2]}}},
+ {first + "," + second, map[string][]string{"foo": {ids[0], ids[1]}, "bar": {ids[0], ids[2]}, "baz": {ids[2]}, "foreign": {foreign}}},
+ } {
+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/metadata/flattened", nil, apiObject{"dataset_ids": tc.datasetIDs}))["data"])
+ tags := object(t, data["tags"])
+ if len(tags) != len(tc.expected) {
+ t.Fatalf("flattened tags contain unexpected values: %v", tags)
+ }
+ for value, expectedIDs := range tc.expected {
+ var actual []string
+ for _, id := range array(t, tags[value]) {
+ actual = append(actual, id.(string))
+ }
+ slices.Sort(actual)
+ slices.Sort(expectedIDs)
+ if !slices.Equal(actual, expectedIDs) {
+ t.Errorf("tag %s documents=%v want=%v", value, actual, expectedIDs)
+ }
+ }
+ }
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/metadata/flattened", nil, nil), 200, 102)
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/metadata/flattened", nil, apiObject{"dataset_ids": "invalid"}), 200, 404)
+}
+
+func TestDatasetFreshGraphTagsAndIngestionState(t *testing.T) {
+ id := createRESTDataset(t, nil)["id"].(string)
+ graph := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/graph", nil, nil))["data"])
+ for _, key := range []string{"graph", "mind_map"} {
+ if len(object(t, graph[key])) != 0 {
+ t.Fatalf("fresh dataset has %s entries: %v", key, graph)
+ }
+ }
+ tags := restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/tags/aggregation", nil, apiObject{"dataset_ids": id}))["data"]
+ if len(array(t, tags)) != 0 {
+ t.Fatalf("fresh dataset has tags: %v", tags)
+ }
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/tags/aggregation", nil, nil), 200, 102)
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/tags/aggregation", nil, apiObject{"dataset_ids": "invalid"}), 200, 102)
+ summary := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/ingestions/summary", nil, nil))["data"])
+ for _, key := range []string{"doc_num", "chunk_num", "token_num"} {
+ if summary[key] != float64(0) {
+ t.Errorf("fresh dataset %s=%v", key, summary[key])
+ }
+ }
+ for _, key := range []string{"status", "download_status"} {
+ counts := object(t, summary[key])
+ expectedKeys := []string{"running_count", "done_count", "fail_count"}
+ if key == "status" {
+ expectedKeys = append(expectedKeys, "unstart_count", "cancel_count")
+ }
+ for _, field := range expectedKeys {
+ if counts[field] != float64(0) {
+ t.Errorf("fresh %s.%s=%v", key, field, counts[field])
+ }
+ }
+ }
+ for _, query := range []apiObject{
+ nil,
+ {"log_type": "file", "keywords": uniqueName("missing"), "page": 1, "page_size": 2},
+ {"create_date_from": "2999-12-31", "create_date_to": "2000-01-01"},
+ } {
+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/ingestions", nil, query))["data"])
+ if data["total"] != float64(0) || len(array(t, data["logs"])) != 0 {
+ t.Fatalf("fresh dataset logs=%v", data)
+ }
+ }
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/ingestions/nonexistent", nil, nil), 200, 102)
+ for _, query := range []apiObject{{"page": "invalid"}, {"page_size": "invalid"}} {
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/ingestions", nil, query), 200, 101)
+ }
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/ingestions", nil, apiObject{"log_type": "unknown"}), 200, 102)
+ for _, suffix := range []string{"/ingestions/summary", "/ingestions", "/ingestions/nonexistent", "/graph", "/compilation/status"} {
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/00000000000040008000000000000000"+suffix, nil, nil), 200, 404)
+ }
+}
+
+func TestDatasetCompilationStatusAndEmbeddingRequirements(t *testing.T) {
+ id := createRESTDataset(t, nil)["id"].(string)
+ // Compilation status is the supported replacement for the retired /index routes.
+ for _, kind := range []string{"", "graph", "raptor", "mindmap", "wiki"} {
+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/compilation/status", nil, apiObject{"kind": kind}))["data"])
+ for field, expected := range (apiObject{"kind": kind, "state": "idle", "progress": float64(0), "inflight": float64(0), "backlog": float64(0)}) {
+ if data[field] != expected {
+ t.Errorf("fresh compilation status %s=%v want=%v", field, data[field], expected)
+ }
+ }
+ }
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/compilation/status", nil, apiObject{"kind": "invalid"}), 200, 102)
+ result := restCall(t, TestConfig, http.MethodPost, "/datasets/"+id+"/embedding/check", apiObject{}, nil)
+ restCode(t, result, 200, 102)
+ if message, _ := result.JSON["message"].(string); !strings.Contains(strings.ToLower(message), "embedding") || !strings.Contains(message, "required") {
+ t.Fatalf("missing embedding error does not identify required model: %v", result.JSON)
+ }
+}
+
+func TestDatasetSearchParameterValidation(t *testing.T) {
+ id := createRESTDataset(t, nil)["id"].(string)
+ // Existing retrieval_api_test.go verifies nonempty matching chunks,
+ // document isolation and parity with this dataset-specific search route.
+ for _, tc := range []struct {
+ name string
+ fields apiObject
+ }{
+ {"missing_question", apiObject{"question": nil}},
+ {"blank_question", apiObject{"question": " "}},
+ {"typed_question", apiObject{"question": 1}},
+ {"zero_page", apiObject{"page": 0}},
+ {"negative_page_size", apiObject{"page_size": -1}},
+ {"zero_size", apiObject{"size": 0}},
+ {"low_top_k", apiObject{"knn_top_k": 0}},
+ {"high_top_k", apiObject{"knn_top_k": 2049}},
+ {"low_candidates", apiObject{"knn_num_candidates": 0}},
+ {"high_candidates", apiObject{"knn_num_candidates": 10001}},
+ {"candidates_below_top_k", apiObject{"knn_top_k": 5, "knn_num_candidates": 4}},
+ {"low_similarity", apiObject{"similarity_threshold": -0.01}},
+ {"high_similarity", apiObject{"similarity_threshold": 1.01}},
+ {"low_vector_weight", apiObject{"vector_similarity_weight": -0.01}},
+ {"high_vector_weight", apiObject{"vector_similarity_weight": 1.01}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ body := apiObject{"question": "contract validation"}
+ for key, value := range tc.fields {
+ body[key] = value
+ }
+ restCode(t, restCall(t, TestConfig, http.MethodPost, "/datasets/"+id+"/search", body, nil), 200, 101)
+ })
+ }
+}
diff --git a/test/api/dataset_list_delete_test.go b/test/api/dataset_list_delete_test.go
new file mode 100644
index 0000000000..e093b40d11
--- /dev/null
+++ b/test/api/dataset_list_delete_test.go
@@ -0,0 +1,188 @@
+//go:build integration
+
+package api
+
+import (
+ "net/http"
+ "reflect"
+ "strings"
+ "sync"
+ "testing"
+)
+
+func TestDatasetListPaginationFilteringAndTotals(t *testing.T) {
+ prefix := uniqueName("list")
+ ids := make([]string, 3)
+ names := make([]string, 3)
+ for i, suffix := range []string{"alpha", "bravo", "charlie"} {
+ data := createRESTDataset(t, apiObject{"name": prefix + suffix})
+ ids[i] = data["id"].(string)
+ names[i] = data["name"].(string)
+ }
+ scoped := strings.Join(ids, ",")
+ for _, tc := range []struct {
+ name string
+ query apiObject
+ want []string
+ total int
+ }{
+ {"first_page", apiObject{"ids": scoped, "sort": "name:asc", "page_size": 2, "page": 1}, ids[:2], 3},
+ {"second_page", apiObject{"ids": scoped, "sort": "name:asc", "page_size": 2, "page": 2}, ids[2:], 3},
+ {"past_end", apiObject{"ids": scoped, "sort": "name:asc", "page_size": 2, "page": 3}, []string{}, 3},
+ {"descending", apiObject{"ids": scoped, "sort": "name:desc"}, []string{ids[2], ids[1], ids[0]}, 3},
+ {"name", apiObject{"ids": scoped, "name": names[1]}, []string{ids[1]}, 1},
+ {"keywords", apiObject{"ids": scoped, "keywords": "bravo"}, []string{ids[1]}, 1},
+ {"missing_keyword", apiObject{"ids": scoped, "keywords": uniqueName("absent")}, []string{}, 0},
+ {"id_name_match", apiObject{"id": ids[0], "name": names[0]}, []string{ids[0]}, 1},
+ {"id_name_mismatch", apiObject{"id": ids[0], "name": names[1]}, []string{}, 0},
+ {"parser_mismatch", apiObject{"ids": scoped, "parser_id": "qa"}, []string{}, 0},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ result := restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, tc.query))
+ got := make([]string, 0)
+ for _, item := range array(t, result["data"]) {
+ got = append(got, object(t, item)["id"].(string))
+ }
+ if !reflect.DeepEqual(got, tc.want) || result["total_datasets"] != float64(tc.total) {
+ t.Fatalf("listed ids=%v total=%v want ids=%v total=%d", got, result["total_datasets"], tc.want, tc.total)
+ }
+ })
+ }
+ for _, query := range []apiObject{{"page": 0}, {"page": -1}, {"page": "a"}, {"page_size": 0}, {"page_size": "a"}, {"orderby": "unknown"}, {"orderby": "CREATE_TIME"}, {"desc": "unknown"}, {"id": ""}, {"id": "not_uuid"}, {"ids": ids[0] + "," + ids[0]}, {"ids": ids[0] + ",invalid"}, {"unknown_field": "x"}} {
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, query), 200, 101)
+ }
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, apiObject{"id": "00000000000040008000000000000000"}), 200, 404)
+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, apiObject{"id": ids[0], "ids": scoped}), 200, 102)
+}
+
+func TestDatasetDeleteValidationAtomicityAndIsolation(t *testing.T) {
+ first := createRESTDataset(t, nil)
+ second := createRESTDataset(t, nil)
+ untouched := createRESTDataset(t, nil)
+ firstID, secondID := first["id"].(string), second["id"].(string)
+ missingID := "00000000000040008000000000000000"
+ for _, tc := range []struct {
+ name string
+ body apiObject
+ code int
+ }{
+ {"empty", apiObject{"ids": []string{}}, 0},
+ {"null", apiObject{"ids": nil}, 0},
+ {"unset", apiObject{}, 0},
+ {"duplicate", apiObject{"ids": []string{firstID, firstID}}, 101},
+ {"malformed_id", apiObject{"ids": []string{"not_uuid"}}, 101},
+ {"type_error", apiObject{"ids": "bad"}, 102},
+ {"missing_id", apiObject{"ids": []string{missingID}}, 403},
+ {"invalid_first", apiObject{"ids": []string{missingID, firstID, secondID}}, 403},
+ {"invalid_middle", apiObject{"ids": []string{firstID, missingID, secondID}}, 403},
+ {"invalid_last", apiObject{"ids": []string{firstID, secondID, missingID}}, 403},
+ {"extra", apiObject{"unknown_field": true}, 101},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ restCode(t, restCall(t, TestConfig, http.MethodDelete, "/datasets", tc.body, nil), 200, tc.code)
+ requireDatasetFields(t, firstID, apiObject{"name": first["name"]})
+ requireDatasetFields(t, secondID, apiObject{"name": second["name"]})
+ })
+ }
+ deleted := object(t, restOK(t, restCall(t, TestConfig, http.MethodDelete, "/datasets", apiObject{"ids": []string{firstID, secondID}}, nil))["data"])
+ if deleted["success_count"] != float64(2) || len(array(t, deleted["errors"])) != 0 {
+ t.Fatalf("batch result=%v", deleted)
+ }
+ requireDatasetAbsent(t, firstID)
+ requireDatasetAbsent(t, secondID)
+ restCode(t, restCall(t, TestConfig, http.MethodDelete, "/datasets", apiObject{"ids": []string{firstID, secondID}}, nil), 200, 403)
+ requireDatasetFields(t, untouched["id"].(string), apiObject{"name": untouched["name"]})
+}
+
+func TestDatasetAuthorizationPreservesResources(t *testing.T) {
+ dataset := createRESTDataset(t, nil)
+ id := dataset["id"].(string)
+ for _, key := range []string{"", TestConfig.InvalidAPIKey} {
+ config := *TestConfig
+ config.APIKey = key
+ for _, tc := range []struct {
+ method, path string
+ body any
+ }{
+ {http.MethodGet, "/datasets", nil}, {http.MethodGet, "/datasets/" + id, nil},
+ {http.MethodPost, "/datasets", apiObject{"name": uniqueName("unauthorized"), "parse_type": 1, "parser_id": "general"}},
+ {http.MethodPut, "/datasets/" + id, apiObject{"name": "unauthorized"}},
+ {http.MethodDelete, "/datasets", apiObject{"ids": []string{id}}},
+ {http.MethodGet, "/datasets/" + id + "/metadata/config", nil},
+ {http.MethodPut, "/datasets/" + id + "/metadata/config", apiObject{"enabled": true}},
+ } {
+ result := restCall(t, &config, tc.method, tc.path, tc.body, nil)
+ if tc.method == http.MethodPost {
+ if data, ok := result.JSON["data"].(apiObject); ok {
+ if createdID, ok := data["id"].(string); ok && createdID != "" {
+ cleanupResource(t, TestConfig, "datasets", createdID)
+ }
+ }
+ }
+ restCode(t, result, 401, 401)
+ }
+ requireDatasetFields(t, id, apiObject{"name": dataset["name"], "parser_config": dataset["parser_config"]})
+ }
+}
+
+func TestDatasetConcurrentReadsAndUpdates(t *testing.T) {
+ dataset := createRESTDataset(t, nil)
+ id := dataset["id"].(string)
+ // A small deterministic set proves each request completes and the final
+ // value is one complete committed update; hundreds add load, not coverage.
+ names := make([]string, 4)
+ var wg sync.WaitGroup
+ for i := range names {
+ names[i] = uniqueName("concurrent")
+ wg.Go(func() {
+ result := restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": names[i]}, nil)
+ restCode(t, result, 200, 0)
+ if object(t, result.JSON["data"])["name"] != names[i] {
+ t.Errorf("update response did not identify the submitted name")
+ }
+ })
+ }
+ wg.Wait()
+ final := getRESTDataset(t, id)["name"]
+ found := false
+ for _, name := range names {
+ found = found || final == name
+ }
+ if !found {
+ t.Fatalf("torn final name=%v", final)
+ }
+ for range 4 {
+ wg.Go(func() { requireDatasetFields(t, id, apiObject{"name": final}) })
+ }
+ wg.Wait()
+}
+
+func TestDatasetConcurrentCreationAndDeletion(t *testing.T) {
+ ids := make([]string, 4)
+ names := make([]string, len(ids))
+ var wg sync.WaitGroup
+ for i := range ids {
+ names[i] = uniqueName("concurrent_create")
+ wg.Go(func() {
+ dataset := createRESTDataset(t, apiObject{"name": names[i]})
+ ids[i] = dataset["id"].(string)
+ if dataset["name"] != names[i] {
+ t.Errorf("concurrent create returned name=%v want=%s", dataset["name"], names[i])
+ }
+ })
+ }
+ wg.Wait()
+ seen := map[string]bool{}
+ for i, id := range ids {
+ if id == "" || seen[id] {
+ t.Fatalf("concurrent creates did not return distinct IDs: %v", ids)
+ }
+ seen[id] = true
+ requireDatasetFields(t, id, apiObject{"name": names[i]})
+ wg.Go(func() {
+ restOK(t, restCall(t, TestConfig, http.MethodDelete, "/datasets", apiObject{"ids": []string{id}}, nil))
+ requireDatasetAbsent(t, id)
+ })
+ }
+ wg.Wait()
+}
diff --git a/test/api/dataset_validation_test.go b/test/api/dataset_validation_test.go
new file mode 100644
index 0000000000..04bc550289
--- /dev/null
+++ b/test/api/dataset_validation_test.go
@@ -0,0 +1,296 @@
+//go:build integration
+
+package api
+
+import (
+ "net/http"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func TestDatasetCreateValidation(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ fields apiObject
+ code int
+ fragment string
+ }{
+ {"missing_name", apiObject{"name": nil}, 102, "required"},
+ {"empty_name", apiObject{"name": ""}, 102, "required"},
+ {"blank_name", apiObject{"name": " \t "}, 102, "empty"},
+ {"long_name", apiObject{"name": strings.Repeat("a", 129)}, 102, "128"},
+ {"number_name", apiObject{"name": 42}, 102, "cannot unmarshal"},
+ {"missing_mode", apiObject{"parse_type": nil}, 102, "parse_type"},
+ {"invalid_mode", apiObject{"parse_type": 3}, 102, "parse_type"},
+ {"empty_parser", apiObject{"parser_id": ""}, 102, "parser_id"},
+ {"unknown_parser", apiObject{"parser_id": "unknown"}, 102, "input should be"},
+ {"typed_parser", apiObject{"parser_id": []string{}}, 102, "cannot unmarshal"},
+ {"empty_permission", apiObject{"permission": ""}, 102, "'me' or 'team'"},
+ {"invalid_permission", apiObject{"permission": "ME"}, 102, "'me' or 'team'"},
+ {"typed_permission", apiObject{"permission": []string{}}, 102, "cannot unmarshal"},
+ {"invalid_language", apiObject{"language": " "}, 102, "character"},
+ {"typed_config", apiObject{"parser_config": []string{}}, 102, "cannot unmarshal"},
+ {"config_size", apiObject{"parser_config": apiObject{"GeneralChunker:bad": apiObject{"delimiter": strings.Repeat("x", 65536)}}}, 101, "size limit"},
+ {"extra_field", apiObject{"unknown_field": true}, 101, "Extra inputs"},
+ {"server_id", apiObject{"id": "injected"}, 101, "Extra inputs"},
+ {"pagerank_creation", apiObject{"pagerank": 50}, 101, "Extra inputs"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ name := uniqueName("invalid_create")
+ fields := apiObject{"name": name, "parse_type": 1, "parser_id": "general"}
+ for key, value := range tc.fields {
+ fields[key] = value
+ }
+ result := restCall(t, TestConfig, http.MethodPost, "/datasets", fields, nil)
+ if data, ok := result.JSON["data"].(apiObject); ok {
+ if id, ok := data["id"].(string); ok && id != "" {
+ cleanupResource(t, TestConfig, "datasets", id)
+ }
+ }
+ restCode(t, result, 200, tc.code)
+ if message, _ := result.JSON["message"].(string); !strings.Contains(strings.ToLower(message), strings.ToLower(tc.fragment)) {
+ t.Errorf("message=%q should contain %q", message, tc.fragment)
+ }
+ // Failed validation must not have silently created the original fixture.
+ listed := restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, apiObject{"name": name}))
+ if len(array(t, listed["data"])) != 0 {
+ t.Fatal("failed create persisted a dataset")
+ }
+ })
+ }
+}
+
+func TestDatasetCreateDefaultsDeduplicationAndBoundaries(t *testing.T) {
+ name := uniqueName("case")
+ first := createRESTDataset(t, apiObject{"name": strings.ToUpper(name)})
+ second := createRESTDataset(t, apiObject{"name": strings.ToLower(name)})
+ if second["name"] != strings.ToLower(name)+"(1)" || first["id"] == second["id"] {
+ t.Fatalf("case-insensitive deduplication failed: %v %v", first, second)
+ }
+ boundary := uniqueName("boundary")
+ boundary += strings.Repeat("x", 128-len(boundary))
+ createRESTDataset(t, apiObject{"name": boundary})
+ for _, tc := range []struct {
+ name string
+ fields apiObject
+ }{{"unset", nil}, {"empty", apiObject{"parser_config": apiObject{}}}, {"null", apiObject{"parser_config": nil, "embedding_model": nil}}} {
+ t.Run(tc.name, func(t *testing.T) {
+ data := createRESTDataset(t, tc.fields)
+ if data["permission"] != "me" || data["parser_id"] != "general" {
+ t.Fatalf("unexpected defaults: %v", data)
+ }
+ requirePipelineParserConfig(t, data["parser_config"], "GeneralChunker:")
+ if embedding, ok := data["embedding_model"].(string); !ok || strings.TrimSpace(embedding) == "" {
+ t.Fatalf("missing configured default embedding: %v", data["embedding_model"])
+ }
+ })
+ }
+}
+
+func TestDatasetEmbeddingValidationAndRoundtrip(t *testing.T) {
+ dataset := createRESTDataset(t, nil)
+ id := dataset["id"].(string)
+ for _, model := range []string{"", " ", "missing-provider", "@Builtin", "model@", " @Builtin", "model@ ", "not-a-real-model@not-a-real-provider"} {
+ t.Run(model, func(t *testing.T) {
+ fields := apiObject{"embedding_model": model}
+ restCode(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, fields, nil), 200, 102)
+ requireDatasetFields(t, id, apiObject{"embedding_model": dataset["embedding_model"]})
+ fields["name"] = uniqueName("bad_model")
+ fields["parse_type"] = 1
+ fields["parser_id"] = "general"
+ result := restCall(t, TestConfig, http.MethodPost, "/datasets", fields, nil)
+ if data, ok := result.JSON["data"].(apiObject); ok {
+ if createdID, ok := data["id"].(string); ok && createdID != "" {
+ cleanupResource(t, TestConfig, "datasets", createdID)
+ }
+ }
+ restCode(t, result, 200, 102)
+ })
+ }
+ // Use the tenant's configured model, rather than a hardcoded provider that may
+ // not be authorized in the environment running these integration tests.
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"embedding_model": dataset["embedding_model"]}, nil))
+ requireDatasetFields(t, id, apiObject{"embedding_model": dataset["embedding_model"]})
+ createRESTDataset(t, apiObject{"embedding_model": dataset["embedding_model"]})
+}
+
+func TestDatasetUpdateValidationPreservesState(t *testing.T) {
+ data := createRESTDataset(t, nil)
+ id := data["id"].(string)
+ for _, tc := range []struct {
+ name string
+ fields apiObject
+ code int
+ }{
+ {"empty", apiObject{}, 102},
+ {"empty_name", apiObject{"name": ""}, 102},
+ {"blank_name", apiObject{"name": " \t "}, 102},
+ {"long_name", apiObject{"name": strings.Repeat("x", 129)}, 102},
+ {"typed_name", apiObject{"name": 42}, 102},
+ {"description_limit", apiObject{"description": strings.Repeat("x", 65536)}, 102},
+ {"typed_description", apiObject{"description": []string{}}, 102},
+ {"avatar_limit", apiObject{"avatar": strings.Repeat("x", 65536)}, 102},
+ {"invalid_avatar", apiObject{"avatar": "not-an-image"}, 102},
+ {"avatar_mime", apiObject{"avatar": "data:image/gif;base64,eA=="}, 102},
+ {"permission", apiObject{"permission": "unknown"}, 102},
+ {"permission_type", apiObject{"permission": []string{}}, 102},
+ {"parser_empty", apiObject{"parse_type": 1, "parser_id": ""}, 102},
+ {"parser_unknown", apiObject{"parse_type": 1, "parser_id": "unknown"}, 102},
+ {"parser_missing_mode", apiObject{"parser_id": "qa"}, 102},
+ {"pagerank_negative", apiObject{"pagerank": -1}, 102},
+ {"pagerank_large", apiObject{"pagerank": 101}, 102},
+ {"pagerank_type", apiObject{"pagerank": "50"}, 102},
+ {"connectors_invalid", apiObject{"connectors": []any{apiObject{"id": ""}}}, 102},
+ {"config_type", apiObject{"parser_config": "invalid"}, 102},
+ {"config_size", apiObject{"parser_config": apiObject{"GeneralChunker:invalid": apiObject{"delimiter": strings.Repeat("x", 65536)}}}, 101},
+ {"config_ext", apiObject{"parser_config": apiObject{"ext": apiObject{}}}, 101},
+ {"unsupported", apiObject{"unknown_field": true}, 101},
+ {"ownership", apiObject{"tenant_id": "other"}, 101},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ restCode(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, tc.fields, nil), 200, tc.code)
+ actual := getRESTDataset(t, id)
+ for _, key := range []string{"name", "description", "avatar", "permission", "parser_id", "parser_config", "embedding_model", "pagerank", "language", "connectors"} {
+ if !reflect.DeepEqual(data[key], actual[key]) {
+ t.Errorf("failed update changed %s: before=%v after=%v", key, data[key], actual[key])
+ }
+ }
+ })
+ }
+ for _, tc := range []struct {
+ id string
+ code int
+ }{{"not_uuid", 101}, {"00000000000040008000000000000000", 404}} {
+ restCode(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+tc.id, apiObject{"description": "missing"}, nil), 200, tc.code)
+ }
+}
+
+func TestDatasetPermissionAndPartialUpdatePreservation(t *testing.T) {
+ data := createRESTDataset(t, apiObject{"permission": "team"})
+ id := data["id"].(string)
+ for _, permission := range []string{"me", "team"} {
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"permission": permission}, nil))
+ requireDatasetFields(t, id, apiObject{"permission": permission})
+ }
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"description": "preserve me"}, nil))
+ expected := getRESTDataset(t, id)
+ newName := uniqueName("partial")
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": newName}, nil))
+ actual := getRESTDataset(t, id)
+ for _, key := range []string{"description", "avatar", "permission", "parser_id", "parser_config", "embedding_model", "pagerank", "language", "connectors"} {
+ if !reflect.DeepEqual(expected[key], actual[key]) {
+ t.Errorf("partial update changed unprovided %s", key)
+ }
+ }
+ if actual["name"] != newName {
+ t.Errorf("name=%v want=%s", actual["name"], newName)
+ }
+}
+
+func TestDatasetChunkMethodMatrix(t *testing.T) {
+ for _, method := range []string{"general", "naive", "book", "email", "laws", "manual", "one", "paper", "picture", "presentation", "qa", "table"} {
+ t.Run(method, func(t *testing.T) {
+ data := createRESTDataset(t, apiObject{"parser_id": method})
+ id := data["id"].(string)
+ expected := method
+ if method == "naive" {
+ expected = "general"
+ }
+ if data["parser_id"] != expected {
+ t.Fatalf("parser=%v want=%s", data["parser_id"], expected)
+ }
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"parse_type": 1, "parser_id": "general"}, nil))
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"parse_type": 1, "parser_id": method}, nil))
+ requireDatasetFields(t, id, apiObject{"parser_id": expected})
+ })
+ }
+}
+
+func TestDatasetMalformedRequestPreservesState(t *testing.T) {
+ dataset := createRESTDataset(t, nil)
+ id := dataset["id"].(string)
+ for _, tc := range []struct {
+ name, body, contentType string
+ }{
+ {"empty", "", "application/json"},
+ {"truncated", `{"name":`, "application/json"},
+ {"array", `[]`, "application/json"},
+ {"scalar", `"name"`, "application/json"},
+ {"null", `null`, "application/json"},
+ {"wrong_content_type", `{"name":"invalid"}`, "text/plain"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ for _, target := range []struct{ method, path string }{
+ {http.MethodPost, "/datasets"},
+ {http.MethodPut, "/datasets/" + id},
+ {http.MethodDelete, "/datasets"},
+ } {
+ result := rawRESTCall(t, TestConfig, target.method, target.path, []byte(tc.body), tc.contentType)
+ if data, ok := result.JSON["data"].(apiObject); ok && target.method == http.MethodPost {
+ if createdID, ok := data["id"].(string); ok && createdID != "" {
+ cleanupResource(t, TestConfig, "datasets", createdID)
+ }
+ }
+ restCode(t, result, 200, 101)
+ requireDatasetFields(t, id, apiObject{"name": dataset["name"], "parser_config": dataset["parser_config"]})
+ }
+ })
+ }
+}
+
+func TestDatasetParserNormalization(t *testing.T) {
+ data := createRESTDataset(t, apiObject{"language": "Klingon", "parser_config": apiObject{"chunk_token_num": 19, "GeneralChunker:unknown": "ignored"}})
+ id := data["id"].(string)
+ requireDatasetFields(t, id, apiObject{"language": "Klingon"})
+ config := requirePipelineParserConfig(t, data["parser_config"], "GeneralChunker:")
+ if _, exists := config["GeneralChunker:unknown"]; exists {
+ t.Fatalf("unknown component survived pipeline normalization: %v", config)
+ }
+}
+
+func TestDatasetPagerankBoundariesAndReset(t *testing.T) {
+ dataset := createRESTDataset(t, nil)
+ id := dataset["id"].(string)
+ status := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/system/status", nil, nil))["data"])
+ engine := object(t, status["doc_engine"])["type"]
+ if engine == "" || engine == nil {
+ t.Fatal("system status did not identify the document engine")
+ }
+ for _, value := range []int{0, 100, 50, 0} {
+ result := restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"pagerank": value}, nil)
+ if engine == "infinity" && value != 0 {
+ restCode(t, result, 200, 102)
+ requireDatasetFields(t, id, apiObject{"pagerank": float64(0)})
+ } else {
+ restOK(t, result)
+ requireDatasetFields(t, id, apiObject{"pagerank": float64(value)})
+ }
+ }
+}
+
+func TestDatasetEmbeddingUpdatePreservesExistingChunks(t *testing.T) {
+ dataset := createRESTDataset(t, nil)
+ id := dataset["id"].(string)
+ document := object(t, uploadRESTDocuments(t, TestConfig, id, []documentUpload{{Name: "embedding.txt", Content: []byte("existing embedding chunk")}})[0])
+ path := "/datasets/" + id + "/documents/" + document["id"].(string) + "/chunks"
+ content := "Dataset embedding roundtrip must preserve this existing chunk."
+ added := object(t, restOK(t, restCall(t, TestConfig, http.MethodPost, path, apiObject{"content": content}, nil))["data"])
+ chunk := object(t, added["chunk"])
+ chunkID := chunk["id"].(string)
+ persisted := getRESTDataset(t, id)
+ if count, ok := persisted["chunk_count"].(float64); !ok || count < 1 {
+ t.Fatalf("fixture has no persisted chunks: %v", persisted)
+ }
+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"embedding_model": dataset["embedding_model"]}, nil))
+ requireDatasetFields(t, id, apiObject{"embedding_model": dataset["embedding_model"], "chunk_count": persisted["chunk_count"]})
+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, path, nil, nil))["data"])
+ chunks := array(t, data["chunks"])
+ if len(chunks) != 1 {
+ t.Fatalf("embedding update changed document chunks: %v", chunks)
+ }
+ actual := object(t, chunks[0])
+ if actual["chunk_id"] != chunkID || actual["content_with_weight"] != content {
+ t.Fatalf("embedding update corrupted existing chunk: %v", actual)
+ }
+}
diff --git a/test/api/rest_request_test.go b/test/api/rest_request_test.go
index 828a090248..15242b852b 100644
--- a/test/api/rest_request_test.go
+++ b/test/api/rest_request_test.go
@@ -3,11 +3,14 @@
package api
import (
+ "bytes"
"context"
+ "crypto/sha256"
"encoding/json/v2"
"errors"
"fmt"
"io"
+ "mime/multipart"
"net/http"
"net/http/httptest"
"net/url"
@@ -61,12 +64,17 @@ func restCallContext(t *testing.T, ctx context.Context, c *ServerTestConfig, met
}
requestURL.RawQuery = params.Encode()
resp, err := c.RequestJSONContext(ctx, method, path, body, query, headers)
+ return readRESTResponse(t, c, method, requestURL.String(), diagnosticBody(requestBody, c.APIKey, c.Password), started, resp, err)
+}
+
+func readRESTResponse(t *testing.T, c *ServerTestConfig, method, requestURL, requestSummary string, started time.Time, resp *http.Response, err error) restResult {
+ t.Helper()
if err != nil {
// url.Error includes the unsanitized URL; diagnostics already include its safe form.
if urlErr, ok := errors.AsType[*url.Error](err); ok {
err = urlErr.Err
}
- t.Fatalf("%s %s payload=%s duration=%s transport error=%v", method, diagnosticURL(requestURL.String()), diagnosticBody(requestBody, c.APIKey, c.Password), time.Since(started), err)
+ t.Fatalf("%s %s payload=%s duration=%s transport error=%v", method, diagnosticURL(requestURL), requestSummary, time.Since(started), err)
}
defer resp.Body.Close()
result := restResult{Status: resp.StatusCode, Header: resp.Header}
@@ -81,6 +89,11 @@ func restCallContext(t *testing.T, ctx context.Context, c *ServerTestConfig, met
_ = json.Unmarshal(result.Body, &result.JSON)
}
responseLog := diagnosticBody(result.Body, c.APIKey, c.Password)
+ contentType := resp.Header.Get("Content-Type")
+ binary := strings.HasPrefix(contentType, "image/") || strings.Contains(contentType, "application/octet-stream") || resp.Header.Get("Content-Disposition") != ""
+ if binary || result.JSON == nil && !strings.Contains(contentType, "text/event-stream") {
+ responseLog = fmt.Sprintf("bytes=%d content_type=%q disposition=%q sha256=%x", len(result.Body), resp.Header.Get("Content-Type"), resp.Header.Get("Content-Disposition"), sha256.Sum256(result.Body))
+ }
if strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") {
events, parseErr := parseSSE(strings.NewReader(string(result.Body)))
preview := make([]any, 0, min(len(events), 5))
@@ -97,10 +110,82 @@ func restCallContext(t *testing.T, ctx context.Context, c *ServerTestConfig, met
encoded, _ := json.Marshal(preview)
responseLog = fmt.Sprintf("SSE bytes=%d events=%d parse_error=%v preview=%s", len(result.Body), len(events), parseErr, diagnosticBody(encoded, c.APIKey, c.Password))
}
- t.Logf("%s %s payload=%s HTTP=%d code=%v response=%s duration=%s", method, diagnosticURL(resp.Request.URL.String()), diagnosticBody(requestBody, c.APIKey, c.Password), result.Status, result.JSON["code"], responseLog, time.Since(started))
+ t.Logf("%s %s payload=%s HTTP=%d code=%v response=%s duration=%s", method, diagnosticURL(requestURL), requestSummary, result.Status, result.JSON["code"], responseLog, time.Since(started))
return result
}
+// rawRESTCall shares bounded response handling and redaction with JSON requests.
+func rawRESTCall(t *testing.T, c *ServerTestConfig, method, path string, body []byte, contentType string) restResult {
+ t.Helper()
+ summary := fmt.Sprintf("content_type=%q bytes=%d sha256=%x body=%s", contentType, len(body), sha256.Sum256(body), diagnosticBody(body, c.APIKey, c.Password))
+ return rawRESTRequest(t, c, method, path, body, contentType, summary)
+}
+
+func rawRESTRequest(t *testing.T, c *ServerTestConfig, method, path string, body []byte, contentType, summary string) restResult {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+ defer cancel()
+ started := time.Now()
+ requestURL := c.apiBase() + "/" + strings.TrimLeft(path, "/")
+ req, err := http.NewRequestWithContext(ctx, method, requestURL, bytes.NewReader(body))
+ if err != nil {
+ t.Fatalf("%s %s payload=%s create request: %v", method, diagnosticURL(requestURL), summary, err)
+ }
+ req.Header.Set("Content-Type", contentType)
+ if c.APIKey != "" {
+ req.Header.Set("Authorization", "Bearer "+c.APIKey)
+ }
+ resp, err := chunkHTTPClient.Do(req)
+ return readRESTResponse(t, c, method, requestURL, summary, started, resp, err)
+}
+
+type documentUpload struct {
+ Name string
+ Content []byte
+}
+
+func multipartRESTDocuments(t *testing.T, c *ServerTestConfig, datasetID string, files []documentUpload) restResult {
+ t.Helper()
+ var body bytes.Buffer
+ writer := multipart.NewWriter(&body)
+ summary := make([]apiObject, 0, len(files))
+ for _, file := range files {
+ part, err := writer.CreateFormFile("file", file.Name)
+ if err != nil {
+ t.Fatalf("multipart file %q: %v", file.Name, err)
+ }
+ if _, err := part.Write(file.Content); err != nil {
+ t.Fatalf("write multipart file %q: %v", file.Name, err)
+ }
+ summary = append(summary, apiObject{"name": file.Name, "size": len(file.Content)})
+ }
+ if err := writer.Close(); err != nil {
+ t.Fatal(err)
+ }
+ encoded, err := json.Marshal(apiObject{"files": summary, "field": "file"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return rawRESTRequest(t, c, http.MethodPost, "/datasets/"+datasetID+"/documents", body.Bytes(), writer.FormDataContentType(), diagnosticBody(encoded, c.APIKey, c.Password))
+}
+
+func uploadRESTDocuments(t *testing.T, c *ServerTestConfig, datasetID string, files []documentUpload) []any {
+ t.Helper()
+ items := array(t, restOK(t, multipartRESTDocuments(t, c, datasetID, files))["data"])
+ if len(items) != len(files) {
+ t.Fatalf("uploaded documents=%d want=%d", len(items), len(files))
+ }
+ seen := map[string]bool{}
+ for _, item := range items {
+ id, ok := object(t, item)["id"].(string)
+ if !ok || id == "" || seen[id] {
+ t.Fatalf("invalid or duplicate uploaded document ID %v", object(t, item)["id"])
+ }
+ seen[id] = true
+ }
+ return items
+}
+
func restCode(t *testing.T, r restResult, status, code int) {
t.Helper()
if r.Status != status {
@@ -129,7 +214,10 @@ func cleanupResource(t *testing.T, c *ServerTestConfig, family, id string) {
r := restCall(t, c, http.MethodDelete, path, body, nil)
// Tests exercise deletion themselves; a subsequently missing resource is expected.
code, ok := r.JSON["code"].(float64)
- if r.Status != 200 || !ok || (code != 0 && code != 102 && code != 103 && code != 404) {
+ // Dataset deletion reports 403 for an already absent ID. The read below
+ // must still prove absence, so an actual permission failure cannot pass.
+ missingDataset := family == "datasets" && code == 403
+ if r.Status != 200 || !ok || (code != 0 && code != 102 && code != 103 && code != 404 && !missingDataset) {
t.Errorf("cleanup %s/%s failed (see diagnostic)", family, id)
}
// A bulk-delete envelope can report partial failures. Prove absence rather
@@ -184,10 +272,58 @@ func TestRESTHelperTransport(t *testing.T) {
}
}
+func TestRESTHelperMultipartAndBinary(t *testing.T) {
+ files := []documentUpload{{Name: "first.txt", Content: []byte("first\x00document")}, {Name: "second.md", Content: []byte("# Second\n")}}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "Bearer test-secret" {
+ t.Error("raw request did not use API key")
+ }
+ if r.Method == http.MethodGet {
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Header().Set("Content-Disposition", `attachment; filename="first.txt"`)
+ _, _ = w.Write(files[0].Content)
+ return
+ }
+ if err := r.ParseMultipartForm(1 << 20); err != nil {
+ t.Errorf("parse multipart: %v", err)
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ defer r.MultipartForm.RemoveAll()
+ parts := r.MultipartForm.File["file"]
+ if len(parts) != len(files) {
+ t.Errorf("multipart files=%d want=%d", len(parts), len(files))
+ return
+ }
+ for i, part := range parts {
+ f, err := part.Open()
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ content, err := io.ReadAll(f)
+ f.Close()
+ if err != nil || part.Filename != files[i].Name || !bytes.Equal(content, files[i].Content) {
+ t.Errorf("multipart file %d changed: name=%q read_error=%v", i, part.Filename, err)
+ }
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = io.WriteString(w, `{"code":0,"data":[{"id":"first"},{"id":"second"}]}`)
+ }))
+ t.Cleanup(server.Close)
+ c := &ServerTestConfig{HostAddress: server.URL, APIVersion: "v1", APIKey: "test-secret", AuthToken: "must-not-be-used"}
+ uploadRESTDocuments(t, c, "dataset", files)
+ r := rawRESTCall(t, c, http.MethodGet, "/documents/first", nil, "")
+ if r.Status != http.StatusOK || r.JSON != nil || !bytes.Equal(r.Body, files[0].Content) || r.Header.Get("Content-Disposition") != `attachment; filename="first.txt"` {
+ t.Fatal("binary response bytes or headers were lost")
+ }
+}
+
func TestRESTHelperDatasetCleanup(t *testing.T) {
- for _, tc := range []struct{ name, body string }{
- {"empty_list", `{"code":0,"data":[]}`},
- {"missing_id", `{"code":404,"message":"Resource not found"}`},
+ for _, tc := range []struct{ name, deleted, body string }{
+ {"empty_list", `{"code":0,"data":{"success_count":1,"errors":[]}}`, `{"code":0,"data":[]}`},
+ {"missing_id", `{"code":0,"data":{"success_count":1,"errors":[]}}`, `{"code":404,"message":"Resource not found"}`},
+ {"already_deleted", `{"code":403,"message":"Resource not found"}`, `{"code":404,"message":"Resource not found"}`},
} {
t.Run(tc.name, func(t *testing.T) {
var deleted, verified atomic.Bool
@@ -195,7 +331,7 @@ func TestRESTHelperDatasetCleanup(t *testing.T) {
w.Header().Set("Content-Type", "application/json")
if r.Method == http.MethodDelete && r.URL.Path == "/api/v1/datasets" {
deleted.Store(true)
- _, _ = io.WriteString(w, `{"code":0,"data":{"success_count":1,"errors":[]}}`)
+ _, _ = io.WriteString(w, tc.deleted)
return
}
if !deleted.Load() || r.Method != http.MethodGet || r.URL.Query().Get("id") != "dataset-id" {