RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
test(api): migrate Python dataset API coverage to Go (#21063)
Sbaaoui Idriss012a59e26e80
diff --git a/test/api/chunk_api_test.go b/test/api/chunk_api_test.goindex 39aa3fafc4..93bc6fec6b 100644--- a/test/api/chunk_api_test.go+++ b/test/api/chunk_api_test.go@@ -23,7 +23,6 @@ import ("encoding/json""fmt""io"- "mime/multipart""net/http""net/url""slices"@@ -215,51 +214,8 @@ func createChunkTestDocument(t *testing.T, filename string) (string, string) {func uploadChunkTestDocument(t *testing.T, datasetID, filename string) string {t.Helper()- const fileContents = "ragflow test upload"- var uploadBody bytes.Buffer- writer := multipart.NewWriter(&uploadBody)- file, err := writer.CreateFormFile("file", filename)- if err != nil {- t.Fatalf("create multipart file field: %v", err)- }- if _, err := io.WriteString(file, fileContents); err != nil {- t.Fatalf("write multipart file: %v", err)- }- if err := writer.Close(); err != nil {- t.Fatalf("close multipart writer: %v", err)- }-- requestURL := fmt.Sprintf("%s/datasets/%s/documents", TestConfig.apiBase(), datasetID)- t.Logf("request: %s %s\npayload: multipart file %q (%d bytes), content=%q",- http.MethodPost, requestURL, filename, len(fileContents), fileContents)- request, err := http.NewRequest(http.MethodPost, requestURL, &uploadBody)- if err != nil {- t.Fatalf("create document upload request: %v", err)- }- request.Header = TestConfig.buildHeaders(nil)- request.Header.Set("Content-Type", writer.FormDataContentType())- response, err := chunkHTTPClient.Do(request)- if err != nil {- t.Fatalf("upload test document: %v", err)- }- uploadResult := readChunkAPIResponse(t, requestURL, response)- if uploadResult.err != nil {- t.Fatalf("decode document upload response: %v; body: %s", uploadResult.err, uploadResult.body)- }- if uploadResult.status != http.StatusOK {- t.Fatalf("document upload status = %d, want %d; response: %s", uploadResult.status, http.StatusOK, uploadResult.body)- }- requireChunkCode(t, uploadResult.payload, 0)-- items, ok := uploadResult.payload["data"].([]interface{})- if !ok || len(items) == 0 {- t.Fatalf("document upload data is not a non-empty array: %v", uploadResult.payload)- }- document, ok := items[0].(map[string]interface{})- if !ok {- t.Fatalf("document upload item is not an object: %v", items[0])- }- documentID := chunkString(t, document, "id")+ items := uploadRESTDocuments(t, TestConfig, datasetID, []documentUpload{{Name: filename, Content: []byte("ragflow test upload")}})+ documentID := chunkString(t, object(t, items[0]), "id")t.Cleanup(func() {deleted := requestChunkAPI(t, TestConfig, http.MethodDelete,fmt.Sprintf("/datasets/%s/documents", datasetID),diff --git a/test/api/dataset_api_test.go b/test/api/dataset_api_test.goindex 36c8b39ca6..b4ae611b0b 100644--- a/test/api/dataset_api_test.go+++ b/test/api/dataset_api_test.go@@ -21,561 +21,195 @@ package apiimport ("bytes""encoding/base64"- "encoding/json"- "fmt""image""image/color"- "image/draw""image/png"- "io""net/http"+ "reflect""strings""testing")-func decodeResponseJSON(resp *http.Response) (map[string]interface{}, error) {- defer resp.Body.Close()- body, err := io.ReadAll(resp.Body)- if err != nil {- return nil, err- }- var payload map[string]interface{}- if err = json.Unmarshal(body, &payload); err != nil {- return nil, err- }- return payload, nil-}--func requireStatusCode(t *testing.T, resp *http.Response, expected int) map[string]interface{} {+// createRESTDataset registers ownership before checking the persisted resource.+// Explicit names are useful for duplicate and boundary tests; all callers own+// only the returned ID, even when the server deduplicates the requested name.+func createRESTDataset(t *testing.T, fields apiObject) apiObject {t.Helper()- if resp.StatusCode != expected {- t.Fatalf("expected status code %d, got %d", expected, resp.StatusCode)+ payload := apiObject{"name": uniqueName("go_dataset"), "parse_type": 1, "parser_id": "general"}+ for key, value := range fields {+ payload[key] = value+ }+ result := restCall(t, TestConfig, http.MethodPost, "/datasets", payload, nil)+ if data, ok := result.JSON["data"].(apiObject); ok {+ if id, ok := data["id"].(string); ok && id != "" {+ cleanupResource(t, TestConfig, "datasets", id)+ }}- payload, err := decodeResponseJSON(resp)- if err != nil {- t.Fatalf("failed to decode response: %v", err)+ data := object(t, restOK(t, result)["data"])+ id, ok := data["id"].(string)+ if !ok || id == "" {+ t.Fatalf("created dataset has invalid id: %v", data)+ }+ persisted := getRESTDataset(t, id)+ for _, key := range []string{"id", "name", "permission", "parser_id", "embedding_model", "parser_config"} {+ if !reflect.DeepEqual(data[key], persisted[key]) {+ t.Fatalf("created %s differs from persisted value: response=%v stored=%v", key, data[key], persisted[key])+ }}- return payload+ return persisted}-func requireCodeZero(t *testing.T, payload map[string]interface{}) {+func getRESTDataset(t *testing.T, id string) apiObject {t.Helper()- code, ok := payload["code"].(float64)- if !ok || int(code) != 0 {- t.Fatalf("expected code 0, got %v, payload: %v", payload["code"], payload)- }+ return object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id, nil, nil))["data"])}-func clearDatasets(t *testing.T) {+func requireDatasetFields(t *testing.T, id string, expected apiObject) apiObject {t.Helper()- resp, err := TestConfig.DeleteJSON("/datasets", map[string]interface{}{"ids": nil, "delete_all": true}, nil)- if err != nil {- t.Fatalf("clear datasets request failed: %v", err)- }- payload := requireStatusCode(t, resp, http.StatusOK)- code, _ := payload["code"].(float64)- if int(code) != 0 && int(code) != 102 {- t.Fatalf("clear datasets failed: code=%v, payload=%v", payload["code"], payload)+ actual := getRESTDataset(t, id)+ for key, want := range expected {+ if !reflect.DeepEqual(actual[key], want) {+ t.Errorf("dataset %s field %s=%#v want=%#v", id, key, actual[key], want)+ }}+ return actual}-func jsonDeepEqual(a, b interface{}) bool {- switch av := a.(type) {- case float64:- switch bv := b.(type) {- case float64:- return av == bv- case int:- return av == float64(bv)- case int64:- return av == float64(bv)- default:- return false- }- case int:- return jsonDeepEqual(float64(av), b)- case int64:- return jsonDeepEqual(float64(av), b)- case string, bool, nil:- return a == b- case []interface{}:- bv, ok := b.([]interface{})- if !ok || len(av) != len(bv) {- return false- }- for i := range av {- if !jsonDeepEqual(av[i], bv[i]) {- return false- }- }- return true- case map[string]interface{}:- bv, ok := b.(map[string]interface{})- if !ok || len(av) != len(bv) {- return false- }- for k, v := range av {- bvVal, ok := bv[k]- if !ok || !jsonDeepEqual(v, bvVal) {- return false- }- }- return true- default:- return a == b- }+func requireDatasetAbsent(t *testing.T, id string) {+ t.Helper()+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id, nil, nil), 200, 404)+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, apiObject{"id": id}), 200, 404)}func TestDatasetCRUDCycle(t *testing.T) {- clearDatasets(t)- // Create- createResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": "restful_dataset_crud"}, nil)- if err != nil {- t.Fatalf("create dataset request failed: %v", err)- }- createPayload := requireStatusCode(t, createResp, http.StatusOK)- requireCodeZero(t, createPayload)- createData, ok := createPayload["data"].(map[string]interface{})- if !ok {- t.Fatalf("create response data is not an object: %v", createPayload)- }- datasetID, ok := createData["id"].(string)- if !ok || datasetID == "" {- t.Fatalf("create response data.id is not a valid string: %v", createData)- }-- // Get by id- getResp, err := TestConfig.GetJSON(fmt.Sprintf("/datasets/%s", datasetID), nil, nil)- if err != nil {- t.Fatalf("get dataset request failed: %v", err)- }- getPayload := requireStatusCode(t, getResp, http.StatusOK)- requireCodeZero(t, getPayload)- getData, ok := getPayload["data"].(map[string]interface{})- if !ok || getData["id"] != datasetID {- t.Fatalf("get response data.id mismatch: %v", getPayload)- }-- // Update- updateResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", datasetID), map[string]interface{}{"name": "restful_dataset_crud_updated"}, nil)- if err != nil {- t.Fatalf("update dataset request failed: %v", err)- }- updatePayload := requireStatusCode(t, updateResp, http.StatusOK)- requireCodeZero(t, updatePayload)- updateData, ok := updatePayload["data"].(map[string]interface{})- if !ok || updateData["name"] != "restful_dataset_crud_updated" {- t.Fatalf("update response data.name mismatch: %v", updatePayload)- }-- // List with id filter- listResp, err := TestConfig.GetJSON("/datasets", map[string]interface{}{"id": datasetID}, nil)- if err != nil {- t.Fatalf("list dataset request failed: %v", err)- }- listPayload := requireStatusCode(t, listResp, http.StatusOK)- requireCodeZero(t, listPayload)- listData, ok := listPayload["data"].([]interface{})- if !ok || len(listData) != 1 {- t.Fatalf("expected 1 dataset in list, got %v", listPayload)- }- firstItem, ok := listData[0].(map[string]interface{})- if !ok || firstItem["id"] != datasetID {- t.Fatalf("list response data[0].id mismatch: %v", listPayload)- }-- // Delete- deleteResp, err := TestConfig.DeleteJSON("/datasets", map[string]interface{}{"ids": []interface{}{datasetID}}, nil)- if err != nil {- t.Fatalf("delete dataset request failed: %v", err)- }- deletePayload := requireStatusCode(t, deleteResp, http.StatusOK)- requireCodeZero(t, deletePayload)-- // List after delete- listAfterResp, err := TestConfig.GetJSON("/datasets", nil, nil)- if err != nil {- t.Fatalf("list after delete request failed: %v", err)- }- listAfterPayload := requireStatusCode(t, listAfterResp, http.StatusOK)- requireCodeZero(t, listAfterPayload)- listAfterData, ok := listAfterPayload["data"].([]interface{})- if !ok {- t.Fatalf("list after delete data is not an array: %v", listAfterPayload)- }- for _, item := range listAfterData {- dataset, ok := item.(map[string]interface{})- if !ok {- continue- }- if dataset["id"] == datasetID {- t.Fatalf("deleted dataset %s still present in list: %v", datasetID, listAfterPayload)- }- }+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ name := uniqueName("renamed")+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": name}, nil))+ requireDatasetFields(t, id, apiObject{"name": name})+ listed := restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, apiObject{"id": id}))+ items := array(t, listed["data"])+ if len(items) != 1 || object(t, items[0])["id"] != id || listed["total_datasets"] != float64(1) {+ t.Fatalf("id-filtered list=%v", listed)+ }+ restOK(t, restCall(t, TestConfig, http.MethodDelete, "/datasets", apiObject{"ids": []string{id}}, nil))+ requireDatasetAbsent(t, id)}func TestDatasetUpdateNameAndCaseInsensitiveContract(t *testing.T) {- clearDatasets(t)-- // Create first dataset- firstResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": "dataset_update_name_source"}, nil)- if err != nil {- t.Fatalf("create first dataset request failed: %v", err)- }- firstPayload := requireStatusCode(t, firstResp, http.StatusOK)- requireCodeZero(t, firstPayload)- firstData, ok := firstPayload["data"].(map[string]interface{})- if !ok {- t.Fatalf("first response data is not an object: %v", firstPayload)- }- firstDatasetID, ok := firstData["id"].(string)- if !ok || firstDatasetID == "" {- t.Fatalf("first dataset id is invalid: %v", firstData)- }-- // Create second dataset- secondResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": "dataset_update_name_target"}, nil)- if err != nil {- t.Fatalf("create second dataset request failed: %v", err)- }- secondPayload := requireStatusCode(t, secondResp, http.StatusOK)- requireCodeZero(t, secondPayload)- secondData, ok := secondPayload["data"].(map[string]interface{})- if !ok {- t.Fatalf("second response data is not an object: %v", secondPayload)- }- secondDatasetName, ok := secondData["name"].(string)- if !ok || secondDatasetName == "" {- t.Fatalf("second dataset name is invalid: %v", secondData)- }-- // Rename first dataset- renameResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", firstDatasetID), map[string]interface{}{"name": "dataset_update_name_renamed"}, nil)- if err != nil {- t.Fatalf("rename dataset request failed: %v", err)- }- renamePayload := requireStatusCode(t, renameResp, http.StatusOK)- requireCodeZero(t, renamePayload)- renameData, ok := renamePayload["data"].(map[string]interface{})- if !ok || renameData["name"] != "dataset_update_name_renamed" {- t.Fatalf("rename response data.name mismatch: %v", renamePayload)- }-- // List with id filter- listResp, err := TestConfig.GetJSON("/datasets", map[string]interface{}{"id": firstDatasetID}, nil)- if err != nil {- t.Fatalf("list dataset request failed: %v", err)- }- listPayload := requireStatusCode(t, listResp, http.StatusOK)- requireCodeZero(t, listPayload)- listData, ok := listPayload["data"].([]interface{})- if !ok || len(listData) != 1 {- t.Fatalf("expected 1 dataset in list, got %v", listPayload)- }- firstItem, ok := listData[0].(map[string]interface{})- if !ok || firstItem["name"] != "dataset_update_name_renamed" {- t.Fatalf("list response data[0].name mismatch: %v", listPayload)- }-- // Try duplicate name (case-insensitive)- duplicateResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", firstDatasetID), map[string]interface{}{"name": strings.ToUpper(secondDatasetName)}, nil)- if err != nil {- t.Fatalf("duplicate case rename request failed: %v", err)- }- duplicatePayload := requireStatusCode(t, duplicateResp, http.StatusOK)- duplicateCode, ok := duplicatePayload["code"].(float64)- if !ok || int(duplicateCode) != 102 {- t.Fatalf("expected code 102 for duplicate case rename, got %v, payload: %v", duplicatePayload["code"], duplicatePayload)- }- duplicateMessage, _ := duplicatePayload["message"].(string)- if !strings.Contains(duplicateMessage, "already exists") {- t.Fatalf("expected message to contain 'already exists', got %q, payload: %v", duplicateMessage, duplicatePayload)- }+ first := createRESTDataset(t, nil)+ second := createRESTDataset(t, nil)+ id := first["id"].(string)+ name := uniqueName("renamed")+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": name}, nil))+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": strings.ToUpper(name)}, nil))+ requireDatasetFields(t, id, apiObject{"name": strings.ToUpper(name)})+ restCode(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": strings.ToUpper(second["name"].(string))}, nil), 200, 102)+ requireDatasetFields(t, id, apiObject{"name": strings.ToUpper(name)})}func encodeAvatar() (string, error) {- img := image.NewRGBA(image.Rectangle{- Min: image.Point{X: 0, Y: 0},- Max: image.Point{X: 100, Y: 100},- })- draw.Draw(img, img.Bounds(), &image.Uniform{color.RGBA{R: 0, G: 0, B: 255, A: 255}}, image.Point{}, draw.Src)+ img := image.NewRGBA(image.Rect(0, 0, 1, 1))+ img.Set(0, 0, color.RGBA{B: 255, A: 255})var buf bytes.Bufferif err := png.Encode(&buf, img); err != nil {return "", err}- encoded := base64.StdEncoding.EncodeToString(buf.Bytes())- return fmt.Sprintf("data:image/png;base64,%s", encoded), nil+ return "data:image/png;base64," + base64.StdEncoding.EncodeToString(buf.Bytes()), nil}func TestDatasetUpdateLanguageConnectorsAvatarAndDescriptionContract(t *testing.T) {- clearDatasets(t)-- createResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": "dataset_update_lang_connectors"}, nil)- if err != nil {- t.Fatalf("create dataset request failed: %v", err)- }- createPayload := requireStatusCode(t, createResp, http.StatusOK)- requireCodeZero(t, createPayload)- createData, ok := createPayload["data"].(map[string]interface{})- if !ok {- t.Fatalf("create response data is not an object: %v", createPayload)- }- datasetID, ok := createData["id"].(string)- if !ok || datasetID == "" {- t.Fatalf("dataset id is invalid: %v", createData)- }-- avatarValue, err := encodeAvatar()+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ avatar, err := encodeAvatar()if err != nil {- t.Fatalf("failed to encode avatar: %v", err)- }-- updateResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", datasetID), map[string]interface{}{- "name": "dataset_update_lang_connectors",- "description": "",- "parser_id": "naive",- "parse_type": 1,- "language": "English",- "connectors": []interface{}{},- "avatar": avatarValue,- }, nil)- if err != nil {- t.Fatalf("update dataset request failed: %v", err)- }- updatePayload := requireStatusCode(t, updateResp, http.StatusOK)- requireCodeZero(t, updatePayload)- updateData, ok := updatePayload["data"].(map[string]interface{})- if !ok {- t.Fatalf("update response data is not an object: %v", updatePayload)- }- if updateData["language"] != "English" {- t.Fatalf("language mismatch: expected English, got %v", updatePayload)- }- connectors, ok := updateData["connectors"].([]interface{})- if !ok {- t.Fatalf("connectors is not an array: %v", updatePayload)- }- if len(connectors) != 0 {- t.Fatalf("connectors should be empty: %v", updatePayload)- }- if updateData["avatar"] != avatarValue {- t.Fatalf("avatar mismatch: %v", updatePayload)- }-- descriptionResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", datasetID), map[string]interface{}{"description": "description"}, nil)- if err != nil {- t.Fatalf("update description request failed: %v", err)- }- descriptionPayload := requireStatusCode(t, descriptionResp, http.StatusOK)- requireCodeZero(t, descriptionPayload)- descriptionData, ok := descriptionPayload["data"].(map[string]interface{})- if !ok || descriptionData["description"] != "description" {- t.Fatalf("description mismatch: %v", descriptionPayload)+ t.Fatal(err)}+ fields := apiObject{"language": "English", "connectors": []any{}, "avatar": avatar, "description": "description"}+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, fields, nil))+ requireDatasetFields(t, id, fields)+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"description": ""}, nil))+ requireDatasetFields(t, id, apiObject{"description": "", "language": "English", "avatar": avatar, "connectors": []any{}})}-func TestDatasetUpdateParserConfigValidMatrixContract(t *testing.T) {- clearDatasets(t)- t.Skip("Go validation or response contract does not match the established API contract")-- cases := []struct {- name string- parserConfig map[string]interface{}- }{- {name: "auto_keywords_min", parserConfig: map[string]interface{}{"auto_keywords": 0}},- {name: "auto_keywords_mid", parserConfig: map[string]interface{}{"auto_keywords": 16}},- {name: "auto_keywords_max", parserConfig: map[string]interface{}{"auto_keywords": 32}},- {name: "auto_questions_min", parserConfig: map[string]interface{}{"auto_questions": 0}},- {name: "auto_questions_mid", parserConfig: map[string]interface{}{"auto_questions": 5}},- {name: "auto_questions_max", parserConfig: map[string]interface{}{"auto_questions": 10}},- {name: "chunk_token_num_min", parserConfig: map[string]interface{}{"chunk_token_num": 1}},- {name: "chunk_token_num_mid", parserConfig: map[string]interface{}{"chunk_token_num": 1024}},- {name: "chunk_token_num_max", parserConfig: map[string]interface{}{"chunk_token_num": 2048}},- {name: "delimiter", parserConfig: map[string]interface{}{"delimiter": "\n"}},- {name: "delimiter_space", parserConfig: map[string]interface{}{"delimiter": " "}},- {name: "html4excel_true", parserConfig: map[string]interface{}{"html4excel": true}},- {name: "html4excel_false", parserConfig: map[string]interface{}{"html4excel": false}},- {name: "layout_recognize_DeepDOC", parserConfig: map[string]interface{}{"layout_recognize": "DeepDOC"}},- {name: "layout_recognize_navie", parserConfig: map[string]interface{}{"layout_recognize": "Plain Text"}},- {name: "topn_tags_min", parserConfig: map[string]interface{}{"topn_tags": 1}},- {name: "topn_tags_mid", parserConfig: map[string]interface{}{"topn_tags": 5}},- {name: "topn_tags_max", parserConfig: map[string]interface{}{"topn_tags": 10}},- {name: "tags_tag_file_id", parserConfig: map[string]interface{}{"tags": map[string]interface{}{"tag_file_id": "file-123"}}},- {name: "filename_embd_weight_min", parserConfig: map[string]interface{}{"filename_embd_weight": 0.1}},- {name: "filename_embd_weight_mid", parserConfig: map[string]interface{}{"filename_embd_weight": 0.5}},- {name: "filename_embd_weight_max", parserConfig: map[string]interface{}{"filename_embd_weight": 1.0}},- {name: "task_page_size_min", parserConfig: map[string]interface{}{"task_page_size": 1}},- {name: "task_page_size_None", parserConfig: map[string]interface{}{"task_page_size": nil}},- {name: "pages", parserConfig: map[string]interface{}{"pages": []interface{}{[]interface{}{1, 100}}}},- {name: "pages_none", parserConfig: map[string]interface{}{"pages": nil}},- {name: "graphrag_true", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"use_graphrag": true}}},- {name: "graphrag_false", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"use_graphrag": false}}},- {name: "graphrag_entity_types", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"entity_types": []interface{}{"age", "sex", "height", "weight"}}}},- {name: "graphrag_method_general", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"method": "general"}}},- {name: "graphrag_method_light", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"method": "light"}}},- {name: "graphrag_community_true", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"community": true}}},- {name: "graphrag_community_false", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"community": false}}},- {name: "graphrag_resolution_true", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"resolution": true}}},- {name: "graphrag_resolution_false", parserConfig: map[string]interface{}{"graphrag": map[string]interface{}{"resolution": false}}},- {name: "raptor_true", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"use_raptor": true}}},- {name: "raptor_false", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"use_raptor": false}}},- {name: "raptor_prompt", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"prompt": "Who are you?"}}},- {name: "raptor_max_token_min", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_token": 512}}},- {name: "raptor_max_token_mid", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_token": 1024}}},- {name: "raptor_max_token_max", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_token": 2048}}},- {name: "raptor_clustering_threshold_min", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"clustering_threshold": 0.0}}},- {name: "raptor_clustering_threshold_mid", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"clustering_threshold": 0.5}}},- {name: "raptor_clustering_threshold_max", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"clustering_threshold": 1.0}}},- {name: "raptor_max_cluster_min", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_cluster": 1}}},- {name: "raptor_max_cluster_mid", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_cluster": 512}}},- {name: "raptor_max_cluster_max", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"max_cluster": 1024}}},- {name: "raptor_random_seed_min", parserConfig: map[string]interface{}{"raptor": map[string]interface{}{"random_seed": 0}}},+func requirePipelineParserConfig(t *testing.T, value any, chunkerPrefix string) apiObject {+ t.Helper()+ config := object(t, value)+ if _, ok := config["File"]; !ok {+ t.Fatalf("parser config missing File node: %v", config)+ }+ parser, chunker := false, false+ for id, params := range config {+ object(t, params)+ parser = parser || strings.HasPrefix(id, "Parser:")+ chunker = chunker || strings.HasPrefix(id, chunkerPrefix)+ }+ if !parser || !chunker {+ t.Fatalf("parser config missing parser/chunker nodes: %v", config)+ }+ for _, key := range []string{"chunk_token_num", "raptor", "graphrag"} {+ if _, ok := config[key]; ok {+ t.Fatalf("unexpected flat setting %s: %v", key, config)+ }}+ return config+}- for _, tc := range cases {- tc := tc+func TestDatasetUpdateParserConfigWithChunkMethodChangeContract(t *testing.T) {+ for _, tc := range []struct {+ name string+ provided bool+ config any+ }{{"empty", true, apiObject{}}, {"null", true, nil}, {"unset", false, nil}} {t.Run(tc.name, func(t *testing.T) {- createResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": fmt.Sprintf("dataset_update_parser_%s", tc.name)}, nil)- if err != nil {- t.Fatalf("create dataset request failed: %v", err)- }- createPayload := requireStatusCode(t, createResp, http.StatusOK)- requireCodeZero(t, createPayload)- createData, ok := createPayload["data"].(map[string]interface{})- if !ok {- t.Fatalf("create response data is not an object: %v", createPayload)- }- datasetID, ok := createData["id"].(string)- if !ok || datasetID == "" {- t.Fatalf("dataset id is invalid: %v", createData)- }-- updateResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", datasetID), map[string]interface{}{"parser_config": tc.parserConfig}, nil)- if err != nil {- t.Fatalf("update parser_config request failed: %v", err)- }- updatePayload := requireStatusCode(t, updateResp, http.StatusOK)- requireCodeZero(t, updatePayload)-- listResp, err := TestConfig.GetJSON("/datasets", map[string]interface{}{"id": datasetID}, nil)- if err != nil {- t.Fatalf("list dataset request failed: %v", err)- }- listPayload := requireStatusCode(t, listResp, http.StatusOK)- requireCodeZero(t, listPayload)- listData, ok := listPayload["data"].([]interface{})- if !ok || len(listData) != 1 {- t.Fatalf("expected 1 dataset in list, got %v", listPayload)- }- firstItem, ok := listData[0].(map[string]interface{})- if !ok {- t.Fatalf("list data[0] is not an object: %v", listPayload)- }- actualParserConfig, ok := firstItem["parser_config"].(map[string]interface{})- if !ok {- t.Fatalf("parser_config is not an object: %v", firstItem)- }-- for key, expectedValue := range tc.parserConfig {- if key == "graphrag" || key == "raptor" {- if _, exists := actualParserConfig[key]; exists {- t.Fatalf("expected %s not in parser_config, but got %v", key, actualParserConfig)- }- continue- }- actualValue, ok := actualParserConfig[key]- if !ok {- t.Fatalf("parser_config missing key %s: %v", key, actualParserConfig)- }- if !jsonDeepEqual(actualValue, expectedValue) {- t.Fatalf("parser_config.%s mismatch: expected %v, got %v", key, expectedValue, actualValue)+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ fields := apiObject{"parse_type": 1, "parser_id": "qa"}+ if tc.provided {+ fields["parser_config"] = tc.config+ }+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, fields, nil))+ persisted := requireDatasetFields(t, id, apiObject{"parser_id": "qa"})+ config := object(t, persisted["parser_config"])+ if len(config) == 0 {+ t.Fatal("parser config is empty after method change")+ }+ for _, key := range []string{"raptor", "graphrag"} {+ if _, ok := config[key]; ok {+ t.Fatalf("unexpected flat key %s", key)}}})}}-func TestDatasetUpdateParserConfigWithChunkMethodChangeContract(t *testing.T) {- clearDatasets(t)-- cases := []struct {- name string- updatePayload map[string]interface{}- }{- {- name: "parser_config_empty",- updatePayload: map[string]interface{}{- "parser_id": "qa",- "parse_type": 1,- "parser_config": map[string]interface{}{},- },- },- {- name: "parser_config_none",- updatePayload: map[string]interface{}{- "parser_id": "qa",- "parse_type": 1,- "parser_config": nil,- },- },- {- name: "parser_config_unset",- updatePayload: map[string]interface{}{- "parser_id": "qa",- "parse_type": 1,- },- },+func TestDatasetComponentScopedParserConfigRoundtrip(t *testing.T) {+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ config := requirePipelineParserConfig(t, dataset["parser_config"], "GeneralChunker:")+ chunkerID := ""+ for key := range config {+ if strings.HasPrefix(key, "GeneralChunker:") {+ chunkerID = key+ break+ }}-- for _, tc := range cases {- tc := tc- t.Run(tc.name, func(t *testing.T) {- createResp, err := TestConfig.PostJSON("/datasets", map[string]interface{}{"name": fmt.Sprintf("dataset_update_%s", tc.name)}, nil)- if err != nil {- t.Fatalf("create dataset request failed: %v", err)- }- createPayload := requireStatusCode(t, createResp, http.StatusOK)- requireCodeZero(t, createPayload)- createData, ok := createPayload["data"].(map[string]interface{})- if !ok {- t.Fatalf("create response data is not an object: %v", createPayload)- }- datasetID, ok := createData["id"].(string)- if !ok || datasetID == "" {- t.Fatalf("dataset id is invalid: %v", createData)- }-- updateResp, err := TestConfig.PutJSON(fmt.Sprintf("/datasets/%s", datasetID), tc.updatePayload, nil)- if err != nil {- t.Fatalf("update dataset request failed: %v", err)- }- updatePayload := requireStatusCode(t, updateResp, http.StatusOK)- requireCodeZero(t, updatePayload)-- listResp, err := TestConfig.GetJSON("/datasets", map[string]interface{}{"id": datasetID}, nil)- if err != nil {- t.Fatalf("list dataset request failed: %v", err)- }- listPayload := requireStatusCode(t, listResp, http.StatusOK)- requireCodeZero(t, listPayload)- listData, ok := listPayload["data"].([]interface{})- if !ok || len(listData) != 1 {- t.Fatalf("expected 1 dataset in list, got %v", listPayload)- }- firstItem, ok := listData[0].(map[string]interface{})- if !ok {- t.Fatalf("list data[0] is not an object: %v", listPayload)- }- actualParserConfig, ok := firstItem["parser_config"].(map[string]interface{})- if !ok || len(actualParserConfig) == 0 {- t.Fatalf("parser_config should be non-empty map: %v", firstItem)- }- if _, exists := actualParserConfig["raptor"]; exists {- t.Fatalf("raptor should not be in parser_config: %v", actualParserConfig)- }- if _, exists := actualParserConfig["graphrag"]; exists {- t.Fatalf("graphrag should not be in parser_config: %v", actualParserConfig)+ for _, delimiter := range []string{"|", ";", "\n"} {+ t.Run(strings.ReplaceAll(delimiter, "\n", "newline"), func(t *testing.T) {+ fields := apiObject{"parser_config": apiObject{chunkerID: apiObject{"parent_child": apiObject{"use_parent_child": true, "children_delimiter": delimiter}}}}+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, fields, nil))+ updated := object(t, getRESTDataset(t, id)["parser_config"])+ chunker := object(t, updated[chunkerID])+ if object(t, chunker["parent_child"])["use_parent_child"] != true || !reflect.DeepEqual(chunker["children_delimiters"], []any{delimiter}) {+ t.Fatalf("component settings did not reach chunker: %v", chunker)+ }+ for key, params := range config {+ if key != chunkerID && !reflect.DeepEqual(params, updated[key]) {+ t.Errorf("unrelated parser node %s changed", key)+ }}})}+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"parser_config": apiObject{"File": apiObject{"layout_recognize": "DeepDOC"}}}, nil))+ requirePipelineParserConfig(t, getRESTDataset(t, id)["parser_config"], "GeneralChunker:")}diff --git a/test/api/dataset_features_test.go b/test/api/dataset_features_test.gonew file mode 100644index 0000000000..d58a23626d--- /dev/null+++ b/test/api/dataset_features_test.go@@ -0,0 +1,271 @@+//go:build integration++package api++import (+ "context"+ "net/http"+ "reflect"+ "slices"+ "strings"+ "testing"+ "time"+)++func TestDatasetMetadataConfigPersistenceAndValidation(t *testing.T) {+ dataset := createRESTDataset(t, nil)+ path := "/datasets/" + dataset["id"].(string) + "/metadata/config"+ empty := apiObject{"enabled": false, "metadata": []any{}, "built_in_metadata": []any{}}+ read := func() apiObject {+ return object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, path, nil, nil))["data"])+ }+ if got := read(); !reflect.DeepEqual(got, empty) {+ t.Fatalf("fresh metadata config=%v want=%v", got, empty)+ }+ fields := []any{+ apiObject{"key": "author", "type": "string", "description": "Author name"},+ apiObject{"key": "tags", "type": "list", "description": "Tag list", "enum": []string{"foo", "bar"}},+ }+ body := apiObject{"metadata": fields, "built_in_metadata": []any{apiObject{"key": "size", "type": "number", "description": "File size"}}}+ expected := apiObject{+ "enabled": false,+ "metadata": []any{+ apiObject{"key": "author", "type": "string", "description": "Author name", "enum": nil},+ apiObject{"key": "tags", "type": "list", "description": "Tag list", "enum": []any{"foo", "bar"}},+ },+ "built_in_metadata": []any{apiObject{"key": "size", "type": "number", "description": "File size", "enum": nil}},+ }+ for _, enabled := range []bool{false, true} {+ if enabled {+ body["enabled"] = true+ }+ expected["enabled"] = enabled+ result := restOK(t, restCall(t, TestConfig, http.MethodPut, path, body, nil))+ if !reflect.DeepEqual(result["data"], expected) || !reflect.DeepEqual(read(), expected) {+ t.Fatalf("metadata config did not persist normalized fields: response=%v stored=%v want=%v", result["data"], read(), expected)+ }+ }+ for _, invalid := range []apiObject{+ {"metadata": []any{apiObject{"key": " ", "type": "string"}}},+ {"metadata": []any{apiObject{"key": "author", "type": "unknown"}}},+ {"metadata": "invalid"},+ {"enabled": "true"},+ } {+ restCode(t, restCall(t, TestConfig, http.MethodPut, path, invalid, nil), 200, 102)+ if got := read(); !reflect.DeepEqual(got, expected) {+ t.Fatalf("invalid metadata update changed config: got=%v want=%v", got, expected)+ }+ }+ restOK(t, restCall(t, TestConfig, http.MethodPut, path, apiObject{}, nil))+ if got := read(); !reflect.DeepEqual(got, empty) {+ t.Fatalf("empty update did not reset metadata config: %v", got)+ }+ for _, method := range []string{http.MethodGet, http.MethodPut} {+ restCode(t, restCall(t, TestConfig, method, "/datasets/00000000000040008000000000000000/metadata/config", apiObject{}, nil), 200, 404)+ }+}++func TestDatasetMetadataSummaryAndFlattenedIdentities(t *testing.T) {+ first := createRESTDataset(t, nil)["id"].(string)+ second := createRESTDataset(t, nil)["id"].(string)+ documents := uploadRESTDocuments(t, TestConfig, first, []documentUpload{+ {Name: "one.txt", Content: []byte("metadata one")},+ {Name: "two.txt", Content: []byte("metadata two")},+ {Name: "three.txt", Content: []byte("metadata three")},+ })+ metadata := []apiObject{+ {"tags": []string{"foo", "bar"}, "author": "alice"},+ {"tags": []string{"foo"}, "author": "bob"},+ {"tags": []string{"bar", "baz"}, "author": ""},+ }+ var ids []string+ for i, doc := range documents {+ id := object(t, doc)["id"].(string)+ ids = append(ids, id)+ restOK(t, restCall(t, TestConfig, http.MethodPatch, "/datasets/"+first+"/documents/"+id, apiObject{"meta_fields": metadata[i]}, nil))+ }+ foreign := object(t, uploadRESTDocuments(t, TestConfig, second, []documentUpload{{Name: "other.txt", Content: []byte("foreign metadata")}})[0])["id"].(string)+ restOK(t, restCall(t, TestConfig, http.MethodPatch, "/datasets/"+second+"/documents/"+foreign, apiObject{"meta_fields": apiObject{"tags": []string{"foreign"}}}, nil))+ // Metadata lives in the document engine. Poll for the precise aggregate,+ // allowing index refresh latency without accepting an empty result.+ for _, tc := range []struct {+ name string+ query apiObject+ tags apiObject+ authors apiObject+ }{+ {"all_documents", nil, apiObject{"foo": float64(2), "bar": float64(2), "baz": float64(1)}, apiObject{"alice": float64(1), "bob": float64(1), "": float64(1)}},+ {"explicit_documents", apiObject{"doc_ids": strings.Join(ids[:2], ",")}, apiObject{"foo": float64(2), "bar": float64(1)}, apiObject{"alice": float64(1), "bob": float64(1)}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second)+ defer cancel()+ wantCounts := tc.tags+ for {+ data := object(t, restOK(t, restCallContext(t, ctx, TestConfig, http.MethodGet, "/datasets/"+first+"/metadata/summary", nil, tc.query))["data"])+ summary := object(t, data["summary"])+ counts := apiObject{}+ if tags, ok := summary["tags"].(apiObject); ok {+ for _, value := range array(t, tags["values"]) {+ pair := array(t, value)+ if len(pair) != 2 {+ t.Fatalf("invalid metadata value/count pair: %v", pair)+ }+ counts[pair[0].(string)] = pair[1]+ }+ }+ if reflect.DeepEqual(counts, wantCounts) {+ authors := object(t, summary["author"])+ got := apiObject{}+ for _, value := range array(t, authors["values"]) {+ pair := array(t, value)+ if len(pair) != 2 {+ t.Fatalf("invalid author value/count: %v", pair)+ }+ got[pair[0].(string)] = pair[1]+ }+ if !reflect.DeepEqual(got, tc.authors) {+ t.Fatalf("author counts differ from stored values: %v", got)+ }+ break+ }+ select {+ case <-ctx.Done():+ t.Fatalf("metadata summary never matched owned documents: got=%v want=%v", counts, wantCounts)+ case <-time.After(100 * time.Millisecond):+ }+ }+ })+ }+ for _, tc := range []struct {+ datasetIDs string+ expected map[string][]string+ }{+ {first, map[string][]string{"foo": {ids[0], ids[1]}, "bar": {ids[0], ids[2]}, "baz": {ids[2]}}},+ {first + "," + second, map[string][]string{"foo": {ids[0], ids[1]}, "bar": {ids[0], ids[2]}, "baz": {ids[2]}, "foreign": {foreign}}},+ } {+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/metadata/flattened", nil, apiObject{"dataset_ids": tc.datasetIDs}))["data"])+ tags := object(t, data["tags"])+ if len(tags) != len(tc.expected) {+ t.Fatalf("flattened tags contain unexpected values: %v", tags)+ }+ for value, expectedIDs := range tc.expected {+ var actual []string+ for _, id := range array(t, tags[value]) {+ actual = append(actual, id.(string))+ }+ slices.Sort(actual)+ slices.Sort(expectedIDs)+ if !slices.Equal(actual, expectedIDs) {+ t.Errorf("tag %s documents=%v want=%v", value, actual, expectedIDs)+ }+ }+ }+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/metadata/flattened", nil, nil), 200, 102)+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/metadata/flattened", nil, apiObject{"dataset_ids": "invalid"}), 200, 404)+}++func TestDatasetFreshGraphTagsAndIngestionState(t *testing.T) {+ id := createRESTDataset(t, nil)["id"].(string)+ graph := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/graph", nil, nil))["data"])+ for _, key := range []string{"graph", "mind_map"} {+ if len(object(t, graph[key])) != 0 {+ t.Fatalf("fresh dataset has %s entries: %v", key, graph)+ }+ }+ tags := restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/tags/aggregation", nil, apiObject{"dataset_ids": id}))["data"]+ if len(array(t, tags)) != 0 {+ t.Fatalf("fresh dataset has tags: %v", tags)+ }+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/tags/aggregation", nil, nil), 200, 102)+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/tags/aggregation", nil, apiObject{"dataset_ids": "invalid"}), 200, 102)+ summary := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/ingestions/summary", nil, nil))["data"])+ for _, key := range []string{"doc_num", "chunk_num", "token_num"} {+ if summary[key] != float64(0) {+ t.Errorf("fresh dataset %s=%v", key, summary[key])+ }+ }+ for _, key := range []string{"status", "download_status"} {+ counts := object(t, summary[key])+ expectedKeys := []string{"running_count", "done_count", "fail_count"}+ if key == "status" {+ expectedKeys = append(expectedKeys, "unstart_count", "cancel_count")+ }+ for _, field := range expectedKeys {+ if counts[field] != float64(0) {+ t.Errorf("fresh %s.%s=%v", key, field, counts[field])+ }+ }+ }+ for _, query := range []apiObject{+ nil,+ {"log_type": "file", "keywords": uniqueName("missing"), "page": 1, "page_size": 2},+ {"create_date_from": "2999-12-31", "create_date_to": "2000-01-01"},+ } {+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/ingestions", nil, query))["data"])+ if data["total"] != float64(0) || len(array(t, data["logs"])) != 0 {+ t.Fatalf("fresh dataset logs=%v", data)+ }+ }+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/ingestions/nonexistent", nil, nil), 200, 102)+ for _, query := range []apiObject{{"page": "invalid"}, {"page_size": "invalid"}} {+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/ingestions", nil, query), 200, 101)+ }+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/ingestions", nil, apiObject{"log_type": "unknown"}), 200, 102)+ for _, suffix := range []string{"/ingestions/summary", "/ingestions", "/ingestions/nonexistent", "/graph", "/compilation/status"} {+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/00000000000040008000000000000000"+suffix, nil, nil), 200, 404)+ }+}++func TestDatasetCompilationStatusAndEmbeddingRequirements(t *testing.T) {+ id := createRESTDataset(t, nil)["id"].(string)+ // Compilation status is the supported replacement for the retired /index routes.+ for _, kind := range []string{"", "graph", "raptor", "mindmap", "wiki"} {+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/compilation/status", nil, apiObject{"kind": kind}))["data"])+ for field, expected := range (apiObject{"kind": kind, "state": "idle", "progress": float64(0), "inflight": float64(0), "backlog": float64(0)}) {+ if data[field] != expected {+ t.Errorf("fresh compilation status %s=%v want=%v", field, data[field], expected)+ }+ }+ }+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets/"+id+"/compilation/status", nil, apiObject{"kind": "invalid"}), 200, 102)+ result := restCall(t, TestConfig, http.MethodPost, "/datasets/"+id+"/embedding/check", apiObject{}, nil)+ restCode(t, result, 200, 102)+ if message, _ := result.JSON["message"].(string); !strings.Contains(strings.ToLower(message), "embedding") || !strings.Contains(message, "required") {+ t.Fatalf("missing embedding error does not identify required model: %v", result.JSON)+ }+}++func TestDatasetSearchParameterValidation(t *testing.T) {+ id := createRESTDataset(t, nil)["id"].(string)+ // Existing retrieval_api_test.go verifies nonempty matching chunks,+ // document isolation and parity with this dataset-specific search route.+ for _, tc := range []struct {+ name string+ fields apiObject+ }{+ {"missing_question", apiObject{"question": nil}},+ {"blank_question", apiObject{"question": " "}},+ {"typed_question", apiObject{"question": 1}},+ {"zero_page", apiObject{"page": 0}},+ {"negative_page_size", apiObject{"page_size": -1}},+ {"zero_size", apiObject{"size": 0}},+ {"low_top_k", apiObject{"knn_top_k": 0}},+ {"high_top_k", apiObject{"knn_top_k": 2049}},+ {"low_candidates", apiObject{"knn_num_candidates": 0}},+ {"high_candidates", apiObject{"knn_num_candidates": 10001}},+ {"candidates_below_top_k", apiObject{"knn_top_k": 5, "knn_num_candidates": 4}},+ {"low_similarity", apiObject{"similarity_threshold": -0.01}},+ {"high_similarity", apiObject{"similarity_threshold": 1.01}},+ {"low_vector_weight", apiObject{"vector_similarity_weight": -0.01}},+ {"high_vector_weight", apiObject{"vector_similarity_weight": 1.01}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ body := apiObject{"question": "contract validation"}+ for key, value := range tc.fields {+ body[key] = value+ }+ restCode(t, restCall(t, TestConfig, http.MethodPost, "/datasets/"+id+"/search", body, nil), 200, 101)+ })+ }+}diff --git a/test/api/dataset_list_delete_test.go b/test/api/dataset_list_delete_test.gonew file mode 100644index 0000000000..e093b40d11--- /dev/null+++ b/test/api/dataset_list_delete_test.go@@ -0,0 +1,188 @@+//go:build integration++package api++import (+ "net/http"+ "reflect"+ "strings"+ "sync"+ "testing"+)++func TestDatasetListPaginationFilteringAndTotals(t *testing.T) {+ prefix := uniqueName("list")+ ids := make([]string, 3)+ names := make([]string, 3)+ for i, suffix := range []string{"alpha", "bravo", "charlie"} {+ data := createRESTDataset(t, apiObject{"name": prefix + suffix})+ ids[i] = data["id"].(string)+ names[i] = data["name"].(string)+ }+ scoped := strings.Join(ids, ",")+ for _, tc := range []struct {+ name string+ query apiObject+ want []string+ total int+ }{+ {"first_page", apiObject{"ids": scoped, "sort": "name:asc", "page_size": 2, "page": 1}, ids[:2], 3},+ {"second_page", apiObject{"ids": scoped, "sort": "name:asc", "page_size": 2, "page": 2}, ids[2:], 3},+ {"past_end", apiObject{"ids": scoped, "sort": "name:asc", "page_size": 2, "page": 3}, []string{}, 3},+ {"descending", apiObject{"ids": scoped, "sort": "name:desc"}, []string{ids[2], ids[1], ids[0]}, 3},+ {"name", apiObject{"ids": scoped, "name": names[1]}, []string{ids[1]}, 1},+ {"keywords", apiObject{"ids": scoped, "keywords": "bravo"}, []string{ids[1]}, 1},+ {"missing_keyword", apiObject{"ids": scoped, "keywords": uniqueName("absent")}, []string{}, 0},+ {"id_name_match", apiObject{"id": ids[0], "name": names[0]}, []string{ids[0]}, 1},+ {"id_name_mismatch", apiObject{"id": ids[0], "name": names[1]}, []string{}, 0},+ {"parser_mismatch", apiObject{"ids": scoped, "parser_id": "qa"}, []string{}, 0},+ } {+ t.Run(tc.name, func(t *testing.T) {+ result := restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, tc.query))+ got := make([]string, 0)+ for _, item := range array(t, result["data"]) {+ got = append(got, object(t, item)["id"].(string))+ }+ if !reflect.DeepEqual(got, tc.want) || result["total_datasets"] != float64(tc.total) {+ t.Fatalf("listed ids=%v total=%v want ids=%v total=%d", got, result["total_datasets"], tc.want, tc.total)+ }+ })+ }+ for _, query := range []apiObject{{"page": 0}, {"page": -1}, {"page": "a"}, {"page_size": 0}, {"page_size": "a"}, {"orderby": "unknown"}, {"orderby": "CREATE_TIME"}, {"desc": "unknown"}, {"id": ""}, {"id": "not_uuid"}, {"ids": ids[0] + "," + ids[0]}, {"ids": ids[0] + ",invalid"}, {"unknown_field": "x"}} {+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, query), 200, 101)+ }+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, apiObject{"id": "00000000000040008000000000000000"}), 200, 404)+ restCode(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, apiObject{"id": ids[0], "ids": scoped}), 200, 102)+}++func TestDatasetDeleteValidationAtomicityAndIsolation(t *testing.T) {+ first := createRESTDataset(t, nil)+ second := createRESTDataset(t, nil)+ untouched := createRESTDataset(t, nil)+ firstID, secondID := first["id"].(string), second["id"].(string)+ missingID := "00000000000040008000000000000000"+ for _, tc := range []struct {+ name string+ body apiObject+ code int+ }{+ {"empty", apiObject{"ids": []string{}}, 0},+ {"null", apiObject{"ids": nil}, 0},+ {"unset", apiObject{}, 0},+ {"duplicate", apiObject{"ids": []string{firstID, firstID}}, 101},+ {"malformed_id", apiObject{"ids": []string{"not_uuid"}}, 101},+ {"type_error", apiObject{"ids": "bad"}, 102},+ {"missing_id", apiObject{"ids": []string{missingID}}, 403},+ {"invalid_first", apiObject{"ids": []string{missingID, firstID, secondID}}, 403},+ {"invalid_middle", apiObject{"ids": []string{firstID, missingID, secondID}}, 403},+ {"invalid_last", apiObject{"ids": []string{firstID, secondID, missingID}}, 403},+ {"extra", apiObject{"unknown_field": true}, 101},+ } {+ t.Run(tc.name, func(t *testing.T) {+ restCode(t, restCall(t, TestConfig, http.MethodDelete, "/datasets", tc.body, nil), 200, tc.code)+ requireDatasetFields(t, firstID, apiObject{"name": first["name"]})+ requireDatasetFields(t, secondID, apiObject{"name": second["name"]})+ })+ }+ deleted := object(t, restOK(t, restCall(t, TestConfig, http.MethodDelete, "/datasets", apiObject{"ids": []string{firstID, secondID}}, nil))["data"])+ if deleted["success_count"] != float64(2) || len(array(t, deleted["errors"])) != 0 {+ t.Fatalf("batch result=%v", deleted)+ }+ requireDatasetAbsent(t, firstID)+ requireDatasetAbsent(t, secondID)+ restCode(t, restCall(t, TestConfig, http.MethodDelete, "/datasets", apiObject{"ids": []string{firstID, secondID}}, nil), 200, 403)+ requireDatasetFields(t, untouched["id"].(string), apiObject{"name": untouched["name"]})+}++func TestDatasetAuthorizationPreservesResources(t *testing.T) {+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ for _, key := range []string{"", TestConfig.InvalidAPIKey} {+ config := *TestConfig+ config.APIKey = key+ for _, tc := range []struct {+ method, path string+ body any+ }{+ {http.MethodGet, "/datasets", nil}, {http.MethodGet, "/datasets/" + id, nil},+ {http.MethodPost, "/datasets", apiObject{"name": uniqueName("unauthorized"), "parse_type": 1, "parser_id": "general"}},+ {http.MethodPut, "/datasets/" + id, apiObject{"name": "unauthorized"}},+ {http.MethodDelete, "/datasets", apiObject{"ids": []string{id}}},+ {http.MethodGet, "/datasets/" + id + "/metadata/config", nil},+ {http.MethodPut, "/datasets/" + id + "/metadata/config", apiObject{"enabled": true}},+ } {+ result := restCall(t, &config, tc.method, tc.path, tc.body, nil)+ if tc.method == http.MethodPost {+ if data, ok := result.JSON["data"].(apiObject); ok {+ if createdID, ok := data["id"].(string); ok && createdID != "" {+ cleanupResource(t, TestConfig, "datasets", createdID)+ }+ }+ }+ restCode(t, result, 401, 401)+ }+ requireDatasetFields(t, id, apiObject{"name": dataset["name"], "parser_config": dataset["parser_config"]})+ }+}++func TestDatasetConcurrentReadsAndUpdates(t *testing.T) {+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ // A small deterministic set proves each request completes and the final+ // value is one complete committed update; hundreds add load, not coverage.+ names := make([]string, 4)+ var wg sync.WaitGroup+ for i := range names {+ names[i] = uniqueName("concurrent")+ wg.Go(func() {+ result := restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": names[i]}, nil)+ restCode(t, result, 200, 0)+ if object(t, result.JSON["data"])["name"] != names[i] {+ t.Errorf("update response did not identify the submitted name")+ }+ })+ }+ wg.Wait()+ final := getRESTDataset(t, id)["name"]+ found := false+ for _, name := range names {+ found = found || final == name+ }+ if !found {+ t.Fatalf("torn final name=%v", final)+ }+ for range 4 {+ wg.Go(func() { requireDatasetFields(t, id, apiObject{"name": final}) })+ }+ wg.Wait()+}++func TestDatasetConcurrentCreationAndDeletion(t *testing.T) {+ ids := make([]string, 4)+ names := make([]string, len(ids))+ var wg sync.WaitGroup+ for i := range ids {+ names[i] = uniqueName("concurrent_create")+ wg.Go(func() {+ dataset := createRESTDataset(t, apiObject{"name": names[i]})+ ids[i] = dataset["id"].(string)+ if dataset["name"] != names[i] {+ t.Errorf("concurrent create returned name=%v want=%s", dataset["name"], names[i])+ }+ })+ }+ wg.Wait()+ seen := map[string]bool{}+ for i, id := range ids {+ if id == "" || seen[id] {+ t.Fatalf("concurrent creates did not return distinct IDs: %v", ids)+ }+ seen[id] = true+ requireDatasetFields(t, id, apiObject{"name": names[i]})+ wg.Go(func() {+ restOK(t, restCall(t, TestConfig, http.MethodDelete, "/datasets", apiObject{"ids": []string{id}}, nil))+ requireDatasetAbsent(t, id)+ })+ }+ wg.Wait()+}diff --git a/test/api/dataset_validation_test.go b/test/api/dataset_validation_test.gonew file mode 100644index 0000000000..04bc550289--- /dev/null+++ b/test/api/dataset_validation_test.go@@ -0,0 +1,296 @@+//go:build integration++package api++import (+ "net/http"+ "reflect"+ "strings"+ "testing"+)++func TestDatasetCreateValidation(t *testing.T) {+ for _, tc := range []struct {+ name string+ fields apiObject+ code int+ fragment string+ }{+ {"missing_name", apiObject{"name": nil}, 102, "required"},+ {"empty_name", apiObject{"name": ""}, 102, "required"},+ {"blank_name", apiObject{"name": " \t "}, 102, "empty"},+ {"long_name", apiObject{"name": strings.Repeat("a", 129)}, 102, "128"},+ {"number_name", apiObject{"name": 42}, 102, "cannot unmarshal"},+ {"missing_mode", apiObject{"parse_type": nil}, 102, "parse_type"},+ {"invalid_mode", apiObject{"parse_type": 3}, 102, "parse_type"},+ {"empty_parser", apiObject{"parser_id": ""}, 102, "parser_id"},+ {"unknown_parser", apiObject{"parser_id": "unknown"}, 102, "input should be"},+ {"typed_parser", apiObject{"parser_id": []string{}}, 102, "cannot unmarshal"},+ {"empty_permission", apiObject{"permission": ""}, 102, "'me' or 'team'"},+ {"invalid_permission", apiObject{"permission": "ME"}, 102, "'me' or 'team'"},+ {"typed_permission", apiObject{"permission": []string{}}, 102, "cannot unmarshal"},+ {"invalid_language", apiObject{"language": " "}, 102, "character"},+ {"typed_config", apiObject{"parser_config": []string{}}, 102, "cannot unmarshal"},+ {"config_size", apiObject{"parser_config": apiObject{"GeneralChunker:bad": apiObject{"delimiter": strings.Repeat("x", 65536)}}}, 101, "size limit"},+ {"extra_field", apiObject{"unknown_field": true}, 101, "Extra inputs"},+ {"server_id", apiObject{"id": "injected"}, 101, "Extra inputs"},+ {"pagerank_creation", apiObject{"pagerank": 50}, 101, "Extra inputs"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ name := uniqueName("invalid_create")+ fields := apiObject{"name": name, "parse_type": 1, "parser_id": "general"}+ for key, value := range tc.fields {+ fields[key] = value+ }+ result := restCall(t, TestConfig, http.MethodPost, "/datasets", fields, nil)+ if data, ok := result.JSON["data"].(apiObject); ok {+ if id, ok := data["id"].(string); ok && id != "" {+ cleanupResource(t, TestConfig, "datasets", id)+ }+ }+ restCode(t, result, 200, tc.code)+ if message, _ := result.JSON["message"].(string); !strings.Contains(strings.ToLower(message), strings.ToLower(tc.fragment)) {+ t.Errorf("message=%q should contain %q", message, tc.fragment)+ }+ // Failed validation must not have silently created the original fixture.+ listed := restOK(t, restCall(t, TestConfig, http.MethodGet, "/datasets", nil, apiObject{"name": name}))+ if len(array(t, listed["data"])) != 0 {+ t.Fatal("failed create persisted a dataset")+ }+ })+ }+}++func TestDatasetCreateDefaultsDeduplicationAndBoundaries(t *testing.T) {+ name := uniqueName("case")+ first := createRESTDataset(t, apiObject{"name": strings.ToUpper(name)})+ second := createRESTDataset(t, apiObject{"name": strings.ToLower(name)})+ if second["name"] != strings.ToLower(name)+"(1)" || first["id"] == second["id"] {+ t.Fatalf("case-insensitive deduplication failed: %v %v", first, second)+ }+ boundary := uniqueName("boundary")+ boundary += strings.Repeat("x", 128-len(boundary))+ createRESTDataset(t, apiObject{"name": boundary})+ for _, tc := range []struct {+ name string+ fields apiObject+ }{{"unset", nil}, {"empty", apiObject{"parser_config": apiObject{}}}, {"null", apiObject{"parser_config": nil, "embedding_model": nil}}} {+ t.Run(tc.name, func(t *testing.T) {+ data := createRESTDataset(t, tc.fields)+ if data["permission"] != "me" || data["parser_id"] != "general" {+ t.Fatalf("unexpected defaults: %v", data)+ }+ requirePipelineParserConfig(t, data["parser_config"], "GeneralChunker:")+ if embedding, ok := data["embedding_model"].(string); !ok || strings.TrimSpace(embedding) == "" {+ t.Fatalf("missing configured default embedding: %v", data["embedding_model"])+ }+ })+ }+}++func TestDatasetEmbeddingValidationAndRoundtrip(t *testing.T) {+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ for _, model := range []string{"", " ", "missing-provider", "@Builtin", "model@", " @Builtin", "model@ ", "not-a-real-model@not-a-real-provider"} {+ t.Run(model, func(t *testing.T) {+ fields := apiObject{"embedding_model": model}+ restCode(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, fields, nil), 200, 102)+ requireDatasetFields(t, id, apiObject{"embedding_model": dataset["embedding_model"]})+ fields["name"] = uniqueName("bad_model")+ fields["parse_type"] = 1+ fields["parser_id"] = "general"+ result := restCall(t, TestConfig, http.MethodPost, "/datasets", fields, nil)+ if data, ok := result.JSON["data"].(apiObject); ok {+ if createdID, ok := data["id"].(string); ok && createdID != "" {+ cleanupResource(t, TestConfig, "datasets", createdID)+ }+ }+ restCode(t, result, 200, 102)+ })+ }+ // Use the tenant's configured model, rather than a hardcoded provider that may+ // not be authorized in the environment running these integration tests.+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"embedding_model": dataset["embedding_model"]}, nil))+ requireDatasetFields(t, id, apiObject{"embedding_model": dataset["embedding_model"]})+ createRESTDataset(t, apiObject{"embedding_model": dataset["embedding_model"]})+}++func TestDatasetUpdateValidationPreservesState(t *testing.T) {+ data := createRESTDataset(t, nil)+ id := data["id"].(string)+ for _, tc := range []struct {+ name string+ fields apiObject+ code int+ }{+ {"empty", apiObject{}, 102},+ {"empty_name", apiObject{"name": ""}, 102},+ {"blank_name", apiObject{"name": " \t "}, 102},+ {"long_name", apiObject{"name": strings.Repeat("x", 129)}, 102},+ {"typed_name", apiObject{"name": 42}, 102},+ {"description_limit", apiObject{"description": strings.Repeat("x", 65536)}, 102},+ {"typed_description", apiObject{"description": []string{}}, 102},+ {"avatar_limit", apiObject{"avatar": strings.Repeat("x", 65536)}, 102},+ {"invalid_avatar", apiObject{"avatar": "not-an-image"}, 102},+ {"avatar_mime", apiObject{"avatar": "data:image/gif;base64,eA=="}, 102},+ {"permission", apiObject{"permission": "unknown"}, 102},+ {"permission_type", apiObject{"permission": []string{}}, 102},+ {"parser_empty", apiObject{"parse_type": 1, "parser_id": ""}, 102},+ {"parser_unknown", apiObject{"parse_type": 1, "parser_id": "unknown"}, 102},+ {"parser_missing_mode", apiObject{"parser_id": "qa"}, 102},+ {"pagerank_negative", apiObject{"pagerank": -1}, 102},+ {"pagerank_large", apiObject{"pagerank": 101}, 102},+ {"pagerank_type", apiObject{"pagerank": "50"}, 102},+ {"connectors_invalid", apiObject{"connectors": []any{apiObject{"id": ""}}}, 102},+ {"config_type", apiObject{"parser_config": "invalid"}, 102},+ {"config_size", apiObject{"parser_config": apiObject{"GeneralChunker:invalid": apiObject{"delimiter": strings.Repeat("x", 65536)}}}, 101},+ {"config_ext", apiObject{"parser_config": apiObject{"ext": apiObject{}}}, 101},+ {"unsupported", apiObject{"unknown_field": true}, 101},+ {"ownership", apiObject{"tenant_id": "other"}, 101},+ } {+ t.Run(tc.name, func(t *testing.T) {+ restCode(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, tc.fields, nil), 200, tc.code)+ actual := getRESTDataset(t, id)+ for _, key := range []string{"name", "description", "avatar", "permission", "parser_id", "parser_config", "embedding_model", "pagerank", "language", "connectors"} {+ if !reflect.DeepEqual(data[key], actual[key]) {+ t.Errorf("failed update changed %s: before=%v after=%v", key, data[key], actual[key])+ }+ }+ })+ }+ for _, tc := range []struct {+ id string+ code int+ }{{"not_uuid", 101}, {"00000000000040008000000000000000", 404}} {+ restCode(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+tc.id, apiObject{"description": "missing"}, nil), 200, tc.code)+ }+}++func TestDatasetPermissionAndPartialUpdatePreservation(t *testing.T) {+ data := createRESTDataset(t, apiObject{"permission": "team"})+ id := data["id"].(string)+ for _, permission := range []string{"me", "team"} {+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"permission": permission}, nil))+ requireDatasetFields(t, id, apiObject{"permission": permission})+ }+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"description": "preserve me"}, nil))+ expected := getRESTDataset(t, id)+ newName := uniqueName("partial")+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"name": newName}, nil))+ actual := getRESTDataset(t, id)+ for _, key := range []string{"description", "avatar", "permission", "parser_id", "parser_config", "embedding_model", "pagerank", "language", "connectors"} {+ if !reflect.DeepEqual(expected[key], actual[key]) {+ t.Errorf("partial update changed unprovided %s", key)+ }+ }+ if actual["name"] != newName {+ t.Errorf("name=%v want=%s", actual["name"], newName)+ }+}++func TestDatasetChunkMethodMatrix(t *testing.T) {+ for _, method := range []string{"general", "naive", "book", "email", "laws", "manual", "one", "paper", "picture", "presentation", "qa", "table"} {+ t.Run(method, func(t *testing.T) {+ data := createRESTDataset(t, apiObject{"parser_id": method})+ id := data["id"].(string)+ expected := method+ if method == "naive" {+ expected = "general"+ }+ if data["parser_id"] != expected {+ t.Fatalf("parser=%v want=%s", data["parser_id"], expected)+ }+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"parse_type": 1, "parser_id": "general"}, nil))+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"parse_type": 1, "parser_id": method}, nil))+ requireDatasetFields(t, id, apiObject{"parser_id": expected})+ })+ }+}++func TestDatasetMalformedRequestPreservesState(t *testing.T) {+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ for _, tc := range []struct {+ name, body, contentType string+ }{+ {"empty", "", "application/json"},+ {"truncated", `{"name":`, "application/json"},+ {"array", `[]`, "application/json"},+ {"scalar", `"name"`, "application/json"},+ {"null", `null`, "application/json"},+ {"wrong_content_type", `{"name":"invalid"}`, "text/plain"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ for _, target := range []struct{ method, path string }{+ {http.MethodPost, "/datasets"},+ {http.MethodPut, "/datasets/" + id},+ {http.MethodDelete, "/datasets"},+ } {+ result := rawRESTCall(t, TestConfig, target.method, target.path, []byte(tc.body), tc.contentType)+ if data, ok := result.JSON["data"].(apiObject); ok && target.method == http.MethodPost {+ if createdID, ok := data["id"].(string); ok && createdID != "" {+ cleanupResource(t, TestConfig, "datasets", createdID)+ }+ }+ restCode(t, result, 200, 101)+ requireDatasetFields(t, id, apiObject{"name": dataset["name"], "parser_config": dataset["parser_config"]})+ }+ })+ }+}++func TestDatasetParserNormalization(t *testing.T) {+ data := createRESTDataset(t, apiObject{"language": "Klingon", "parser_config": apiObject{"chunk_token_num": 19, "GeneralChunker:unknown": "ignored"}})+ id := data["id"].(string)+ requireDatasetFields(t, id, apiObject{"language": "Klingon"})+ config := requirePipelineParserConfig(t, data["parser_config"], "GeneralChunker:")+ if _, exists := config["GeneralChunker:unknown"]; exists {+ t.Fatalf("unknown component survived pipeline normalization: %v", config)+ }+}++func TestDatasetPagerankBoundariesAndReset(t *testing.T) {+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ status := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, "/system/status", nil, nil))["data"])+ engine := object(t, status["doc_engine"])["type"]+ if engine == "" || engine == nil {+ t.Fatal("system status did not identify the document engine")+ }+ for _, value := range []int{0, 100, 50, 0} {+ result := restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"pagerank": value}, nil)+ if engine == "infinity" && value != 0 {+ restCode(t, result, 200, 102)+ requireDatasetFields(t, id, apiObject{"pagerank": float64(0)})+ } else {+ restOK(t, result)+ requireDatasetFields(t, id, apiObject{"pagerank": float64(value)})+ }+ }+}++func TestDatasetEmbeddingUpdatePreservesExistingChunks(t *testing.T) {+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ document := object(t, uploadRESTDocuments(t, TestConfig, id, []documentUpload{{Name: "embedding.txt", Content: []byte("existing embedding chunk")}})[0])+ path := "/datasets/" + id + "/documents/" + document["id"].(string) + "/chunks"+ content := "Dataset embedding roundtrip must preserve this existing chunk."+ added := object(t, restOK(t, restCall(t, TestConfig, http.MethodPost, path, apiObject{"content": content}, nil))["data"])+ chunk := object(t, added["chunk"])+ chunkID := chunk["id"].(string)+ persisted := getRESTDataset(t, id)+ if count, ok := persisted["chunk_count"].(float64); !ok || count < 1 {+ t.Fatalf("fixture has no persisted chunks: %v", persisted)+ }+ restOK(t, restCall(t, TestConfig, http.MethodPut, "/datasets/"+id, apiObject{"embedding_model": dataset["embedding_model"]}, nil))+ requireDatasetFields(t, id, apiObject{"embedding_model": dataset["embedding_model"], "chunk_count": persisted["chunk_count"]})+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, path, nil, nil))["data"])+ chunks := array(t, data["chunks"])+ if len(chunks) != 1 {+ t.Fatalf("embedding update changed document chunks: %v", chunks)+ }+ actual := object(t, chunks[0])+ if actual["chunk_id"] != chunkID || actual["content_with_weight"] != content {+ t.Fatalf("embedding update corrupted existing chunk: %v", actual)+ }+}diff --git a/test/api/rest_request_test.go b/test/api/rest_request_test.goindex 828a090248..15242b852b 100644--- a/test/api/rest_request_test.go+++ b/test/api/rest_request_test.go@@ -3,11 +3,14 @@package apiimport (+ "bytes""context"+ "crypto/sha256""encoding/json/v2""errors""fmt""io"+ "mime/multipart""net/http""net/http/httptest""net/url"@@ -61,12 +64,17 @@ func restCallContext(t *testing.T, ctx context.Context, c *ServerTestConfig, met}requestURL.RawQuery = params.Encode()resp, err := c.RequestJSONContext(ctx, method, path, body, query, headers)+ return readRESTResponse(t, c, method, requestURL.String(), diagnosticBody(requestBody, c.APIKey, c.Password), started, resp, err)+}++func readRESTResponse(t *testing.T, c *ServerTestConfig, method, requestURL, requestSummary string, started time.Time, resp *http.Response, err error) restResult {+ t.Helper()if err != nil {// url.Error includes the unsanitized URL; diagnostics already include its safe form.if urlErr, ok := errors.AsType[*url.Error](err); ok {err = urlErr.Err}- t.Fatalf("%s %s payload=%s duration=%s transport error=%v", method, diagnosticURL(requestURL.String()), diagnosticBody(requestBody, c.APIKey, c.Password), time.Since(started), err)+ t.Fatalf("%s %s payload=%s duration=%s transport error=%v", method, diagnosticURL(requestURL), requestSummary, time.Since(started), err)}defer resp.Body.Close()result := restResult{Status: resp.StatusCode, Header: resp.Header}@@ -81,6 +89,11 @@ func restCallContext(t *testing.T, ctx context.Context, c *ServerTestConfig, met_ = json.Unmarshal(result.Body, &result.JSON)}responseLog := diagnosticBody(result.Body, c.APIKey, c.Password)+ contentType := resp.Header.Get("Content-Type")+ binary := strings.HasPrefix(contentType, "image/") || strings.Contains(contentType, "application/octet-stream") || resp.Header.Get("Content-Disposition") != ""+ if binary || result.JSON == nil && !strings.Contains(contentType, "text/event-stream") {+ responseLog = fmt.Sprintf("bytes=%d content_type=%q disposition=%q sha256=%x", len(result.Body), resp.Header.Get("Content-Type"), resp.Header.Get("Content-Disposition"), sha256.Sum256(result.Body))+ }if strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") {events, parseErr := parseSSE(strings.NewReader(string(result.Body)))preview := make([]any, 0, min(len(events), 5))@@ -97,10 +110,82 @@ func restCallContext(t *testing.T, ctx context.Context, c *ServerTestConfig, metencoded, _ := json.Marshal(preview)responseLog = fmt.Sprintf("SSE bytes=%d events=%d parse_error=%v preview=%s", len(result.Body), len(events), parseErr, diagnosticBody(encoded, c.APIKey, c.Password))}- t.Logf("%s %s payload=%s HTTP=%d code=%v response=%s duration=%s", method, diagnosticURL(resp.Request.URL.String()), diagnosticBody(requestBody, c.APIKey, c.Password), result.Status, result.JSON["code"], responseLog, time.Since(started))+ t.Logf("%s %s payload=%s HTTP=%d code=%v response=%s duration=%s", method, diagnosticURL(requestURL), requestSummary, result.Status, result.JSON["code"], responseLog, time.Since(started))return result}+// rawRESTCall shares bounded response handling and redaction with JSON requests.+func rawRESTCall(t *testing.T, c *ServerTestConfig, method, path string, body []byte, contentType string) restResult {+ t.Helper()+ summary := fmt.Sprintf("content_type=%q bytes=%d sha256=%x body=%s", contentType, len(body), sha256.Sum256(body), diagnosticBody(body, c.APIKey, c.Password))+ return rawRESTRequest(t, c, method, path, body, contentType, summary)+}++func rawRESTRequest(t *testing.T, c *ServerTestConfig, method, path string, body []byte, contentType, summary string) restResult {+ t.Helper()+ ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)+ defer cancel()+ started := time.Now()+ requestURL := c.apiBase() + "/" + strings.TrimLeft(path, "/")+ req, err := http.NewRequestWithContext(ctx, method, requestURL, bytes.NewReader(body))+ if err != nil {+ t.Fatalf("%s %s payload=%s create request: %v", method, diagnosticURL(requestURL), summary, err)+ }+ req.Header.Set("Content-Type", contentType)+ if c.APIKey != "" {+ req.Header.Set("Authorization", "Bearer "+c.APIKey)+ }+ resp, err := chunkHTTPClient.Do(req)+ return readRESTResponse(t, c, method, requestURL, summary, started, resp, err)+}++type documentUpload struct {+ Name string+ Content []byte+}++func multipartRESTDocuments(t *testing.T, c *ServerTestConfig, datasetID string, files []documentUpload) restResult {+ t.Helper()+ var body bytes.Buffer+ writer := multipart.NewWriter(&body)+ summary := make([]apiObject, 0, len(files))+ for _, file := range files {+ part, err := writer.CreateFormFile("file", file.Name)+ if err != nil {+ t.Fatalf("multipart file %q: %v", file.Name, err)+ }+ if _, err := part.Write(file.Content); err != nil {+ t.Fatalf("write multipart file %q: %v", file.Name, err)+ }+ summary = append(summary, apiObject{"name": file.Name, "size": len(file.Content)})+ }+ if err := writer.Close(); err != nil {+ t.Fatal(err)+ }+ encoded, err := json.Marshal(apiObject{"files": summary, "field": "file"})+ if err != nil {+ t.Fatal(err)+ }+ return rawRESTRequest(t, c, http.MethodPost, "/datasets/"+datasetID+"/documents", body.Bytes(), writer.FormDataContentType(), diagnosticBody(encoded, c.APIKey, c.Password))+}++func uploadRESTDocuments(t *testing.T, c *ServerTestConfig, datasetID string, files []documentUpload) []any {+ t.Helper()+ items := array(t, restOK(t, multipartRESTDocuments(t, c, datasetID, files))["data"])+ if len(items) != len(files) {+ t.Fatalf("uploaded documents=%d want=%d", len(items), len(files))+ }+ seen := map[string]bool{}+ for _, item := range items {+ id, ok := object(t, item)["id"].(string)+ if !ok || id == "" || seen[id] {+ t.Fatalf("invalid or duplicate uploaded document ID %v", object(t, item)["id"])+ }+ seen[id] = true+ }+ return items+}+func restCode(t *testing.T, r restResult, status, code int) {t.Helper()if r.Status != status {@@ -129,7 +214,10 @@ func cleanupResource(t *testing.T, c *ServerTestConfig, family, id string) {r := restCall(t, c, http.MethodDelete, path, body, nil)// Tests exercise deletion themselves; a subsequently missing resource is expected.code, ok := r.JSON["code"].(float64)- if r.Status != 200 || !ok || (code != 0 && code != 102 && code != 103 && code != 404) {+ // Dataset deletion reports 403 for an already absent ID. The read below+ // must still prove absence, so an actual permission failure cannot pass.+ missingDataset := family == "datasets" && code == 403+ if r.Status != 200 || !ok || (code != 0 && code != 102 && code != 103 && code != 404 && !missingDataset) {t.Errorf("cleanup %s/%s failed (see diagnostic)", family, id)}// A bulk-delete envelope can report partial failures. Prove absence rather@@ -184,10 +272,58 @@ func TestRESTHelperTransport(t *testing.T) {}}+func TestRESTHelperMultipartAndBinary(t *testing.T) {+ files := []documentUpload{{Name: "first.txt", Content: []byte("first\x00document")}, {Name: "second.md", Content: []byte("# Second\n")}}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Header.Get("Authorization") != "Bearer test-secret" {+ t.Error("raw request did not use API key")+ }+ if r.Method == http.MethodGet {+ w.Header().Set("Content-Type", "application/octet-stream")+ w.Header().Set("Content-Disposition", `attachment; filename="first.txt"`)+ _, _ = w.Write(files[0].Content)+ return+ }+ if err := r.ParseMultipartForm(1 << 20); err != nil {+ t.Errorf("parse multipart: %v", err)+ w.WriteHeader(http.StatusBadRequest)+ return+ }+ defer r.MultipartForm.RemoveAll()+ parts := r.MultipartForm.File["file"]+ if len(parts) != len(files) {+ t.Errorf("multipart files=%d want=%d", len(parts), len(files))+ return+ }+ for i, part := range parts {+ f, err := part.Open()+ if err != nil {+ t.Error(err)+ return+ }+ content, err := io.ReadAll(f)+ f.Close()+ if err != nil || part.Filename != files[i].Name || !bytes.Equal(content, files[i].Content) {+ t.Errorf("multipart file %d changed: name=%q read_error=%v", i, part.Filename, err)+ }+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = io.WriteString(w, `{"code":0,"data":[{"id":"first"},{"id":"second"}]}`)+ }))+ t.Cleanup(server.Close)+ c := &ServerTestConfig{HostAddress: server.URL, APIVersion: "v1", APIKey: "test-secret", AuthToken: "must-not-be-used"}+ uploadRESTDocuments(t, c, "dataset", files)+ r := rawRESTCall(t, c, http.MethodGet, "/documents/first", nil, "")+ if r.Status != http.StatusOK || r.JSON != nil || !bytes.Equal(r.Body, files[0].Content) || r.Header.Get("Content-Disposition") != `attachment; filename="first.txt"` {+ t.Fatal("binary response bytes or headers were lost")+ }+}+func TestRESTHelperDatasetCleanup(t *testing.T) {- for _, tc := range []struct{ name, body string }{- {"empty_list", `{"code":0,"data":[]}`},- {"missing_id", `{"code":404,"message":"Resource not found"}`},+ for _, tc := range []struct{ name, deleted, body string }{+ {"empty_list", `{"code":0,"data":{"success_count":1,"errors":[]}}`, `{"code":0,"data":[]}`},+ {"missing_id", `{"code":0,"data":{"success_count":1,"errors":[]}}`, `{"code":404,"message":"Resource not found"}`},+ {"already_deleted", `{"code":403,"message":"Resource not found"}`, `{"code":404,"message":"Resource not found"}`},} {t.Run(tc.name, func(t *testing.T) {var deleted, verified atomic.Bool@@ -195,7 +331,7 @@ func TestRESTHelperDatasetCleanup(t *testing.T) {w.Header().Set("Content-Type", "application/json")if r.Method == http.MethodDelete && r.URL.Path == "/api/v1/datasets" {deleted.Store(true)- _, _ = io.WriteString(w, `{"code":0,"data":{"success_count":1,"errors":[]}}`)+ _, _ = io.WriteString(w, tc.deleted)return}if !deleted.Load() || r.Method != http.MethodGet || r.URL.Query().Get("id") != "dataset-id" {