RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
test(api): migrate Python document API coverage to Go (#21066)
Sbaaoui Idriss26e6acc2174e
diff --git a/test/api/document_api_test.go b/test/api/document_api_test.gonew file mode 100644index 0000000000..0b03443ed2--- /dev/null+++ b/test/api/document_api_test.go@@ -0,0 +1,775 @@+//go:build integration++package api++import (+ "context"+ "fmt"+ "net/http"+ "reflect"+ "slices"+ "strings"+ "testing"+ "time"+)++func documentRESTPath(datasetID string) string { return "/datasets/" + datasetID + "/documents" }++func seedRESTDocuments(t *testing.T, count int) (string, []apiObject) {+ t.Helper()+ dataset := createRESTDataset(t, apiObject{"parse_type": 1, "parser_id": "general"})+ id := dataset["id"].(string)+ files := make([]documentUpload, count)+ for i := range count {+ files[i] = documentUpload{Name: fmt.Sprintf("record_%02d.txt", i), Content: []byte(fmt.Sprintf("Botanical evidence record %d contains coriander leaves and seeds.\n", i))}+ }+ uploaded := uploadRESTDocuments(t, TestConfig, id, files)+ if len(uploaded) != count {+ t.Fatalf("uploaded=%d want=%d", len(uploaded), count)+ }+ docs := make([]apiObject, count)+ for i, item := range uploaded {+ docs[i] = object(t, item)+ }+ return id, docs+}++func listRESTDocuments(t *testing.T, datasetID string, query apiObject) (apiObject, []apiObject) {+ t.Helper()+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, documentRESTPath(datasetID), nil, query))["data"])+ items := array(t, data["docs"])+ docs := make([]apiObject, 0, len(items))+ seen := map[string]bool{}+ for _, item := range items {+ doc := object(t, item)+ id, ok := doc["id"].(string)+ if !ok || id == "" || seen[id] || doc["dataset_id"] != datasetID {+ t.Fatalf("invalid document identity: %v", doc)+ }+ seen[id] = true+ docs = append(docs, doc)+ }+ if total, ok := data["total"].(float64); !ok || total < float64(len(docs)) {+ t.Fatalf("invalid total: %v", data["total"])+ }+ return data, docs+}++func getRESTDocument(t *testing.T, datasetID, documentID string) apiObject {+ t.Helper()+ data, docs := listRESTDocuments(t, datasetID, apiObject{"id": documentID})+ if data["total"] != float64(1) || len(docs) != 1 || docs[0]["id"] != documentID {+ t.Fatalf("document %s not uniquely retrievable: %v", documentID, data)+ }+ return docs[0]+}++func documentRESTIDs(docs []apiObject) []string {+ ids := make([]string, len(docs))+ for i, doc := range docs {+ ids[i] = doc["id"].(string)+ }+ return ids+}++func requireRESTDocumentIDs(t *testing.T, docs []apiObject, expected []string) {+ t.Helper()+ got := documentRESTIDs(docs)+ slices.Sort(got)+ want := slices.Clone(expected)+ slices.Sort(want)+ if !slices.Equal(got, want) {+ t.Fatalf("document IDs=%v want=%v", got, want)+ }+}++func TestRESTDocumentUploadPersistenceAndNames(t *testing.T) {+ dataset := createRESTDataset(t, apiObject{"parse_type": 1, "parser_id": "general"})+ id := dataset["id"].(string)+ files := []documentUpload{+ {Name: "same.txt", Content: []byte("first content")},+ {Name: "same.txt", Content: []byte("second content is longer")},+ {Name: "empty.txt"},+ {Name: strings.Repeat("a", 251) + ".txt", Content: []byte("maximum byte length")},+ {Name: "notes.md", Content: []byte("# Notes\nCoriander evidence.")},+ {Name: "record.json", Content: []byte(`{"plant":"coriander"}`)},+ {Name: "page.html", Content: []byte("<!doctype html><title>Evidence</title><p>Coriander</p>")},+ {Name: "mail.eml", Content: []byte("From: [email protected]\r\nSubject: Evidence\r\n\r\nCoriander seeds.")},+ }+ uploaded := uploadRESTDocuments(t, TestConfig, id, files)+ if len(uploaded) != len(files) {+ t.Fatalf("uploaded=%d want=%d", len(uploaded), len(files))+ }+ for i, item := range uploaded {+ doc := object(t, item)+ expectedName := files[i].Name+ if i == 1 {+ expectedName = "same(1).txt"+ }+ if doc["name"] != expectedName || doc["dataset_id"] != id || doc["size"] != float64(len(files[i].Content)) {+ t.Fatalf("upload metadata mismatch: %v", doc)+ }+ persisted := getRESTDocument(t, id, doc["id"].(string))+ for _, key := range []string{"id", "name", "dataset_id", "size"} {+ if persisted[key] != doc[key] {+ t.Fatalf("persisted %s=%v upload=%v", key, persisted[key], doc[key])+ }+ }+ if persisted["ingestion_status"] != "UNSTART" {+ t.Fatalf("upload unexpectedly started ingestion: %v", persisted)+ }+ }+ repeated := object(t, uploadRESTDocuments(t, TestConfig, id, files[:1])[0])+ if repeated["name"] != "same(2).txt" {+ t.Fatalf("repeated upload name=%v", repeated["name"])+ }+ // Unique row names alone cannot prove duplicate uploads kept separate blobs.+ for i := range 2 {+ doc := object(t, uploaded[i])+ file := files[i]+ file.Name = doc["name"].(string)+ requireDocumentDownload(t, rawRESTCall(t, TestConfig, http.MethodGet, documentRESTPath(id)+"/"+doc["id"].(string), nil, ""), file)+ }+ repeatedFile := files[0]+ repeatedFile.Name = repeated["name"].(string)+ requireDocumentDownload(t, rawRESTCall(t, TestConfig, http.MethodGet, documentRESTPath(id)+"/"+repeated["id"].(string), nil, ""), repeatedFile)+ data, docs := listRESTDocuments(t, id, apiObject{"page_size": 100})+ if data["total"] != float64(len(files)+1) || len(docs) != len(files)+1 {+ t.Fatalf("persisted upload total=%v", data)+ }+}++func TestRESTDocumentUploadValidation(t *testing.T) {+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ path := documentRESTPath(id)+ for _, tc := range []struct {+ name string+ files []documentUpload+ code int+ message string+ }{+ {"missing_file", nil, 101, "No file part!"},+ {"empty_filename", []documentUpload{{Name: "", Content: []byte("x")}}, 101, "No file part!"},+ {"overlong_filename", []documentUpload{{Name: strings.Repeat("a", 252) + ".txt"}}, 101, "255 bytes"},+ {"multibyte_overlong_filename", []documentUpload{{Name: strings.Repeat("界", 84) + ".txt"}}, 101, "255 bytes"},+ {"unsupported_format", []documentUpload{{Name: "executable.exe", Content: []byte("not executable")}}, 500, "not been supported"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ r := multipartRESTDocuments(t, TestConfig, id, tc.files)+ restCode(t, r, 200, tc.code)+ if !strings.Contains(fmt.Sprint(r.JSON["message"]), tc.message) {+ t.Fatalf("message=%v want substring %q", r.JSON["message"], tc.message)+ }+ data, docs := listRESTDocuments(t, id, nil)+ if data["total"] != float64(0) || len(docs) != 0 {+ t.Fatalf("failed upload persisted documents: %v", data)+ }+ })+ }+ restCode(t, rawRESTCall(t, TestConfig, http.MethodPost, path, []byte("--broken\r\n"), "multipart/form-data; boundary=broken"), 200, 101)+ restCode(t, multipartRESTDocuments(t, TestConfig, TestConfig.InvalidID, []documentUpload{{Name: "valid.txt", Content: []byte("x")}}), 200, 102)+ // Mixed uploads explicitly report the rejected file while retaining the valid file.+ partial := multipartRESTDocuments(t, TestConfig, id, []documentUpload{{Name: "valid.txt", Content: []byte("valid content")}, {Name: "invalid.exe"}})+ restCode(t, partial, 200, 500)+ accepted := array(t, partial.JSON["data"])+ if len(accepted) != 1 || object(t, accepted[0])["name"] != "valid.txt" || !strings.Contains(fmt.Sprint(partial.JSON["message"]), "invalid.exe") {+ t.Fatalf("partial upload=%v", partial.JSON)+ }+ data, docs := listRESTDocuments(t, id, nil)+ if data["total"] != float64(1) {+ t.Fatalf("partial upload total=%v", data)+ }+ requireRESTDocumentIDs(t, docs, []string{object(t, accepted[0])["id"].(string)})+}++func TestRESTDocumentListingFiltersPaginationAndReads(t *testing.T) {+ id, uploaded := seedRESTDocuments(t, 3)+ ids := documentRESTIDs(uploaded)+ cases := []struct {+ name string+ query apiObject+ expected []string+ total int+ }{+ {"all", nil, ids, 3},+ {"first_page", apiObject{"orderby": "name", "desc": false, "page": 1, "page_size": 2}, ids[:2], 3},+ {"last_page", apiObject{"orderby": "name", "desc": false, "page": 2, "page_size": 2}, ids[2:], 3},+ {"past_end", apiObject{"page": 3, "page_size": 2}, nil, 3},+ {"keyword", apiObject{"keywords": "record_01"}, ids[1:2], 1},+ {"keyword_no_match", apiObject{"keywords": "does_not_exist"}, nil, 0},+ {"id", apiObject{"id": ids[0]}, ids[:1], 1},+ {"name", apiObject{"name": uploaded[1]["name"]}, ids[1:2], 1},+ {"mismatched_existing_id_name", apiObject{"id": ids[0], "name": uploaded[1]["name"]}, nil, 0},+ {"unstarted", apiObject{"run": "UNSTART"}, ids, 3},+ {"future_creation", apiObject{"create_time_from": "9999999999000"}, nil, 3},+ {"past_creation", apiObject{"create_time_to": "1"}, nil, 3},+ {"invalid_page_defaults", apiObject{"page": "invalid", "page_size": -1}, ids, 3},+ {"unknown_query_ignored", apiObject{"unexpected": "ignored"}, ids, 3},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ data, docs := listRESTDocuments(t, id, tc.query)+ if data["total"] != float64(tc.total) {+ t.Fatalf("total=%v want=%d", data["total"], tc.total)+ }+ requireRESTDocumentIDs(t, docs, tc.expected)+ })+ }+ for _, order := range []string{"name", "create_time", "update_time"} {+ for _, desc := range []bool{false, true} {+ _, docs := listRESTDocuments(t, id, apiObject{"orderby": order, "desc": desc})+ for i := 1; i < len(docs); i++ {+ less := false+ if order == "name" {+ less = docs[i-1][order].(string) > docs[i][order].(string)+ } else {+ less = docs[i-1][order].(float64) > docs[i][order].(float64)+ }+ equal := docs[i-1][order] == docs[i][order]+ if !equal && less != desc {+ t.Fatalf("incorrect order field=%s desc=%v documents=%v", order, desc, docs)+ }+ }+ }+ }+ for _, tc := range []struct {+ query apiObject+ code int+ }{+ {apiObject{"orderby": "name; DROP TABLE document"}, 101},+ {apiObject{"metadata_condition": "{bad"}, 102},+ {apiObject{"metadata_condition": "[]"}, 102},+ {apiObject{"run": "DONE"}, 102},+ {apiObject{"id": "missing-document"}, 102},+ {apiObject{"name": "missing.txt"}, 102},+ {apiObject{"create_time_from": "invalid"}, 102},+ {apiObject{"create_time_to": "invalid"}, 102},+ {apiObject{"id": ids[0], "ids": ids[1]}, 102},+ } {+ restCode(t, restCall(t, TestConfig, http.MethodGet, documentRESTPath(id), nil, tc.query), 200, tc.code)+ }+ restCode(t, restCall(t, TestConfig, http.MethodGet, documentRESTPath(TestConfig.InvalidID), nil, nil), 200, 404)+ t.Run("concurrent_reads", func(t *testing.T) {+ for i := range 4 {+ t.Run(fmt.Sprint(i), func(t *testing.T) {+ t.Parallel()+ data, docs := listRESTDocuments(t, id, nil)+ if data["total"] != float64(3) {+ t.Fatalf("total=%v", data["total"])+ }+ requireRESTDocumentIDs(t, docs, ids)+ })+ }+ })+}++func TestRESTDocumentUpdatesPersistAndRejectAtomically(t *testing.T) {+ id, docs := seedRESTDocuments(t, 2)+ docID := docs[0]["id"].(string)+ path := documentRESTPath(id) + "/" + docID+ for _, name := range []string{"renamed.txt", strings.Repeat("a", 251) + ".txt", "UPPER.TXT"} {+ restOK(t, restCall(t, TestConfig, http.MethodPatch, path, apiObject{"name": name}, nil))+ if got := getRESTDocument(t, id, docID)["name"]; got != name {+ t.Fatalf("name=%v want=%s", got, name)+ }+ }+ for _, tc := range []struct {+ name string+ body apiObject+ code int+ }{+ {"null_name", apiObject{"name": nil}, 102}, {"numeric_name", apiObject{"name": 1}, 102},+ {"empty_name", apiObject{"name": ""}, 101}, {"extension_change", apiObject{"name": "renamed.pdf"}, 101},+ {"too_long", apiObject{"name": strings.Repeat("a", 252) + ".txt"}, 101},+ {"duplicate", apiObject{"name": docs[1]["name"]}, 102},+ {"chunk_counter", apiObject{"name": "must_not_persist.txt", "chunk_count": 7}, 102},+ {"token_counter", apiObject{"token_count": 5}, 102}, {"progress", apiObject{"progress": 1}, 102},+ {"invalid_enabled", apiObject{"enabled": 2}, 102}, {"null_enabled", apiObject{"enabled": nil}, 102},+ {"meta_not_object", apiObject{"meta_fields": []any{}}, 102},+ {"nested_meta", apiObject{"meta_fields": apiObject{"nested": apiObject{"x": 1}}}, 102},+ {"nested_meta_list", apiObject{"meta_fields": apiObject{"nested": []any{apiObject{"x": 1}}}}, 102},+ {"invalid_chunk_method", apiObject{"chunk_method": "unknown"}, 102}, {"empty_chunk_method", apiObject{"chunk_method": ""}, 102},+ {"parser_not_object", apiObject{"parser_config": []any{}}, 102},+ } {+ t.Run(tc.name, func(t *testing.T) {+ before := getRESTDocument(t, id, docID)+ restCode(t, restCall(t, TestConfig, http.MethodPatch, path, tc.body, nil), 200, tc.code)+ after := getRESTDocument(t, id, docID)+ for _, key := range []string{"id", "dataset_id", "name", "size", "chunk_count", "token_count", "progress", "parser_config", "meta_fields", "status"} {+ if !reflect.DeepEqual(before[key], after[key]) {+ t.Fatalf("rejected update changed %s: before=%v after=%v", key, before[key], after[key])+ }+ }+ })+ }+ for _, method := range []string{"naive", "manual", "qa", "table", "paper", "book", "laws", "presentation", "picture", "one", "knowledge_graph", "email", "tag"} {+ restOK(t, restCall(t, TestConfig, http.MethodPatch, path, apiObject{"chunk_method": method}, nil))+ persisted := getRESTDocument(t, id, docID)+ if persisted["chunk_method"] != method || persisted["parser_id"] != method {+ t.Fatalf("parser change not persisted: %v", persisted)+ }+ }+ restOK(t, restCall(t, TestConfig, http.MethodPatch, path, apiObject{"enabled": 0}, nil))+ if getRESTDocument(t, id, docID)["status"] != "0" {+ t.Fatal("disable not persisted")+ }+ restOK(t, restCall(t, TestConfig, http.MethodPatch, path, apiObject{"enabled": 1}, nil))+ if getRESTDocument(t, id, docID)["status"] != "1" {+ t.Fatal("enable not persisted")+ }+ before := getRESTDocument(t, id, docID)+ restOK(t, restCall(t, TestConfig, http.MethodPatch, path, apiObject{"unexpected": "ignored", "id": "forged", "dataset_id": "forged", "size": 1}, nil))+ after := getRESTDocument(t, id, docID)+ for _, key := range []string{"id", "dataset_id", "size", "name", "parser_id"} {+ if before[key] != after[key] {+ t.Fatalf("unsupported field changed %s", key)+ }+ }+ otherID, _ := seedRESTDocuments(t, 1)+ restCode(t, restCall(t, TestConfig, http.MethodPatch, documentRESTPath(otherID)+"/"+docID, apiObject{"name": "foreign.txt"}, nil), 200, 102)+ restCode(t, restCall(t, TestConfig, http.MethodPatch, documentRESTPath(id)+"/"+TestConfig.InvalidID, apiObject{"name": "missing.txt"}, nil), 200, 102)+ restCode(t, restCall(t, TestConfig, http.MethodPatch, documentRESTPath(TestConfig.InvalidID)+"/"+docID, apiObject{"name": "missing.txt"}, nil), 200, 404)+ if getRESTDocument(t, id, docID)["name"] != before["name"] {+ t.Fatal("cross-dataset update renamed original")+ }+}++func TestRESTDocumentParserConfigurationAndMetadataConfig(t *testing.T) {+ id, docs := seedRESTDocuments(t, 1)+ docID := docs[0]["id"].(string)+ path := documentRESTPath(id) + "/" + docID+ original := getRESTDocument(t, id, docID)+ config := object(t, original["parser_config"])+ var component string+ for key, value := range config {+ if strings.Contains(key, "Chunker:") {+ if _, ok := value.(map[string]any); ok {+ component = key+ break+ }+ }+ }+ if component == "" {+ t.Fatalf("general parser has no component-scoped chunker: %v", config)+ }+ restOK(t, restCall(t, TestConfig, http.MethodPatch, path, apiObject{"parser_config": apiObject{component: apiObject{"chunk_token_size": 128}}}, nil))+ persisted := getRESTDocument(t, id, docID)+ if object(t, object(t, persisted["parser_config"])[component])["chunk_token_size"] != float64(128) {+ t.Fatalf("component config not persisted: %v", persisted["parser_config"])+ }+ if persisted["name"] != original["name"] || persisted["size"] != original["size"] {+ t.Fatal("parser update changed unrelated fields")+ }+ for node := range config {+ if !strings.HasPrefix(node, "Parser:") {+ continue+ }+ for _, pages := range []any{"not-an-array", []any{[]any{0, 1}}, []any{[]any{3, 1}}, []any{[]any{1, "bad"}}} {+ before := getRESTDocument(t, id, docID)+ restCode(t, restCall(t, TestConfig, http.MethodPatch, path, apiObject{"parser_config": apiObject{node: apiObject{"pdf": apiObject{"pages": pages}}}}, nil), 200, 102)+ after := getRESTDocument(t, id, docID)+ if !reflect.DeepEqual(before["parser_config"], after["parser_config"]) {+ t.Fatalf("invalid page ranges changed parser config: %v", after)+ }+ }+ }+ // Flat Python-era options are discarded by the component-scoped Go contract.+ restOK(t, restCall(t, TestConfig, http.MethodPatch, path, apiObject{"parser_config": apiObject{"chunk_token_num": -1, "raptor": apiObject{"use_raptor": true}}}, nil))+ persisted = getRESTDocument(t, id, docID)+ pc := object(t, persisted["parser_config"])+ for _, key := range []string{"chunk_token_num", "raptor"} {+ if _, ok := pc[key]; ok {+ t.Fatalf("flat parser key retained: %s", key)+ }+ }+ if object(t, pc[component])["chunk_token_size"] != object(t, config[component])["chunk_token_size"] {+ t.Fatalf("flat options did not restore component defaults: got=%v defaults=%v", pc[component], config[component])+ }+ metadata := apiObject{"author": "alice", "tags": []any{"one", "two"}}+ metadataPath := path + "/metadata/config"+ restCode(t, restCall(t, TestConfig, http.MethodPut, metadataPath, apiObject{}, nil), 200, 101)+ restOK(t, restCall(t, TestConfig, http.MethodPut, metadataPath, apiObject{"metadata": metadata}, nil))+ pc = object(t, getRESTDocument(t, id, docID)["parser_config"])+ if !reflect.DeepEqual(object(t, pc["Extractor:AutoExtractDefault"])["metadata"], metadata) {+ t.Fatalf("extractor metadata not persisted: %v", pc)+ }+ restCode(t, restCall(t, TestConfig, http.MethodPut, documentRESTPath(id)+"/"+TestConfig.InvalidID+"/metadata/config", apiObject{"metadata": metadata}, nil), 200, 102)+}++func waitRESTDocumentMetadata(t *testing.T, datasetID, documentID string, want apiObject) apiObject {+ t.Helper()+ ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)+ defer cancel()+ ticker := time.NewTicker(250 * time.Millisecond)+ defer ticker.Stop()+ for {+ r := restCallContext(t, ctx, TestConfig, http.MethodGet, documentRESTPath(datasetID), nil, apiObject{"id": documentID})+ data := object(t, restOK(t, r)["data"])+ docs := array(t, data["docs"])+ if len(docs) != 1 {+ t.Fatalf("metadata document missing: %v", data)+ }+ doc := object(t, docs[0])+ if reflect.DeepEqual(doc["meta_fields"], want) {+ return doc+ }+ select {+ case <-ctx.Done():+ t.Fatalf("metadata did not persist: got=%v want=%v", doc["meta_fields"], want)+ case <-ticker.C:+ }+ }+}++func TestRESTDocumentMetadataReplacementAndBatchUpdates(t *testing.T) {+ id, docs := seedRESTDocuments(t, 3)+ ids := documentRESTIDs(docs)+ path := documentRESTPath(id)+ initial := apiObject{"author": "alice", "tags": []any{"one", "two"}, "year": float64(2026)}+ restOK(t, restCall(t, TestConfig, http.MethodPatch, path+"/"+ids[0], apiObject{"meta_fields": initial}, nil))+ waitRESTDocumentMetadata(t, id, ids[0], initial)+ for _, empty := range []any{nil, apiObject{}} {+ restOK(t, restCall(t, TestConfig, http.MethodPatch, path+"/"+ids[0], apiObject{"meta_fields": empty}, nil))+ waitRESTDocumentMetadata(t, id, ids[0], apiObject{})+ }+ replacement := apiObject{"author": "bob"}+ restOK(t, restCall(t, TestConfig, http.MethodPatch, path+"/"+ids[0], apiObject{"meta_fields": replacement}, nil))+ waitRESTDocumentMetadata(t, id, ids[0], replacement)+ batch := func(selector apiObject, updates, deletes []any, matched, updated int) {+ t.Helper()+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodPatch, path+"/metadatas", apiObject{"selector": selector, "updates": updates, "deletes": deletes}, nil))["data"])+ if data["matched_docs"] != float64(matched) || data["updated"] != float64(updated) {+ t.Fatalf("batch counts=%v want matched=%d updated=%d", data, matched, updated)+ }+ }+ batch(apiObject{"document_ids": ids[:2]}, []any{apiObject{"key": "status", "value": "reviewed"}}, nil, 2, 2)+ waitRESTDocumentMetadata(t, id, ids[0], apiObject{"author": "bob", "status": "reviewed"})+ waitRESTDocumentMetadata(t, id, ids[1], apiObject{"status": "reviewed"})+ waitRESTDocumentMetadata(t, id, ids[2], apiObject{})+ batch(apiObject{"document_ids": ids[:2]}, []any{apiObject{"key": "status", "value": "reviewed"}}, nil, 2, 0)+ batch(apiObject{"document_ids": ids, "metadata_condition": apiObject{"conditions": []any{apiObject{"comparison_operator": "is", "name": "author", "value": "bob"}}}}, []any{apiObject{"key": "selected", "value": "yes"}}, []any{apiObject{"key": "status"}}, 1, 1)+ waitRESTDocumentMetadata(t, id, ids[0], apiObject{"author": "bob", "selected": "yes"})+ waitRESTDocumentMetadata(t, id, ids[1], apiObject{"status": "reviewed"})+ batch(apiObject{"document_ids": []string{}}, []any{apiObject{"key": "forbidden", "value": "no"}}, nil, 0, 0)+ batch(apiObject{"document_ids": ids, "metadata_condition": apiObject{"conditions": []any{apiObject{"comparison_operator": "is", "name": "missing", "value": "never"}}}}, []any{apiObject{"key": "forbidden", "value": "no"}}, nil, 0, 0)+ for _, body := range []apiObject{+ {"selector": []any{1}}, {"selector": apiObject{"document_ids": "not-list"}}, {"selector": apiObject{"document_ids": []any{1}}},+ {"selector": apiObject{"metadata_condition": []any{1}}}, {"updates": apiObject{"key": "x"}}, {"deletes": []any{apiObject{"x": "y"}}},+ {"updates": []any{apiObject{"key": ""}}}, {"updates": []any{apiObject{"key": "missing-value"}}},+ {"selector": apiObject{"document_ids": []string{ids[0], TestConfig.InvalidID}}, "updates": []any{apiObject{"key": "author", "value": "must_not_persist"}}},+ } {+ restCode(t, restCall(t, TestConfig, http.MethodPatch, path+"/metadatas", body, nil), 200, 102)+ }+ waitRESTDocumentMetadata(t, id, ids[0], apiObject{"author": "bob", "selected": "yes"})+ _, filtered := listRESTDocuments(t, id, apiObject{"metadata": "{\"author\":[\"bob\"]}"})+ requireRESTDocumentIDs(t, filtered, ids[:1])+ batch(apiObject{"document_ids": ids[1:2]}, []any{apiObject{"key": "tags", "value": []any{"one"}}}, nil, 1, 1)+ batch(apiObject{"document_ids": ids[1:2]}, []any{apiObject{"key": "tags", "value": []any{"one", "two"}}}, nil, 1, 1)+ waitRESTDocumentMetadata(t, id, ids[1], apiObject{"status": "reviewed", "tags": []any{"one", "two"}})+ batch(apiObject{"document_ids": ids[1:2]}, []any{apiObject{"key": "tags", "value": []any{"one", "two"}}}, nil, 1, 0)+}++func TestRESTDocumentAuthenticationCannotMutate(t *testing.T) {+ id, docs := seedRESTDocuments(t, 1)+ docID := docs[0]["id"].(string)+ path := documentRESTPath(id)+ before := getRESTDocument(t, id, docID)+ for _, token := range []string{"", TestConfig.InvalidAPIKey} {+ c := *TestConfig+ c.APIKey = token+ c.AuthToken = ""+ for _, tc := range []struct {+ method, path string+ body any+ }{+ {http.MethodGet, path, nil}, {http.MethodPatch, path + "/" + docID, apiObject{"name": "forbidden.txt"}},+ {http.MethodDelete, path, apiObject{"ids": []string{docID}}},+ {http.MethodPost, path + "/parse", apiObject{"document_ids": []string{docID}}},+ {http.MethodPost, path + "/stop", apiObject{"document_ids": []string{docID}}},+ {http.MethodPatch, path + "/metadatas", apiObject{"selector": apiObject{"document_ids": []string{docID}}, "updates": []any{apiObject{"key": "forbidden", "value": "yes"}}}},+ {http.MethodPut, path + "/" + docID + "/metadata/config", apiObject{"metadata": apiObject{"forbidden": "yes"}}},+ } {+ restCode(t, restCall(t, &c, tc.method, tc.path, tc.body, nil), 401, 401)+ }+ restCode(t, multipartRESTDocuments(t, &c, id, []documentUpload{{Name: "forbidden.txt", Content: []byte("unauthorized")}}), 401, 401)+ }+ data, remaining := listRESTDocuments(t, id, nil)+ if data["total"] != float64(1) {+ t.Fatalf("unauthorized upload mutated total: %v", data)+ }+ requireRESTDocumentIDs(t, remaining, []string{docID})+ after := getRESTDocument(t, id, docID)+ for _, key := range []string{"name", "parser_config", "meta_fields", "ingestion_status"} {+ if !reflect.DeepEqual(before[key], after[key]) {+ t.Fatalf("unauthorized mutation changed %s", key)+ }+ }+}++func TestRESTDocumentDeletionValidationIsolationAndAbsence(t *testing.T) {+ id, docs := seedRESTDocuments(t, 3)+ ids := documentRESTIDs(docs)+ otherID, otherDocs := seedRESTDocuments(t, 1)+ otherDocID := otherDocs[0]["id"].(string)+ path := documentRESTPath(id)+ for _, tc := range []struct {+ body any+ code int+ }{+ {apiObject{}, 102}, {apiObject{"ids": []string{}}, 102}, {apiObject{"ids": "wrong"}, 102},+ {apiObject{"ids": []string{ids[0], ids[0]}}, 101}, {apiObject{"ids": []string{ids[0]}, "delete_all": true}, 102},+ {apiObject{"ids": []string{ids[0], TestConfig.InvalidID}}, 102},+ {apiObject{"ids": []string{TestConfig.InvalidID, ids[0]}}, 102},+ {apiObject{"ids": []string{ids[0], otherDocID}}, 102}, {"not an object", 101},+ } {+ restCode(t, restCall(t, TestConfig, http.MethodDelete, path, tc.body, nil), 200, tc.code)+ data, remaining := listRESTDocuments(t, id, nil)+ if data["total"] != float64(3) {+ t.Fatalf("rejected delete changed count: %v", data)+ }+ requireRESTDocumentIDs(t, remaining, ids)+ getRESTDocument(t, otherID, otherDocID)+ }+ restCode(t, restCall(t, TestConfig, http.MethodDelete, documentRESTPath(TestConfig.InvalidID), apiObject{"ids": ids}, nil), 200, 404)+ for _, target := range [][]string{ids[:1], ids[1:]} {+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodDelete, path, apiObject{"ids": target}, nil))["data"])+ if data["deleted"] != float64(len(target)) {+ t.Fatalf("deleted=%v want=%d", data["deleted"], len(target))+ }+ for _, docID := range target {+ restCode(t, restCall(t, TestConfig, http.MethodGet, path, nil, apiObject{"id": docID}), 200, 102)+ restCode(t, restCall(t, TestConfig, http.MethodGet, path+"/"+docID, nil, nil), 200, 404)+ }+ }+ data, remaining := listRESTDocuments(t, id, nil)+ if data["total"] != float64(0) || len(remaining) != 0 {+ t.Fatalf("delete left documents: %v", data)+ }+ restCode(t, restCall(t, TestConfig, http.MethodDelete, path, apiObject{"ids": ids}, nil), 200, 102)+ getRESTDocument(t, otherID, otherDocID)+}++func waitRESTDocumentStatus(t *testing.T, datasetID string, ids []string, expected string) []apiObject {+ t.Helper()+ ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)+ defer cancel()+ ticker := time.NewTicker(500 * time.Millisecond)+ defer ticker.Stop()+ for {+ data := object(t, restOK(t, restCallContext(t, ctx, TestConfig, http.MethodGet, documentRESTPath(datasetID), nil, apiObject{"page_size": 100}))["data"])+ found := map[string]apiObject{}+ for _, item := range array(t, data["docs"]) {+ doc := object(t, item)+ found[doc["id"].(string)] = doc+ }+ all := true+ result := make([]apiObject, 0, len(ids))+ for _, id := range ids {+ doc, ok := found[id]+ if !ok {+ t.Fatalf("ingesting document %s disappeared", id)+ }+ status := fmt.Sprint(doc["ingestion_status"])+ switch status {+ case expected:+ case "FAILED", "STOPPED", "COMPLETED":+ t.Fatalf("document %s terminal=%s want=%s detail=%v", id, status, expected, doc)+ case "CREATED", "SCHEDULED", "RUNNING", "STOPPING":+ all = false+ default:+ t.Fatalf("unexpected ingestion status %q document=%v", status, doc)+ }+ result = append(result, doc)+ }+ if all {+ return result+ }+ select {+ case <-ctx.Done():+ t.Fatalf("ingestion timeout waiting for %s: %v", expected, result)+ case <-ticker.C:+ }+ }+}++func TestRESTDocumentParsingValidationCompletionAndRepeatedStart(t *testing.T) {+ id, docs := seedRESTDocuments(t, 3)+ ids := documentRESTIDs(docs)+ path := documentRESTPath(id)+ for _, operation := range []string{"parse", "stop"} {+ for _, body := range []any{apiObject{}, apiObject{"document_ids": []string{}}, apiObject{"document_ids": "wrong"}, "not an object"} {+ restCode(t, restCall(t, TestConfig, http.MethodPost, path+"/"+operation, body, nil), 200, 102)+ }+ restCode(t, restCall(t, TestConfig, http.MethodPost, documentRESTPath(TestConfig.InvalidID)+"/"+operation, apiObject{"document_ids": ids}, nil), 200, 404)+ }+ // The start endpoint reports a count of accepted documents, including partial batches.+ invalid := object(t, restOK(t, restCall(t, TestConfig, http.MethodPost, path+"/parse", apiObject{"document_ids": []string{TestConfig.InvalidID}}, nil))["data"])+ if invalid["success_count"] != float64(0) {+ t.Fatalf("invalid document accepted: %v", invalid)+ }+ for _, docID := range ids {+ if getRESTDocument(t, id, docID)["ingestion_status"] != "UNSTART" {+ t.Fatal("invalid parse mutated existing document")+ }+ }+ stopped := object(t, restOK(t, restCall(t, TestConfig, http.MethodPost, path+"/stop", apiObject{"document_ids": []string{ids[0], ids[0]}}, nil))["data"])+ if stopped["success_count"] != float64(0) || len(array(t, stopped["errors"])) != 1 {+ t.Fatalf("stop unstarted=%v", stopped)+ }+ if !strings.Contains(fmt.Sprint(stopped["errors"]), "has not started or already completed") {+ t.Fatalf("unexpected stop rejection: %v", stopped)+ }+ restCode(t, restCall(t, TestConfig, http.MethodPost, path+"/stop", apiObject{"document_ids": []string{ids[0], TestConfig.InvalidID}}, nil), 200, 100)+ parsed := object(t, restOK(t, restCall(t, TestConfig, http.MethodPost, path+"/parse", apiObject{"document_ids": append(slices.Clone(ids), ids[0], TestConfig.InvalidID)}, nil))["data"])+ if parsed["success_count"] != float64(3) {+ t.Fatalf("accepted unique valid count=%v", parsed)+ }+ completed := waitRESTDocumentStatus(t, id, ids, "COMPLETED")+ for _, doc := range completed {+ docID := doc["id"].(string)+ if doc["chunk_count"].(float64) <= 0 || doc["token_count"].(float64) <= 0 || doc["progress"].(float64) != 1 {+ t.Fatalf("completed document lacks parsed content: %v", doc)+ }+ chunks := object(t, restOK(t, restCall(t, TestConfig, http.MethodGet, path+"/"+docID+"/chunks", nil, nil))["data"])+ items := array(t, chunks["chunks"])+ if len(items) == 0 {+ t.Fatalf("completed document %s has no chunks", docID)+ }+ var content strings.Builder+ for _, item := range items {+ chunk := object(t, item)+ text, ok := chunk["content_with_weight"].(string)+ if !ok || text == "" {+ t.Fatalf("parsed chunk has no source text: %v", chunk)+ }+ content.WriteString(text)+ }+ if !strings.Contains(strings.ToLower(content.String()), "coriander") {+ t.Fatalf("parsed chunks lost source content: %s", content.String())+ }+ }+ before := completed[0]+ rerun := object(t, restOK(t, restCall(t, TestConfig, http.MethodPost, path+"/parse", apiObject{"document_ids": ids[:1]}, nil))["data"])+ if rerun["success_count"] != float64(0) {+ t.Fatalf("completed document unexpectedly queued again: %v", rerun)+ }+ after := waitRESTDocumentStatus(t, id, ids[:1], "COMPLETED")[0]+ if after["chunk_count"] != before["chunk_count"] || !reflect.DeepEqual(after["latest_ingestion_event"], before["latest_ingestion_event"]) {+ t.Fatalf("repeated start changed chunks: before=%v after=%v", before["chunk_count"], after["chunk_count"])+ }+}++func TestRESTDocumentStopReachesTerminalState(t *testing.T) {+ dataset := createRESTDataset(t, apiObject{"parse_type": 1, "parser_id": "general"})+ id := dataset["id"].(string)+ path := documentRESTPath(id)+ // A moderate text fixture leaves a useful stop window without making the+ // retry expensive. Only a proved completion race permits another attempt.+ content := []byte(strings.Repeat("Coriander leaves and seeds provide botanical evidence.\n", 1280))+ for attempt := range 3 {+ doc := object(t, uploadRESTDocuments(t, TestConfig, id, []documentUpload{{Name: fmt.Sprintf("stop_%d.txt", attempt), Content: content}})[0])+ docID := doc["id"].(string)+ started := object(t, restOK(t, restCall(t, TestConfig, http.MethodPost, path+"/parse", apiObject{"document_ids": []string{docID}}, nil))["data"])+ if started["success_count"] != float64(1) {+ t.Fatalf("parse not accepted: %v", started)+ }+ result := object(t, restOK(t, restCall(t, TestConfig, http.MethodPost, path+"/stop", apiObject{"document_ids": []string{docID, docID}}, nil))["data"])+ switch result["success_count"] {+ case float64(1):+ stopped := waitRESTDocumentStatus(t, id, []string{docID}, "STOPPED")[0]+ repeated := object(t, restOK(t, restCall(t, TestConfig, http.MethodPost, path+"/stop", apiObject{"document_ids": []string{docID}}, nil))["data"])+ if repeated["success_count"] != float64(1) {+ t.Fatalf("repeated stopped request=%v", repeated)+ }+ restarted := object(t, restOK(t, restCall(t, TestConfig, http.MethodPost, path+"/parse", apiObject{"document_ids": []string{docID}}, nil))["data"])+ if restarted["success_count"] != float64(1) {+ t.Fatalf("stopped document was not accepted for retry: %v", restarted)+ }+ after := waitRESTDocumentStatus(t, id, []string{docID}, "COMPLETED")[0]+ if after["chunk_count"].(float64) <= 0 || after["token_count"].(float64) <= 0 || after["progress"] != float64(1) || reflect.DeepEqual(stopped["latest_ingestion_event"], after["latest_ingestion_event"]) {+ t.Fatalf("retry failed to produce a new completed run: stopped=%v completed=%v", stopped, after)+ }+ return+ case float64(0):+ errors := array(t, result["errors"])+ if len(errors) != 1 || errors[0] != "Can't stop parsing document that has not started or already completed" {+ t.Fatalf("stop rejection=%v", result)+ }+ waitRESTDocumentStatus(t, id, []string{docID}, "COMPLETED")+ t.Logf("attempt %d completed before stop; retrying with a fresh document", attempt+1)+ default:+ t.Fatalf("unexpected stop count: %v", result)+ }+ }+ t.Fatal("no document reached STOPPED in three attempts; stopping was not demonstrated")+}++func TestRESTDocumentConcurrentUploadAndDeletion(t *testing.T) {+ dataset := createRESTDataset(t, nil)+ id := dataset["id"].(string)+ ids := make([]string, 3)+ t.Run("upload", func(t *testing.T) {+ for i := range ids {+ t.Run(fmt.Sprintf("file_%d", i), func(t *testing.T) {+ t.Parallel()+ name := fmt.Sprintf("parallel_%d.txt", i)+ content := []byte(fmt.Sprintf("parallel content %d", i))+ doc := object(t, uploadRESTDocuments(t, TestConfig, id, []documentUpload{{Name: name, Content: content}})[0])+ if doc["name"] != name || doc["size"] != float64(len(content)) {+ t.Fatalf("concurrent upload metadata=%v", doc)+ }+ ids[i] = doc["id"].(string)+ })+ }+ })+ data, docs := listRESTDocuments(t, id, nil)+ if data["total"] != float64(len(ids)) {+ t.Fatalf("concurrent upload lost documents: %v", data)+ }+ requireRESTDocumentIDs(t, docs, ids)+ t.Run("delete", func(t *testing.T) {+ for _, docID := range ids {+ t.Run(docID, func(t *testing.T) {+ t.Parallel()+ deleted := object(t, restOK(t, restCall(t, TestConfig, http.MethodDelete, documentRESTPath(id), apiObject{"ids": []string{docID}}, nil))["data"])+ if deleted["deleted"] != float64(1) {+ t.Fatalf("concurrent deletion=%v", deleted)+ }+ restCode(t, restCall(t, TestConfig, http.MethodGet, documentRESTPath(id), nil, apiObject{"id": docID}), 200, 102)+ })+ }+ })+ data, docs = listRESTDocuments(t, id, nil)+ if data["total"] != float64(0) || len(docs) != 0 {+ t.Fatalf("concurrent deletion left documents: %v", data)+ }+}++func TestRESTDocumentRejectedParserUpdatePreservesName(t *testing.T) {+ id, docs := seedRESTDocuments(t, 1)+ docID := docs[0]["id"].(string)+ before := getRESTDocument(t, id, docID)+ config := object(t, before["parser_config"])+ var parser string+ for key := range config {+ if strings.HasPrefix(key, "Parser:") {+ parser = key+ break+ }+ }+ if parser == "" {+ t.Fatalf("general parser component missing: %v", config)+ }+ payload := apiObject{+ "name": "must_not_persist.txt",+ "parser_config": apiObject{parser: apiObject{"pdf": apiObject{"pages": []any{[]any{3, 1}}}}},+ }+ restCode(t, restCall(t, TestConfig, http.MethodPatch, documentRESTPath(id)+"/"+docID, payload, nil), 200, 102)+ after := getRESTDocument(t, id, docID)+ for _, key := range []string{"name", "parser_config", "meta_fields", "size", "dataset_id"} {+ if !reflect.DeepEqual(before[key], after[key]) {+ t.Errorf("rejected parser update changed %s: before=%v after=%v", key, before[key], after[key])+ }+ }+}diff --git a/test/api/document_raw_routes_test.go b/test/api/document_raw_routes_test.gonew file mode 100644index 0000000000..58479bb907--- /dev/null+++ b/test/api/document_raw_routes_test.go@@ -0,0 +1,301 @@+//go:build integration++package api++import (+ "archive/zip"+ "bytes"+ "context"+ "crypto/sha256"+ "encoding/base64"+ "image"+ "image/color"+ "image/png"+ "mime"+ "net/http"+ "net/url"+ "os"+ "strings"+ "testing"+ "time"+)++func documentDownloadFixtures(t *testing.T) []documentUpload {+ t.Helper()+ var imageBytes bytes.Buffer+ picture := image.NewRGBA(image.Rect(0, 0, 2, 2))+ picture.Set(0, 0, color.RGBA{R: 255, A: 255})+ if err := png.Encode(&imageBytes, picture); err != nil {+ t.Fatal(err)+ }+ pdf, err := os.ReadFile("../benchmark/test_docs/Doc1.pdf")+ if err != nil {+ t.Fatal(err)+ }+ files := []documentUpload{+ {Name: "plain.txt", Content: []byte("Download integrity: 中文\r\nsecond line\x00end")},+ {Name: "notes.md", Content: []byte("# Download\n\n**unaltered** content\n")},+ {Name: "record.json", Content: []byte("{\"code\":102,\"message\":\"this is uploaded content, not an API error\"}\n")},+ {Name: "message.eml", Content: []byte("From: [email protected]\r\nTo: [email protected]\r\nSubject: integrity\r\n\r\nDownload bytes.\r\n")},+ {Name: "page.html", Content: []byte("<!doctype html><html><body>Download integrity</body></html>")},+ {Name: "pixels.png", Content: imageBytes.Bytes()},+ {Name: "source.pdf", Content: pdf},+ }+ for _, office := range []struct{ name, member, contentType, xml string }{+ {"text.docx", "word/document.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml", `<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>Download integrity</w:t></w:r></w:p></w:body></w:document>`},+ {"cells.xlsx", "xl/workbook.xml", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml", `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheets/></workbook>`},+ {"slides.pptx", "ppt/presentation.xml", "application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml", `<p:presentation xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"><p:sldIdLst/></p:presentation>`},+ } {+ var content bytes.Buffer+ writer := zip.NewWriter(&content)+ for _, part := range []struct{ name, value string }{+ {"[Content_Types].xml", `<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Override PartName="/` + office.member + `" ContentType="` + office.contentType + `"/></Types>`},+ {"_rels/.rels", `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="r1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="` + office.member + `"/></Relationships>`},+ {office.member, office.xml},+ } {+ member, err := writer.Create(part.name)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := member.Write([]byte(part.value)); err != nil {+ t.Fatal(err)+ }+ }+ if err := writer.Close(); err != nil {+ t.Fatal(err)+ }+ files = append(files, documentUpload{Name: office.name, Content: content.Bytes()})+ }+ return files+}++func requireDocumentDownload(t *testing.T, result restResult, file documentUpload) {+ t.Helper()+ if result.Status != http.StatusOK {+ t.Fatalf("download HTTP=%d want=200 (see request diagnostic)", result.Status)+ }+ mediaType, _, err := mime.ParseMediaType(result.Header.Get("Content-Type"))+ if err != nil || mediaType != "application/octet-stream" {+ t.Fatalf("download Content-Type=%q error=%v", result.Header.Get("Content-Type"), err)+ }+ disposition, params, err := mime.ParseMediaType(result.Header.Get("Content-Disposition"))+ if err != nil || disposition != "attachment" || params["filename"] != file.Name {+ t.Fatalf("download disposition=%q filename=%q want=%q error=%v", disposition, params["filename"], file.Name, err)+ }+ if !bytes.Equal(result.Body, file.Content) {+ t.Fatalf("download %q bytes=%d sha256=%x want bytes=%d sha256=%x", file.Name, len(result.Body), sha256.Sum256(result.Body), len(file.Content), sha256.Sum256(file.Content))+ }+}++func TestRESTDocumentDownloadIntegrity(t *testing.T) {+ dataset := createRESTDataset(t, apiObject{"name": uniqueName("download")})+ datasetID := dataset["id"].(string)+ files := documentDownloadFixtures(t)+ documents := uploadRESTDocuments(t, TestConfig, datasetID, files)+ paths := make([]string, len(files))+ for i, file := range files {+ document := object(t, documents[i])+ if document["name"] != file.Name {+ t.Fatalf("uploaded filename=%v want=%q", document["name"], file.Name)+ }+ paths[i] = "/datasets/" + datasetID + "/documents/" + document["id"].(string)+ t.Run(file.Name, func(t *testing.T) {+ for range 2 {+ requireDocumentDownload(t, rawRESTCall(t, TestConfig, http.MethodGet, paths[i], nil, ""), file)+ }+ })+ }+ t.Run("concurrent", func(t *testing.T) {+ for i, file := range files {+ t.Run(file.Name, func(t *testing.T) {+ t.Parallel()+ requireDocumentDownload(t, rawRESTCall(t, TestConfig, http.MethodGet, paths[i], nil, ""), file)+ })+ }+ })+ t.Run("authentication", func(t *testing.T) {+ for _, token := range []string{"", TestConfig.InvalidAPIKey} {+ client := *TestConfig+ client.APIKey, client.AuthToken = token, ""+ restCode(t, rawRESTCall(t, &client, http.MethodGet, paths[0], nil, ""), http.StatusUnauthorized, 401)+ }+ requireDocumentDownload(t, rawRESTCall(t, TestConfig, http.MethodGet, paths[0], nil, ""), files[0])+ })+ t.Run("missing_or_wrong_dataset", func(t *testing.T) {+ other := createRESTDataset(t, apiObject{"name": uniqueName("download_other")})+ otherID := other["id"].(string)+ for _, path := range []string{+ "/datasets/" + datasetID + "/documents/" + TestConfig.InvalidID,+ "/datasets/" + TestConfig.InvalidID + "/documents/" + object(t, documents[0])["id"].(string),+ "/datasets/" + otherID + "/documents/" + object(t, documents[0])["id"].(string),+ } {+ result := rawRESTCall(t, TestConfig, http.MethodGet, path, nil, "")+ restCode(t, result, http.StatusOK, 404)+ if result.Header.Get("Content-Disposition") != "" {+ t.Fatal("missing document returned an attachment")+ }+ }+ requireDocumentDownload(t, rawRESTCall(t, TestConfig, http.MethodGet, paths[0], nil, ""), files[0])+ })+ t.Run("document_id_route_is_metadata", func(t *testing.T) {+ id := object(t, documents[0])["id"].(string)+ result := restCall(t, TestConfig, http.MethodGet, "/documents/"+id, nil, nil)+ if result.Status != http.StatusOK || object(t, result.JSON["data"])["id"] != id {+ t.Fatalf("document metadata missing uploaded ID %q", id)+ }+ missing := restCall(t, TestConfig, http.MethodGet, "/documents/"+TestConfig.InvalidID, nil, nil)+ if missing.Status != http.StatusNotFound || missing.JSON["error"] != "document not found" {+ t.Fatalf("missing document metadata contract=%d %v", missing.Status, missing.JSON)+ }+ for _, token := range []string{"", TestConfig.InvalidAPIKey} {+ client := *TestConfig+ client.APIKey, client.AuthToken = token, ""+ restCode(t, restCall(t, &client, http.MethodGet, "/documents/"+id, nil, nil), http.StatusUnauthorized, 401)+ }+ })+}++func TestRESTPrivateDocumentRawRoutes(t *testing.T) {+ paths := []string{+ "/documents/images/not-a-valid-image-id",+ "/documents/" + TestConfig.InvalidID + "/images/imagetemps-page-1.png",+ "/documents/" + TestConfig.InvalidID + "/thumbnail",+ }+ for _, path := range paths {+ t.Run(strings.TrimPrefix(path, "/documents/"), func(t *testing.T) {+ for _, token := range []string{"", TestConfig.InvalidAPIKey} {+ client := *TestConfig+ client.APIKey, client.AuthToken = token, ""+ result := rawRESTCall(t, &client, http.MethodGet, path, nil, "")+ restCode(t, result, http.StatusOK, 102)+ if result.JSON["message"] != "Authorization is not valid!" {+ t.Fatalf("private image authorization message=%v", result.JSON["message"])+ }+ }+ result := rawRESTCall(t, TestConfig, http.MethodGet, path, nil, "")+ restCode(t, result, http.StatusOK, 102)+ if result.JSON["message"] != "Image not found." {+ t.Fatalf("missing image message=%v", result.JSON["message"])+ }+ })+ }+}++func TestRESTDocumentRawImagesRequireIndexedOwnership(t *testing.T) {+ dataset := createRESTDataset(t, apiObject{"name": uniqueName("raw_image")})+ datasetID := dataset["id"].(string)+ file := documentDownloadFixtures(t)[5]+ documents := uploadRESTDocuments(t, TestConfig, datasetID, []documentUpload{file})+ id := object(t, documents[0])["id"].(string)+ imageID := datasetID + "-" + file.Name+ for _, path := range []string{+ "/documents/images/" + imageID,+ "/documents/" + id + "/images/" + imageID,+ "/documents/" + id + "/thumbnail",+ } {+ result := rawRESTCall(t, TestConfig, http.MethodGet, path, nil, "")+ restCode(t, result, http.StatusOK, 102)+ if result.JSON["message"] != "Image not found." {+ t.Fatalf("unindexed image message=%v", result.JSON["message"])+ }+ }+ requireDocumentDownload(t, rawRESTCall(t, TestConfig, http.MethodGet, "/datasets/"+datasetID+"/documents/"+id, nil, ""), file)+}++func TestRESTDocumentIndexedImageIntegrity(t *testing.T) {+ dataset := createRESTDataset(t, apiObject{"name": uniqueName("indexed_image")})+ datasetID := dataset["id"].(string)+ file := documentDownloadFixtures(t)[5]+ docs := uploadRESTDocuments(t, TestConfig, datasetID, []documentUpload{+ file, {Name: "other.txt", Content: []byte("This document does not own the image.")},+ })+ id := object(t, docs[0])["id"].(string)+ otherID := object(t, docs[1])["id"].(string)+ data := object(t, restOK(t, restCall(t, TestConfig, http.MethodPost, "/datasets/"+datasetID+"/documents/"+id+"/chunks", apiObject{+ "content": "A red pixel in an image used to test document ownership.",+ "image_base64": base64.StdEncoding.EncodeToString(file.Content),+ }, nil))["data"])+ chunk := object(t, data["chunk"])+ imageID, ok := chunk["image_id"].(string)+ if !ok || imageID != datasetID+"-"+chunk["id"].(string) {+ t.Fatalf("chunk image ID=%v", chunk["image_id"])+ }+ paths := []string{"/documents/images/" + imageID, "/documents/" + id + "/images/" + imageID}+ ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second)+ defer cancel()+ ticker := time.NewTicker(100 * time.Millisecond)+ defer ticker.Stop()+ for {+ result := restCallContext(t, ctx, TestConfig, http.MethodGet, paths[0], nil, nil)+ if result.JSON == nil {+ if result.Status != http.StatusOK || !bytes.Equal(result.Body, file.Content) {+ t.Fatal("indexed image does not match uploaded PNG")+ }+ break+ }+ restCode(t, result, http.StatusOK, 102)+ if result.JSON["message"] != "Image not found." {+ t.Fatalf("unexpected image response=%v", result.JSON)+ }+ select {+ case <-ctx.Done():+ t.Fatalf("indexed image was not readable before deadline: %v", ctx.Err())+ case <-ticker.C:+ }+ }+ for _, path := range paths {+ t.Run(path, func(t *testing.T) {+ for range 2 {+ result := rawRESTCall(t, TestConfig, http.MethodGet, path, nil, "")+ if result.Status != http.StatusOK || result.Header.Get("Content-Type") != "image/png" || result.Header.Get("Cache-Control") != "no-store" || !bytes.Equal(result.Body, file.Content) {+ t.Fatalf("image HTTP=%d type=%q cache=%q bytes=%d checksum=%x want=%x", result.Status, result.Header.Get("Content-Type"), result.Header.Get("Cache-Control"), len(result.Body), sha256.Sum256(result.Body), sha256.Sum256(file.Content))+ }+ }+ for _, token := range []string{"", TestConfig.InvalidAPIKey} {+ client := *TestConfig+ client.APIKey, client.AuthToken = token, ""+ result := rawRESTCall(t, &client, http.MethodGet, path, nil, "")+ restCode(t, result, http.StatusOK, 102)+ if result.JSON["message"] != "Authorization is not valid!" {+ t.Fatalf("existing image auth error=%v", result.JSON)+ }+ }+ })+ }+ result := rawRESTCall(t, TestConfig, http.MethodGet, "/documents/"+otherID+"/images/"+imageID, nil, "")+ restCode(t, result, http.StatusOK, 102)+ if result.JSON["message"] != "Image not found." {+ t.Fatalf("foreign document image response=%v", result.JSON)+ }+}++func TestRESTDocumentArtifactSafety(t *testing.T) {+ for _, token := range []string{"", TestConfig.InvalidAPIKey} {+ client := *TestConfig+ client.APIKey, client.AuthToken = token, ""+ restCode(t, rawRESTCall(t, &client, http.MethodGet, "/documents/artifact/not-an-artifact.png", nil, ""), http.StatusUnauthorized, 401)+ }+ for _, tc := range []struct{ filename, message string }{+ {"payload.exe", "invalid file type"},+ {"payload.png.exe", "invalid file type"},+ {"no-extension", "invalid file type"},+ {`..\escape.png`, "invalid filename"},+ {`folder\escape.json`, "invalid filename"},+ {uniqueName("missing") + ".png", "artifact not found"},+ {uniqueName("missing") + ".PNG", "artifact not found"},+ {uniqueName("missing") + ".json", "artifact not found"},+ {uniqueName("missing") + ".html", "artifact not found"},+ } {+ t.Run(tc.filename, func(t *testing.T) {+ path := "/documents/artifact/" + url.PathEscape(tc.filename)+ result := rawRESTCall(t, TestConfig, http.MethodGet, path, nil, "")+ restCode(t, result, http.StatusOK, 102)+ if result.JSON["message"] != tc.message {+ t.Fatalf("artifact message=%v want=%q", result.JSON["message"], tc.message)+ }+ if result.Header.Get("Content-Disposition") != "" {+ t.Fatal("rejected artifact returned file headers")+ }+ })+ }+}