Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(datasource/dingtalk): tighten Validate sampling edges (#4079)
fight for you57f8a65fec7b
diff --git a/internal/datasource/connector/dingtalk/connector.go b/internal/datasource/connector/dingtalk/connector.goindex d3a77d334..4bb0a3d0f 100644--- a/internal/datasource/connector/dingtalk/connector.go+++ b/internal/datasource/connector/dingtalk/connector.go@@ -116,9 +116,10 @@ const maxValidateProbes = 5//// The operator is not expected to reach every workspace in the tenant: app// credentials are valid as long as one reachable workspace yields one readable-// document. Reading the first document of the first workspace and failing on it-// rejects a working configuration whenever an unrelated workspace — one the-// operator was never granted — happens to be listed first.+// document. A failed probe does not condemn that workspace; the walk keeps+// looking, inside the probe cap, for another document the operator can read.+// When the cap itself stops a conclusive walk, the error says how many+// documents were read.func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSourceConfig) error {cfg, err := parseConfig(dataSourceConfig)if err != nil {@@ -138,14 +139,16 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSoworkspaceFail intprobes intlistings int+ probeCapHit bool)- for _, item := range workspaces {+ for i, item := range workspaces {rootNodeID := strings.TrimSpace(item.RootNodeID)if rootNodeID == "" {continue}- document, found, unexplored, err := firstValidateDocument(- ctx, api, rootNodeID, settings, &listings,+ readable, unexplored, hitCap, err := sampleWorkspace(+ ctx, api, item.Name, rootNodeID, settings,+ &listings, &probes, &sawDocument, &lastErr,)if err != nil {lastErr = fmt.Errorf("workspace %q: %w", item.Name, err)@@ -157,29 +160,24 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSoworkspaceFail++continue}+ if readable {+ return nil+ }if unexplored {inconclusive = true}- if !found {- continue- }- sawDocument = true- probes++- if err := verifyDocument(ctx, api, document); err != nil {- lastErr = fmt.Errorf("workspace %q document %q: %w", item.Name, document.Name, err)- if ctx.Err() != nil || isContextError(err) {- return fmt.Errorf("validate DingTalk data source: %w", lastErr)- }- } else {- return nil- }+ // Once the cap is spent no further probe can succeed, so walking on+ // could only find an unlistable folder and turn the failed probes+ // into acceptance.if probes >= maxValidateProbes {+ probeCapHit = hitCap || i+1 < len(workspaces)break}}// Folders were left unexplored, because the listing budget ran out or a// folder could not be listed, so an unreadable document elsewhere does not- // prove the data source is unusable.+ // prove the data source is unusable. A probe cap hit in that situation+ // stays quiet too: the walk cannot claim it sampled every visible document.if inconclusive {return nil}@@ -194,6 +192,12 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSoif lastErr == nil {return nil}+ if probeCapHit {+ return fmt.Errorf(+ "validate DingTalk data source: %w; only the first %d documents were probed",+ lastErr, maxValidateProbes,+ )+ }return fmt.Errorf("validate DingTalk data source: %w", lastErr)}@@ -205,45 +209,83 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo// client's retries of each).const maxValidateListings = 20-// firstValidateDocument walks a workspace breadth-first, the way scanWorkspace-// does, and returns the first document it finds. It requests one page at a-// time instead of listing whole folders, so the budget bounds real requests.-// unexplored reports that some nodes were never seen, because the budget ran-// out or a folder could not be listed, so the walk cannot prove the workspace-// holds no readable document.+// sampleWorkspace walks one workspace breadth-first, the way scanWorkspace+// does, and probes ingestible documents until one is readable, the probe cap+// is spent, or the walk runs out of nodes. It requests one page at a time+// instead of listing whole folders, so the budget bounds real requests.+//+// unexplored reports that some folder was never opened or could not be listed,+// so a failed probe elsewhere must not reject the data source. readable reports+// that a probe succeeded. The walk stops as soon as the failed probe that+// spends maxValidateProbes returns, without another listing; capHit then+// reports that nodes of this workspace were still left to visit. The caller+// turns that into an error only when the walk is conclusive.+//+// A folder's later pages are held back once the listings still in the budget+// are no longer enough to open every sibling already queued, so each of those+// siblings still gets one page. The unread tail is not itself "unexplored"+// when a document was probed after it was skipped: the reservation is what+// made room for that probe. A tail skipped with no probe after it stays+// inconclusive, because the only readable document may sit on a page the+// sample never requested. A folder that receives no page at all, a subfolder+// listing error, a repeated page token, or a tail with no sibling waiting all+// stay inconclusive too.+//+// Any failed page of the workspace root is a hard error, matching listNodes+// during sync. Subfolder listing errors stay inconclusive unless the request+// was cancelled or timed out.//// The walk selects nodes with the same settings-aware predicate the picker and// the sync use, so a type the data source does not ingest is never probed.-func firstValidateDocument(+func sampleWorkspace(ctx context.Context,api dingTalkAPI,+ workspaceName string,rootNodeID string,settings documentSettings,listings *int,-) (document node, found, unexplored bool, err error) {+ probes *int,+ sawDocument *bool,+ lastErr *error,+) (readable, unexplored, capHit bool, err error) {queue := []string{rootNodeID}visited := map[string]struct{}{rootNodeID: {}}+ uncoveredSkip := falsefor len(queue) > 0 {parent := queue[0]queue = queue[1:]// Finish every page of a folder before descending, like listNodes// does for scanWorkspace: a document on a later page of this folder// must be found before the budget is spent on its subfolders.+ // Siblings already on the queue stay ahead of those subfolders.var subfolders []stringpageToken := ""seenTokens := map[string]struct{}{}for {+ // Reserve one listing per sibling already queued before spending+ // another page on this folder. The root's own pages have an empty+ // queue here — its children are not queued until the root is+ // finished — so a document on the root's next page is still read+ // before the walk descends.+ if pageToken != "" && *listings+len(queue) >= maxValidateListings {+ if len(queue) == 0 {+ unexplored = true+ } else {+ uncoveredSkip = true+ }+ break+ }rootPage := parent == rootNodeID && pageToken == ""if !rootPage {if *listings >= maxValidateListings {- return node{}, false, true, nil+ return false, true, false, nil}*listings++}children, next, listErr := api.listNodesPage(ctx, parent, pageToken)if listErr != nil {- if rootPage || ctx.Err() != nil || isContextError(listErr) {- return node{}, false, false, listErr+ if parent == rootNodeID || ctx.Err() != nil || isContextError(listErr) {+ return false, false, false, listErr}// The documents this operator can read may live in exactly// this folder, so its failure leaves the walk inconclusive@@ -251,9 +293,27 @@ func firstValidateDocument(unexplored = truebreak}- for _, child := range children {+ for i, child := range children {if child.isIngestible(settings) {- return child, true, unexplored, nil+ *probes+++ *sawDocument = true+ uncoveredSkip = false+ probeErr := verifyDocument(ctx, api, child)+ if probeErr != nil {+ if ctx.Err() != nil || isContextError(probeErr) {+ return false, false, false, fmt.Errorf("document %q: %w", child.Name, probeErr)+ }+ *lastErr = fmt.Errorf(+ "workspace %q document %q: %w", workspaceName, child.Name, probeErr,+ )+ if *probes >= maxValidateProbes {+ more := i+1 < len(children) || len(subfolders) > 0 ||+ next != "" || len(queue) > 0+ return false, unexplored, more, nil+ }+ continue+ }+ return true, false, false, nil}if child.isFolder() || child.HasChildren {if _, seen := visited[child.ID]; seen {@@ -277,7 +337,10 @@ func firstValidateDocument(}queue = append(queue, subfolders...)}- return node{}, false, unexplored, nil+ if uncoveredSkip {+ unexplored = true+ }+ return false, unexplored, false, nil}// verifyDocument proves one visible document is readable. Native documents arediff --git a/internal/datasource/connector/dingtalk/validate_test.go b/internal/datasource/connector/dingtalk/validate_test.goindex 86ac79c63..02121e3c9 100644--- a/internal/datasource/connector/dingtalk/validate_test.go+++ b/internal/datasource/connector/dingtalk/validate_test.go@@ -147,6 +147,11 @@ func TestValidateReportsWhenNoVisibleDocumentIsReadable(t *testing.T) {t.Fatalf("Validate error should carry %s, got: %v", want, err)}}+ // Both documents were probed, so the cap did not stop the walk and the+ // error must not claim that it did.+ if strings.Contains(err.Error(), "only the first") {+ t.Fatalf("Validate error should not mention the probe cap, got: %v", err)+ }}// A tenant with no documents anywhere leaves nothing to read, so the@@ -194,9 +199,10 @@ func TestValidateSurvivesWorkspaceListingFailure(t *testing.T) {}}-// An app without the document read permission fails every probe. Validate must-// probe at most one document per workspace and stop after maxValidateProbes,-// instead of firing one call per document across the whole tenant.+// An app without the document read permission fails every probe. Validate keeps+// going to the next document in the same workspace, but it still stops after+// maxValidateProbes reads and says so, instead of firing one call per document+// across the whole tenant.func TestValidateCapsProbesWhenEveryDocumentIsUnreadable(t *testing.T) {api := &fakeAPI{nodes: map[string][]node{},@@ -216,20 +222,32 @@ func TestValidateCapsProbesWhenEveryDocumentIsUnreadable(t *testing.T) {}c := testConnector(api)- if err := c.Validate(context.Background(), testConfig()); err == nil {+ err := c.Validate(context.Background(), testConfig())+ if err == nil {t.Fatal("Validate must fail when no visible document can be read")}total := 0for _, n := range api.blockCalls {total += n}- if total > maxValidateProbes {- t.Fatalf("Validate made %d document probes, want at most %d", total, maxValidateProbes)+ if total != maxValidateProbes {+ t.Fatalf("Validate made %d document probes, want %d", total, maxValidateProbes)}- for w := 0; w < workspaces; w++ {- if api.blockCalls[fmt.Sprintf("doc-%d-1", w)] != 0 {- t.Fatalf("workspace root-%d was probed past its first document", w)- }+ // The first workspace has four unreadable documents, so the cap must be+ // spent there and on the next workspace's first document — not one probe+ // per workspace, and not on a third workspace.+ if api.blockCalls["doc-0-1"] != 1 {+ t.Fatal("a failed probe must not stop the workspace from trying its next document")+ }+ if api.blockCalls["doc-1-0"] != 1 || api.blockCalls["doc-1-1"] != 0 || api.blockCalls["doc-2-0"] != 0 {+ t.Fatalf("probe cap was not applied across workspaces, calls=%v", api.blockCalls)+ }+ wantNote := fmt.Sprintf("only the first %d documents were probed", maxValidateProbes)+ if !strings.Contains(err.Error(), wantNote) {+ t.Fatalf("Validate error should say the probe cap stopped the walk, got: %v", err)+ }+ if !strings.Contains(err.Error(), `document "doc-1-0"`) {+ t.Fatalf("Validate error should name the last probed document, got: %v", err)}}@@ -554,10 +572,14 @@ func TestValidateBudgetCountsNodeListingPages(t *testing.T) {// A folder's later pages must be read before its subfolders: otherwise a root// whose first page is all folders spends the budget below them and never sees-// the document on the root's second page.+// the document on the root's second page. The document is unreadable and the+// twenty subfolders cannot all be listed after that page spends a budget unit,+// so the rest of the tree is unknown and Validate accepts.func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) {var mu sync.Mutexprobed := false+ folderLists := 0+ folderListsBeforeProbe := 0server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {w.Header().Set("Content-Type", "application/json")switch {@@ -578,6 +600,12 @@ func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) {_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc","name":"Doc","type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))default:+ mu.Lock()+ folderLists+++ if !probed {+ folderListsBeforeProbe+++ }+ mu.Unlock()_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf","type":"FILE","category":"FILE","extension":"pdf"}]}`))}case strings.HasPrefix(r.URL.Path, "/v1.0/doc/suites/documents/"):@@ -598,8 +626,14 @@ func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) {if !probed {t.Fatalf("the document on the root's second page was never probed (err=%v)", err)}- if err == nil {- t.Fatal("Validate must report the unreadable document instead of accepting")+ if folderListsBeforeProbe != 0 {+ t.Fatalf("listed %d subfolders before probing the root's second page", folderListsBeforeProbe)+ }+ if folderLists == 0 {+ t.Fatal("Validate stopped at the unreadable document instead of continuing the walk")+ }+ if err != nil {+ t.Fatalf("Validate must accept once the listing budget leaves folders unexplored, got: %v", err)}}@@ -640,3 +674,530 @@ func TestValidateStopsOnRepeatedPageToken(t *testing.T) {t.Fatalf("folder was listed %d times, want 2 (first page + the repeated token once)", folderRequests)}}++// The first document of a workspace can be unreadable while a later one in the+// same folder is not. Stopping at the first probe would reject a data source+// the operator can actually sync.+func TestValidateTriesTheNextDocumentAfterAFailedProbe(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},+ nodes: map[string][]node{+ "root-a": {+ {ID: "doc-bad", Name: "Old", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ {ID: "doc-good", Name: "New", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ },+ blocks: map[string][]json.RawMessage{+ "doc-good": {rawJSON(`{"paragraph":{"text":"hello"}}`)},+ },+ blockErrors: map[string]error{+ "doc-bad": errors.New("forbidden.accessDenied: the operator has no permission"),+ },+ }++ if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil {+ t.Fatalf("Validate must accept once a later document in the folder is readable, got: %v", err)+ }+ if api.blockCalls["doc-bad"] != 1 || api.blockCalls["doc-good"] != 1 {+ t.Fatalf("both documents should be probed, calls=%v", api.blockCalls)+ }+}++// A readable document that lives in a later sibling folder must still be+// reached after the first folder's document fails. The same walk, when every+// document fails, reports the later one: the workspace is conclusive.+func TestValidateTriesADocumentInALaterSiblingFolder(t *testing.T) {+ denied := errors.New("forbidden.accessDenied: the operator has no permission")+ t.Run("later folder readable", func(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},+ nodes: map[string][]node{+ "root-a": {+ {ID: "folder-a", Name: "A", Type: "FOLDER"},+ {ID: "folder-b", Name: "B", Type: "FOLDER"},+ },+ "folder-a": {+ {ID: "doc-bad", Name: "Old", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ "folder-b": {+ {ID: "doc-good", Name: "New", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ },+ blocks: map[string][]json.RawMessage{+ "doc-good": {rawJSON(`{"paragraph":{"text":"hello"}}`)},+ },+ blockErrors: map[string]error{"doc-bad": denied},+ }+ if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil {+ t.Fatalf("Validate must accept the readable document in the later folder, got: %v", err)+ }+ if api.blockCalls["doc-bad"] != 1 || api.blockCalls["doc-good"] != 1 {+ t.Fatalf("both sibling documents should be probed, calls=%v", api.blockCalls)+ }+ })+ t.Run("every document unreadable", func(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},+ nodes: map[string][]node{+ "root-a": {+ {ID: "folder-a", Name: "A", Type: "FOLDER"},+ {ID: "folder-b", Name: "B", Type: "FOLDER"},+ },+ "folder-a": {+ {ID: "doc-first", Name: "First", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ "folder-b": {+ {ID: "doc-second", Name: "Second", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ },+ blockErrors: map[string]error{"doc-first": denied, "doc-second": denied},+ }+ err := testConnector(api).Validate(context.Background(), testConfig())+ if err == nil || !strings.Contains(err.Error(), `document "Second"`) {+ t.Fatalf("Validate must report the later unreadable document, got: %v", err)+ }+ if api.blockCalls["doc-first"] != 1 || api.blockCalls["doc-second"] != 1 {+ t.Fatalf("both sibling documents should be probed, calls=%v", api.blockCalls)+ }+ })+}++// An unreadable document on a later root page is conclusive when the rest of+// the workspace fits in the listing budget. The page is still read before any+// subfolder, and the walk then finishes those subfolders instead of stopping+// on the first failed probe.+func TestValidateReportsUnreadableDocumentOnALaterRootPage(t *testing.T) {+ var mu sync.Mutex+ probed := false+ folderLists := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/v1.0/oauth2/accessToken":+ _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))+ case "/v2.0/wiki/workspaces":+ _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))+ case "/v2.0/wiki/nodes":+ parent := r.URL.Query().Get("parentNodeId")+ switch {+ case parent == "root-a" && r.URL.Query().Get("nextToken") == "":+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"folder","type":"FOLDER"}],"nextToken":"p2"}`))+ case parent == "root-a":+ _, _ = w.Write([]byte(+ `{"nodes":[{"nodeId":"doc","name":"Doc","type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))+ default:+ mu.Lock()+ folderLists+++ mu.Unlock()+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf","type":"FILE","category":"FILE","extension":"pdf"}]}`))+ }+ case "/v1.0/doc/suites/documents/doc/blocks":+ mu.Lock()+ probed = true+ mu.Unlock()+ w.WriteHeader(http.StatusForbidden)+ _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ err := testConnector(testClient(server)).Validate(context.Background(), testConfig())+ mu.Lock()+ defer mu.Unlock()+ if !probed || folderLists != 1 {+ t.Fatalf("probed=%v folderLists=%d, want the page probed and its subfolder listed", probed, folderLists)+ }+ if err == nil || !strings.Contains(err.Error(), `document "Doc"`) {+ t.Fatalf("Validate must report the unreadable document, got: %v", err)+ }+ if strings.Contains(err.Error(), "only the first") {+ t.Fatalf("one probed document must not be described as the probe cap, got: %v", err)+ }+}++// A root listing that fails on a later page is the same failure sync sees from+// listNodes: the workspace is not "unexplored", even when an earlier page+// already named a folder that holds a readable document. Another workspace can+// still prove the credentials, as a failed first page already can.+func TestValidateRejectsAFailedLaterRootPage(t *testing.T) {+ const workspaces = `{"workspaces":[` ++ `{"workspaceId":"a","rootNodeId":"root-a","name":"A"},` ++ `{"workspaceId":"b","rootNodeId":"root-b","name":"B"}]}`+ handler := func(extraWorkspace bool) http.HandlerFunc {+ return func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/v1.0/oauth2/accessToken":+ _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))+ case "/v2.0/wiki/workspaces":+ if extraWorkspace {+ _, _ = w.Write([]byte(workspaces))+ return+ }+ _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))+ case "/v2.0/wiki/nodes":+ parent := r.URL.Query().Get("parentNodeId")+ switch {+ case parent == "root-a" && r.URL.Query().Get("nextToken") == "":+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"folder","type":"FOLDER"}],"nextToken":"p2"}`))+ case parent == "root-a":+ w.WriteHeader(http.StatusBadRequest)+ _, _ = w.Write([]byte(`{"code":"invalidRequest","message":"bad page"}`))+ case parent == "folder":+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"hidden","name":"Hidden",` ++ `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))+ default:+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Handbook",` ++ `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))+ }+ case "/v1.0/doc/suites/documents/hidden/blocks", "/v1.0/doc/suites/documents/doc-b/blocks":+ _, _ = w.Write([]byte(`{"success":true,"result":{"data":[{"blockType":"paragraph"}]}}`))+ default:+ http.NotFound(w, r)+ }+ }+ }+ t.Run("only workspace", func(t *testing.T) {+ server := httptest.NewServer(handler(false))+ defer server.Close()+ err := testConnector(testClient(server)).Validate(context.Background(), testConfig())+ if err == nil || !strings.Contains(err.Error(), "status=400") ||+ !strings.Contains(err.Error(), `workspace "A"`) {+ t.Fatalf("Validate must surface the root page failure, got: %v", err)+ }+ })+ t.Run("later workspace readable", func(t *testing.T) {+ server := httptest.NewServer(handler(true))+ defer server.Close()+ if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {+ t.Fatalf("Validate must survive one workspace whose later root page fails, got: %v", err)+ }+ })+}++// The first folder under the root can hold more non-document pages than the+// listing budget. Those pages must not be spent to the end while a sibling+// folder is still queued: the sibling is opened, and a document there is+// probed. A short folder is still finished first, so a document on its second+// page is not skipped just because a sibling is waiting.+func TestValidateOpensSiblingFolderBeforeSpendingTheListingBudget(t *testing.T) {+ const fatPages = 30+ type listingStats struct {+ fatPages int+ probed map[string]int+ }+ start := func(t *testing.T, siblingReadable, docOnSecondPage bool) (*httptest.Server, *listingStats) {+ t.Helper()+ stats := &listingStats{probed: map[string]int{}}+ var mu sync.Mutex+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/v1.0/oauth2/accessToken":+ _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))+ case "/v2.0/wiki/workspaces":+ _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))+ case "/v2.0/wiki/nodes":+ parent := r.URL.Query().Get("parentNodeId")+ token := r.URL.Query().Get("nextToken")+ switch parent {+ case "root-a":+ _, _ = w.Write([]byte(`{"nodes":[` ++ `{"nodeId":"folder-a","type":"FOLDER"},` ++ `{"nodeId":"folder-b","type":"FOLDER"}]}`))+ case "folder-a":+ mu.Lock()+ stats.fatPages+++ page := stats.fatPages+ mu.Unlock()+ if token != "" && token != strconv.Itoa(page-1) {+ t.Errorf("folder-a page token = %q, previous page was %d", token, page-1)+ }+ item := fmt.Sprintf(+ `{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}`, page,+ )+ if docOnSecondPage && page == 2 {+ item = `{"nodeId":"doc-early","name":"Early",` ++ `"type":"FILE","category":"ALIDOC","extension":"adoc"}`+ }+ next := ""+ if page < fatPages {+ next = strconv.Itoa(page)+ }+ _, _ = fmt.Fprintf(w, `{"nodes":[%s],"nextToken":%q}`, item, next)+ case "folder-b":+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Sibling",` ++ `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))+ default:+ http.NotFound(w, r)+ }+ default:+ const prefix = "/v1.0/doc/suites/documents/"+ if !strings.HasPrefix(r.URL.Path, prefix) || !strings.HasSuffix(r.URL.Path, "/blocks") {+ http.NotFound(w, r)+ return+ }+ id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, prefix), "/blocks")+ mu.Lock()+ stats.probed[id]+++ mu.Unlock()+ if id == "doc-b" && !siblingReadable {+ w.WriteHeader(http.StatusForbidden)+ _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))+ return+ }+ _, _ = w.Write([]byte(`{"success":true,"result":{"data":[{"blockType":"paragraph"}]}}`))+ }+ }))+ return server, stats+ }++ t.Run("sibling document readable", func(t *testing.T) {+ server, stats := start(t, true, false)+ defer server.Close()+ if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {+ t.Fatalf("Validate must accept the sibling document, got: %v", err)+ }+ if stats.probed["doc-b"] != 1 {+ t.Fatalf("sibling document probes=%d, want 1 (fat folder pages=%d)", stats.probed["doc-b"], stats.fatPages)+ }+ if stats.fatPages >= maxValidateListings {+ t.Fatalf("fat folder used %d pages, want fewer than %d so the sibling still fits",+ stats.fatPages, maxValidateListings)+ }+ })+ t.Run("sibling document unreadable", func(t *testing.T) {+ server, stats := start(t, false, false)+ defer server.Close()+ err := testConnector(testClient(server)).Validate(context.Background(), testConfig())+ if err == nil || !strings.Contains(err.Error(), `document "Sibling"`) {+ t.Fatalf("Validate must report the sibling document, got: %v", err)+ }+ if stats.probed["doc-b"] != 1 {+ t.Fatalf("sibling document probes=%d, want 1 (fat folder pages=%d)", stats.probed["doc-b"], stats.fatPages)+ }+ if stats.fatPages >= maxValidateListings {+ t.Fatalf("fat folder used %d pages, want fewer than %d so the sibling still fits",+ stats.fatPages, maxValidateListings)+ }+ })+ t.Run("short folder still yields its second page", func(t *testing.T) {+ server, stats := start(t, false, true)+ defer server.Close()+ if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {+ t.Fatalf("Validate must accept the document on the short folder's second page, got: %v", err)+ }+ if stats.probed["doc-early"] != 1 || stats.probed["doc-b"] != 0 {+ t.Fatalf("probes=%v, want only the document on page 2", stats.probed)+ }+ if stats.fatPages != 2 {+ t.Fatalf("folder-a was listed %d times, want 2", stats.fatPages)+ }+ })+}++// Skipping the unread tail of a fat folder is what leaves room for its sibling.+// When that sample contains no document, the tail might still hold the only+// readable one, so an unreadable document in another workspace must not fail+// the data source.+func TestValidateSkippedTailWithoutAProbeStaysInconclusive(t *testing.T) {+ var mu sync.Mutex+ probed := false+ fatPages := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/v1.0/oauth2/accessToken":+ _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))+ case "/v2.0/wiki/workspaces":+ _, _ = w.Write([]byte(`{"workspaces":[` ++ `{"workspaceId":"a","rootNodeId":"root-a","name":"A"},` ++ `{"workspaceId":"b","rootNodeId":"root-b","name":"B"}]}`))+ case "/v2.0/wiki/nodes":+ parent := r.URL.Query().Get("parentNodeId")+ switch parent {+ case "root-a":+ _, _ = w.Write([]byte(`{"nodes":[` ++ `{"nodeId":"folder-a","type":"FOLDER"},` ++ `{"nodeId":"folder-b","type":"FOLDER"}]}`))+ case "folder-a":+ mu.Lock()+ fatPages+++ page := fatPages+ mu.Unlock()+ next := ""+ if page < 30 {+ next = strconv.Itoa(page)+ }+ _, _ = fmt.Fprintf(w,+ `{"nodes":[{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}],"nextToken":%q}`,+ page, next)+ case "folder-b":+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf-b","type":"FILE",` ++ `"category":"FILE","extension":"pdf"}]}`))+ default:+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Secret",` ++ `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))+ }+ case "/v1.0/doc/suites/documents/doc-b/blocks":+ mu.Lock()+ probed = true+ mu.Unlock()+ w.WriteHeader(http.StatusForbidden)+ _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ err := testConnector(testClient(server)).Validate(context.Background(), testConfig())+ mu.Lock()+ defer mu.Unlock()+ if err != nil {+ t.Fatalf("Validate must accept when the skipped tail was never probed, got: %v", err)+ }+ if !probed {+ t.Fatal("the other workspace's document was never probed")+ }+ if fatPages >= maxValidateListings {+ t.Fatalf("fat folder used %d pages, want room left for its sibling", fatPages)+ }+}++// An unlistable folder already makes the walk inconclusive. Hitting the probe+// cap afterwards must not turn that into a failure: a readable document may+// still sit in the folder that could not be listed.+func TestValidateProbeCapDoesNotOverrideAnInconclusiveWalk(t *testing.T) {+ api := &fakeAPI{+ nodes: map[string][]node{+ "root-deep": {{ID: "folder", Name: "Folder", Type: "FOLDER"}},+ },+ nodeErrors: map[string]error{"folder": errors.New("DingTalk API status=500")},+ blockErrors: map[string]error{},+ }+ api.workspaces = []workspace{{ID: "deep", RootNodeID: "root-deep", Name: "Deep"}}+ denied := errors.New("forbidden.accessDenied: the operator has no permission")+ for w := 0; w < maxValidateProbes+2; w++ {+ root := fmt.Sprintf("root-%d", w)+ id := fmt.Sprintf("doc-%d", w)+ api.workspaces = append(api.workspaces, workspace{ID: root, RootNodeID: root, Name: root})+ api.nodes[root] = []node{{ID: id, Name: id, Type: "FILE", Category: "ALIDOC", Extension: "adoc"}}+ api.blockErrors[id] = denied+ }++ if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil {+ t.Fatalf("Validate must accept an inconclusive walk even after the probe cap, got: %v", err)+ }+ total := 0+ for _, n := range api.blockCalls {+ total += n+ }+ if total != maxValidateProbes {+ t.Fatalf("Validate made %d document probes, want %d", total, maxValidateProbes)+ }+}++// Once the probe cap is spent no later probe can succeed. A workspace walked+// after that point must not get the chance to find an unlistable folder and+// turn every failed probe into acceptance.+func TestValidateStopsWalkingOnceTheProbeCapIsSpent(t *testing.T) {+ api := &fakeAPI{+ nodes: map[string][]node{},+ nodeErrors: map[string]error{"folder": errors.New("DingTalk API status=500")},+ blockErrors: map[string]error{},+ }+ denied := errors.New("forbidden.accessDenied: the operator has no permission")+ for w := 0; w < maxValidateProbes; w++ {+ root := fmt.Sprintf("root-%d", w)+ id := fmt.Sprintf("doc-%d", w)+ api.workspaces = append(api.workspaces, workspace{ID: root, RootNodeID: root, Name: root})+ api.nodes[root] = []node{{ID: id, Name: id, Type: "FILE", Category: "ALIDOC", Extension: "adoc"}}+ api.blockErrors[id] = denied+ }+ api.workspaces = append(api.workspaces, workspace{ID: "late", RootNodeID: "root-late", Name: "Late"})+ api.nodes["root-late"] = []node{{ID: "folder", Name: "Folder", Type: "FOLDER"}}++ err := testConnector(api).Validate(context.Background(), testConfig())+ if err == nil || !strings.Contains(err.Error(), "no permission") {+ t.Fatalf("Validate must report the failed probes, got: %v", err)+ }+ wantNote := fmt.Sprintf("only the first %d documents were probed", maxValidateProbes)+ if !strings.Contains(err.Error(), wantNote) {+ t.Fatalf("Validate error should say the probe cap stopped the walk, got: %v", err)+ }+}++// A tail skipped to make room for siblings is only covered by a probe made+// after it. A document probed before the skip says nothing about the skipped+// pages, so an empty sibling must leave the walk inconclusive.+func TestValidateSkippedTailIsNotCoveredByAnEarlierProbe(t *testing.T) {+ var mu sync.Mutex+ fatPages := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/v1.0/oauth2/accessToken":+ _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))+ case "/v2.0/wiki/workspaces":+ _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))+ case "/v2.0/wiki/nodes":+ switch r.URL.Query().Get("parentNodeId") {+ case "root-a":+ _, _ = w.Write([]byte(`{"nodes":[` ++ `{"nodeId":"doc-x","name":"X","type":"FILE","category":"ALIDOC","extension":"adoc"},` ++ `{"nodeId":"folder-a","type":"FOLDER"},` ++ `{"nodeId":"folder-b","type":"FOLDER"}]}`))+ case "folder-a":+ mu.Lock()+ fatPages+++ page := fatPages+ mu.Unlock()+ next := ""+ if page < 30 {+ next = strconv.Itoa(page)+ }+ _, _ = fmt.Fprintf(w,+ `{"nodes":[{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}],"nextToken":%q}`,+ page, next)+ case "folder-b":+ _, _ = w.Write([]byte(`{"nodes":[]}`))+ default:+ http.NotFound(w, r)+ }+ case "/v1.0/doc/suites/documents/doc-x/blocks":+ w.WriteHeader(http.StatusForbidden)+ _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {+ t.Fatalf("Validate must accept when the skipped tail was never covered by a probe, got: %v", err)+ }+ mu.Lock()+ defer mu.Unlock()+ if fatPages >= maxValidateListings {+ t.Fatalf("fat folder used %d pages, want room left for its sibling", fatPages)+ }+}++// A probe cancelled by the caller's deadline still names the document, so the+// operator can tell which read timed out.+func TestValidateContextErrorNamesTheProbedDocument(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},+ nodes: map[string][]node{+ "root-a": {{ID: "doc", Name: "Handbook", Type: "FILE", Category: "ALIDOC", Extension: "adoc"}},+ },+ blockErrors: map[string]error{"doc": context.DeadlineExceeded},+ }+ err := testConnector(api).Validate(context.Background(), testConfig())+ if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), `document "Handbook"`) {+ t.Fatalf("Validate must surface the timed-out probe with its document, got: %v", err)+ }+}