Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(datasource/dingtalk): tighten Validate sampling edges (#4079)

fight for you57f8a65fec7b
diff --git a/internal/datasource/connector/dingtalk/connector.go b/internal/datasource/connector/dingtalk/connector.go
index d3a77d334..4bb0a3d0f 100644
--- a/internal/datasource/connector/dingtalk/connector.go
+++ b/internal/datasource/connector/dingtalk/connector.go
@@ -116,9 +116,10 @@ const maxValidateProbes = 5
//
// The operator is not expected to reach every workspace in the tenant: app
// credentials are valid as long as one reachable workspace yields one readable
-// document. Reading the first document of the first workspace and failing on it
-// rejects a working configuration whenever an unrelated workspace — one the
-// operator was never granted — happens to be listed first.
+// document. A failed probe does not condemn that workspace; the walk keeps
+// looking, inside the probe cap, for another document the operator can read.
+// When the cap itself stops a conclusive walk, the error says how many
+// documents were read.
func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSourceConfig) error {
cfg, err := parseConfig(dataSourceConfig)
if err != nil {
@@ -138,14 +139,16 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo
workspaceFail int
probes int
listings int
+ probeCapHit bool
)
- for _, item := range workspaces {
+ for i, item := range workspaces {
rootNodeID := strings.TrimSpace(item.RootNodeID)
if rootNodeID == "" {
continue
}
- document, found, unexplored, err := firstValidateDocument(
- ctx, api, rootNodeID, settings, &listings,
+ readable, unexplored, hitCap, err := sampleWorkspace(
+ ctx, api, item.Name, rootNodeID, settings,
+ &listings, &probes, &sawDocument, &lastErr,
)
if err != nil {
lastErr = fmt.Errorf("workspace %q: %w", item.Name, err)
@@ -157,29 +160,24 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo
workspaceFail++
continue
}
+ if readable {
+ return nil
+ }
if unexplored {
inconclusive = true
}
- if !found {
- continue
- }
- sawDocument = true
- probes++
- if err := verifyDocument(ctx, api, document); err != nil {
- lastErr = fmt.Errorf("workspace %q document %q: %w", item.Name, document.Name, err)
- if ctx.Err() != nil || isContextError(err) {
- return fmt.Errorf("validate DingTalk data source: %w", lastErr)
- }
- } else {
- return nil
- }
+ // Once the cap is spent no further probe can succeed, so walking on
+ // could only find an unlistable folder and turn the failed probes
+ // into acceptance.
if probes >= maxValidateProbes {
+ probeCapHit = hitCap || i+1 < len(workspaces)
break
}
}
// Folders were left unexplored, because the listing budget ran out or a
// folder could not be listed, so an unreadable document elsewhere does not
- // prove the data source is unusable.
+ // prove the data source is unusable. A probe cap hit in that situation
+ // stays quiet too: the walk cannot claim it sampled every visible document.
if inconclusive {
return nil
}
@@ -194,6 +192,12 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo
if lastErr == nil {
return nil
}
+ if probeCapHit {
+ return fmt.Errorf(
+ "validate DingTalk data source: %w; only the first %d documents were probed",
+ lastErr, maxValidateProbes,
+ )
+ }
return fmt.Errorf("validate DingTalk data source: %w", lastErr)
}
@@ -205,45 +209,83 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo
// client's retries of each).
const maxValidateListings = 20
-// firstValidateDocument walks a workspace breadth-first, the way scanWorkspace
-// does, and returns the first document it finds. It requests one page at a
-// time instead of listing whole folders, so the budget bounds real requests.
-// unexplored reports that some nodes were never seen, because the budget ran
-// out or a folder could not be listed, so the walk cannot prove the workspace
-// holds no readable document.
+// sampleWorkspace walks one workspace breadth-first, the way scanWorkspace
+// does, and probes ingestible documents until one is readable, the probe cap
+// is spent, or the walk runs out of nodes. It requests one page at a time
+// instead of listing whole folders, so the budget bounds real requests.
+//
+// unexplored reports that some folder was never opened or could not be listed,
+// so a failed probe elsewhere must not reject the data source. readable reports
+// that a probe succeeded. The walk stops as soon as the failed probe that
+// spends maxValidateProbes returns, without another listing; capHit then
+// reports that nodes of this workspace were still left to visit. The caller
+// turns that into an error only when the walk is conclusive.
+//
+// A folder's later pages are held back once the listings still in the budget
+// are no longer enough to open every sibling already queued, so each of those
+// siblings still gets one page. The unread tail is not itself "unexplored"
+// when a document was probed after it was skipped: the reservation is what
+// made room for that probe. A tail skipped with no probe after it stays
+// inconclusive, because the only readable document may sit on a page the
+// sample never requested. A folder that receives no page at all, a subfolder
+// listing error, a repeated page token, or a tail with no sibling waiting all
+// stay inconclusive too.
+//
+// Any failed page of the workspace root is a hard error, matching listNodes
+// during sync. Subfolder listing errors stay inconclusive unless the request
+// was cancelled or timed out.
//
// The walk selects nodes with the same settings-aware predicate the picker and
// the sync use, so a type the data source does not ingest is never probed.
-func firstValidateDocument(
+func sampleWorkspace(
ctx context.Context,
api dingTalkAPI,
+ workspaceName string,
rootNodeID string,
settings documentSettings,
listings *int,
-) (document node, found, unexplored bool, err error) {
+ probes *int,
+ sawDocument *bool,
+ lastErr *error,
+) (readable, unexplored, capHit bool, err error) {
queue := []string{rootNodeID}
visited := map[string]struct{}{rootNodeID: {}}
+ uncoveredSkip := false
for len(queue) > 0 {
parent := queue[0]
queue = queue[1:]
// Finish every page of a folder before descending, like listNodes
// does for scanWorkspace: a document on a later page of this folder
// must be found before the budget is spent on its subfolders.
+ // Siblings already on the queue stay ahead of those subfolders.
var subfolders []string
pageToken := ""
seenTokens := map[string]struct{}{}
for {
+ // Reserve one listing per sibling already queued before spending
+ // another page on this folder. The root's own pages have an empty
+ // queue here — its children are not queued until the root is
+ // finished — so a document on the root's next page is still read
+ // before the walk descends.
+ if pageToken != "" && *listings+len(queue) >= maxValidateListings {
+ if len(queue) == 0 {
+ unexplored = true
+ } else {
+ uncoveredSkip = true
+ }
+ break
+ }
rootPage := parent == rootNodeID && pageToken == ""
if !rootPage {
if *listings >= maxValidateListings {
- return node{}, false, true, nil
+ return false, true, false, nil
}
*listings++
}
children, next, listErr := api.listNodesPage(ctx, parent, pageToken)
if listErr != nil {
- if rootPage || ctx.Err() != nil || isContextError(listErr) {
- return node{}, false, false, listErr
+ if parent == rootNodeID || ctx.Err() != nil || isContextError(listErr) {
+ return false, false, false, listErr
}
// The documents this operator can read may live in exactly
// this folder, so its failure leaves the walk inconclusive
@@ -251,9 +293,27 @@ func firstValidateDocument(
unexplored = true
break
}
- for _, child := range children {
+ for i, child := range children {
if child.isIngestible(settings) {
- return child, true, unexplored, nil
+ *probes++
+ *sawDocument = true
+ uncoveredSkip = false
+ probeErr := verifyDocument(ctx, api, child)
+ if probeErr != nil {
+ if ctx.Err() != nil || isContextError(probeErr) {
+ return false, false, false, fmt.Errorf("document %q: %w", child.Name, probeErr)
+ }
+ *lastErr = fmt.Errorf(
+ "workspace %q document %q: %w", workspaceName, child.Name, probeErr,
+ )
+ if *probes >= maxValidateProbes {
+ more := i+1 < len(children) || len(subfolders) > 0 ||
+ next != "" || len(queue) > 0
+ return false, unexplored, more, nil
+ }
+ continue
+ }
+ return true, false, false, nil
}
if child.isFolder() || child.HasChildren {
if _, seen := visited[child.ID]; seen {
@@ -277,7 +337,10 @@ func firstValidateDocument(
}
queue = append(queue, subfolders...)
}
- return node{}, false, unexplored, nil
+ if uncoveredSkip {
+ unexplored = true
+ }
+ return false, unexplored, false, nil
}
// verifyDocument proves one visible document is readable. Native documents are
diff --git a/internal/datasource/connector/dingtalk/validate_test.go b/internal/datasource/connector/dingtalk/validate_test.go
index 86ac79c63..02121e3c9 100644
--- a/internal/datasource/connector/dingtalk/validate_test.go
+++ b/internal/datasource/connector/dingtalk/validate_test.go
@@ -147,6 +147,11 @@ func TestValidateReportsWhenNoVisibleDocumentIsReadable(t *testing.T) {
t.Fatalf("Validate error should carry %s, got: %v", want, err)
}
}
+ // Both documents were probed, so the cap did not stop the walk and the
+ // error must not claim that it did.
+ if strings.Contains(err.Error(), "only the first") {
+ t.Fatalf("Validate error should not mention the probe cap, got: %v", err)
+ }
}
// A tenant with no documents anywhere leaves nothing to read, so the
@@ -194,9 +199,10 @@ func TestValidateSurvivesWorkspaceListingFailure(t *testing.T) {
}
}
-// An app without the document read permission fails every probe. Validate must
-// probe at most one document per workspace and stop after maxValidateProbes,
-// instead of firing one call per document across the whole tenant.
+// An app without the document read permission fails every probe. Validate keeps
+// going to the next document in the same workspace, but it still stops after
+// maxValidateProbes reads and says so, instead of firing one call per document
+// across the whole tenant.
func TestValidateCapsProbesWhenEveryDocumentIsUnreadable(t *testing.T) {
api := &fakeAPI{
nodes: map[string][]node{},
@@ -216,20 +222,32 @@ func TestValidateCapsProbesWhenEveryDocumentIsUnreadable(t *testing.T) {
}
c := testConnector(api)
- if err := c.Validate(context.Background(), testConfig()); err == nil {
+ err := c.Validate(context.Background(), testConfig())
+ if err == nil {
t.Fatal("Validate must fail when no visible document can be read")
}
total := 0
for _, n := range api.blockCalls {
total += n
}
- if total > maxValidateProbes {
- t.Fatalf("Validate made %d document probes, want at most %d", total, maxValidateProbes)
+ if total != maxValidateProbes {
+ t.Fatalf("Validate made %d document probes, want %d", total, maxValidateProbes)
}
- for w := 0; w < workspaces; w++ {
- if api.blockCalls[fmt.Sprintf("doc-%d-1", w)] != 0 {
- t.Fatalf("workspace root-%d was probed past its first document", w)
- }
+ // The first workspace has four unreadable documents, so the cap must be
+ // spent there and on the next workspace's first document — not one probe
+ // per workspace, and not on a third workspace.
+ if api.blockCalls["doc-0-1"] != 1 {
+ t.Fatal("a failed probe must not stop the workspace from trying its next document")
+ }
+ if api.blockCalls["doc-1-0"] != 1 || api.blockCalls["doc-1-1"] != 0 || api.blockCalls["doc-2-0"] != 0 {
+ t.Fatalf("probe cap was not applied across workspaces, calls=%v", api.blockCalls)
+ }
+ wantNote := fmt.Sprintf("only the first %d documents were probed", maxValidateProbes)
+ if !strings.Contains(err.Error(), wantNote) {
+ t.Fatalf("Validate error should say the probe cap stopped the walk, got: %v", err)
+ }
+ if !strings.Contains(err.Error(), `document "doc-1-0"`) {
+ t.Fatalf("Validate error should name the last probed document, got: %v", err)
}
}
@@ -554,10 +572,14 @@ func TestValidateBudgetCountsNodeListingPages(t *testing.T) {
// A folder's later pages must be read before its subfolders: otherwise a root
// whose first page is all folders spends the budget below them and never sees
-// the document on the root's second page.
+// the document on the root's second page. The document is unreadable and the
+// twenty subfolders cannot all be listed after that page spends a budget unit,
+// so the rest of the tree is unknown and Validate accepts.
func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) {
var mu sync.Mutex
probed := false
+ folderLists := 0
+ folderListsBeforeProbe := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
@@ -578,6 +600,12 @@ func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) {
_, _ = w.Write([]byte(
`{"nodes":[{"nodeId":"doc","name":"Doc","type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
default:
+ mu.Lock()
+ folderLists++
+ if !probed {
+ folderListsBeforeProbe++
+ }
+ mu.Unlock()
_, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf","type":"FILE","category":"FILE","extension":"pdf"}]}`))
}
case strings.HasPrefix(r.URL.Path, "/v1.0/doc/suites/documents/"):
@@ -598,8 +626,14 @@ func TestValidateReadsFolderPagesBeforeDescending(t *testing.T) {
if !probed {
t.Fatalf("the document on the root's second page was never probed (err=%v)", err)
}
- if err == nil {
- t.Fatal("Validate must report the unreadable document instead of accepting")
+ if folderListsBeforeProbe != 0 {
+ t.Fatalf("listed %d subfolders before probing the root's second page", folderListsBeforeProbe)
+ }
+ if folderLists == 0 {
+ t.Fatal("Validate stopped at the unreadable document instead of continuing the walk")
+ }
+ if err != nil {
+ t.Fatalf("Validate must accept once the listing budget leaves folders unexplored, got: %v", err)
}
}
@@ -640,3 +674,530 @@ func TestValidateStopsOnRepeatedPageToken(t *testing.T) {
t.Fatalf("folder was listed %d times, want 2 (first page + the repeated token once)", folderRequests)
}
}
+
+// The first document of a workspace can be unreadable while a later one in the
+// same folder is not. Stopping at the first probe would reject a data source
+// the operator can actually sync.
+func TestValidateTriesTheNextDocumentAfterAFailedProbe(t *testing.T) {
+ api := &fakeAPI{
+ workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},
+ nodes: map[string][]node{
+ "root-a": {
+ {ID: "doc-bad", Name: "Old", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
+ {ID: "doc-good", Name: "New", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
+ },
+ },
+ blocks: map[string][]json.RawMessage{
+ "doc-good": {rawJSON(`{"paragraph":{"text":"hello"}}`)},
+ },
+ blockErrors: map[string]error{
+ "doc-bad": errors.New("forbidden.accessDenied: the operator has no permission"),
+ },
+ }
+
+ if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil {
+ t.Fatalf("Validate must accept once a later document in the folder is readable, got: %v", err)
+ }
+ if api.blockCalls["doc-bad"] != 1 || api.blockCalls["doc-good"] != 1 {
+ t.Fatalf("both documents should be probed, calls=%v", api.blockCalls)
+ }
+}
+
+// A readable document that lives in a later sibling folder must still be
+// reached after the first folder's document fails. The same walk, when every
+// document fails, reports the later one: the workspace is conclusive.
+func TestValidateTriesADocumentInALaterSiblingFolder(t *testing.T) {
+ denied := errors.New("forbidden.accessDenied: the operator has no permission")
+ t.Run("later folder readable", func(t *testing.T) {
+ api := &fakeAPI{
+ workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},
+ nodes: map[string][]node{
+ "root-a": {
+ {ID: "folder-a", Name: "A", Type: "FOLDER"},
+ {ID: "folder-b", Name: "B", Type: "FOLDER"},
+ },
+ "folder-a": {
+ {ID: "doc-bad", Name: "Old", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
+ },
+ "folder-b": {
+ {ID: "doc-good", Name: "New", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
+ },
+ },
+ blocks: map[string][]json.RawMessage{
+ "doc-good": {rawJSON(`{"paragraph":{"text":"hello"}}`)},
+ },
+ blockErrors: map[string]error{"doc-bad": denied},
+ }
+ if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil {
+ t.Fatalf("Validate must accept the readable document in the later folder, got: %v", err)
+ }
+ if api.blockCalls["doc-bad"] != 1 || api.blockCalls["doc-good"] != 1 {
+ t.Fatalf("both sibling documents should be probed, calls=%v", api.blockCalls)
+ }
+ })
+ t.Run("every document unreadable", func(t *testing.T) {
+ api := &fakeAPI{
+ workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},
+ nodes: map[string][]node{
+ "root-a": {
+ {ID: "folder-a", Name: "A", Type: "FOLDER"},
+ {ID: "folder-b", Name: "B", Type: "FOLDER"},
+ },
+ "folder-a": {
+ {ID: "doc-first", Name: "First", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
+ },
+ "folder-b": {
+ {ID: "doc-second", Name: "Second", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},
+ },
+ },
+ blockErrors: map[string]error{"doc-first": denied, "doc-second": denied},
+ }
+ err := testConnector(api).Validate(context.Background(), testConfig())
+ if err == nil || !strings.Contains(err.Error(), `document "Second"`) {
+ t.Fatalf("Validate must report the later unreadable document, got: %v", err)
+ }
+ if api.blockCalls["doc-first"] != 1 || api.blockCalls["doc-second"] != 1 {
+ t.Fatalf("both sibling documents should be probed, calls=%v", api.blockCalls)
+ }
+ })
+}
+
+// An unreadable document on a later root page is conclusive when the rest of
+// the workspace fits in the listing budget. The page is still read before any
+// subfolder, and the walk then finishes those subfolders instead of stopping
+// on the first failed probe.
+func TestValidateReportsUnreadableDocumentOnALaterRootPage(t *testing.T) {
+ var mu sync.Mutex
+ probed := false
+ folderLists := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/v1.0/oauth2/accessToken":
+ _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))
+ case "/v2.0/wiki/workspaces":
+ _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))
+ case "/v2.0/wiki/nodes":
+ parent := r.URL.Query().Get("parentNodeId")
+ switch {
+ case parent == "root-a" && r.URL.Query().Get("nextToken") == "":
+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"folder","type":"FOLDER"}],"nextToken":"p2"}`))
+ case parent == "root-a":
+ _, _ = w.Write([]byte(
+ `{"nodes":[{"nodeId":"doc","name":"Doc","type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
+ default:
+ mu.Lock()
+ folderLists++
+ mu.Unlock()
+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf","type":"FILE","category":"FILE","extension":"pdf"}]}`))
+ }
+ case "/v1.0/doc/suites/documents/doc/blocks":
+ mu.Lock()
+ probed = true
+ mu.Unlock()
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ err := testConnector(testClient(server)).Validate(context.Background(), testConfig())
+ mu.Lock()
+ defer mu.Unlock()
+ if !probed || folderLists != 1 {
+ t.Fatalf("probed=%v folderLists=%d, want the page probed and its subfolder listed", probed, folderLists)
+ }
+ if err == nil || !strings.Contains(err.Error(), `document "Doc"`) {
+ t.Fatalf("Validate must report the unreadable document, got: %v", err)
+ }
+ if strings.Contains(err.Error(), "only the first") {
+ t.Fatalf("one probed document must not be described as the probe cap, got: %v", err)
+ }
+}
+
+// A root listing that fails on a later page is the same failure sync sees from
+// listNodes: the workspace is not "unexplored", even when an earlier page
+// already named a folder that holds a readable document. Another workspace can
+// still prove the credentials, as a failed first page already can.
+func TestValidateRejectsAFailedLaterRootPage(t *testing.T) {
+ const workspaces = `{"workspaces":[` +
+ `{"workspaceId":"a","rootNodeId":"root-a","name":"A"},` +
+ `{"workspaceId":"b","rootNodeId":"root-b","name":"B"}]}`
+ handler := func(extraWorkspace bool) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/v1.0/oauth2/accessToken":
+ _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))
+ case "/v2.0/wiki/workspaces":
+ if extraWorkspace {
+ _, _ = w.Write([]byte(workspaces))
+ return
+ }
+ _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))
+ case "/v2.0/wiki/nodes":
+ parent := r.URL.Query().Get("parentNodeId")
+ switch {
+ case parent == "root-a" && r.URL.Query().Get("nextToken") == "":
+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"folder","type":"FOLDER"}],"nextToken":"p2"}`))
+ case parent == "root-a":
+ w.WriteHeader(http.StatusBadRequest)
+ _, _ = w.Write([]byte(`{"code":"invalidRequest","message":"bad page"}`))
+ case parent == "folder":
+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"hidden","name":"Hidden",` +
+ `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
+ default:
+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Handbook",` +
+ `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
+ }
+ case "/v1.0/doc/suites/documents/hidden/blocks", "/v1.0/doc/suites/documents/doc-b/blocks":
+ _, _ = w.Write([]byte(`{"success":true,"result":{"data":[{"blockType":"paragraph"}]}}`))
+ default:
+ http.NotFound(w, r)
+ }
+ }
+ }
+ t.Run("only workspace", func(t *testing.T) {
+ server := httptest.NewServer(handler(false))
+ defer server.Close()
+ err := testConnector(testClient(server)).Validate(context.Background(), testConfig())
+ if err == nil || !strings.Contains(err.Error(), "status=400") ||
+ !strings.Contains(err.Error(), `workspace "A"`) {
+ t.Fatalf("Validate must surface the root page failure, got: %v", err)
+ }
+ })
+ t.Run("later workspace readable", func(t *testing.T) {
+ server := httptest.NewServer(handler(true))
+ defer server.Close()
+ if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {
+ t.Fatalf("Validate must survive one workspace whose later root page fails, got: %v", err)
+ }
+ })
+}
+
+// The first folder under the root can hold more non-document pages than the
+// listing budget. Those pages must not be spent to the end while a sibling
+// folder is still queued: the sibling is opened, and a document there is
+// probed. A short folder is still finished first, so a document on its second
+// page is not skipped just because a sibling is waiting.
+func TestValidateOpensSiblingFolderBeforeSpendingTheListingBudget(t *testing.T) {
+ const fatPages = 30
+ type listingStats struct {
+ fatPages int
+ probed map[string]int
+ }
+ start := func(t *testing.T, siblingReadable, docOnSecondPage bool) (*httptest.Server, *listingStats) {
+ t.Helper()
+ stats := &listingStats{probed: map[string]int{}}
+ var mu sync.Mutex
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/v1.0/oauth2/accessToken":
+ _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))
+ case "/v2.0/wiki/workspaces":
+ _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))
+ case "/v2.0/wiki/nodes":
+ parent := r.URL.Query().Get("parentNodeId")
+ token := r.URL.Query().Get("nextToken")
+ switch parent {
+ case "root-a":
+ _, _ = w.Write([]byte(`{"nodes":[` +
+ `{"nodeId":"folder-a","type":"FOLDER"},` +
+ `{"nodeId":"folder-b","type":"FOLDER"}]}`))
+ case "folder-a":
+ mu.Lock()
+ stats.fatPages++
+ page := stats.fatPages
+ mu.Unlock()
+ if token != "" && token != strconv.Itoa(page-1) {
+ t.Errorf("folder-a page token = %q, previous page was %d", token, page-1)
+ }
+ item := fmt.Sprintf(
+ `{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}`, page,
+ )
+ if docOnSecondPage && page == 2 {
+ item = `{"nodeId":"doc-early","name":"Early",` +
+ `"type":"FILE","category":"ALIDOC","extension":"adoc"}`
+ }
+ next := ""
+ if page < fatPages {
+ next = strconv.Itoa(page)
+ }
+ _, _ = fmt.Fprintf(w, `{"nodes":[%s],"nextToken":%q}`, item, next)
+ case "folder-b":
+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Sibling",` +
+ `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
+ default:
+ http.NotFound(w, r)
+ }
+ default:
+ const prefix = "/v1.0/doc/suites/documents/"
+ if !strings.HasPrefix(r.URL.Path, prefix) || !strings.HasSuffix(r.URL.Path, "/blocks") {
+ http.NotFound(w, r)
+ return
+ }
+ id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, prefix), "/blocks")
+ mu.Lock()
+ stats.probed[id]++
+ mu.Unlock()
+ if id == "doc-b" && !siblingReadable {
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))
+ return
+ }
+ _, _ = w.Write([]byte(`{"success":true,"result":{"data":[{"blockType":"paragraph"}]}}`))
+ }
+ }))
+ return server, stats
+ }
+
+ t.Run("sibling document readable", func(t *testing.T) {
+ server, stats := start(t, true, false)
+ defer server.Close()
+ if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {
+ t.Fatalf("Validate must accept the sibling document, got: %v", err)
+ }
+ if stats.probed["doc-b"] != 1 {
+ t.Fatalf("sibling document probes=%d, want 1 (fat folder pages=%d)", stats.probed["doc-b"], stats.fatPages)
+ }
+ if stats.fatPages >= maxValidateListings {
+ t.Fatalf("fat folder used %d pages, want fewer than %d so the sibling still fits",
+ stats.fatPages, maxValidateListings)
+ }
+ })
+ t.Run("sibling document unreadable", func(t *testing.T) {
+ server, stats := start(t, false, false)
+ defer server.Close()
+ err := testConnector(testClient(server)).Validate(context.Background(), testConfig())
+ if err == nil || !strings.Contains(err.Error(), `document "Sibling"`) {
+ t.Fatalf("Validate must report the sibling document, got: %v", err)
+ }
+ if stats.probed["doc-b"] != 1 {
+ t.Fatalf("sibling document probes=%d, want 1 (fat folder pages=%d)", stats.probed["doc-b"], stats.fatPages)
+ }
+ if stats.fatPages >= maxValidateListings {
+ t.Fatalf("fat folder used %d pages, want fewer than %d so the sibling still fits",
+ stats.fatPages, maxValidateListings)
+ }
+ })
+ t.Run("short folder still yields its second page", func(t *testing.T) {
+ server, stats := start(t, false, true)
+ defer server.Close()
+ if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {
+ t.Fatalf("Validate must accept the document on the short folder's second page, got: %v", err)
+ }
+ if stats.probed["doc-early"] != 1 || stats.probed["doc-b"] != 0 {
+ t.Fatalf("probes=%v, want only the document on page 2", stats.probed)
+ }
+ if stats.fatPages != 2 {
+ t.Fatalf("folder-a was listed %d times, want 2", stats.fatPages)
+ }
+ })
+}
+
+// Skipping the unread tail of a fat folder is what leaves room for its sibling.
+// When that sample contains no document, the tail might still hold the only
+// readable one, so an unreadable document in another workspace must not fail
+// the data source.
+func TestValidateSkippedTailWithoutAProbeStaysInconclusive(t *testing.T) {
+ var mu sync.Mutex
+ probed := false
+ fatPages := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/v1.0/oauth2/accessToken":
+ _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))
+ case "/v2.0/wiki/workspaces":
+ _, _ = w.Write([]byte(`{"workspaces":[` +
+ `{"workspaceId":"a","rootNodeId":"root-a","name":"A"},` +
+ `{"workspaceId":"b","rootNodeId":"root-b","name":"B"}]}`))
+ case "/v2.0/wiki/nodes":
+ parent := r.URL.Query().Get("parentNodeId")
+ switch parent {
+ case "root-a":
+ _, _ = w.Write([]byte(`{"nodes":[` +
+ `{"nodeId":"folder-a","type":"FOLDER"},` +
+ `{"nodeId":"folder-b","type":"FOLDER"}]}`))
+ case "folder-a":
+ mu.Lock()
+ fatPages++
+ page := fatPages
+ mu.Unlock()
+ next := ""
+ if page < 30 {
+ next = strconv.Itoa(page)
+ }
+ _, _ = fmt.Fprintf(w,
+ `{"nodes":[{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}],"nextToken":%q}`,
+ page, next)
+ case "folder-b":
+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"pdf-b","type":"FILE",` +
+ `"category":"FILE","extension":"pdf"}]}`))
+ default:
+ _, _ = w.Write([]byte(`{"nodes":[{"nodeId":"doc-b","name":"Secret",` +
+ `"type":"FILE","category":"ALIDOC","extension":"adoc"}]}`))
+ }
+ case "/v1.0/doc/suites/documents/doc-b/blocks":
+ mu.Lock()
+ probed = true
+ mu.Unlock()
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ err := testConnector(testClient(server)).Validate(context.Background(), testConfig())
+ mu.Lock()
+ defer mu.Unlock()
+ if err != nil {
+ t.Fatalf("Validate must accept when the skipped tail was never probed, got: %v", err)
+ }
+ if !probed {
+ t.Fatal("the other workspace's document was never probed")
+ }
+ if fatPages >= maxValidateListings {
+ t.Fatalf("fat folder used %d pages, want room left for its sibling", fatPages)
+ }
+}
+
+// An unlistable folder already makes the walk inconclusive. Hitting the probe
+// cap afterwards must not turn that into a failure: a readable document may
+// still sit in the folder that could not be listed.
+func TestValidateProbeCapDoesNotOverrideAnInconclusiveWalk(t *testing.T) {
+ api := &fakeAPI{
+ nodes: map[string][]node{
+ "root-deep": {{ID: "folder", Name: "Folder", Type: "FOLDER"}},
+ },
+ nodeErrors: map[string]error{"folder": errors.New("DingTalk API status=500")},
+ blockErrors: map[string]error{},
+ }
+ api.workspaces = []workspace{{ID: "deep", RootNodeID: "root-deep", Name: "Deep"}}
+ denied := errors.New("forbidden.accessDenied: the operator has no permission")
+ for w := 0; w < maxValidateProbes+2; w++ {
+ root := fmt.Sprintf("root-%d", w)
+ id := fmt.Sprintf("doc-%d", w)
+ api.workspaces = append(api.workspaces, workspace{ID: root, RootNodeID: root, Name: root})
+ api.nodes[root] = []node{{ID: id, Name: id, Type: "FILE", Category: "ALIDOC", Extension: "adoc"}}
+ api.blockErrors[id] = denied
+ }
+
+ if err := testConnector(api).Validate(context.Background(), testConfig()); err != nil {
+ t.Fatalf("Validate must accept an inconclusive walk even after the probe cap, got: %v", err)
+ }
+ total := 0
+ for _, n := range api.blockCalls {
+ total += n
+ }
+ if total != maxValidateProbes {
+ t.Fatalf("Validate made %d document probes, want %d", total, maxValidateProbes)
+ }
+}
+
+// Once the probe cap is spent no later probe can succeed. A workspace walked
+// after that point must not get the chance to find an unlistable folder and
+// turn every failed probe into acceptance.
+func TestValidateStopsWalkingOnceTheProbeCapIsSpent(t *testing.T) {
+ api := &fakeAPI{
+ nodes: map[string][]node{},
+ nodeErrors: map[string]error{"folder": errors.New("DingTalk API status=500")},
+ blockErrors: map[string]error{},
+ }
+ denied := errors.New("forbidden.accessDenied: the operator has no permission")
+ for w := 0; w < maxValidateProbes; w++ {
+ root := fmt.Sprintf("root-%d", w)
+ id := fmt.Sprintf("doc-%d", w)
+ api.workspaces = append(api.workspaces, workspace{ID: root, RootNodeID: root, Name: root})
+ api.nodes[root] = []node{{ID: id, Name: id, Type: "FILE", Category: "ALIDOC", Extension: "adoc"}}
+ api.blockErrors[id] = denied
+ }
+ api.workspaces = append(api.workspaces, workspace{ID: "late", RootNodeID: "root-late", Name: "Late"})
+ api.nodes["root-late"] = []node{{ID: "folder", Name: "Folder", Type: "FOLDER"}}
+
+ err := testConnector(api).Validate(context.Background(), testConfig())
+ if err == nil || !strings.Contains(err.Error(), "no permission") {
+ t.Fatalf("Validate must report the failed probes, got: %v", err)
+ }
+ wantNote := fmt.Sprintf("only the first %d documents were probed", maxValidateProbes)
+ if !strings.Contains(err.Error(), wantNote) {
+ t.Fatalf("Validate error should say the probe cap stopped the walk, got: %v", err)
+ }
+}
+
+// A tail skipped to make room for siblings is only covered by a probe made
+// after it. A document probed before the skip says nothing about the skipped
+// pages, so an empty sibling must leave the walk inconclusive.
+func TestValidateSkippedTailIsNotCoveredByAnEarlierProbe(t *testing.T) {
+ var mu sync.Mutex
+ fatPages := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/v1.0/oauth2/accessToken":
+ _, _ = w.Write([]byte(`{"accessToken":"token","expireIn":7200}`))
+ case "/v2.0/wiki/workspaces":
+ _, _ = w.Write([]byte(`{"workspaces":[{"workspaceId":"a","rootNodeId":"root-a","name":"A"}]}`))
+ case "/v2.0/wiki/nodes":
+ switch r.URL.Query().Get("parentNodeId") {
+ case "root-a":
+ _, _ = w.Write([]byte(`{"nodes":[` +
+ `{"nodeId":"doc-x","name":"X","type":"FILE","category":"ALIDOC","extension":"adoc"},` +
+ `{"nodeId":"folder-a","type":"FOLDER"},` +
+ `{"nodeId":"folder-b","type":"FOLDER"}]}`))
+ case "folder-a":
+ mu.Lock()
+ fatPages++
+ page := fatPages
+ mu.Unlock()
+ next := ""
+ if page < 30 {
+ next = strconv.Itoa(page)
+ }
+ _, _ = fmt.Fprintf(w,
+ `{"nodes":[{"nodeId":"pdf-%d","type":"FILE","category":"FILE","extension":"pdf"}],"nextToken":%q}`,
+ page, next)
+ case "folder-b":
+ _, _ = w.Write([]byte(`{"nodes":[]}`))
+ default:
+ http.NotFound(w, r)
+ }
+ case "/v1.0/doc/suites/documents/doc-x/blocks":
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = w.Write([]byte(`{"code":"forbidden.accessDenied","message":"no permission"}`))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ if err := testConnector(testClient(server)).Validate(context.Background(), testConfig()); err != nil {
+ t.Fatalf("Validate must accept when the skipped tail was never covered by a probe, got: %v", err)
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if fatPages >= maxValidateListings {
+ t.Fatalf("fat folder used %d pages, want room left for its sibling", fatPages)
+ }
+}
+
+// A probe cancelled by the caller's deadline still names the document, so the
+// operator can tell which read timed out.
+func TestValidateContextErrorNamesTheProbedDocument(t *testing.T) {
+ api := &fakeAPI{
+ workspaces: []workspace{{ID: "a", RootNodeID: "root-a", Name: "Alpha"}},
+ nodes: map[string][]node{
+ "root-a": {{ID: "doc", Name: "Handbook", Type: "FILE", Category: "ALIDOC", Extension: "adoc"}},
+ },
+ blockErrors: map[string]error{"doc": context.DeadlineExceeded},
+ }
+ err := testConnector(api).Validate(context.Background(), testConfig())
+ if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), `document "Handbook"`) {
+ t.Fatalf("Validate must surface the timed-out probe with its document, got: %v", err)
+ }
+}