Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(datasource/dingtalk): Validate 只校验所选范围,不再扫全租户 (#4421)
linus-liu-webd92728d88c6d
diff --git a/internal/datasource/connector/dingtalk/connector.go b/internal/datasource/connector/dingtalk/connector.goindex cc29835e4..262fbe2e8 100644--- a/internal/datasource/connector/dingtalk/connector.go+++ b/internal/datasource/connector/dingtalk/connector.go@@ -112,11 +112,19 @@ func (c *Connector) Type() string {const maxValidateProbes = 5// Validate checks the application credentials and operator access, including-// node listing and a sample document read when one is visible in a workspace.+// node listing and a sample document read when one is visible.//-// The operator is not expected to reach every workspace in the tenant: app-// credentials are valid as long as one reachable workspace yields one readable-// document. A failed probe does not condemn that workspace; the walk keeps+// A data source that selects resources is validated against those resources+// only. They are what the operator asked to sync, so an unrelated workspace+// that happens to be listed first, and that the operator cannot read, must not+// decide whether the selection is usable: a selected folder or workspace is+// walked for one readable document, and a selected document is probed+// directly. Without a selection there is nothing narrower to sample, so the+// whole tenant is walked as before.+//+// In both modes the operator is not expected to reach every object: app+// credentials are valid as long as one reachable document can be read. A+// failed probe does not condemn the object it was found in; the walk keeps// looking, inside the probe cap, for another document the operator can read.// When the cap itself stops a conclusive walk, the error says how many// documents were read.@@ -127,6 +135,15 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSo}api := c.api(cfg)settings := parseDocumentSettings(dataSourceConfig)+ if selected := uniqueIDs(dataSourceConfig.ResourceIDs); len(selected) > 0 {+ return validateSelectedResources(ctx, api, selected, settings)+ }+ return validateTenantCredentials(ctx, api, settings)+}++// validateTenantCredentials samples every workspace in the tenant, which is+// what a data source without a selection is going to sync.+func validateTenantCredentials(ctx context.Context, api dingTalkAPI, settings documentSettings) error {workspaces, err := api.listWorkspaces(ctx)if err != nil {return fmt.Errorf("validate DingTalk data source: %w", err)@@ -174,19 +191,168 @@ func (c *Connector) Validate(ctx context.Context, dataSourceConfig *types.DataSobreak}}- // Folders were left unexplored, because the listing budget ran out or a- // folder could not be listed, so an unreadable document elsewhere does not- // prove the data source is unusable. A probe cap hit in that situation- // stays quiet too: the walk cannot claim it sampled every visible document.+ return validateOutcome(inconclusive, sawDocument, workspaceFail, probeCapHit, lastErr)+}++// validateSelectedResources checks only the scopes a data source selects.+// References are resolved through the same code path a sync uses, so Validate+// answers the question the operator asked — would this selection sync+// anything? — instead of sampling workspaces the selection never mentions.+//+// A reference this build cannot resolve is reported rather than quietly+// replaced by the tenant walk: the data source would fail on that resource,+// and a selection the picker can no longer resolve is exactly what the+// operator needs to see.+func validateSelectedResources(+ ctx context.Context,+ api dingTalkAPI,+ selected []string,+ settings documentSettings,+) error {+ if err := ctx.Err(); err != nil {+ return fmt.Errorf("validate DingTalk data source: %w", err)+ }+ workspaces, err := api.listWorkspaces(ctx)+ if err != nil {+ return fmt.Errorf("validate DingTalk data source: %w", err)+ }+ scopes, failures, err := resolveSyncScopes(ctx, api, workspaces, selected, settings)+ if err != nil {+ return fmt.Errorf("validate DingTalk data source: %w", err)+ }+ names := make(map[string]string, len(workspaces))+ for _, item := range workspaces {+ names[item.ID] = item.Name+ }++ var (+ lastErr error+ resolveErr error+ sawDocument bool+ inconclusive bool+ scopeFail int+ probes int+ listings int+ probeCapHit bool+ )+ for _, resourceID := range selected {+ failure := failures[resourceID]+ if failure == nil {+ continue+ }+ scopeFail+++ if resolveErr == nil {+ resolveErr = fmt.Errorf("resource %s: %w", resourceID, failure)+ }+ }+ for i, scope := range scopes {+ // A failed probe names the workspace it was found in, the way the+ // tenant-wide sample does, so it stays actionable.+ name := names[scope.Reference.WorkspaceID]+ if name == "" {+ name = scope.Reference.WorkspaceID+ }+ readable, unexplored, hitCap, err := sampleScope(+ ctx, api, scope, name, settings,+ &listings, &probes, &sawDocument, &lastErr,+ )+ if err != nil {+ // Name the selected node when the selection is one: a folder that+ // cannot be listed is what the operator has to look at, and the+ // workspace alone would not say which folder.+ label := fmt.Sprintf("workspace %q", name)+ if scope.Reference.NodeID != "" {+ label = fmt.Sprintf("workspace %q node %q", name, scope.Reference.NodeID)+ }+ lastErr = fmt.Errorf("%s: %w", label, err)+ // A cancelled or timed-out request proves nothing about the+ // credentials, so it must never fall through to acceptance.+ if ctx.Err() != nil || isContextError(err) {+ return fmt.Errorf("validate DingTalk data source: %w", lastErr)+ }+ scopeFail+++ continue+ }+ if readable {+ return nil+ }+ if unexplored {+ inconclusive = true+ }+ if probes >= maxValidateProbes {+ probeCapHit = hitCap || i+1 < len(scopes)+ break+ }+ }+ if lastErr == nil {+ // A failed probe names a workspace and a document the operator can act+ // on; a reference that does not resolve can only name itself.+ lastErr = resolveErr+ }+ return validateOutcome(inconclusive, sawDocument, scopeFail, probeCapHit, lastErr)+}++// sampleScope probes one resolved selection scope. A selected document is read+// through its own read API, while a selected folder or workspace is walked the+// way the tenant sample walks a workspace root, so a readable document below+// the selected folder is still found.+func sampleScope(+ ctx context.Context,+ api dingTalkAPI,+ scope syncScope,+ workspaceName string,+ settings documentSettings,+ listings *int,+ probes *int,+ sawDocument *bool,+ lastErr *error,+) (readable, unexplored, capHit bool, err error) {+ if scope.Document == nil {+ return sampleWorkspace(+ ctx, api, workspaceName, scope.StartNodeID, settings,+ listings, probes, sawDocument, lastErr,+ )+ }+ document := *scope.Document+ *probes+++ *sawDocument = true+ if probeErr := verifyDocument(ctx, api, document); probeErr != nil {+ if ctx.Err() != nil || isContextError(probeErr) {+ return false, false, false, fmt.Errorf("document %q: %w", document.title(), probeErr)+ }+ *lastErr = fmt.Errorf(+ "workspace %q document %q: %w", workspaceName, document.title(), probeErr,+ )+ return false, false, false, nil+ }+ return true, false, false, nil+}++// validateOutcome folds the counters of a validate walk into its result. The+// tenant-wide sample and the selection-scoped one both end here, because they+// answer the same question: is there a document in what this data source syncs+// that the operator can read?+//+// Folders were left unexplored, because the listing budget ran out or a folder+// could not be listed, so an unreadable document elsewhere does not prove the+// data source is unusable. A probe cap hit in that situation stays quiet too:+// the walk cannot claim it sampled every visible document.+//+// Nothing readable anywhere. Only a scope that exposed no document at all is+// accepted: there was nothing to read, so credentials could not be disproved.+// A document that exists but cannot be read means the data source would sync+// nothing, which is worth reporting.+func validateOutcome(+ inconclusive bool,+ sawDocument bool,+ failures int,+ probeCapHit bool,+ lastErr error,+) error {if inconclusive {return nil}-- // Nothing readable anywhere. Only a tenant that exposed no document at all- // is accepted: there was nothing to read, so credentials could not be- // disproved. A document that exists but cannot be read means the data- // source would sync nothing, which is worth reporting.- if !sawDocument && workspaceFail == 0 {+ if !sawDocument && failures == 0 {return nil}if lastErr == nil {diff --git a/internal/datasource/connector/dingtalk/validate_selection_test.go b/internal/datasource/connector/dingtalk/validate_selection_test.gonew file mode 100644index 000000000..56f35f5bc--- /dev/null+++ b/internal/datasource/connector/dingtalk/validate_selection_test.go@@ -0,0 +1,628 @@+package dingtalk++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "slices"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/datasource"+)++// A data source that names resources syncs those resources, so Validate has to+// check them instead of the whole tenant. Sampling the tenant answers a+// different question, and an unreadable workspace listed first can spend the+// probe budget before the selection is ever reached.++// recordingAPI records every node listing, so a test can prove which+// workspaces a validation walked. The fakeAPI counters only say which+// documents were probed.+type recordingAPI struct {+ *fakeAPI+ listed []string+}++func (a *recordingAPI) listNodes(ctx context.Context, parentID string) ([]node, error) {+ a.listed = append(a.listed, parentID)+ return a.fakeAPI.listNodes(ctx, parentID)+}++func (a *recordingAPI) listNodesPage(ctx context.Context, parentID, pageToken string) ([]node, string, error) {+ a.listed = append(a.listed, parentID)+ return a.fakeAPI.listNodesPage(ctx, parentID, pageToken)+}++// mustEncodeResource encodes a picker reference the way ListResources does, so+// the tests exercise the resource IDs a saved data source really carries.+func mustEncodeResource(t *testing.T, ref resourceReference) string {+ t.Helper()+ id, err := encodeResourceReference(ref)+ if err != nil {+ t.Fatalf("encode resource reference %+v: %v", ref, err)+ }+ return id+}++// realTenant mirrors the tenant shape this scoping was reported from: eight+// workspaces in the listing, the first ones full of documents the operator+// cannot read, and the reachable one — the fifth — behind them. The probe+// budget was spent on the first workspace, so the reachable documents were+// never probed and the data source could not be created, even though a sync of+// the same selection read every one of its documents.+func realTenant(t *testing.T, denied error) *fakeAPI {+ t.Helper()+ api := &fakeAPI{+ workspaces: []workspace{+ {ID: "security", RootNodeID: "root-security", Name: "Security"},+ {ID: "ops", RootNodeID: "root-ops", Name: "Operations"},+ {ID: "sales", RootNodeID: "root-sales", Name: "Sales"},+ {ID: "hr", RootNodeID: "root-hr", Name: "People"},+ {ID: "faq", RootNodeID: "root-faq", Name: "Product FAQ"},+ {ID: "pm", RootNodeID: "root-pm", Name: "Projects"},+ {ID: "design", RootNodeID: "root-design", Name: "Design"},+ {ID: "archive", RootNodeID: "root-archive", Name: "Archive"},+ },+ nodes: map[string][]node{+ "root-faq": {+ {ID: "faq-folder", Name: "FAQ", Type: "FOLDER"},+ },+ "faq-folder": {+ {ID: "faq-doc", Name: "Product FAQ", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ "root-design": {+ {ID: "design-folder", Name: "Design", Type: "FOLDER"},+ },+ "root-archive": {+ {ID: "archive-folder", Name: "Archive", Type: "FOLDER"},+ },+ },+ blocks: map[string][]json.RawMessage{+ "faq-doc": {rawJSON(`{"paragraph":{"text":"hello"}}`)},+ },+ blockErrors: map[string]error{},+ }+ // Every other workspace exposes documents this operator cannot read,+ // several of them enough to spend the whole probe budget.+ for _, workspaceID := range []string{"security", "ops", "sales", "hr", "pm"} {+ root := "root-" + workspaceID+ for i := 0; i < 6; i++ {+ id := fmt.Sprintf("%s-doc-%d", workspaceID, i)+ api.nodes[root] = append(api.nodes[root], node{+ ID: id, Name: id, Type: "FILE", Category: "ALIDOC", Extension: "adoc",+ })+ api.blockErrors[id] = denied+ }+ }+ return api+}++// A selected workspace or folder must be validated on its own: the readable+// document inside it decides the outcome, whatever the rest of the tenant+// looks like. On the tenant this was reported from, Validate failed here and+// the data source could not be created at all.+func TestValidateSamplesOnlyTheSelectedResources(t *testing.T) {+ denied := errors.New("forbidden.accessDenied: the operator has no permission")+ cases := []struct {+ name string+ resourceID func(t *testing.T) string+ }{+ {+ name: "the selected workspace",+ resourceID: func(*testing.T) string { return "faq" },+ },+ {+ name: "a folder inside the selected workspace",+ resourceID: func(t *testing.T) string {+ return mustEncodeResource(t, resourceReference{+ WorkspaceID: "faq", NodeID: "faq-folder",+ })+ },+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ api := realTenant(t, denied)+ recorder := &recordingAPI{fakeAPI: api}+ err := testConnector(recorder).Validate(+ context.Background(), testConfig(tc.resourceID(t)),+ )+ if err != nil {+ t.Fatalf("Validate must accept a selection whose documents are readable, got: %v", err)+ }+ if api.blockCalls["faq-doc"] != 1 {+ t.Fatalf("the selected document was probed %d times, want 1", api.blockCalls["faq-doc"])+ }+ // The unrelated workspaces must not be walked at all: sampling+ // them is what spent the budget before the selection was reached.+ if len(recorder.listed) == 0 {+ t.Fatal("Validate never listed the selection")+ }+ for _, parent := range recorder.listed {+ if parent != "root-faq" && parent != "faq-folder" {+ t.Fatalf("Validate listed %q, which the selection does not name", parent)+ }+ }+ })+ }+}++// The mirror image: a readable document outside the selection must not vouch+// for a selection whose own documents are all unreadable. The data source+// would sync nothing, so Validate has to reject it.+func TestValidateRejectsASelectionWhoseDocumentsAreAllUnreadable(t *testing.T) {+ denied := errors.New("forbidden.accessDenied: the operator has no permission")+ api := &fakeAPI{+ workspaces: []workspace{+ {ID: "open", RootNodeID: "root-open", Name: "Open"},+ {ID: "locked", RootNodeID: "root-locked", Name: "Locked"},+ },+ nodes: map[string][]node{+ "root-open": {+ {ID: "doc-open", Name: "Public", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ "root-locked": {+ {ID: "locked-folder", Name: "Contracts", Type: "FOLDER"},+ },+ "locked-folder": {+ {ID: "locked-a", Name: "Contract A", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ {ID: "locked-b", Name: "Contract B", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ },+ blocks: map[string][]json.RawMessage{+ "doc-open": {rawJSON(`{"paragraph":{"text":"hello"}}`)},+ },+ blockErrors: map[string]error{"locked-a": denied, "locked-b": denied},+ }+ selected := mustEncodeResource(t, resourceReference{+ WorkspaceID: "locked", NodeID: "locked-folder",+ })++ err := testConnector(api).Validate(context.Background(), testConfig(selected))+ if err == nil {+ t.Fatal("Validate must fail: every document in the selection is unreadable, " ++ "so the data source would sync nothing")+ }+ for _, want := range []string{`workspace "Locked"`, `document "Contract B"`, "no permission"} {+ if !strings.Contains(err.Error(), want) {+ t.Fatalf("Validate error should carry %s, got: %v", want, err)+ }+ }+ if strings.Contains(err.Error(), "only the first") {+ t.Fatalf("two probed documents must not be described as the probe cap, got: %v", err)+ }+ if api.blockCalls["doc-open"] != 0 {+ t.Fatalf("a document outside the selection was probed %d times", api.blockCalls["doc-open"])+ }+ if api.blockCalls["locked-a"] != 1 || api.blockCalls["locked-b"] != 1 {+ t.Fatalf("both selected documents should be probed, calls=%v", api.blockCalls)+ }+}++// A selection that holds no document at all has nothing to read, so the+// credentials cannot be disproved and Validate accepts. Folders and types with+// no ingest path do not count as documents.+func TestValidateAcceptsASelectionWithoutDocuments(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "empty", RootNodeID: "root-empty", Name: "Empty"}},+ nodes: map[string][]node{+ "root-empty": {+ {ID: "empty-folder", Name: "Empty", Type: "FOLDER"},+ },+ "empty-folder": {+ {ID: "sub", Name: "Sub", Type: "FOLDER"},+ {ID: "clip", Name: "Clip", Type: "FILE", Category: "VIDEO", Extension: "mp4"},+ },+ "sub": {},+ },+ }+ selected := mustEncodeResource(t, resourceReference{+ WorkspaceID: "empty", NodeID: "empty-folder",+ })++ if err := testConnector(api).Validate(context.Background(), testConfig(selected)); err != nil {+ t.Fatalf("Validate must accept a selection that holds no document to read, got: %v", err)+ }+ if len(api.blockCalls) != 0 || len(api.verifyCalls) != 0 {+ t.Fatalf("a selection without ingestible documents must not be probed, blocks=%v verifies=%v",+ api.blockCalls, api.verifyCalls)+ }+}++// A cancelled or timed-out request proves nothing about the credentials, so it+// must never fall through to acceptance, not even when another selected scope+// would have proven them.+func TestValidateFailsWhenTheSelectionWalkIsCancelled(t *testing.T) {+ denied := errors.New("forbidden.accessDenied: the operator has no permission")+ t.Run("cancelled before the first call", func(t *testing.T) {+ api := realTenant(t, denied)+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ err := testConnector(api).Validate(ctx, testConfig("faq"))+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("Validate must report the cancellation, got: %v", err)+ }+ if api.blockCalls["faq-doc"] != 0 {+ t.Fatal("Validate must not probe anything under a cancelled context")+ }+ })+ t.Run("document probe times out", func(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "faq", RootNodeID: "root-faq", Name: "FAQ"}},+ nodes: map[string][]node{+ "root-faq": {+ {ID: "faq-doc", Name: "FAQ", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ },+ blockErrors: map[string]error{"faq-doc": context.DeadlineExceeded},+ }+ err := testConnector(api).Validate(context.Background(), testConfig("faq"))+ if !errors.Is(err, context.DeadlineExceeded) ||+ !strings.Contains(err.Error(), `document "FAQ"`) {+ t.Fatalf("Validate must surface the timed-out probe with its document, got: %v", err)+ }+ })+ t.Run("resolution listing is cancelled", func(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "faq", RootNodeID: "root-faq", Name: "FAQ"}},+ nodes: map[string][]node{"root-faq": {{ID: "faq-folder", Type: "FOLDER"}}},+ nodeErrors: map[string]error{"root-faq": context.Canceled},+ }+ selected := mustEncodeResource(t, resourceReference{+ WorkspaceID: "faq", NodeID: "faq-folder",+ })+ err := testConnector(api).Validate(context.Background(), testConfig(selected))+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("Validate must report the cancellation, got: %v", err)+ }+ })+ t.Run("a cancelled scope is not rescued by a readable one", func(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{+ {ID: "locked", RootNodeID: "root-locked", Name: "Locked"},+ {ID: "open", RootNodeID: "root-open", Name: "Open"},+ },+ nodes: map[string][]node{+ "root-locked": {+ {+ ID: "cancelled-doc", Name: "Cancelled", Type: "FILE",+ Category: "ALIDOC", Extension: "adoc",+ },+ },+ "root-open": {+ {ID: "open-folder", Name: "Handbook", Type: "FOLDER"},+ },+ "open-folder": {+ {+ ID: "open-doc", Name: "Handbook", Type: "FILE",+ Category: "ALIDOC", Extension: "adoc",+ },+ },+ },+ blocks: map[string][]json.RawMessage{"open-doc": {rawJSON(`{"paragraph":{"text":"hi"}}`)}},+ blockErrors: map[string]error{"cancelled-doc": context.Canceled},+ }+ selected := []string{+ "locked",+ mustEncodeResource(t, resourceReference{WorkspaceID: "open", NodeID: "open-folder"}),+ }+ err := testConnector(api).Validate(context.Background(), testConfig(selected...))+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("Validate must report the cancellation, got: %v", err)+ }+ if api.blockCalls["open-doc"] != 0 {+ t.Fatal("Validate must stop at the cancellation instead of accepting on the next scope")+ }+ })+}++// Without a selection there is nothing narrower to sample, so the whole tenant+// is walked exactly as before.+func TestValidateWithoutASelectionKeepsSamplingTheTenant(t *testing.T) {+ denied := errors.New("forbidden.accessDenied: the operator has no permission")+ t.Run("a readable later workspace still proves the credentials", func(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{+ {ID: "locked", RootNodeID: "root-locked", Name: "Locked"},+ {ID: "open", RootNodeID: "root-open", Name: "Open"},+ },+ nodes: map[string][]node{+ "root-locked": {+ {ID: "doc-locked", Name: "Secret", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ "root-open": {+ {ID: "doc-open", Name: "Public", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ },+ blocks: map[string][]json.RawMessage{+ "doc-open": {rawJSON(`{"paragraph":{"text":"hello"}}`)},+ },+ blockErrors: map[string]error{"doc-locked": denied},+ }+ recorder := &recordingAPI{fakeAPI: api}++ if err := testConnector(recorder).Validate(context.Background(), testConfig()); err != nil {+ t.Fatalf("Validate must sample the whole tenant without a selection, got: %v", err)+ }+ if api.blockCalls["doc-locked"] != 1 || api.blockCalls["doc-open"] != 1 {+ t.Fatalf("every workspace should be sampled, calls=%v", api.blockCalls)+ }+ if !slices.Contains(recorder.listed, "root-locked") ||+ !slices.Contains(recorder.listed, "root-open") {+ t.Fatalf("the tenant walk must list every workspace, listed=%v", recorder.listed)+ }+ })+ t.Run("the probe cap still stops the tenant walk", func(t *testing.T) {+ // Deliberately unchanged: with no selection, the probe budget is still+ // shared across the tenant, so a workspace that spends all of it+ // still hides the readable workspace behind it. A selection is what+ // makes Validate answer for the operator's own scope.+ api := realTenant(t, denied)+ err := testConnector(api).Validate(context.Background(), testConfig())+ if err == nil {+ t.Fatal("Validate must fail when the probe cap stops the tenant walk")+ }+ wantNote := fmt.Sprintf("only the first %d documents were probed", maxValidateProbes)+ if !strings.Contains(err.Error(), wantNote) {+ t.Fatalf("Validate error should say the probe cap stopped the walk, got: %v", err)+ }+ if api.blockCalls["faq-doc"] != 0 {+ t.Fatal("the tenant walk must not have reached the later workspace")+ }+ })+}++// A reference this build cannot resolve must be reported, never quietly+// replaced by the tenant walk: the data source would fail on that resource,+// and a selection the picker can no longer resolve is what the operator has to+// see. The first case is the shape of a Base reference+// (dingtalk:v1?base=<id>) on a build whose resource IDs do not model Bases:+// the reference does not decode, so it cannot be validated and must not be+// silently dropped.+func TestValidateRejectsASelectionItCannotResolve(t *testing.T) {+ readableTenant := func() *fakeAPI {+ return &fakeAPI{+ workspaces: []workspace{{ID: "open", RootNodeID: "root-open", Name: "Open"}},+ nodes: map[string][]node{+ "root-open": {+ {ID: "doc-open", Name: "Public", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ },+ blocks: map[string][]json.RawMessage{+ "doc-open": {rawJSON(`{"paragraph":{"text":"hello"}}`)},+ },+ }+ }+ cases := []struct {+ name string+ resource string+ want string+ wantErr error+ }{+ {+ name: "a reference field this build does not model",+ resource: "dingtalk:v1?base=base-1",+ want: "unsupported field",+ },+ {+ name: "a workspace that is gone",+ resource: "gone",+ want: "is unavailable",+ wantErr: datasource.ErrResourceNotFound,+ },+ {+ name: "a node that is gone",+ resource: mustEncodeResource(t, resourceReference{+ WorkspaceID: "open", NodeID: "gone-node",+ }),+ want: "is unavailable",+ wantErr: datasource.ErrResourceNotFound,+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ api := readableTenant()+ err := testConnector(api).Validate(context.Background(), testConfig(tc.resource))+ if err == nil {+ t.Fatal("Validate must not accept a selection it cannot resolve")+ }+ if !strings.Contains(err.Error(), "resource "+tc.resource) {+ t.Fatalf("Validate error should name the resource, got: %v", err)+ }+ if !strings.Contains(err.Error(), tc.want) {+ t.Fatalf("Validate error should carry %q, got: %v", tc.want, err)+ }+ if tc.wantErr != nil && !errors.Is(err, tc.wantErr) {+ t.Fatalf("Validate error should wrap %v, got: %v", tc.wantErr, err)+ }+ if len(api.blockCalls) != 0 {+ t.Fatalf("Validate must not fall back to the tenant walk, probed %v", api.blockCalls)+ }+ })+ }+ t.Run("a node with no ingest path", func(t *testing.T) {+ // The upload switch is off for testConfig, so the uploaded file the+ // reference names is not ingestible in this data source: the selection+ // would sync nothing, and a sync would report the resource as failed.+ api := readableTenant()+ api.nodes["root-open"] = []node{{+ ID: "doc-open", Name: "Upload", Type: "FILE", Category: "DOCUMENT", Extension: "pdf",+ }}+ selected := mustEncodeResource(t, resourceReference{WorkspaceID: "open", NodeID: "doc-open"})++ err := testConnector(api).Validate(context.Background(), testConfig(selected))+ if err == nil {+ t.Fatal("Validate must not accept a selection it cannot resolve")+ }+ if !strings.Contains(err.Error(), "not a supported online document or folder") {+ t.Fatalf("Validate error should name the unsupported node, got: %v", err)+ }+ if len(api.blockCalls) != 0 || len(api.verifyCalls) != 0 {+ t.Fatalf("Validate must not fall back to the tenant walk, blocks=%v verifies=%v",+ api.blockCalls, api.verifyCalls)+ }+ })+}++// A selected folder that cannot be listed cannot be scanned, so Validate+// reports it instead of accepting. The error names the folder, which is what+// the operator has to look at: the workspace alone would not say which one.+func TestValidateReportsASelectedFolderThatCannotBeListed(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "team", RootNodeID: "root-team", Name: "Team"}},+ nodes: map[string][]node{+ "root-team": {+ {ID: "broken", Name: "Broken", Type: "FOLDER"},+ },+ },+ nodeErrors: map[string]error{"broken": errors.New("DingTalk API status=500")},+ }+ selected := mustEncodeResource(t, resourceReference{WorkspaceID: "team", NodeID: "broken"})++ err := testConnector(api).Validate(context.Background(), testConfig(selected))+ if err == nil {+ t.Fatal("Validate must not accept a selection it cannot scan")+ }+ for _, want := range []string{`workspace "Team"`, `node "broken"`, "status=500"} {+ if !strings.Contains(err.Error(), want) {+ t.Fatalf("Validate error should carry %s, got: %v", want, err)+ }+ }+}++// A selected document is a leaf: it is probed through its own read API instead+// of being walked, and a readable one is enough to accept the data source.+func TestValidateProbesASelectedDocumentDirectly(t *testing.T) {+ denied := errors.New("forbidden.accessDenied: the operator has no permission")+ t.Run("readable document", func(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "open", RootNodeID: "root-open", Name: "Open"}},+ nodes: map[string][]node{+ "root-open": {+ {ID: "doc", Name: "Handbook", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ },+ blocks: map[string][]json.RawMessage{+ "doc": {rawJSON(`{"paragraph":{"text":"hello"}}`)},+ },+ }+ recorder := &recordingAPI{fakeAPI: api}+ selected := mustEncodeResource(t, resourceReference{WorkspaceID: "open", NodeID: "doc"})++ if err := testConnector(recorder).Validate(context.Background(), testConfig(selected)); err != nil {+ t.Fatalf("Validate must accept a readable selected document, got: %v", err)+ }+ if api.blockCalls["doc"] != 1 {+ t.Fatalf("the selected document was probed %d times, want 1", api.blockCalls["doc"])+ }+ if len(recorder.listed) != 1 || recorder.listed[0] != "root-open" {+ t.Fatalf("a selected document is resolved through its parent only, listed=%v", recorder.listed)+ }+ })+ t.Run("unreadable document", func(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "open", RootNodeID: "root-open", Name: "Open"}},+ nodes: map[string][]node{+ "root-open": {+ {ID: "doc", Name: "Handbook", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ },+ blockErrors: map[string]error{"doc": denied},+ }+ selected := mustEncodeResource(t, resourceReference{WorkspaceID: "open", NodeID: "doc"})++ err := testConnector(api).Validate(context.Background(), testConfig(selected))+ if err == nil || !strings.Contains(err.Error(), `document "Handbook"`) {+ t.Fatalf("Validate must report the unreadable selected document, got: %v", err)+ }+ })+ t.Run("readable uploaded file", func(t *testing.T) {+ api := selectedUpload(nil)+ selected := mustEncodeResource(t, resourceReference{WorkspaceID: "open", NodeID: "file"})++ err := testConnector(api).Validate(context.Background(), testConfigWithUploadedFiles(selected))+ if err != nil {+ t.Fatalf("Validate must accept a readable selected upload, got: %v", err)+ }+ // An uploaded file has no blocks: its download location is resolved,+ // which exercises the same permissions without transferring the bytes.+ if api.verifyCalls["file"] != 1 || api.dlCalls["file"] != 0 {+ t.Fatalf("upload should be verified through its download location, verify=%v download=%v",+ api.verifyCalls, api.dlCalls)+ }+ })+ t.Run("unreadable uploaded file", func(t *testing.T) {+ api := selectedUpload(denied)+ selected := mustEncodeResource(t, resourceReference{WorkspaceID: "open", NodeID: "file"})++ err := testConnector(api).Validate(context.Background(), testConfigWithUploadedFiles(selected))+ if err == nil || !strings.Contains(err.Error(), `document "Handbook.pdf"`) {+ t.Fatalf("Validate must report the unreadable selected upload, got: %v", err)+ }+ })+}++// selectedUpload is a tenant whose only selected node is an uploaded PDF, so+// the selection walk has to use the download verification path.+func selectedUpload(verifyErr error) *fakeAPI {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "open", RootNodeID: "root-open", Name: "Open"}},+ nodes: map[string][]node{+ "root-open": {+ {+ ID: "file", Name: "Handbook.pdf", Type: "FILE",+ Category: "DOCUMENT", Extension: "pdf",+ },+ },+ },+ }+ if verifyErr != nil {+ api.verifyErrors = map[string]error{"file": verifyErr}+ }+ return api+}++// One selected scope that yields a readable document proves the credentials,+// exactly as one readable workspace does for the tenant walk: the other+// selected scopes stay in the report of the sync that will follow, they do not+// reject the data source.+func TestValidateAcceptsASelectionWhenOneScopeIsReadable(t *testing.T) {+ denied := errors.New("forbidden.accessDenied: the operator has no permission")+ api := &fakeAPI{+ workspaces: []workspace{+ {ID: "locked", RootNodeID: "root-locked", Name: "Locked"},+ {ID: "open", RootNodeID: "root-open", Name: "Open"},+ },+ nodes: map[string][]node{+ "root-locked": {+ {ID: "locked-folder", Name: "Contracts", Type: "FOLDER"},+ },+ "locked-folder": {+ {ID: "locked-doc", Name: "Contract", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ "root-open": {+ {ID: "open-folder", Name: "Handbook", Type: "FOLDER"},+ },+ "open-folder": {+ {ID: "open-doc", Name: "Handbook", Type: "FILE", Category: "ALIDOC", Extension: "adoc"},+ },+ },+ blocks: map[string][]json.RawMessage{+ "open-doc": {rawJSON(`{"paragraph":{"text":"hello"}}`)},+ },+ blockErrors: map[string]error{"locked-doc": denied},+ }+ selected := []string{+ mustEncodeResource(t, resourceReference{WorkspaceID: "locked", NodeID: "locked-folder"}),+ mustEncodeResource(t, resourceReference{WorkspaceID: "open", NodeID: "open-folder"}),+ }++ if err := testConnector(api).Validate(context.Background(), testConfig(selected...)); err != nil {+ t.Fatalf("Validate must accept when one selected scope yields a readable document, got: %v", err)+ }+ if api.blockCalls["locked-doc"] != 1 || api.blockCalls["open-doc"] != 1 {+ t.Fatalf("both selected scopes should be probed, calls=%v", api.blockCalls)+ }+}