Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(datasource/dingtalk): 跳过不再静默,展开叶子不再报错
linus-liu-web08f7e702cfb3
diff --git a/internal/datasource/connector/dingtalk/connector.go b/internal/datasource/connector/dingtalk/connector.goindex 1a8f3b83e..798f5fd5c 100644--- a/internal/datasource/connector/dingtalk/connector.go+++ b/internal/datasource/connector/dingtalk/connector.go@@ -13,6 +13,7 @@ import ("unicode/utf8""github.com/Tencent/WeKnora/internal/datasource"+ "github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/types")@@ -21,6 +22,53 @@ const (maxTraversalNodes = 1_000_000)+// mediaExtensions are the file types this connector deliberately never+// downloads: media carries no text WeKnora could index.+var mediaExtensions = map[string]struct{}{+ "mp4": {}, "mov": {}, "avi": {}, "mkv": {}, "flv": {}, "wmv": {}, "m4v": {}, "webm": {},+ "mp3": {}, "wav": {}, "m4a": {}, "aac": {}, "flac": {},+}++// unsupportedDocumentExtensions names the native DingTalk document types the+// wiki API lists but this connector has no ingest path for, so a skip can name+// the concrete type instead of only saying the node is unsupported. The+// remaining types expose no read API at all.+var unsupportedDocumentExtensions = map[string]string{+ "able": "DingTalk multi-dimensional table",+ "amind": "DingTalk mind map",+ "appt": "DingTalk presentation",+ "adraw": "DingTalk drawing",+}++// skipReason explains, in the words of the sync log, why a node the connector+// just listed will not be ingested. Both answers are deterministic: retrying+// the sync can never turn such a node into a document, which is what separates+// a skip from a failed read that must be retried.+func skipReason(n node) string {+ extension := strings.ToLower(strings.TrimSpace(n.Extension))+ _, knownMediaExtension := mediaExtensions[extension]+ if strings.EqualFold(n.Category, "VIDEO") || knownMediaExtension {+ return "video/media files are deliberately not downloaded by this connector"+ }+ if label := unsupportedDocumentExtensions[extension]; label != "" {+ return label + " has no ingest path in this connector yet"+ }+ return "no ingest path for this DingTalk node type in this connector yet"+}++// scopeLabel names a sync scope the way an operator can match it in DingTalk:+// the workspace followed by the node the scan starts from.+func scopeLabel(scope syncScope) string {+ nodeID := scope.StartNodeID+ if scope.Document != nil {+ nodeID = scope.Document.ID+ }+ if nodeID == "" {+ return scope.Reference.WorkspaceID+ }+ return scope.Reference.WorkspaceID + "/" + nodeID+}+var (_ datasource.Connector = (*Connector)(nil)_ datasource.FullSyncWithCursor = (*Connector)(nil)@@ -162,9 +210,27 @@ func (c *Connector) ListResources(if failure := failures[parentID]; failure != nil {return nil, failure}- if len(scopes) != 1 || scopes[0].Document != nil {- return nil, fmt.Errorf("%w: DingTalk resource %q is not an expandable folder",- datasource.ErrResourceNotFound, parentID)+ if len(scopes) != 1 {+ // One reference resolves to exactly one scope. Any other count means+ // there is nothing behind it that could be listed; the picker reads+ // an error as a failed expansion, so the honest empty listing is+ // returned instead.+ logger.Warnf(ctx,+ "[DingTalk] expand %s: resolved %d scopes, want one; reporting no children",+ parentID, len(scopes))+ return []types.Resource{}, nil+ }+ if scopes[0].Document != nil {+ // A document is a leaf: it genuinely has no children. Returning an+ // error here (the previous "is not an expandable folder") surfaces+ // in the picker as a failure toast for a selection that syncs fine,+ // so the empty listing is the correct answer. The picker is told+ // HasChildren=false for a document, which is what keeps it from+ // offering the expander in the first place.+ logger.Warnf(ctx,+ "[DingTalk] expand %s: the reference is a document, not a folder; reporting no children",+ parentID)+ return []types.Resource{}, nil}parentNodeID = scopes[0].StartNodeID}@@ -431,7 +497,7 @@ func (c *Connector) sync(oldRevisions = stored}}- documents, err := scanScope(ctx, api, scope)+ documents, skipped, err := scanScope(ctx, api, scope)if err != nil {if isContextError(err) {return nil, nil, err@@ -439,13 +505,26 @@ func (c *Connector) sync(// Never infer deletions from an incomplete tree. Other independent// selections may still complete, while this scope keeps its previous// cursor and is retried on the next run.+ logger.Warnf(ctx, "[DingTalk] scan scope %s failed, will retry next sync: %v",+ scopeLabel(scope), err)complete = falsenext.Resources[scope.ResourceID] = cloneRevisions(oldRevisions)items = append(items, failedResource(scope.ResourceID, err))continue}++ // Skipped nodes are deterministic: this connector has no ingest path for+ // them, so they are reported once and never retried — unlike a failed+ // read below, which must stay retryable.+ for _, node := range skipped {+ logger.Infof(ctx, "[DingTalk] skip node %s (name=%q type=%s category=%s extension=%s): %s",+ node.ID, node.title(), node.Type, node.Category, node.Extension, skipReason(node))+ }+newRevisions := make(map[string]string, len(documents))currentDocuments := make(map[string]struct{}, len(documents))+ synced := 0+ failed := 0for _, document := range documents {if document.ID == "" {@@ -465,10 +544,13 @@ func (c *Connector) sync(if isContextError(err) {return nil, nil, err}+ logger.Warnf(ctx,+ "[DingTalk] read document %s (name=%q extension=%s) failed, will retry next sync: %v",+ document.ID, document.title(), document.Extension, err)items = append(items, failedDocument(scope.ResourceID, scope.Reference.WorkspaceID, document, err,))- failedDocuments+++ failed++if existed {// Do not advance failed documents. The next incremental run// must retry them even if modifiedTime remains unchanged.@@ -480,6 +562,7 @@ func (c *Connector) sync(items = append(items, fetchedDocument(scope.ResourceID, scope.Reference.WorkspaceID, document, rendered,))+ synced++newRevisions[document.ID] = revision}@@ -492,6 +575,14 @@ func (c *Connector) sync(}}next.Resources[scope.ResourceID] = newRevisions++ // One line per scope, in the same shape as the IMA connector's. total+ // counts every file the scope listed — synced + still-unchanged ++ // failed + skipped — so a scope that quietly loses two thirds of its+ // nodes can no longer look like a clean run.+ logger.Infof(ctx, "[DingTalk] scope %s: total=%d synced=%d skipped=%d failed=%d",+ scopeLabel(scope), len(documents)+len(skipped), synced, len(skipped), failed)+ failedDocuments += failed}// Reconcile the union of all selections. Moving a document between two@@ -670,27 +761,34 @@ func childByID(children []node, nodeID string) (node, bool) {return node{}, false}-func scanScope(ctx context.Context, api dingTalkAPI, scope syncScope) ([]node, error) {+// scanScope lists every ingestible document in one scope, together with the+// nodes it saw but cannot ingest. The skipped nodes are returned rather than+// discarded so the caller can report them: a full sync must never look clean+// while silently dropping part of the tree.+func scanScope(ctx context.Context, api dingTalkAPI, scope syncScope) ([]node, []node, error) {if scope.Document != nil {- return []node{*scope.Document}, nil+ return []node{*scope.Document}, nil, nil}return scanWorkspace(ctx, api, scope.Reference.WorkspaceID, scope.StartNodeID)}+// scanWorkspace walks a workspace subtree breadth-first. Folders are traversal+// only; every file is either an ingestible document or a skip.func scanWorkspace(ctx context.Context,api dingTalkAPI,workspaceID string,rootNodeID string,-) ([]node, error) {+) ([]node, []node, error) {queue := []string{rootNodeID}visitedParents := make(map[string]struct{})seenNodes := make(map[string]struct{})var documents []node+ var skipped []nodefor len(queue) > 0 {if err := ctx.Err(); err != nil {- return nil, err+ return nil, nil, err}parentID := queue[0]queue = queue[1:]@@ -701,24 +799,30 @@ func scanWorkspace(children, err := api.listNodes(ctx, parentID)if err != nil {- return nil, err+ return nil, nil, err}for _, child := range children {if child.ID == "" {continue}if child.WorkspaceID != "" && child.WorkspaceID != workspaceID {- return nil, fmt.Errorf("DingTalk node %q belongs to a different workspace", child.ID)+ return nil, nil, fmt.Errorf("DingTalk node %q belongs to a different workspace", child.ID)}if _, seen := seenNodes[child.ID]; seen {continue}seenNodes[child.ID] = struct{}{}if len(seenNodes) > maxTraversalNodes {- return nil, fmt.Errorf("DingTalk workspace exceeds %d nodes", maxTraversalNodes)+ return nil, nil, fmt.Errorf("DingTalk workspace exceeds %d nodes", maxTraversalNodes)}- if child.isDocument() {+ switch {+ case child.isDocument():documents = append(documents, child)+ case child.isFolder():+ // Containers hold no content of their own, so they are never+ // reported as skipped.+ default:+ skipped = append(skipped, child)}if child.isFolder() || child.HasChildren {queue = append(queue, child.ID)@@ -728,7 +832,10 @@ func scanWorkspace(sort.SliceStable(documents, func(i, j int) bool {return documents[i].ID < documents[j].ID})- return documents, nil+ sort.SliceStable(skipped, func(i, j int) bool {+ return skipped[i].ID < skipped[j].ID+ })+ return documents, skipped, nil}func cloneRevisions(revisions map[string]string) map[string]string {diff --git a/internal/datasource/connector/dingtalk/connector_test.go b/internal/datasource/connector/dingtalk/connector_test.goindex 915d37191..cfd3e182d 100644--- a/internal/datasource/connector/dingtalk/connector_test.go+++ b/internal/datasource/connector/dingtalk/connector_test.go@@ -1,6 +1,7 @@package dingtalkimport (+ "bytes""context""encoding/json""errors"@@ -8,6 +9,7 @@ import ("testing""github.com/Tencent/WeKnora/internal/datasource"+ "github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/types")@@ -110,10 +112,14 @@ func TestConnectorListsWorkspacesAndFetchesNestedDocuments(t *testing.T) {if err != nil || len(documents) != 1 || documents[0].Type != "document" {t.Fatalf("folder children = %#v, %v; want one document", documents, err)}- if _, err := connector.ListResources(+ // A document is a leaf: expanding it is answered with no children rather+ // than an error, so a picker expansion cannot turn a selection that syncs+ // fine into a failure.+ leafChildren, err := connector.ListResources(context.Background(), testConfig(), documents[0].ExternalID,- ); !errors.Is(err, datasource.ErrResourceNotFound) {- t.Fatalf("expanding a document error = %v, want ErrResourceNotFound", err)+ )+ if err != nil || len(leafChildren) != 0 {+ t.Fatalf("expanding a document = %#v, %v; want no children", leafChildren, err)}ancestors, err := connector.ResolveResourceAncestors(context.Background(), testConfig(), []string{documents[0].ExternalID},@@ -441,3 +447,323 @@ func TestValidateProbesNodeAndDocumentAccess(t *testing.T) {t.Fatal("Validate() error = nil, want node access failure")}}++// Expanding a document is answered with no children rather than an error: a+// leaf genuinely has none, and the picker renders an error as a failed+// expansion — a toast on a step whose selection syncs perfectly well. The+// listing already reports HasChildren=false for a document, which keeps the+// picker from offering the expander; this is the answer for a client that+// expands anyway, or for a selection saved by a build that did offer it.+func TestListResourcesReportsNoChildrenForALeafDocument(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "team", RootNodeID: "team-root", Name: "Team"}},+ nodes: map[string][]node{+ "team-root": {{+ ID: "doc-1", WorkspaceID: "team", Name: "Plan.adoc",+ Type: "FILE", Category: "ALIDOC", Extension: "adoc",+ }},+ },+ }+ documentID, err := encodeResourceReference(resourceReference{+ WorkspaceID: "team", NodeID: "doc-1",+ })+ if err != nil {+ t.Fatal(err)+ }++ resources, err := testConnector(api).ListResources(context.Background(), testConfig(), documentID)+ if err != nil {+ t.Fatalf("ListResources(%q) error = %v, want an empty listing", documentID, err)+ }+ if len(resources) != 0 {+ t.Fatalf("ListResources(%q) = %#v, want no children", documentID, resources)+ }+}++// captureLogs redirects the process logger into a buffer for one test. The sync+// log is the surface a skipped node has to appear on, so it is asserted on+// directly rather than through an internal counter.+func captureLogs(t *testing.T) *bytes.Buffer {+ t.Helper()+ var buffer bytes.Buffer+ logger.SetOutput(&buffer)+ t.Cleanup(logger.ConfigureFromEnv)+ return &buffer+}++// skippedNodesFixture mixes supported documents with the node types this+// connector deliberately cannot ingest, spread over a folder tree: reporting+// must survive nesting and must not disturb what is fetched.+func skippedNodesFixture() *fakeAPI {+ return &fakeAPI{+ workspaces: []workspace{{ID: "space", RootNodeID: "root", Name: "Space"}},+ nodes: map[string][]node{+ "root": {+ {ID: "docs", WorkspaceID: "space", Name: "Docs", Type: "FOLDER"},+ {+ ID: "video-1", WorkspaceID: "space", Name: "Lesson.mp4", Type: "FILE",+ Category: "VIDEO", Extension: "mp4",+ },+ {+ ID: "table-1", WorkspaceID: "space", Name: "Roadmap.able", Type: "FILE",+ Category: "ALIDOC", Extension: "able",+ },+ },+ "docs": {+ {+ ID: "doc-1", WorkspaceID: "space", Name: "Runbook.adoc", Type: "FILE",+ Category: "ALIDOC", Extension: "adoc", ModifiedTime: "r1",+ },+ {+ ID: "doc-2", WorkspaceID: "space", Name: "Policy.adoc", Type: "FILE",+ Category: "ALIDOC", Extension: "adoc", ModifiedTime: "r2",+ },+ {+ ID: "mind-1", WorkspaceID: "space", Name: "Plan.amind", Type: "FILE",+ Category: "ALIDOC", Extension: "amind",+ },+ {ID: "empty", WorkspaceID: "space", Name: "Empty", Type: "FOLDER"},+ },+ "empty": nil,+ },+ blocks: map[string][]json.RawMessage{+ "doc-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"run"}}`)},+ "doc-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"policy"}}`)},+ },+ nodeErrors: make(map[string]error),+ blockErrors: make(map[string]error),+ }+}++// Seeing an unsupported node must not change what is fetched: the supported+// siblings still sync exactly as before, and nothing is requested for a skip.+func TestFetchAllKeepsSyncingSupportedSiblingsOfSkippedNodes(t *testing.T) {+ api := skippedNodesFixture()+ items, err := testConnector(api).FetchAll(+ context.Background(), testConfig("space"), []string{"space"},+ )+ if err != nil {+ t.Fatalf("FetchAll() error = %v", err)+ }+ if len(items) != 2 {+ t.Fatalf("FetchAll() returned %d items, want 2: %#v", len(items), items)+ }+ byID := make(map[string]types.FetchedItem, len(items))+ for _, item := range items {+ byID[item.ExternalID] = item+ }+ if len(byID) != 2 || byID["doc-1"].ContentType != "text/markdown" ||+ byID["doc-2"].ContentType != "text/markdown" {+ t.Fatalf("supported documents = %#v", byID)+ }+ for _, skipped := range []string{"video-1", "table-1", "mind-1"} {+ if _, exists := byID[skipped]; exists {+ t.Fatalf("unsupported node %q was synced: %#v", skipped, byID[skipped])+ }+ if api.blockCalls[skipped] != 0 {+ t.Fatalf("unsupported node %q was requested: %#v", skipped, api.blockCalls)+ }+ }+ if len(api.blockCalls) != 2 || api.blockCalls["doc-1"] != 1 || api.blockCalls["doc-2"] != 1 {+ t.Fatalf("block calls = %#v, want one per supported document", api.blockCalls)+ }+}++// Every node the connector drops must reach the sync log with its identity and+// the concrete reason it cannot be ingested: media is never downloaded on+// purpose, while a native DingTalk type simply has no ingest path yet.+func TestSyncLogsEverySkippedNodeWithItsReason(t *testing.T) {+ api := skippedNodesFixture()+ logs := captureLogs(t)+ if _, err := testConnector(api).FetchAll(+ context.Background(), testConfig("space"), []string{"space"},+ ); err != nil {+ t.Fatalf("FetchAll() error = %v", err)+ }++ output := logs.String()+ for _, want := range []string{+ `[DingTalk] skip node video-1 (name="Lesson.mp4" type=FILE category=VIDEO extension=mp4): ` ++ "video/media files are deliberately not downloaded by this connector",+ `[DingTalk] skip node table-1 (name="Roadmap.able" type=FILE category=ALIDOC extension=able): ` ++ "DingTalk multi-dimensional table has no ingest path in this connector yet",+ `[DingTalk] skip node mind-1 (name="Plan.amind" type=FILE category=ALIDOC extension=amind): ` ++ "DingTalk mind map has no ingest path in this connector yet",+ } {+ if !strings.Contains(output, want) {+ t.Fatalf("sync log is missing:\n%s\ngot:\n%s", want, output)+ }+ }+ for _, folder := range []string{"skip node docs", "skip node empty"} {+ if strings.Contains(output, folder) {+ t.Fatalf("folder reported as a skipped node: %q in\n%s", folder, output)+ }+ }+}++// The per-scope summary states how much of the scope arrived, in the same shape+// as the IMA connector's per-knowledge-base line.+func TestSyncLogsPerScopeSummary(t *testing.T) {+ api := skippedNodesFixture()+ logs := captureLogs(t)+ if _, err := testConnector(api).FetchAll(+ context.Background(), testConfig("space"), []string{"space"},+ ); err != nil {+ t.Fatalf("FetchAll() error = %v", err)+ }+ want := "[DingTalk] scope space/root: total=5 synced=2 skipped=3 failed=0"+ if !strings.Contains(logs.String(), want) {+ t.Fatalf("sync log is missing %q:\n%s", want, logs.String())+ }+}++// A transient read failure is counted as failed — not as skipped — is logged as+// retryable, and still leaves the document out of the cursor.+func TestSyncSummaryCountsTransientFailuresAndStillRetries(t *testing.T) {+ api := skippedNodesFixture()+ api.blockErrors = map[string]error{"doc-2": errors.New("storage temporarily unavailable")}+ logs := captureLogs(t)++ items, err := testConnector(api).FetchAll(+ context.Background(), testConfig("space"), []string{"space"},+ )+ var partial *datasource.PartialFetchError+ if !errors.As(err, &partial) {+ t.Fatalf("FetchAll() error = %v, want PartialFetchError", err)+ }+ if len(items) != 2 {+ t.Fatalf("FetchAll() items = %#v, want 1 document and 1 failure", items)+ }+ output := logs.String()+ if !strings.Contains(output, "storage temporarily unavailable") ||+ !strings.Contains(output, "failed, will retry next sync") {+ t.Fatalf("transient failure was not logged as retryable:\n%s", output)+ }+ want := "[DingTalk] scope space/root: total=5 synced=1 skipped=3 failed=1"+ if !strings.Contains(output, want) {+ t.Fatalf("sync log is missing %q:\n%s", want, output)+ }+}++// Containers are not content: a tree of folders alone must produce no skip+// report at all, only a zeroed summary.+func TestSyncLogsNoSkipsForFolderOnlyTree(t *testing.T) {+ api := &fakeAPI{+ workspaces: []workspace{{ID: "space", RootNodeID: "root"}},+ nodes: map[string][]node{+ "root": {{ID: "outer", WorkspaceID: "space", Name: "Outer", Type: "FOLDER"}},+ "outer": {{ID: "inner", WorkspaceID: "space", Name: "Inner", Type: "FOLDER"}},+ "inner": nil,+ },+ }+ logs := captureLogs(t)+ items, err := testConnector(api).FetchAll(+ context.Background(), testConfig("space"), []string{"space"},+ )+ if err != nil || len(items) != 0 {+ t.Fatalf("FetchAll() = %#v, %v; want no items", items, err)+ }+ output := logs.String()+ if strings.Contains(output, "skip node") {+ t.Fatalf("folder-only tree reported skips:\n%s", output)+ }+ want := "[DingTalk] scope space/root: total=0 synced=0 skipped=0 failed=0"+ if !strings.Contains(output, want) {+ t.Fatalf("sync log is missing %q:\n%s", want, output)+ }+}++// scanScope hands the caller the skipped nodes with their DingTalk identity+// intact, and a single-document scope has no tree to skip anything from.+func TestScanScopeReportsSkippedNodesAndSingleDocumentScopeHasNone(t *testing.T) {+ api := skippedNodesFixture()+ documents, skipped, err := scanScope(context.Background(), api, syncScope{+ ResourceID: "resource",+ Reference: resourceReference{WorkspaceID: "space"},+ StartNodeID: "root",+ })+ if err != nil {+ t.Fatalf("scanScope() error = %v", err)+ }+ if len(documents) != 2 {+ t.Fatalf("scanScope() documents = %#v, want 2", documents)+ }+ if len(skipped) != 3 {+ t.Fatalf("scanScope() skipped = %#v, want 3", skipped)+ }+ got := make(map[string]node, len(skipped))+ for _, item := range skipped {+ got[item.ID] = item+ }+ if got["video-1"].Category != "VIDEO" || got["video-1"].Extension != "mp4" ||+ got["video-1"].Name != "Lesson.mp4" ||+ got["table-1"].Category != "ALIDOC" || got["table-1"].Extension != "able" ||+ got["mind-1"].Extension != "amind" {+ t.Fatalf("skipped nodes lost their identity: %#v", got)+ }++ document := syncDocument()+ documents, skipped, err = scanScope(context.Background(), api, syncScope{+ ResourceID: "resource",+ Reference: resourceReference{WorkspaceID: "space"},+ Document: &document,+ })+ if err != nil || len(documents) != 1 || documents[0].ID != "doc" || len(skipped) != 0 {+ t.Fatalf("single-document scanScope() = %#v, %#v, %v", documents, skipped, err)+ }+}++// The reason has to say which kind of unsupported a node is: a video is skipped+// on purpose, a native type has simply not been implemented.+func TestSkipReasonDistinguishesMediaFromUnimplementedTypes(t *testing.T) {+ for _, testCase := range []struct {+ label string+ node node+ want string+ }{+ {+ "video category",+ node{Type: "FILE", Category: "VIDEO", Extension: "mp4"},+ "video/media files are deliberately not downloaded by this connector",+ },+ {+ "video extension without a category",+ node{Type: "FILE", Category: "OTHER", Extension: "MOV"},+ "video/media files are deliberately not downloaded by this connector",+ },+ {+ "audio extension",+ node{Type: "FILE", Category: "OTHER", Extension: "mp3"},+ "video/media files are deliberately not downloaded by this connector",+ },+ {+ // Only the types with a name worth printing are in the map; every+ // other unsupported node gets the generic reason. Kept as a+ // regression guard on the map.+ "type without a dedicated label",+ node{Type: "FILE", Category: "ALIDOC", Extension: "axls"},+ "no ingest path for this DingTalk node type in this connector yet",+ },+ {+ "multidimensional table",+ node{Type: "FILE", Category: "ALIDOC", Extension: "able"},+ "DingTalk multi-dimensional table has no ingest path in this connector yet",+ },+ {+ "mind map",+ node{Type: "FILE", Category: "ALIDOC", Extension: "amind"},+ "DingTalk mind map has no ingest path in this connector yet",+ },+ {+ "unknown type",+ node{Type: "FILE", Category: "OTHER", Extension: "bin"},+ "no ingest path for this DingTalk node type in this connector yet",+ },+ } {+ t.Run(testCase.label, func(t *testing.T) {+ if got := skipReason(testCase.node); got != testCase.want {+ t.Fatalf("skipReason() = %q, want %q", got, testCase.want)+ }+ })+ }+}