Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
feat(datasource/dingtalk): 保留来源目录结构(开关控制,默认关) (#4422)
linus-liu-webe849eaad5cc0
diff --git a/internal/datasource/connector/dingtalk/connector.go b/internal/datasource/connector/dingtalk/connector.goindex 4bb0a3d0f..cc29835e4 100644--- a/internal/datasource/connector/dingtalk/connector.go+++ b/internal/datasource/connector/dingtalk/connector.go@@ -647,6 +647,12 @@ type syncScope struct {Reference resourceReferenceStartNodeID stringDocument *node+ // FolderPath is the folder hierarchy between the workspace root and the+ // scope's own node, root-first, with the scope's own folder name last. It+ // anchors every document the scope yields, so selecting a subfolder and+ // selecting the whole workspace file the same document under the same+ // folder instead of under two.+ FolderPath []string}func (s syncScope) contains(candidate syncScope) bool {@@ -688,6 +694,9 @@ func (c *Connector) sync(api := c.api(cfg)settings := parseDocumentSettings(dataSourceConfig)+ if settings.PreserveFolderStructure {+ logger.Infof(ctx, "[DingTalk] source folder structure is encoded into document file names")+ }workspaces, err := api.listWorkspaces(ctx)if err != nil {return nil, nil, err@@ -696,6 +705,13 @@ func (c *Connector) sync(if err != nil {return nil, nil, err}+ // The workspace name is the first folder segment of every encoded path. It+ // is read from the listing the sync already made, so the leading level+ // costs no extra request.+ workspaceFolderNames := make(map[string]string, len(workspaces))+ for _, item := range workspaces {+ workspaceFolderNames[item.ID] = workspaceFolderName(item)+ }next := &cursorState{Version: cursorVersion,@@ -757,7 +773,8 @@ func (c *Connector) sync(synced := 0failed := 0- for _, document := range documents {+ for _, scanned := range documents {+ document := scanned.nodeif document.ID == "" {continue}@@ -770,7 +787,13 @@ func (c *Connector) sync(continue}- item, err := readDocument(ctx, api, scope.ResourceID, scope.Reference.WorkspaceID, document)+ folders := documentFolders(+ settings, workspaceFolderNames[scope.Reference.WorkspaceID],+ scope.FolderPath, scanned.folders,+ )+ item, err := readDocument(+ ctx, api, scope.ResourceID, scope.Reference.WorkspaceID, document, folders,+ )if err != nil {if isContextError(err) {return nil, nil, err@@ -894,6 +917,7 @@ func resolveSyncScopes(}, nil}+ var ancestorFolders []stringparentNodeID := rootNodeIDfor _, ancestorID := range ref.Ancestors {children, err := listChildren(parentNodeID)@@ -908,6 +932,11 @@ func resolveSyncScopes(if ancestor.WorkspaceID != "" && ancestor.WorkspaceID != ref.WorkspaceID {return syncScope{}, fmt.Errorf("DingTalk ancestor %q belongs to a different workspace", ancestorID)}+ // The name is read off a listing this resolution has to make+ // anyway, so anchoring the path at the workspace root costs no+ // extra request. A selection saved without ancestors yields a+ // shallower path rather than an error.+ ancestorFolders = append(ancestorFolders, ancestor.title())parentNodeID = ancestor.ID}children, err := listChildren(parentNodeID)@@ -926,11 +955,13 @@ func resolveSyncScopes(case selectedNode.isFolder():return syncScope{ResourceID: canonicalID, Reference: ref, StartNodeID: selectedNode.ID,+ FolderPath: appendFolder(ancestorFolders, selectedNode.title()),}, nilcase selectedNode.isIngestible(settings):document := selectedNodereturn syncScope{ResourceID: canonicalID, Reference: ref, Document: &document,+ FolderPath: ancestorFolders,}, nildefault:return syncScope{}, fmt.Errorf("DingTalk node %q is not a supported online document or folder",@@ -990,6 +1021,26 @@ func childByID(children []node, nodeID string) (node, bool) {return node{}, false}+// scannedDocument pairs an ingestible node with the folder hierarchy the walk+// found it under, relative to the scan's root node. The hierarchy is carried+// alongside the node rather than recomputed later: the breadth-first walk is+// the only place that knows which parent listed the node.+type scannedDocument struct {+ node+ // folders holds the folder titles between the scan root and the document,+ // root-first. A document sitting directly in the scan root has none.+ folders []string+}++// appendFolder returns parent plus one folder title. The result is always a+// fresh slice: appending in place would let one sibling's name overwrite+// another's inside a shared backing array.+func appendFolder(parent []string, name string) []string {+ folders := make([]string, 0, len(parent)+1)+ folders = append(folders, parent...)+ return append(folders, name)+}+// scanScope lists every ingestible document in one scope, together with the// nodes it saw but cannot ingest. The skipped nodes are returned rather than// discarded so the caller can report them: a full sync must never look clean@@ -999,26 +1050,34 @@ func scanScope(api dingTalkAPI,scope syncScope,settings documentSettings,-) ([]node, []node, error) {+) ([]scannedDocument, []node, error) {if scope.Document != nil {- return []node{*scope.Document}, nil, nil+ // A single-document scope has no tree to walk; the scope's own+ // FolderPath already carries the folders above it.+ return []scannedDocument{{node: *scope.Document}}, nil, nil}return scanWorkspace(ctx, api, scope.Reference.WorkspaceID, scope.StartNodeID, settings)}// scanWorkspace walks a workspace subtree breadth-first. Folders are traversal-// only; every file is either an ingestible document or a skip.+// only; every file is either an ingestible document or a skip. Each document is+// reported with the folder titles the walk descended through to reach it, so+// the caller can mirror the source hierarchy in the knowledge base.func scanWorkspace(ctx context.Context,api dingTalkAPI,workspaceID string,rootNodeID string,settings documentSettings,-) ([]node, []node, error) {+) ([]scannedDocument, []node, error) {queue := []string{rootNodeID}visitedParents := make(map[string]struct{})seenNodes := make(map[string]struct{})- var documents []node+ // foldersByParent maps a node ID to the folder titles between the scan root+ // and that node, root-first. Only nodes the walk descends through get an+ // entry; the scan root itself has none.+ foldersByParent := map[string][]string{rootNodeID: nil}+ var documents []scannedDocumentvar skipped []nodefor len(queue) > 0 {@@ -1052,7 +1111,10 @@ func scanWorkspace(}switch {case child.isIngestible(settings):- documents = append(documents, child)+ documents = append(documents, scannedDocument{+ node: child,+ folders: foldersByParent[parentID],+ })case child.isFolder():// Containers hold no content of their own, so they are never// reported as skipped.@@ -1060,6 +1122,7 @@ func scanWorkspace(skipped = append(skipped, child)}if child.isFolder() || child.HasChildren {+ foldersByParent[child.ID] = appendFolder(foldersByParent[parentID], child.title())queue = append(queue, child.ID)}}@@ -1141,13 +1204,16 @@ func warnUnknownBlockTypes(ctx context.Context, document node, rendered renderRe// readDocument turns one supported wiki node into a FetchedItem. Native adoc// documents are rendered from their blocks, and uploaded files are transferred-// byte-for-byte so WeKnora's own parsers can read them.+// byte-for-byte so WeKnora's own parsers can read them. folders is the source+// folder hierarchy to encode in the item's FileName, or nil to keep the+// historical flat name.func readDocument(ctx context.Context,api dingTalkAPI,sourceResourceID string,workspaceID string,document node,+ folders []string,) (types.FetchedItem, error) {switch {case document.isOnlineDocument():@@ -1160,14 +1226,71 @@ func readDocument(// both the request context and the node identity are known. Uploaded// files take the default branch and have no rendering step to lose.warnUnknownBlockTypes(ctx, document, rendered)- return fetchedDocument(sourceResourceID, workspaceID, document, rendered), nil+ return fetchedDocument(sourceResourceID, workspaceID, document, rendered, folders), nildefault:data, err := api.downloadDocument(ctx, document.ID)if err != nil {return types.FetchedItem{}, err}- return fetchedBinaryDocument(sourceResourceID, workspaceID, document, data), nil+ return fetchedBinaryDocument(sourceResourceID, workspaceID, document, data, folders), nil+ }+}++// documentFolders assembles the folder segments a document's FileName encodes:+// the workspace, the folders between the workspace root and the scope's own+// node, then the folders the scan descended through.+//+// It returns nil when the switch is off, which is what keeps the emitted+// FileName byte-for-byte identical to previous releases for every data source+// that never turned it on.+func documentFolders(+ settings documentSettings,+ workspaceName string,+ scopeFolders []string,+ scanFolders []string,+) []string {+ if !settings.PreserveFolderStructure {+ return nil}+ folders := make([]string, 0, len(scopeFolders)+len(scanFolders)+1)+ if workspaceName != "" {+ folders = append(folders, workspaceName)+ }+ folders = append(folders, scopeFolders...)+ return append(folders, scanFolders...)+}++// workspaceFolderName names a workspace the way the resource picker does: the+// workspace name, falling back to its ID, which is what an operator sees when a+// tenant exposes no name.+func workspaceFolderName(item workspace) string {+ if name := strings.TrimSpace(item.Name); name != "" {+ return name+ }+ return item.ID+}++// buildFolderFileName prepends the folder hierarchy to a document's file name.+//+// folder_path downstream is derived from FileName alone+// (SplitKnowledgeRelativePath → NormalizeKnowledgeFolderPath), so the hierarchy+// has to be encoded here as a relative path.+//+// Every level is sanitised on its own, as the Yuque connector's own folder+// encoding does, so a "/" inside a folder name cannot silently create an extra+// nesting level. Folder names the downstream normaliser rejects outright — ".",+// "..", an empty name — drop their level instead of failing the sync. An empty+// folder list, which is what the default settings produce, returns the file+// name unchanged.+func buildFolderFileName(folders []string, fileName string) string {+ if len(folders) == 0 {+ return fileName+ }+ segments := make([]string, 0, len(folders))+ for _, folder := range folders {+ segments = append(segments, datasource.SanitizeFileName(folder))+ }+ return strings.Join(segments, "/") + "/" + fileName}// documentMetadata describes a wiki node the same way for rendered and@@ -1195,6 +1318,7 @@ func fetchedDocument(workspaceID string,document node,rendered renderResult,+ folders []string,) types.FetchedItem {metadata := documentMetadata(workspaceID, document)if len(rendered.UnknownTypes) > 0 {@@ -1205,7 +1329,7 @@ func fetchedDocument(Title: document.title(),Content: []byte(rendered.Markdown),ContentType: "text/markdown",- FileName: renderedDocumentFileName(document.title()),+ FileName: buildFolderFileName(folders, renderedDocumentFileName(document.title())),URL: documentURL(document),UpdatedAt: document.modifiedAt(),Metadata: metadata,@@ -1234,6 +1358,7 @@ func fetchedBinaryDocument(workspaceID string,document node,data []byte,+ folders []string,) types.FetchedItem {contentType, ok := document.binaryContentType()if !ok {@@ -1244,7 +1369,7 @@ func fetchedBinaryDocument(Title: document.title(),Content: data,ContentType: contentType,- FileName: binaryDocumentFileName(document),+ FileName: buildFolderFileName(folders, binaryDocumentFileName(document)),URL: documentURL(document),UpdatedAt: document.modifiedAt(),Metadata: documentMetadata(workspaceID, document),diff --git a/internal/datasource/connector/dingtalk/folder_path_settings_test.go b/internal/datasource/connector/dingtalk/folder_path_settings_test.gonew file mode 100644index 000000000..90e037015--- /dev/null+++ b/internal/datasource/connector/dingtalk/folder_path_settings_test.go@@ -0,0 +1,198 @@+package dingtalk++import (+ "context"+ "encoding/json"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+)++// --- settings ---++// The folder switch is read through the same helper as the uploaded-files+// switch, and an absent, malformed or unexpected value has to leave the data+// source on the historical flat layout.+func TestParseDocumentSettingsReadsTheFolderSwitch(t *testing.T) {+ for _, testCase := range []struct {+ label string+ settings map[string]interface{}+ want bool+ }{+ {"absent key", map[string]interface{}{}, false},+ {"null settings entry", map[string]interface{}{"preserve_folder_structure": nil}, false},+ {"bool true", map[string]interface{}{"preserve_folder_structure": true}, true},+ {"bool false", map[string]interface{}{"preserve_folder_structure": false}, false},+ {"numeric string", map[string]interface{}{"preserve_folder_structure": "1"}, true},+ {"padded on", map[string]interface{}{"preserve_folder_structure": " on "}, true},+ {"padded OFF", map[string]interface{}{"preserve_folder_structure": " OFF "}, false},+ {"unknown spelling", map[string]interface{}{"preserve_folder_structure": "maybe"}, false},+ {"unexpected type", map[string]interface{}{"preserve_folder_structure": 1}, false},+ {+ "independent of the uploaded-files switch",+ map[string]interface{}{"include_uploaded_files": true},+ false,+ },+ } {+ t.Run(testCase.label, func(t *testing.T) {+ dataSourceConfig := testConfig("ws")+ dataSourceConfig.Settings = testCase.settings+ got := parseDocumentSettings(dataSourceConfig)+ if got.PreserveFolderStructure != testCase.want {+ t.Fatalf("PreserveFolderStructure = %t, want %t", got.PreserveFolderStructure, testCase.want)+ }+ if got.IncludeUploadedFiles != settingBool(testCase.settings["include_uploaded_files"], false) {+ t.Fatalf("IncludeUploadedFiles = %t, the two switches are not independent",+ got.IncludeUploadedFiles)+ }+ })+ }+ if got := parseDocumentSettings(nil); got.PreserveFolderStructure {+ t.Fatal("parseDocumentSettings(nil).PreserveFolderStructure = true, want false")+ }+}++// documentFolders is the single place the switch is honoured, so the nil it+// returns when the switch is off is what keeps an existing data source+// byte-for-byte unchanged.+func TestDocumentFoldersIsNilUnlessTheSwitchIsOn(t *testing.T) {+ scope := []string{"天诛渗透攻防"}+ scan := []string{"2024 年度"}+ off := documentFolders(documentSettings{}, "工作区", scope, scan)+ if off != nil {+ t.Fatalf("documentFolders() with the switch off = %#v, want nil", off)+ }+ on := documentFolders(+ documentSettings{PreserveFolderStructure: true}, "工作区", scope, scan,+ )+ want := []string{"工作区", "天诛渗透攻防", "2024 年度"}+ if strings.Join(on, "|") != strings.Join(want, "|") {+ t.Fatalf("documentFolders() = %#v, want %#v", on, want)+ }+ // A workspace with no name of its own contributes no segment rather than an+ // empty level.+ unnamed := documentFolders(documentSettings{PreserveFolderStructure: true}, "", scope, nil)+ if strings.Join(unnamed, "|") != "天诛渗透攻防" {+ t.Fatalf("documentFolders() without a workspace name = %#v", unnamed)+ }+}++// --- file name assembly ---++// The folder path a synced document ends up in is derived downstream from+// FileName alone, so the encoding is only correct if that split recovers the+// hierarchy.+func TestBuildFolderFileNameSurvivesTheDownstreamFolderSplit(t *testing.T) {+ for _, testCase := range []struct {+ label string+ folders []string+ fileName string+ wantFolder string+ wantName string+ }{+ {+ "no folders keeps the historical shape",+ nil, "FAQ.md", "", "FAQ.md",+ },+ {+ "workspace only",+ []string{"工作区"},+ "FAQ.md", "工作区", "FAQ.md",+ },+ {+ "workspace and nested folders",+ []string{"天穹安全能力中心", "天诛渗透攻防", "2024 年度"},+ "2024 年度复盘 FAQ.md",+ "天穹安全能力中心/天诛渗透攻防/2024 年度",+ "2024 年度复盘 FAQ.md",+ },+ {+ "a separator inside a name cannot add a level",+ []string{"安全/中心", "A/B"},+ "A_B 手册.md",+ "安全_中心/A_B",+ "A_B 手册.md",+ },+ {+ "a level the normaliser rejects is dropped, not fatal",+ []string{"工作区", ".", "子目录"},+ "FAQ.md",+ "工作区/子目录",+ "FAQ.md",+ },+ } {+ t.Run(testCase.label, func(t *testing.T) {+ raw := buildFolderFileName(testCase.folders, testCase.fileName)+ folder, name := types.SplitKnowledgeRelativePath(raw)+ if folder != testCase.wantFolder || name != testCase.wantName {+ t.Fatalf("split(%q) = (%q, %q), want (%q, %q)",+ raw, folder, name, testCase.wantFolder, testCase.wantName)+ }+ })+ }+}++func TestBuildFolderFileNameWithoutFoldersIsTheIdentity(t *testing.T) {+ for _, name := range []string{"FAQ.md", "手册.docx", "untitled"} {+ if got := buildFolderFileName(nil, name); got != name {+ t.Fatalf("buildFolderFileName(nil, %q) = %q, want it unchanged", name, got)+ }+ }+}++// Sibling folders are recorded while the walk visits them, so a name must not+// be written into a slice another sibling is still holding.+func TestAppendFolderNeverSharesABackingArray(t *testing.T) {+ parent := make([]string, 1, 8)+ parent[0] = "工作区"+ first := appendFolder(parent, "天诛渗透攻防")+ second := appendFolder(parent, "天禹漏洞情报")+ if first[len(first)-1] != "天诛渗透攻防" {+ t.Fatalf("first sibling = %#v, its name was overwritten", first)+ }+ if second[len(second)-1] != "天禹漏洞情报" {+ t.Fatalf("second sibling = %#v, its name was overwritten", second)+ }+ if len(parent) != 1 || parent[0] != "工作区" {+ t.Fatalf("appendFolder() mutated its input: %#v", parent)+ }+}++// --- cursor safety ---++// The folder dimension is an ingest-path concern only. If it reached the sync+// cursor, turning the switch on would change what the next incremental sync+// considers unchanged — and, worse, which documents it considers deleted.+func TestFolderPathsNeverReachTheSyncCursor(t *testing.T) {+ _, off, err := testConnector(tenantFolderPathFixture()).FetchIncremental(+ context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID), nil,+ )+ if err != nil {+ t.Fatalf("FetchIncremental() with the switch off error = %v", err)+ }+ _, on, err := testConnector(tenantFolderPathFixture()).FetchIncremental(+ context.Background(), tenantFolderPathTestConfig(tenantWorkspaceID), nil,+ )+ if err != nil {+ t.Fatalf("FetchIncremental() with the switch on error = %v", err)+ }+ encode := func(t *testing.T, cursor *types.SyncCursor) string {+ t.Helper()+ if cursor == nil {+ t.Fatal("FetchIncremental() returned no cursor")+ }+ raw, err := json.Marshal(cursor.ConnectorCursor["resources"])+ if err != nil {+ t.Fatal(err)+ }+ return string(raw)+ }+ offCursor, onCursor := encode(t, off), encode(t, on)+ if offCursor != onCursor {+ t.Fatalf("the cursor depends on the folder switch:\noff: %s\non: %s", offCursor, onCursor)+ }+ if strings.Contains(onCursor, tenantWorkspaceName) {+ t.Fatalf("the cursor carries folder names: %s", onCursor)+ }+}diff --git a/internal/datasource/connector/dingtalk/folder_path_test.go b/internal/datasource/connector/dingtalk/folder_path_test.gonew file mode 100644index 000000000..94374d2c5--- /dev/null+++ b/internal/datasource/connector/dingtalk/folder_path_test.go@@ -0,0 +1,435 @@+package dingtalk++import (+ "context"+ "encoding/json"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+)++// The fixture below reproduces the shape of the tenant that reported the+// problem: one workspace whose root holds five folders, a document that sits+// directly at the root, both formats inside the folders (native adoc FAQ pages+// and uploaded docx/pptx manuals), an empty folder, and one extra nesting level+// so the depth is not uniform. It goes through the public fetch entry points+// only, so every assertion describes what an operator sees in the knowledge+// base rather than how the connector is wired internally.+//+// Node names carry no file extension: DingTalk reports the extension in its own+// field, exactly as the rest of this package's fixtures do.+const (+ tenantWorkspaceID = "ws-tianqiong"+ tenantWorkspaceName = "天穹安全能力中心-天系列使用FAQ"+ tenantRootNodeID = "root-tianqiong"+ // tenantModifiedAt is the millisecond revision DingTalk reports for the+ // fixture's documents. It is what makes an incremental sync skip an+ // untouched document, so the relocation case below is about the file name+ // and not about a missing revision.+ tenantModifiedAt = int64(1760000000000)+)++// tenantUploadedFilesConfig is the state of the reported tenant before this+// change: uploaded manuals are ingested, the folder-path switch does not exist+// yet, so the settings bag has no key for it.+func tenantUploadedFilesConfig(resources ...string) *types.DataSourceConfig {+ cfg := testConfig(resources...)+ cfg.Settings = map[string]interface{}{"include_uploaded_files": true}+ return cfg+}++// tenantFolderPathTestConfig turns the folder-path switch on as well.+func tenantFolderPathTestConfig(resources ...string) *types.DataSourceConfig {+ cfg := tenantUploadedFilesConfig(resources...)+ cfg.Settings["preserve_folder_structure"] = true+ return cfg+}++// tenantFolderPathFixture builds the reported tenant tree.+func tenantFolderPathFixture() *fakeAPI {+ adoc := func(id, name string) node {+ return node{+ ID: id, WorkspaceID: tenantWorkspaceID, Type: "FILE", Category: "ALIDOC",+ Extension: "adoc", Name: name, ModifiedTimestamp: tenantModifiedAt,+ }+ }+ upload := func(id, name, extension string) node {+ return node{+ ID: id, WorkspaceID: tenantWorkspaceID, Type: "FILE", Category: "DOCUMENT",+ Extension: extension, Name: name, ModifiedTimestamp: tenantModifiedAt,+ }+ }+ folder := func(id, name string) node {+ return node{+ ID: id, WorkspaceID: tenantWorkspaceID, Type: "FOLDER", Name: name, HasChildren: true,+ }+ }+ return &fakeAPI{+ workspaces: []workspace{{+ ID: tenantWorkspaceID, RootNodeID: tenantRootNodeID, Name: tenantWorkspaceName,+ }},+ nodes: map[string][]node{+ tenantRootNodeID: {+ folder("f-easm", "EASM-外部攻击面管理模块"),+ adoc("d-cloud-faq", "天穹云端安全能力中枢常见问题 FAQ"),+ folder("f-tianxi", "天析应急分析"),+ folder("f-tianyu", "天禹漏洞情报"),+ folder("f-tianfu", "天伏攻防知识库"),+ folder("f-tianzhu", "天诛渗透攻防"),+ },+ "f-easm": {+ adoc("d-easm-faq", "EASM 模块使用 FAQ"),+ upload("d-easm-manual", "EASM 模块操作手册.docx", "docx"),+ },+ "f-tianxi": {+ adoc("d-tianxi-faq", "天析应急分析 FAQ"),+ upload("d-tianxi-deck", "天析应急分析介绍.pptx", "pptx"),+ },+ "f-tianyu": {+ upload("d-tianyu-manual", "天禹漏洞情报使用手册.docx", "docx"),+ upload("d-tianyu-deck", "天禹漏洞情报介绍.pptx", "pptx"),+ },+ // 天伏攻防知识库 is listed but holds nothing: an empty folder must+ // stay invisible in the knowledge base instead of producing a level+ // with no documents in it.+ "f-tianfu": {},+ "f-tianzhu": {+ upload("d-tianzhu-manual", "天诛渗透攻防手册.docx", "docx"),+ adoc("d-tianzhu-faq-1", "天诛渗透攻防 FAQ 一"),+ adoc("d-tianzhu-faq-2", "天诛渗透攻防 FAQ 二"),+ folder("f-tianzhu-2024", "2024 年度"),+ },+ "f-tianzhu-2024": {+ adoc("d-tianzhu-2024-faq", "2024 年度复盘 FAQ"),+ },+ },+ blocks: map[string][]json.RawMessage{+ "d-cloud-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"cloud"}}`)},+ "d-easm-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"easm"}}`)},+ "d-tianxi-faq": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"tianxi"}}`)},+ "d-tianzhu-faq-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"one"}}`)},+ "d-tianzhu-faq-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"two"}}`)},+ "d-tianzhu-2024-faq": {+ rawJSON(`{"blockType":"paragraph","paragraph":{"text":"review"}}`),+ },+ },+ downloads: map[string][]byte{+ "d-easm-manual": []byte("easm"),+ "d-tianxi-deck": []byte("tianxi"),+ "d-tianyu-manual": []byte("tianyu-manual"),+ "d-tianyu-deck": []byte("tianyu-deck"),+ "d-tianzhu-manual": []byte("tianzhu"),+ },+ nodeErrors: make(map[string]error),+ blockErrors: make(map[string]error),+ }+}++// fileNamesByDocument indexes the fetched items by node ID so an assertion can+// name the document it is talking about instead of an index.+func fileNamesByDocument(t *testing.T, items []types.FetchedItem) map[string]string {+ t.Helper()+ names := make(map[string]string, len(items))+ for _, item := range items {+ if _, seen := names[item.ExternalID]; seen {+ t.Fatalf("document %q was fetched more than once", item.ExternalID)+ }+ names[item.ExternalID] = item.FileName+ }+ return names+}++func assertFileNames(t *testing.T, got map[string]string, want map[string]string) {+ t.Helper()+ for id, wantName := range want {+ if got[id] != wantName {+ t.Errorf("document %s FileName = %q, want %q", id, got[id], wantName)+ }+ }+ if len(got) != len(want) {+ t.Fatalf("fetched %d documents, want %d: %#v", len(got), len(want), got)+ }+}++// Regression guard: a data source that never turned the switch on keeps the+// historical flat file names, byte for byte. Every document of the reported+// tenant is listed, in the same format it was ingested with before.+func TestDingTalkFolderPathsDefaultOffKeepsHistoricalFileNames(t *testing.T) {+ captureLogs(t)+ connector := testConnector(tenantFolderPathFixture())+ items, err := connector.FetchAll(+ context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID),+ []string{tenantWorkspaceID},+ )+ if err != nil {+ t.Fatalf("FetchAll() error = %v", err)+ }+ assertFileNames(t, fileNamesByDocument(t, items), map[string]string{+ "d-easm-faq": "EASM 模块使用 FAQ.md",+ "d-easm-manual": "EASM 模块操作手册.docx",+ "d-cloud-faq": "天穹云端安全能力中枢常见问题 FAQ.md",+ "d-tianxi-faq": "天析应急分析 FAQ.md",+ "d-tianxi-deck": "天析应急分析介绍.pptx",+ "d-tianyu-manual": "天禹漏洞情报使用手册.docx",+ "d-tianyu-deck": "天禹漏洞情报介绍.pptx",+ "d-tianzhu-manual": "天诛渗透攻防手册.docx",+ "d-tianzhu-faq-1": "天诛渗透攻防 FAQ 一.md",+ "d-tianzhu-faq-2": "天诛渗透攻防 FAQ 二.md",+ "d-tianzhu-2024-faq": "2024 年度复盘 FAQ.md",+ })+}++// The reported problem, end to end: with the switch on, every document encodes+// its source folder hierarchy relative to the workspace root, so the five+// top-level folders stay distinguishable in the knowledge base. The workspace+// name is the first segment, and a document sitting at the root carries no+// folder segment beyond it.+func TestDingTalkFolderPathsPreserveTheTenantShapeWhenEnabled(t *testing.T) {+ captureLogs(t)+ connector := testConnector(tenantFolderPathFixture())+ items, err := connector.FetchAll(+ context.Background(),+ tenantFolderPathTestConfig(tenantWorkspaceID),+ []string{tenantWorkspaceID},+ )+ if err != nil {+ t.Fatalf("FetchAll() error = %v", err)+ }+ root := tenantWorkspaceName + "/"+ assertFileNames(t, fileNamesByDocument(t, items), map[string]string{+ "d-easm-faq": root + "EASM-外部攻击面管理模块/EASM 模块使用 FAQ.md",+ "d-easm-manual": root + "EASM-外部攻击面管理模块/EASM 模块操作手册.docx",+ "d-cloud-faq": root + "天穹云端安全能力中枢常见问题 FAQ.md",+ "d-tianxi-faq": root + "天析应急分析/天析应急分析 FAQ.md",+ "d-tianxi-deck": root + "天析应急分析/天析应急分析介绍.pptx",+ "d-tianyu-manual": root + "天禹漏洞情报/天禹漏洞情报使用手册.docx",+ "d-tianyu-deck": root + "天禹漏洞情报/天禹漏洞情报介绍.pptx",+ "d-tianzhu-manual": root + "天诛渗透攻防/天诛渗透攻防手册.docx",+ "d-tianzhu-faq-1": root + "天诛渗透攻防/天诛渗透攻防 FAQ 一.md",+ "d-tianzhu-faq-2": root + "天诛渗透攻防/天诛渗透攻防 FAQ 二.md",+ "d-tianzhu-2024-faq": root + "天诛渗透攻防/2024 年度/2024 年度复盘 FAQ.md",+ })+}++// The switch has to read the same string spellings the connector's other switch+// reads, and anything unrecognised — including an absent key — has to leave the+// historical behaviour alone.+func TestDingTalkFolderPathsReadsTheSameSettingsSpellings(t *testing.T) {+ captureLogs(t)+ for _, testCase := range []struct {+ label string+ settings map[string]interface{}+ want string+ }{+ {+ "key absent",+ map[string]interface{}{"include_uploaded_files": true},+ "天诛渗透攻防 FAQ 一.md",+ },+ {+ "bool true",+ map[string]interface{}{"preserve_folder_structure": true},+ tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md",+ },+ {+ "string yes",+ map[string]interface{}{"preserve_folder_structure": "yes"},+ tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md",+ },+ {+ "string ON with padding",+ map[string]interface{}{"preserve_folder_structure": " ON "},+ tenantWorkspaceName + "/天诛渗透攻防/天诛渗透攻防 FAQ 一.md",+ },+ {+ "bool false",+ map[string]interface{}{"preserve_folder_structure": false},+ "天诛渗透攻防 FAQ 一.md",+ },+ {+ "unknown spelling",+ map[string]interface{}{"preserve_folder_structure": "sometimes"},+ "天诛渗透攻防 FAQ 一.md",+ },+ {+ "unexpected type",+ map[string]interface{}{"preserve_folder_structure": 42},+ "天诛渗透攻防 FAQ 一.md",+ },+ } {+ t.Run(testCase.label, func(t *testing.T) {+ cfg := tenantUploadedFilesConfig(tenantWorkspaceID)+ cfg.Settings = testCase.settings+ connector := testConnector(tenantFolderPathFixture())+ items, err := connector.FetchAll(+ context.Background(), cfg, []string{tenantWorkspaceID},+ )+ if err != nil {+ t.Fatalf("FetchAll() error = %v", err)+ }+ names := fileNamesByDocument(t, items)+ if names["d-tianzhu-faq-1"] != testCase.want {+ t.Fatalf("FileName = %q, want %q", names["d-tianzhu-faq-1"], testCase.want)+ }+ })+ }+}++// Selecting a folder is the common case, and the path a document lands on must+// not depend on how deep the operator drilled into the picker: the same+// document selected through the whole workspace, through its own folder and as+// a single document has to end up in one folder, not three.+func TestDingTalkFolderPathsDoNotDependOnTheSelectedScope(t *testing.T) {+ captureLogs(t)+ connector := testConnector(tenantFolderPathFixture())+ wholeWorkspace, err := connector.FetchAll(+ context.Background(),+ tenantFolderPathTestConfig(tenantWorkspaceID),+ []string{tenantWorkspaceID},+ )+ if err != nil {+ t.Fatalf("workspace scope error = %v", err)+ }+ folderID, err := encodeResourceReference(resourceReference{+ WorkspaceID: tenantWorkspaceID, NodeID: "f-tianzhu-2024", Ancestors: []string{"f-tianzhu"},+ })+ if err != nil {+ t.Fatal(err)+ }+ nestedFolder, err := connector.FetchAll(+ context.Background(),+ tenantFolderPathTestConfig(folderID),+ []string{folderID},+ )+ if err != nil {+ t.Fatalf("folder scope error = %v", err)+ }+ documentID, err := encodeResourceReference(resourceReference{+ WorkspaceID: tenantWorkspaceID, NodeID: "d-tianzhu-2024-faq",+ Ancestors: []string{"f-tianzhu", "f-tianzhu-2024"},+ })+ if err != nil {+ t.Fatal(err)+ }+ singleDocument, err := connector.FetchAll(+ context.Background(),+ tenantFolderPathTestConfig(documentID),+ []string{documentID},+ )+ if err != nil {+ t.Fatalf("document scope error = %v", err)+ }++ want := tenantWorkspaceName + "/天诛渗透攻防/2024 年度/2024 年度复盘 FAQ.md"+ for label, items := range map[string][]types.FetchedItem{+ "workspace": wholeWorkspace,+ "folder": nestedFolder,+ "document": singleDocument,+ } {+ if got := fileNamesByDocument(t, items)["d-tianzhu-2024-faq"]; got != want {+ t.Errorf("%s scope FileName = %q, want %q", label, got, want)+ }+ }+}++// Names are hostile but not tidy: a folder name may contain the path separator,+// punctuation DingTalk allows and file systems do not, and two sibling folders+// may share a name. Each level is sanitised on its own so a "/" inside one name+// cannot invent an extra level, and same-named siblings merge into one folder —+// the same trade-off the Yuque connector documents for same-named groups.+func TestDingTalkFolderPathsSanitizeEachLevelAndMergeSameNamedFolders(t *testing.T) {+ captureLogs(t)+ const workspaceName = "安全/中心"+ adoc := func(id, name string) node {+ return node{+ ID: id, WorkspaceID: "ws-odd", Type: "FILE", Category: "ALIDOC",+ Extension: "adoc", Name: name,+ }+ }+ folder := func(id, name string) node {+ return node{+ ID: id, WorkspaceID: "ws-odd", Type: "FOLDER", Name: name, HasChildren: true,+ }+ }+ api := &fakeAPI{+ workspaces: []workspace{{ID: "ws-odd", RootNodeID: "root-odd", Name: workspaceName}},+ nodes: map[string][]node{+ "root-odd": {+ folder("f-slash", "A/B"),+ folder("f-punct", `<>:"|?*`),+ // Two folders with the same title: DingTalk node IDs are+ // unique, titles are not.+ folder("f-dup-1", "重复"),+ folder("f-dup-2", "重复"),+ // A name of only whitespace falls back to the node ID, which+ // is what the resource picker shows for it too.+ folder("f-unnamed", " "),+ },+ "f-slash": {adoc("d-slash", "A/B 手册")},+ "f-punct": {adoc("d-punct", "符号")},+ "f-dup-1": {adoc("d-dup-1", "第一份")},+ "f-dup-2": {adoc("d-dup-2", "第二份")},+ "f-unnamed": {adoc("d-unnamed", "无名目录")},+ },+ blocks: map[string][]json.RawMessage{+ "d-slash": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"a"}}`)},+ "d-punct": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"b"}}`)},+ "d-dup-1": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"c"}}`)},+ "d-dup-2": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"d"}}`)},+ "d-unnamed": {rawJSON(`{"blockType":"paragraph","paragraph":{"text":"e"}}`)},+ },+ nodeErrors: make(map[string]error),+ blockErrors: make(map[string]error),+ }+ connector := testConnector(api)+ items, err := connector.FetchAll(+ context.Background(), tenantFolderPathTestConfig("ws-odd"), []string{"ws-odd"},+ )+ if err != nil {+ t.Fatalf("FetchAll() error = %v", err)+ }+ // "安全/中心" cannot invent a level either, and neither can "A/B" or the+ // document name: the file name keeps the sanitised form it already had.+ assertFileNames(t, fileNamesByDocument(t, items), map[string]string{+ "d-slash": "安全_中心/A_B/A_B 手册.md",+ "d-punct": "安全_中心/_______/符号.md",+ "d-dup-1": "安全_中心/重复/第一份.md",+ "d-dup-2": "安全_中心/重复/第二份.md",+ "d-unnamed": "安全_中心/f-unnamed/无名目录.md",+ })+}++// Turning the switch on in an existing data source is an ingest-path change,+// not a migration: an incremental sync compares content revisions, so the+// documents already in the knowledge base are not re-read and therefore keep+// the flat file names they were ingested with — and, above all, the folder+// dimension must never make the connector report them as deleted.+func TestDingTalkEnablingFolderPathsDoesNotRelocateOrDeleteExistingDocuments(t *testing.T) {+ captureLogs(t)+ connector := testConnector(tenantFolderPathFixture())+ items, cursor, err := connector.FetchIncremental(+ context.Background(), tenantUploadedFilesConfig(tenantWorkspaceID), nil,+ )+ if err != nil {+ t.Fatalf("first FetchIncremental() error = %v", err)+ }+ if names := fileNamesByDocument(t, items); names["d-tianzhu-faq-1"] != "天诛渗透攻防 FAQ 一.md" {+ t.Fatalf("first sync FileName = %q, want the flat name", names["d-tianzhu-faq-1"])+ }++ items, _, err = connector.FetchIncremental(+ context.Background(), tenantFolderPathTestConfig(tenantWorkspaceID), cursor,+ )+ if err != nil {+ t.Fatalf("second FetchIncremental() error = %v", err)+ }+ for _, item := range items {+ if item.IsDeleted {+ t.Fatalf("enabling folder paths reported document %q as deleted", item.ExternalID)+ }+ }+ if len(items) != 0 {+ t.Fatalf("second sync returned %d items, want none: unchanged documents are not "++ "re-read, so they keep the file name they were ingested with", len(items))+ }+}diff --git a/internal/datasource/connector/dingtalk/settings.go b/internal/datasource/connector/dingtalk/settings.goindex 9f224e674..3a5306592 100644--- a/internal/datasource/connector/dingtalk/settings.go+++ b/internal/datasource/connector/dingtalk/settings.go@@ -37,6 +37,14 @@ type documentSettings struct {// IncludeUploadedFiles admits uploaded Office/PDF files (FILE/DOCUMENT// nodes) that are downloaded from 钉盘 and parsed by WeKnora itself.IncludeUploadedFiles bool+ // PreserveFolderStructure encodes each document's DingTalk folder hierarchy+ // into its FileName, so the document lands in the knowledge base folder+ // that mirrors where it lives in the wiki instead of at the root.+ //+ // It is an ingest-path switch, not a migration. An already-ingested+ // document is only re-read when its content changes, so turning the switch+ // on does not relocate it; a full sync does.+ PreserveFolderStructure bool}// parseDocumentSettings reads the settings bag of a data source. A missing or@@ -48,6 +56,7 @@ func parseDocumentSettings(dataSourceConfig *types.DataSourceConfig) documentSetraw = dataSourceConfig.Settings}return documentSettings{- IncludeUploadedFiles: settingBool(raw["include_uploaded_files"], false),+ IncludeUploadedFiles: settingBool(raw["include_uploaded_files"], false),+ PreserveFolderStructure: settingBool(raw["preserve_folder_structure"], false),}}diff --git a/website-docs/03-features/10-datasource.md b/website-docs/03-features/10-datasource.mdindex 66276dd38..8d90188ba 100644--- a/website-docs/03-features/10-datasource.md+++ b/website-docs/03-features/10-datasource.md@@ -233,6 +233,12 @@- **上传文件(默认关闭)**:`Settings.include_uploaded_files`(默认 `false`)开启后,所选范围内钉盘中的普通上传附件(`FILE/DOCUMENT` 的 `docx` / `pptx` / `xlsx` / `pdf`)会被下载并交给 WeKnora 自带解析器入库,文件名保留原扩展名、按扩展名选择解析器;单个文件上限 64 MiB,超出上限即失败且不重试(重签名只会拿到同样的字节)。下载失败(401/403/404 等)会重新申请签名后重试,最多 3 次。- **开关的默认值与成本**:`include_uploaded_files` 未设置或非法值一律按 `false` 处理,因此**存量数据源升级后的下一次同步(含增量同步)行为不变**,不会突然开始下载和向量化附件。打开后,这些文件的正文会进入知识库,占用存储并产生 embedding 成本;不需要时请保持关闭。关闭时连接校验、资源列表和同步都会跳过上传附件,同步日志给出的原因是 `include_uploaded_files` 未启用,而不是类型不支持。- **开关的作用点**:测试连接(校验)、资源选择列表、全量同步与增量同步共用同一个判据,一处关闭即处处一致。+- **目录结构(默认关闭)**:钉钉知识库本身就是目录树,但连接器历来把文档平铺到知识库根目录,`folder_path` 为空,不同来源目录的文档在知识库里无法区分。`Settings.preserve_folder_structure`(默认 `false`)开启后,`FileName` 编码为 `<工作区名>/<子目录>/…/<标题>.ext`,由下游 `SplitKnowledgeRelativePath` → `NormalizeKnowledgeFolderPath` 拆出 `folder_path`——与语雀 `folder_mode=toc`、飞书云盘目录映射是同一套机制。**工作区名是第一段**:一个知识库可以同时链接多个钉钉数据源,没有这一段,不同工作区里同名的顶层目录会静默合并成一个(语雀用书名作第一段是同样的理由);工作区没有名字时退回用它的 ID。**路径始终以工作区为根**:选整个工作区、选某个子目录、选单篇文档,同一篇文档都落到同一个目录,不会出现两份。每一层目录名单独清洗(名字里的 `/`、`\`、`:` 等替换为 `_`),因此目录名里的斜杠不会凭空多出一层;同名子目录合并为同一个知识库目录(钉钉节点 ID 唯一、标题不唯一),`.`、`..` 这类非法层级由下游归一化丢弃而不是让同步失败。开启后目录由连接器托管:在知识库里手动调整过的目录,会在该文档下次同步时被覆盖回钉钉的结构。+- **目录开关的默认值与作用范围**:`preserve_folder_structure` 未设置或非法值一律按 `false` 处理,因此**存量数据源升级后 `FileName` 逐字节不变、`folder_path` 仍为空**。它只决定**入库路径**,不决定哪些文档入库:测试连接与资源选择列表不受它影响,关掉它也不会让任何文档消失。目录维度不写入同步游标——开关开与关时游标里的"资源 → 文档版本"表完全一致,所以目录永远不会让文档被判定为"源端已删除"。+- **目录相关注意 / 限制**:+ - **开启开关不会自动归位已导入的文档**:增量同步按 `modifiedTimestamp` 判断变化,只重新读取内容有变化的文档,因此**已经在知识库里的文档会保持原来的扁平文件名**——它们并没有被重新读取。只有内容变更过的文档、或手动触发一次全量同步之后,才会整体移到新目录。这与语雀修改 `folder_mode` 后的行为一致。+ - **`include_uploaded_files` 关闭时**,上传附件(`docx` / `pptx` / `xlsx` / `pdf`)不入库,其所在目录自然也不会出现;在线 `adoc` 文档不受该开关影响。+ - **目录深度与长度由下游统一裁剪**:`folder_path` 最多 16 层、单个目录名最多 128 字节、整条路径最多 1024 字节,超出部分被截断(与文件夹上传同一套规则)。- **同步**:按文档 `modifiedTimestamp`(毫秒)增量读取,缺失时回退 `modifiedTime`;合并重叠选择。全量同步也会对照上次游标对账删除,避免 `sync_mode=full` 漏删。目录遍历不完整时暂缓删除。- **正文**:公开 Blocks API 只返回文档根下的一级块;高亮块等容器若响应里带有 `children` 会继续渲染,否则在元数据中标记 `nested_blocks_unavailable`,避免把残缺正文当成完整成功。- **校验**:测试连接会列出知识库、探测根节点列表,并在根下存在在线文档时试读 Blocks;对已开启摄取的上传文件只解析存储位置并申请下载 URL,不下载正文,以便尽早发现缺少 `Wiki.Node.Read` / `Storage.File.Read`,同时让校验对多兆字节文件保持廉价。