Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(cli): preserve directory structure in doc upload --recursive (#3753)
SIRIUS49e262147ca5
diff --git a/cli/cmd/doc/upload.go b/cli/cmd/doc/upload.goindex dc50fea46..f06347017 100644--- a/cli/cmd/doc/upload.go+++ b/cli/cmd/doc/upload.go@@ -83,7 +83,9 @@ has a generic name like "report.pdf" but you want to surface it as e.g.The two input modes (positional file / --recursive directory walk) aremutually exclusive - pass exactly one. Use --recursive --glob to upload a-directory tree (see Examples). To ingest a remote URL use "weknora doc fetch";+directory tree (see Examples); subdirectories are preserved as knowledge-base+folders, so a nested file lands at its path relative to the walked directory.+To ingest a remote URL use "weknora doc fetch";to create an entry from inline text use "weknora doc create".Server-side ingestion knobs:diff --git a/cli/cmd/doc/upload_recursive.go b/cli/cmd/doc/upload_recursive.goindex 111ba29fe..05062dcbc 100644--- a/cli/cmd/doc/upload_recursive.go+++ b/cli/cmd/doc/upload_recursive.go@@ -7,6 +7,7 @@ import ("io/fs""os""path/filepath"+ "strings""github.com/Tencent/WeKnora/cli/internal/cmdutil""github.com/Tencent/WeKnora/cli/internal/iostreams"@@ -84,7 +85,13 @@ func runUploadRecursive(ctx context.Context, opts *UploadOptions, fopts *cmdutilchannel := cmp.Or(opts.Channel, uploadChannel)outcomes, runErr := cmdutil.RunBatch(ctx, matches, func(ctx context.Context, p string) error {- k, err := svc.CreateKnowledgeFromFile(ctx, kbID, p, meta, opts.EnableMultimodel, "", channel, nil)+ // Path-qualified name so the server can rebuild the directory tree.+ // CreateKnowledgeFromFile splits this into folder_path + file_name+ // (types.SplitKnowledgeRelativePath), which is the same channel the web+ // UI's folder upload uses. Passing "" here flattened every file into the+ // KB root, losing the structure --recursive was asked to walk.+ customFileName := knowledgeRelativePath(dir, p)+ k, err := svc.CreateKnowledgeFromFile(ctx, kbID, p, meta, opts.EnableMultimodel, customFileName, channel, nil)if err != nil {// Per-file progress lines are human progress signal; suppress// under --format json so they don't precede the JSON object on stdout.@@ -147,6 +154,29 @@ func runUploadRecursive(ctx context.Context, opts *UploadOptions, fopts *cmdutilreturn nil}+// knowledgeRelativePath returns the path of file relative to root, using+// forward slashes so the server's SplitKnowledgeRelativePath can split it+// into folder_path + file_name.+//+// A file directly under root yields "" so the request stays byte-identical to+// the pre-folder behaviour for the common "upload a flat directory" case --+// only nested files gain a folder prefix. Returns "" if file is not under+// root (defensive: walkMatches only ever yields paths under root).+func knowledgeRelativePath(root, file string) string {+ rel, err := filepath.Rel(root, file)+ if err != nil || rel == "." || escapesRoot(rel) || filepath.Dir(rel) == "." {+ return ""+ }+ return filepath.ToSlash(rel)+}++// escapesRoot reports whether rel is the parent directory or a path under it.+// The check is component-wise, not a ".." prefix match: a legitimate child+// like "..drafts/report.pdf" starts with ".." but is still inside root.+func escapesRoot(rel string) bool {+ return rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator))+}+// walkMatches returns every regular file under root whose base name matches// pattern. Order is filepath.WalkDir's lexical order (stdlib guarantee on// every supported FS), which is deterministic for test assertions.diff --git a/cli/cmd/doc/upload_recursive_test.go b/cli/cmd/doc/upload_recursive_test.goindex d8cb44a7b..f24a040ea 100644--- a/cli/cmd/doc/upload_recursive_test.go+++ b/cli/cmd/doc/upload_recursive_test.go@@ -32,6 +32,9 @@ type scriptedUploadSvc struct {lastMetadata map[string]stringlastEnableMultimodel *boollastChannel string+ // customFileName is the path-qualified name the CLI passes so the server+ // can rebuild the directory tree. Recorded per-call, keyed by base name.+ customFileName map[string]string}func (s *scriptedUploadSvc) CreateKnowledgeFromFile(@@ -39,16 +42,21 @@ func (s *scriptedUploadSvc) CreateKnowledgeFromFile(_, filePath string,metadata map[string]string,enableMultimodel *bool,- _, channel string,+ customFileName, channel string,_ *sdk.KnowledgeProcessOverrides,) (*sdk.Knowledge, error) {- s.called = append(s.called, filepath.Base(filePath))+ base := filepath.Base(filePath)+ s.called = append(s.called, base)+ if s.customFileName == nil {+ s.customFileName = make(map[string]string)+ }+ s.customFileName[base] = customFileNames.lastMetadata = metadatas.lastEnableMultimodel = enableMultimodels.lastChannel = channel- r, ok := s.results[filepath.Base(filePath)]+ r, ok := s.results[base]if !ok {- return &sdk.Knowledge{ID: "doc_" + filepath.Base(filePath), FileName: filepath.Base(filePath)}, nil+ return &sdk.Knowledge{ID: "doc_" + base, FileName: base}, nil}return r.k, r.err}@@ -79,6 +87,39 @@ func TestUploadRecursive_WalksAllFiles(t *testing.T) {}}+// Files directly under the walked directory keep an empty customFileName so the+// request is byte-identical to the pre-folder behaviour; only nested files gain+// a folder prefix, which is what lets the server rebuild the tree.+func TestUploadRecursive_PreservesFolderStructure(t *testing.T) {+ _, _ = iostreams.SetForTest(t)+ dir := t.TempDir()+ mkTree(t, dir, "root.pdf", "sub/nested.pdf", "sub/deeper/leaf.pdf", "..drafts/report.pdf")++ svc := &scriptedUploadSvc{}+ opts := &UploadOptions{Recursive: true, Glob: "*"}+ fopts := &cmdutil.FormatOptions{Mode: cmdutil.FormatText}+ require.NoError(t, runUploadRecursive(context.Background(), opts, fopts, svc, "kb_xxx", dir))++ assert.Equal(t, "", svc.customFileName["root.pdf"])+ assert.Equal(t, "sub/nested.pdf", svc.customFileName["nested.pdf"])+ assert.Equal(t, "sub/deeper/leaf.pdf", svc.customFileName["leaf.pdf"])+ // A child directory whose name starts with ".." is still inside the walked+ // root and must keep its prefix.+ assert.Equal(t, "..drafts/report.pdf", svc.customFileName["report.pdf"])+}++// The prefix must be relative to the walked directory, not the process CWD or+// an absolute path -- an absolute path would leak the uploader's filesystem+// layout into the knowledge base folder tree.+func TestKnowledgeRelativePath(t *testing.T) {+ root := filepath.Join(t.TempDir(), "docs")+ // Files directly under root keep an empty name (server uses the base name).+ assert.Equal(t, "", knowledgeRelativePath(root, filepath.Join(root, "a.pdf")))+ assert.Equal(t, "sub/c.pdf", knowledgeRelativePath(root, filepath.Join(root, "sub", "c.pdf")))+ // Defensive: a path outside root yields no prefix rather than a "../" chain.+ assert.Equal(t, "", knowledgeRelativePath(root, filepath.Join(root, "..", "outside.pdf")))+}+func TestUploadRecursive_GlobFilter(t *testing.T) {_, _ = iostreams.SetForTest(t)dir := t.TempDir()