Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(cli): preserve directory structure in doc upload --recursive (#3753)

SIRIUS49e262147ca5
diff --git a/cli/cmd/doc/upload.go b/cli/cmd/doc/upload.go
index dc50fea46..f06347017 100644
--- a/cli/cmd/doc/upload.go
+++ b/cli/cmd/doc/upload.go
@@ -83,7 +83,9 @@ has a generic name like "report.pdf" but you want to surface it as e.g.
The two input modes (positional file / --recursive directory walk) are
mutually exclusive - pass exactly one. Use --recursive --glob to upload a
-directory tree (see Examples). To ingest a remote URL use "weknora doc fetch";
+directory tree (see Examples); subdirectories are preserved as knowledge-base
+folders, so a nested file lands at its path relative to the walked directory.
+To ingest a remote URL use "weknora doc fetch";
to create an entry from inline text use "weknora doc create".
Server-side ingestion knobs:
diff --git a/cli/cmd/doc/upload_recursive.go b/cli/cmd/doc/upload_recursive.go
index 111ba29fe..05062dcbc 100644
--- a/cli/cmd/doc/upload_recursive.go
+++ b/cli/cmd/doc/upload_recursive.go
@@ -7,6 +7,7 @@ import (
"io/fs"
"os"
"path/filepath"
+ "strings"
"github.com/Tencent/WeKnora/cli/internal/cmdutil"
"github.com/Tencent/WeKnora/cli/internal/iostreams"
@@ -84,7 +85,13 @@ func runUploadRecursive(ctx context.Context, opts *UploadOptions, fopts *cmdutil
channel := cmp.Or(opts.Channel, uploadChannel)
outcomes, runErr := cmdutil.RunBatch(ctx, matches, func(ctx context.Context, p string) error {
- k, err := svc.CreateKnowledgeFromFile(ctx, kbID, p, meta, opts.EnableMultimodel, "", channel, nil)
+ // Path-qualified name so the server can rebuild the directory tree.
+ // CreateKnowledgeFromFile splits this into folder_path + file_name
+ // (types.SplitKnowledgeRelativePath), which is the same channel the web
+ // UI's folder upload uses. Passing "" here flattened every file into the
+ // KB root, losing the structure --recursive was asked to walk.
+ customFileName := knowledgeRelativePath(dir, p)
+ k, err := svc.CreateKnowledgeFromFile(ctx, kbID, p, meta, opts.EnableMultimodel, customFileName, channel, nil)
if err != nil {
// Per-file progress lines are human progress signal; suppress
// under --format json so they don't precede the JSON object on stdout.
@@ -147,6 +154,29 @@ func runUploadRecursive(ctx context.Context, opts *UploadOptions, fopts *cmdutil
return nil
}
+// knowledgeRelativePath returns the path of file relative to root, using
+// forward slashes so the server's SplitKnowledgeRelativePath can split it
+// into folder_path + file_name.
+//
+// A file directly under root yields "" so the request stays byte-identical to
+// the pre-folder behaviour for the common "upload a flat directory" case --
+// only nested files gain a folder prefix. Returns "" if file is not under
+// root (defensive: walkMatches only ever yields paths under root).
+func knowledgeRelativePath(root, file string) string {
+ rel, err := filepath.Rel(root, file)
+ if err != nil || rel == "." || escapesRoot(rel) || filepath.Dir(rel) == "." {
+ return ""
+ }
+ return filepath.ToSlash(rel)
+}
+
+// escapesRoot reports whether rel is the parent directory or a path under it.
+// The check is component-wise, not a ".." prefix match: a legitimate child
+// like "..drafts/report.pdf" starts with ".." but is still inside root.
+func escapesRoot(rel string) bool {
+ return rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator))
+}
+
// walkMatches returns every regular file under root whose base name matches
// pattern. Order is filepath.WalkDir's lexical order (stdlib guarantee on
// every supported FS), which is deterministic for test assertions.
diff --git a/cli/cmd/doc/upload_recursive_test.go b/cli/cmd/doc/upload_recursive_test.go
index d8cb44a7b..f24a040ea 100644
--- a/cli/cmd/doc/upload_recursive_test.go
+++ b/cli/cmd/doc/upload_recursive_test.go
@@ -32,6 +32,9 @@ type scriptedUploadSvc struct {
lastMetadata map[string]string
lastEnableMultimodel *bool
lastChannel string
+ // customFileName is the path-qualified name the CLI passes so the server
+ // can rebuild the directory tree. Recorded per-call, keyed by base name.
+ customFileName map[string]string
}
func (s *scriptedUploadSvc) CreateKnowledgeFromFile(
@@ -39,16 +42,21 @@ func (s *scriptedUploadSvc) CreateKnowledgeFromFile(
_, filePath string,
metadata map[string]string,
enableMultimodel *bool,
- _, channel string,
+ customFileName, channel string,
_ *sdk.KnowledgeProcessOverrides,
) (*sdk.Knowledge, error) {
- s.called = append(s.called, filepath.Base(filePath))
+ base := filepath.Base(filePath)
+ s.called = append(s.called, base)
+ if s.customFileName == nil {
+ s.customFileName = make(map[string]string)
+ }
+ s.customFileName[base] = customFileName
s.lastMetadata = metadata
s.lastEnableMultimodel = enableMultimodel
s.lastChannel = channel
- r, ok := s.results[filepath.Base(filePath)]
+ r, ok := s.results[base]
if !ok {
- return &sdk.Knowledge{ID: "doc_" + filepath.Base(filePath), FileName: filepath.Base(filePath)}, nil
+ return &sdk.Knowledge{ID: "doc_" + base, FileName: base}, nil
}
return r.k, r.err
}
@@ -79,6 +87,39 @@ func TestUploadRecursive_WalksAllFiles(t *testing.T) {
}
}
+// Files directly under the walked directory keep an empty customFileName so the
+// request is byte-identical to the pre-folder behaviour; only nested files gain
+// a folder prefix, which is what lets the server rebuild the tree.
+func TestUploadRecursive_PreservesFolderStructure(t *testing.T) {
+ _, _ = iostreams.SetForTest(t)
+ dir := t.TempDir()
+ mkTree(t, dir, "root.pdf", "sub/nested.pdf", "sub/deeper/leaf.pdf", "..drafts/report.pdf")
+
+ svc := &scriptedUploadSvc{}
+ opts := &UploadOptions{Recursive: true, Glob: "*"}
+ fopts := &cmdutil.FormatOptions{Mode: cmdutil.FormatText}
+ require.NoError(t, runUploadRecursive(context.Background(), opts, fopts, svc, "kb_xxx", dir))
+
+ assert.Equal(t, "", svc.customFileName["root.pdf"])
+ assert.Equal(t, "sub/nested.pdf", svc.customFileName["nested.pdf"])
+ assert.Equal(t, "sub/deeper/leaf.pdf", svc.customFileName["leaf.pdf"])
+ // A child directory whose name starts with ".." is still inside the walked
+ // root and must keep its prefix.
+ assert.Equal(t, "..drafts/report.pdf", svc.customFileName["report.pdf"])
+}
+
+// The prefix must be relative to the walked directory, not the process CWD or
+// an absolute path -- an absolute path would leak the uploader's filesystem
+// layout into the knowledge base folder tree.
+func TestKnowledgeRelativePath(t *testing.T) {
+ root := filepath.Join(t.TempDir(), "docs")
+ // Files directly under root keep an empty name (server uses the base name).
+ assert.Equal(t, "", knowledgeRelativePath(root, filepath.Join(root, "a.pdf")))
+ assert.Equal(t, "sub/c.pdf", knowledgeRelativePath(root, filepath.Join(root, "sub", "c.pdf")))
+ // Defensive: a path outside root yields no prefix rather than a "../" chain.
+ assert.Equal(t, "", knowledgeRelativePath(root, filepath.Join(root, "..", "outside.pdf")))
+}
+
func TestUploadRecursive_GlobFilter(t *testing.T) {
_, _ = iostreams.SetForTest(t)
dir := t.TempDir()