Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(datasource): escape filenames in generated multipart uploads (#4647)

Sun Shihao68dabc25a9dd
diff --git a/internal/application/service/datasource_file_header_test.go b/internal/application/service/datasource_file_header_test.go
new file mode 100644
index 000000000..8df066628
--- /dev/null
+++ b/internal/application/service/datasource_file_header_test.go
@@ -0,0 +1,71 @@
+package service
+
+import (
+ "bytes"
+ "io"
+ "mime"
+ "testing"
+)
+
+func TestBytesToFileHeader(t *testing.T) {
+ content := []byte("# Quarterly report\n\n季度报告\n\x00\xff")
+ tests := []struct {
+ name string
+ filename string
+ wantFilename string
+ content []byte
+ }{
+ {name: "plain", filename: "report.md", wantFilename: "report.md", content: content},
+ {name: "unicode", filename: "季度报告.md", wantFilename: "季度报告.md", content: content},
+ {name: "quotes", filename: `Q4 "Final".md`, wantFilename: `Q4 "Final".md`, content: content},
+ {name: "semicolon", filename: "Q4; Final.md", wantFilename: "Q4; Final.md", content: content},
+ {
+ name: "quoted folder",
+ filename: `Reports "2026"/Q4 "Final".md`,
+ wantFilename: `Q4 "Final".md`,
+ content: content,
+ },
+ {name: "empty content", filename: `Q4 "Empty".md`, wantFilename: `Q4 "Empty".md`},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ file, err := bytesToFileHeader(tt.content, tt.filename)
+ if err != nil {
+ t.Fatalf("bytesToFileHeader(%q): %v", tt.filename, err)
+ }
+ if file.Filename != tt.wantFilename {
+ t.Errorf("filename = %q, want %q", file.Filename, tt.wantFilename)
+ }
+ if file.Size != int64(len(tt.content)) {
+ t.Errorf("size = %d, want %d", file.Size, len(tt.content))
+ }
+ if got := file.Header.Get("Content-Type"); got != "application/octet-stream" {
+ t.Errorf("content type = %q, want application/octet-stream", got)
+ }
+ mediaType, params, err := mime.ParseMediaType(file.Header.Get("Content-Disposition"))
+ if err != nil {
+ t.Fatalf("parse content disposition: %v", err)
+ }
+ if mediaType != "form-data" || params["name"] != "file" || params["filename"] != tt.filename {
+ t.Errorf("content disposition = %q, want file part for %q",
+ file.Header.Get("Content-Disposition"), tt.filename)
+ }
+ opened, err := file.Open()
+ if err != nil {
+ t.Fatalf("open file: %v", err)
+ }
+ t.Cleanup(func() {
+ if err := opened.Close(); err != nil {
+ t.Errorf("close file: %v", err)
+ }
+ })
+ got, err := io.ReadAll(opened)
+ if err != nil {
+ t.Fatalf("read file: %v", err)
+ }
+ if !bytes.Equal(got, tt.content) {
+ t.Errorf("content = %q, want %q", got, tt.content)
+ }
+ })
+ }
+}
diff --git a/internal/application/service/datasource_filename_ingest_test.go b/internal/application/service/datasource_filename_ingest_test.go
new file mode 100644
index 000000000..87f0ed8b9
--- /dev/null
+++ b/internal/application/service/datasource_filename_ingest_test.go
@@ -0,0 +1,82 @@
+package service
+
+import (
+ "bytes"
+ "context"
+ "io"
+ "testing"
+
+ filesvc "github.com/Tencent/WeKnora/internal/application/service/file"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestIngestItem_QuotedFilename(t *testing.T) {
+ // Notion page titles become filenames without replacing double quotes.
+ item := &types.FetchedItem{
+ ExternalID: "notion-page",
+ Title: `Q4 "Final"`,
+ FileName: `Q4 "Final".md`,
+ Content: []byte("# Q4 report\n\n季度报告\n"),
+ ContentType: "text/markdown",
+ }
+ ds := &types.DataSource{
+ ID: "ds-1",
+ TenantID: 1,
+ KnowledgeBaseID: "kb-1",
+ Type: types.ChannelNotion,
+ }
+ repo := &filenameKnowledgeRepo{}
+ fileSvc := filesvc.NewLocalFileService(t.TempDir(), "")
+ task := &createKnowledgeTaskEnqueuerStub{}
+ ks := &knowledgeService{
+ repo: repo,
+ kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ID: ds.KnowledgeBaseID}},
+ fileSvc: fileSvc,
+ task: task,
+ }
+ svc := &DataSourceService{knowledgeService: ks}
+ ctx := newCreateKnowledgeFileContext()
+ isUpdate, err := svc.ingestItem(ctx, ds, item, nil)
+ if err != nil {
+ t.Fatalf("ingestItem: %v", err)
+ }
+ if isUpdate {
+ t.Fatal("new page was reported as an update")
+ }
+ knowledge := repo.createdKnowledge
+ if knowledge == nil {
+ t.Fatal("knowledge was not created")
+ }
+ if knowledge.FileName != item.FileName || knowledge.Title != item.FileName {
+ t.Errorf("filename = %q, title = %q, want %q", knowledge.FileName, knowledge.Title, item.FileName)
+ }
+ if task.calls != 1 {
+ t.Errorf("processing tasks = %d, want 1", task.calls)
+ }
+ file, err := fileSvc.GetFile(ctx, knowledge.FilePath)
+ if err != nil {
+ t.Fatalf("open ingested file: %v", err)
+ }
+ t.Cleanup(func() {
+ if err := file.Close(); err != nil {
+ t.Errorf("close ingested file: %v", err)
+ }
+ })
+ got, err := io.ReadAll(file)
+ if err != nil {
+ t.Fatalf("read ingested file: %v", err)
+ }
+ if !bytes.Equal(got, item.Content) {
+ t.Errorf("ingested content = %q, want %q", got, item.Content)
+ }
+}
+
+type filenameKnowledgeRepo struct {
+ createKnowledgeFileRepoStub
+}
+
+func (*filenameKnowledgeRepo) FindByDataSourceExternalID(
+ context.Context, uint64, string, string, string,
+) (*types.Knowledge, error) {
+ return nil, nil
+}
diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go
index 4e2c0c11b..b8abb4865 100644
--- a/internal/application/service/datasource_service.go
+++ b/internal/application/service/datasource_service.go
@@ -7,7 +7,6 @@ import (
"errors"
"fmt"
"mime/multipart"
- "net/textproto"
"reflect"
"slices"
"strings"
@@ -1600,12 +1599,7 @@ func bytesToFileHeader(data []byte, filename string) (*multipart.FileHeader, err
var buf bytes.Buffer
writer := multipart.NewWriter(&buf)
- // Create a form file part
- partHeader := make(textproto.MIMEHeader)
- partHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename))
- partHeader.Set("Content-Type", "application/octet-stream")
-
- part, err := writer.CreatePart(partHeader)
+ part, err := writer.CreateFormFile("file", filename)
if err != nil {
return nil, fmt.Errorf("create multipart part: %w", err)
}