Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(datasource): escape filenames in generated multipart uploads (#4647)
Sun Shihao68dabc25a9dd
diff --git a/internal/application/service/datasource_file_header_test.go b/internal/application/service/datasource_file_header_test.gonew file mode 100644index 000000000..8df066628--- /dev/null+++ b/internal/application/service/datasource_file_header_test.go@@ -0,0 +1,71 @@+package service++import (+ "bytes"+ "io"+ "mime"+ "testing"+)++func TestBytesToFileHeader(t *testing.T) {+ content := []byte("# Quarterly report\n\n季度报告\n\x00\xff")+ tests := []struct {+ name string+ filename string+ wantFilename string+ content []byte+ }{+ {name: "plain", filename: "report.md", wantFilename: "report.md", content: content},+ {name: "unicode", filename: "季度报告.md", wantFilename: "季度报告.md", content: content},+ {name: "quotes", filename: `Q4 "Final".md`, wantFilename: `Q4 "Final".md`, content: content},+ {name: "semicolon", filename: "Q4; Final.md", wantFilename: "Q4; Final.md", content: content},+ {+ name: "quoted folder",+ filename: `Reports "2026"/Q4 "Final".md`,+ wantFilename: `Q4 "Final".md`,+ content: content,+ },+ {name: "empty content", filename: `Q4 "Empty".md`, wantFilename: `Q4 "Empty".md`},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ file, err := bytesToFileHeader(tt.content, tt.filename)+ if err != nil {+ t.Fatalf("bytesToFileHeader(%q): %v", tt.filename, err)+ }+ if file.Filename != tt.wantFilename {+ t.Errorf("filename = %q, want %q", file.Filename, tt.wantFilename)+ }+ if file.Size != int64(len(tt.content)) {+ t.Errorf("size = %d, want %d", file.Size, len(tt.content))+ }+ if got := file.Header.Get("Content-Type"); got != "application/octet-stream" {+ t.Errorf("content type = %q, want application/octet-stream", got)+ }+ mediaType, params, err := mime.ParseMediaType(file.Header.Get("Content-Disposition"))+ if err != nil {+ t.Fatalf("parse content disposition: %v", err)+ }+ if mediaType != "form-data" || params["name"] != "file" || params["filename"] != tt.filename {+ t.Errorf("content disposition = %q, want file part for %q",+ file.Header.Get("Content-Disposition"), tt.filename)+ }+ opened, err := file.Open()+ if err != nil {+ t.Fatalf("open file: %v", err)+ }+ t.Cleanup(func() {+ if err := opened.Close(); err != nil {+ t.Errorf("close file: %v", err)+ }+ })+ got, err := io.ReadAll(opened)+ if err != nil {+ t.Fatalf("read file: %v", err)+ }+ if !bytes.Equal(got, tt.content) {+ t.Errorf("content = %q, want %q", got, tt.content)+ }+ })+ }+}diff --git a/internal/application/service/datasource_filename_ingest_test.go b/internal/application/service/datasource_filename_ingest_test.gonew file mode 100644index 000000000..87f0ed8b9--- /dev/null+++ b/internal/application/service/datasource_filename_ingest_test.go@@ -0,0 +1,82 @@+package service++import (+ "bytes"+ "context"+ "io"+ "testing"++ filesvc "github.com/Tencent/WeKnora/internal/application/service/file"+ "github.com/Tencent/WeKnora/internal/types"+)++func TestIngestItem_QuotedFilename(t *testing.T) {+ // Notion page titles become filenames without replacing double quotes.+ item := &types.FetchedItem{+ ExternalID: "notion-page",+ Title: `Q4 "Final"`,+ FileName: `Q4 "Final".md`,+ Content: []byte("# Q4 report\n\n季度报告\n"),+ ContentType: "text/markdown",+ }+ ds := &types.DataSource{+ ID: "ds-1",+ TenantID: 1,+ KnowledgeBaseID: "kb-1",+ Type: types.ChannelNotion,+ }+ repo := &filenameKnowledgeRepo{}+ fileSvc := filesvc.NewLocalFileService(t.TempDir(), "")+ task := &createKnowledgeTaskEnqueuerStub{}+ ks := &knowledgeService{+ repo: repo,+ kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ID: ds.KnowledgeBaseID}},+ fileSvc: fileSvc,+ task: task,+ }+ svc := &DataSourceService{knowledgeService: ks}+ ctx := newCreateKnowledgeFileContext()+ isUpdate, err := svc.ingestItem(ctx, ds, item, nil)+ if err != nil {+ t.Fatalf("ingestItem: %v", err)+ }+ if isUpdate {+ t.Fatal("new page was reported as an update")+ }+ knowledge := repo.createdKnowledge+ if knowledge == nil {+ t.Fatal("knowledge was not created")+ }+ if knowledge.FileName != item.FileName || knowledge.Title != item.FileName {+ t.Errorf("filename = %q, title = %q, want %q", knowledge.FileName, knowledge.Title, item.FileName)+ }+ if task.calls != 1 {+ t.Errorf("processing tasks = %d, want 1", task.calls)+ }+ file, err := fileSvc.GetFile(ctx, knowledge.FilePath)+ if err != nil {+ t.Fatalf("open ingested file: %v", err)+ }+ t.Cleanup(func() {+ if err := file.Close(); err != nil {+ t.Errorf("close ingested file: %v", err)+ }+ })+ got, err := io.ReadAll(file)+ if err != nil {+ t.Fatalf("read ingested file: %v", err)+ }+ if !bytes.Equal(got, item.Content) {+ t.Errorf("ingested content = %q, want %q", got, item.Content)+ }+}++type filenameKnowledgeRepo struct {+ createKnowledgeFileRepoStub+}++func (*filenameKnowledgeRepo) FindByDataSourceExternalID(+ context.Context, uint64, string, string, string,+) (*types.Knowledge, error) {+ return nil, nil+}diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.goindex 4e2c0c11b..b8abb4865 100644--- a/internal/application/service/datasource_service.go+++ b/internal/application/service/datasource_service.go@@ -7,7 +7,6 @@ import ("errors""fmt""mime/multipart"- "net/textproto""reflect""slices""strings"@@ -1600,12 +1599,7 @@ func bytesToFileHeader(data []byte, filename string) (*multipart.FileHeader, errvar buf bytes.Bufferwriter := multipart.NewWriter(&buf)- // Create a form file part- partHeader := make(textproto.MIMEHeader)- partHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename))- partHeader.Set("Content-Type", "application/octet-stream")-- part, err := writer.CreatePart(partHeader)+ part, err := writer.CreateFormFile("file", filename)if err != nil {return nil, fmt.Errorf("create multipart part: %w", err)}