RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(file): shorten long URL-upload filenames without dropping the extension (#20814)
Zhengxu (Joshua) Jine5357b3ec2ea
diff --git a/internal/service/file/file_test.go b/internal/service/file/file_test.goindex ab0abcccb7..f7b3394c58 100644--- a/internal/service/file/file_test.go+++ b/internal/service/file/file_test.go@@ -8,10 +8,12 @@ import ("io""net/http""net/http/httptest"+ "net/url""ragflow/internal/common""strings""testing""time"+ "unicode/utf8""ragflow/internal/dao""ragflow/internal/entity"@@ -462,6 +464,75 @@ func TestFileService_UploadFromURL_HTMLNormalizesReadableContent(t *testing.T) {}}+func TestFileService_UploadFromURL_ShortensLongNamesWithinTheLimit(t *testing.T) {+ // The leading "a" puts the 255-byte boundary inside a 3-byte character.+ longName := "a" + strings.Repeat("报", 100)+ cases := []struct {+ name string+ path string+ contentType string+ body string+ wantName string+ wantExt string+ }{+ {+ name: "pdf without extension",+ path: longName,+ contentType: "application/pdf",+ body: "%PDF-1.7 fake pdf",+ wantName: "a" + strings.Repeat("报", 83) + ".pdf",+ wantExt: "pdf",+ },+ {+ name: "docx keeps extension",+ path: longName + ".docx",+ contentType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ body: "PK\x03\x04 fake docx",+ wantName: "a" + strings.Repeat("报", 83) + ".docx",+ wantExt: "docx",+ },+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", c.contentType)+ _, _ = w.Write([]byte(c.body))+ }))+ defer server.Close()++ origAssert := common.AssertURLSafe+ origPinned := utility.PinnedHTTPClient+ common.AssertURLSafe = func(rawURL string) (string, string, error) {+ return "127.0.0.1", "127.0.0.1", nil+ }+ utility.PinnedHTTPClient = func(hostname, resolvedIP string, timeout time.Duration) *http.Client {+ return server.Client()+ }+ t.Cleanup(func() {+ common.AssertURLSafe = origAssert+ utility.PinnedHTTPClient = origPinned+ })++ factory := storage.GetStorageFactory()+ originalStorage := factory.GetStorage()+ factory.SetStorage(&fakeStorage{})+ t.Cleanup(func() { factory.SetStorage(originalStorage) })++ resp, err := testFileService().UploadFromURL(t.Context(), "tenant123", server.URL+"/"+url.PathEscape(c.path))+ if err != nil {+ t.Fatalf("UploadFromURL failed: %v", err)+ }+ name, _ := resp["name"].(string)+ if name != c.wantName || !utf8.ValidString(name) {+ t.Fatalf("name = %q, want %q", name, c.wantName)+ }+ if resp["extension"] != c.wantExt {+ t.Fatalf("extension = %#v, want %q", resp["extension"], c.wantExt)+ }+ })+ }+}+func TestNormalizeUploadInfoContent_PDFTakesPrecedenceOverHTML(t *testing.T) {filename, contentType, data := utility.NormalizeUploadInfoContent("report",diff --git a/internal/service/file/file_url.go b/internal/service/file/file_url.goindex 071048101b..50ee05699f 100644--- a/internal/service/file/file_url.go+++ b/internal/service/file/file_url.go@@ -8,8 +8,12 @@ import ("ragflow/internal/storage""ragflow/internal/utility""strings"+ "unicode/utf8")+// maxUploadFilenameBytes is the longest name checkUploadInfoHealth accepts.+const maxUploadFilenameBytes = 255+// UploadFromURL fetches a remote URL, saves the content to the tenant's root// folder, and returns the file metadata map — mirroring Python// FileService.upload_info(tenant_id, None, url).@@ -61,7 +65,9 @@ func normalizeRemoteUploadFilename(rawURL, contentType string, data []byte) strifilename += ".pdf"}}- return filename+ // The appended ".pdf" can push a name that sanitizeFilename already+ // shortened back over the limit.+ return truncateFilename(filename, maxUploadFilenameBytes)}func sanitizeFilename(name string) string {@@ -87,8 +93,23 @@ func sanitizeFilename(name string) string {if stem := strings.SplitN(strings.ToUpper(name), ".", 2)[0]; reservedDeviceNames[stem] {return "download"}- if len(name) > 255 {- name = name[:255]+ return truncateFilename(name, maxUploadFilenameBytes)+}++// truncateFilename shortens name to at most maxBytes bytes. It keeps the+// extension when it fits and cuts on a rune boundary, so the result stays+// valid UTF-8 and keeps the file type the extension implies.+func truncateFilename(name string, maxBytes int) string {+ if len(name) <= maxBytes {+ return name+ }+ ext := filepath.Ext(name)+ if len(ext) >= maxBytes {+ ext = ""+ }+ end := maxBytes - len(ext)+ for end > 0 && !utf8.RuneStart(name[end]) {+ end--}- return name+ return name[:end] + ext}diff --git a/internal/service/file/file_url_test.go b/internal/service/file/file_url_test.goindex 2e4d71fa18..216185ec90 100644--- a/internal/service/file/file_url_test.go+++ b/internal/service/file/file_url_test.go@@ -1,6 +1,10 @@package file-import "testing"+import (+ "strings"+ "testing"+ "unicode/utf8"+)func TestSanitizeFilename(t *testing.T) {cases := []struct {@@ -14,6 +18,9 @@ func TestSanitizeFilename(t *testing.T) {{"reserved device", "CON", "download"},{"only dots", "...", "download"},{"leading dots", ".hidden", "hidden"},+ {"long name keeps extension", strings.Repeat("a", 300) + ".pdf", strings.Repeat("a", 251) + ".pdf"},+ // The leading "a" puts the 255-byte boundary inside a 3-byte character.+ {"long CJK name cut on a character", "a" + strings.Repeat("报", 100), "a" + strings.Repeat("报", 84)},}for _, c := range cases {got := sanitizeFilename(c.in)@@ -33,5 +40,8 @@ func TestSanitizeFilename(t *testing.T) {if got == "" {t.Errorf("%s: empty result for %q", c.name, c.in)}+ if len(got) > maxUploadFilenameBytes || !utf8.ValidString(got) {+ t.Errorf("%s: %q is longer than %d bytes or not valid UTF-8", c.name, got, maxUploadFilenameBytes)+ }}}