Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(file): shorten long URL-upload filenames without dropping the extension (#20814)

Zhengxu (Joshua) Jine5357b3ec2ea
diff --git a/internal/service/file/file_test.go b/internal/service/file/file_test.go
index ab0abcccb7..f7b3394c58 100644
--- a/internal/service/file/file_test.go
+++ b/internal/service/file/file_test.go
@@ -8,10 +8,12 @@ import (
"io"
"net/http"
"net/http/httptest"
+ "net/url"
"ragflow/internal/common"
"strings"
"testing"
"time"
+ "unicode/utf8"
"ragflow/internal/dao"
"ragflow/internal/entity"
@@ -462,6 +464,75 @@ func TestFileService_UploadFromURL_HTMLNormalizesReadableContent(t *testing.T) {
}
}
+func TestFileService_UploadFromURL_ShortensLongNamesWithinTheLimit(t *testing.T) {
+ // The leading "a" puts the 255-byte boundary inside a 3-byte character.
+ longName := "a" + strings.Repeat("报", 100)
+ cases := []struct {
+ name string
+ path string
+ contentType string
+ body string
+ wantName string
+ wantExt string
+ }{
+ {
+ name: "pdf without extension",
+ path: longName,
+ contentType: "application/pdf",
+ body: "%PDF-1.7 fake pdf",
+ wantName: "a" + strings.Repeat("报", 83) + ".pdf",
+ wantExt: "pdf",
+ },
+ {
+ name: "docx keeps extension",
+ path: longName + ".docx",
+ contentType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ body: "PK\x03\x04 fake docx",
+ wantName: "a" + strings.Repeat("报", 83) + ".docx",
+ wantExt: "docx",
+ },
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", c.contentType)
+ _, _ = w.Write([]byte(c.body))
+ }))
+ defer server.Close()
+
+ origAssert := common.AssertURLSafe
+ origPinned := utility.PinnedHTTPClient
+ common.AssertURLSafe = func(rawURL string) (string, string, error) {
+ return "127.0.0.1", "127.0.0.1", nil
+ }
+ utility.PinnedHTTPClient = func(hostname, resolvedIP string, timeout time.Duration) *http.Client {
+ return server.Client()
+ }
+ t.Cleanup(func() {
+ common.AssertURLSafe = origAssert
+ utility.PinnedHTTPClient = origPinned
+ })
+
+ factory := storage.GetStorageFactory()
+ originalStorage := factory.GetStorage()
+ factory.SetStorage(&fakeStorage{})
+ t.Cleanup(func() { factory.SetStorage(originalStorage) })
+
+ resp, err := testFileService().UploadFromURL(t.Context(), "tenant123", server.URL+"/"+url.PathEscape(c.path))
+ if err != nil {
+ t.Fatalf("UploadFromURL failed: %v", err)
+ }
+ name, _ := resp["name"].(string)
+ if name != c.wantName || !utf8.ValidString(name) {
+ t.Fatalf("name = %q, want %q", name, c.wantName)
+ }
+ if resp["extension"] != c.wantExt {
+ t.Fatalf("extension = %#v, want %q", resp["extension"], c.wantExt)
+ }
+ })
+ }
+}
+
func TestNormalizeUploadInfoContent_PDFTakesPrecedenceOverHTML(t *testing.T) {
filename, contentType, data := utility.NormalizeUploadInfoContent(
"report",
diff --git a/internal/service/file/file_url.go b/internal/service/file/file_url.go
index 071048101b..50ee05699f 100644
--- a/internal/service/file/file_url.go
+++ b/internal/service/file/file_url.go
@@ -8,8 +8,12 @@ import (
"ragflow/internal/storage"
"ragflow/internal/utility"
"strings"
+ "unicode/utf8"
)
+// maxUploadFilenameBytes is the longest name checkUploadInfoHealth accepts.
+const maxUploadFilenameBytes = 255
+
// UploadFromURL fetches a remote URL, saves the content to the tenant's root
// folder, and returns the file metadata map — mirroring Python
// FileService.upload_info(tenant_id, None, url).
@@ -61,7 +65,9 @@ func normalizeRemoteUploadFilename(rawURL, contentType string, data []byte) stri
filename += ".pdf"
}
}
- return filename
+ // The appended ".pdf" can push a name that sanitizeFilename already
+ // shortened back over the limit.
+ return truncateFilename(filename, maxUploadFilenameBytes)
}
func sanitizeFilename(name string) string {
@@ -87,8 +93,23 @@ func sanitizeFilename(name string) string {
if stem := strings.SplitN(strings.ToUpper(name), ".", 2)[0]; reservedDeviceNames[stem] {
return "download"
}
- if len(name) > 255 {
- name = name[:255]
+ return truncateFilename(name, maxUploadFilenameBytes)
+}
+
+// truncateFilename shortens name to at most maxBytes bytes. It keeps the
+// extension when it fits and cuts on a rune boundary, so the result stays
+// valid UTF-8 and keeps the file type the extension implies.
+func truncateFilename(name string, maxBytes int) string {
+ if len(name) <= maxBytes {
+ return name
+ }
+ ext := filepath.Ext(name)
+ if len(ext) >= maxBytes {
+ ext = ""
+ }
+ end := maxBytes - len(ext)
+ for end > 0 && !utf8.RuneStart(name[end]) {
+ end--
}
- return name
+ return name[:end] + ext
}
diff --git a/internal/service/file/file_url_test.go b/internal/service/file/file_url_test.go
index 2e4d71fa18..216185ec90 100644
--- a/internal/service/file/file_url_test.go
+++ b/internal/service/file/file_url_test.go
@@ -1,6 +1,10 @@
package file
-import "testing"
+import (
+ "strings"
+ "testing"
+ "unicode/utf8"
+)
func TestSanitizeFilename(t *testing.T) {
cases := []struct {
@@ -14,6 +18,9 @@ func TestSanitizeFilename(t *testing.T) {
{"reserved device", "CON", "download"},
{"only dots", "...", "download"},
{"leading dots", ".hidden", "hidden"},
+ {"long name keeps extension", strings.Repeat("a", 300) + ".pdf", strings.Repeat("a", 251) + ".pdf"},
+ // The leading "a" puts the 255-byte boundary inside a 3-byte character.
+ {"long CJK name cut on a character", "a" + strings.Repeat("报", 100), "a" + strings.Repeat("报", 84)},
}
for _, c := range cases {
got := sanitizeFilename(c.in)
@@ -33,5 +40,8 @@ func TestSanitizeFilename(t *testing.T) {
if got == "" {
t.Errorf("%s: empty result for %q", c.name, c.in)
}
+ if len(got) > maxUploadFilenameBytes || !utf8.ValidString(got) {
+ t.Errorf("%s: %q is longer than %d bytes or not valid UTF-8", c.name, got, maxUploadFilenameBytes)
+ }
}
}