RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat: configure dataset document upload size (#21517)
buua43618ddc7f87faf
diff --git a/docker/.env b/docker/.envindex a98ff61fe9..ace2a8ba3a 100644--- a/docker/.env+++ b/docker/.env@@ -352,10 +352,12 @@ TZ=Asia/Shanghai# Uncomment the following line if your operating system is MacOS:# MACOS=1-# The maximum file size limit (in bytes) for each upload to your dataset or RAGFlow's File system.-# To change the 1GB file size limit, uncomment the line below and update as needed.-# MAX_CONTENT_LENGTH=1073741824-# After updating, ensure `client_max_body_size` in nginx/nginx.conf is updated accordingly.+# Maximum total request-body size for multipart uploads (default 1 GiB).+# MAX_MULTIPART_CONTENT_LENGTH=1073741824+# MAX_CONTENT_LENGTH=1073741824 # fallback name when MAX_MULTIPART_CONTENT_LENGTH is unset+# Maximum size in bytes for one dataset document upload (default 128 MiB).+# MAX_DOCUMENT_UPLOAD_SIZE=134217728+# When raising the multipart limit, ensure `client_max_body_size` in nginx/nginx.conf is updated accordingly.# Note that neither `MAX_CONTENT_LENGTH` nor `client_max_body_size` sets the maximum size for files uploaded to an agent.# See https://ragflow.io/docs/dev/begin_component for details.@@ -557,4 +559,3 @@ OAUTH_AUTO_REGISTER=true# RAGFLOW_MCP_PORT=9382# RAGFLOW_MCP_LAUNCH_MODE=self-host# RAGFLOW_MCP_HOST_API_KEY=-diff --git a/docs/faq.mdx b/docs/faq.mdxindex 21d053efe8..cf902a8d09 100644--- a/docs/faq.mdx+++ b/docs/faq.mdx@@ -328,7 +328,7 @@ If your model is not currently supported but has APIs compatible with those of OThe Go API limits non-multipart request bodies to 1 MiB by default. Set `MAX_JSON_CONTENT_LENGTH` to change that limit in bytes. Multipart upload requests share a 1 GiB default total-body limit; set `MAX_MULTIPART_CONTENT_LENGTH` to change it in bytes. Upload endpoints can enforce smaller per-file or per-request limits.-The file-manager per-file limit follows `MAX_MULTIPART_CONTENT_LENGTH` (1 GiB by default). Dataset documents allow 128 MiB per file, and document-info uploads allow 100 MiB per file; increasing the shared multipart limit does not raise those per-file limits.+The file-manager per-file limit follows `MAX_MULTIPART_CONTENT_LENGTH` (1 GiB by default). Dataset documents allow 128 MiB per file by default; set `MAX_DOCUMENT_UPLOAD_SIZE` to change that limit in bytes. The total multipart request limit must also be large enough for the file and multipart overhead. Document-info uploads allow 100 MiB per file, and increasing the shared multipart limit does not raise that limit.If you raise the multipart limit above 1 GiB, also increase `client_max_body_size` in **docker/nginx/nginx.conf**. The Go image contains its own copy of this file, so make the change effective by either rebuilding `RAGFLOW_IMAGE` or enabling the `./nginx/nginx.conf:/etc/nginx/nginx.conf` bind mount for the deployed service in **docker/docker-compose.yml**, then recreate that service.diff --git a/internal/common/environments.go b/internal/common/environments.goindex 2d5e673656..40120f83c9 100644--- a/internal/common/environments.go+++ b/internal/common/environments.go@@ -89,6 +89,7 @@ const (EnvMaxContentLength = "MAX_CONTENT_LENGTH"EnvMaxJSONContentLength = "MAX_JSON_CONTENT_LENGTH"EnvMaxMultipartContentLength = "MAX_MULTIPART_CONTENT_LENGTH"+ EnvMaxDocumentUploadSize = "MAX_DOCUMENT_UPLOAD_SIZE"EnvRAGFlowDictPath = "RAGFLOW_DICT_PATH"EnvDefaultSuperuserEmail = "DEFAULT_SUPERUSER_EMAIL"EnvDefaultSuperuserNickname = "DEFAULT_SUPERUSER_NICKNAME"diff --git a/internal/common/http.go b/internal/common/http.goindex d106e7d58a..d05c291bd2 100644--- a/internal/common/http.go+++ b/internal/common/http.go@@ -43,6 +43,8 @@ const (DefaultMaxJSONRequestBodyBytes int64 = 1 << 20// DefaultMaxMultipartRequestBodyBytes is the default total limit for multipart uploads.DefaultMaxMultipartRequestBodyBytes int64 = 1 << 30+ // DefaultMaxDocumentUploadBytes is the default per-file limit for dataset document uploads.+ DefaultMaxDocumentUploadBytes int64 = 128 << 20// MaxWebhookRequestBodyBytes is the default and maximum request body size for agent webhooks.MaxWebhookRequestBodyBytes int64 = 10 * 1_000_000)@@ -50,17 +52,23 @@ const (// GetMaxJSONRequestBodyBytes returns the configured non-multipart request-body// limit in bytes, defaulting to 1 MiB.func GetMaxJSONRequestBodyBytes() int64 {- return getMaxRequestBodyBytes(EnvMaxJSONContentLength, "", DefaultMaxJSONRequestBodyBytes)+ return getMaxConfiguredBytes(EnvMaxJSONContentLength, "", DefaultMaxJSONRequestBodyBytes)}// GetMaxMultipartRequestBodyBytes returns the configured multipart upload// request-body limit in bytes, defaulting to 1 GiB. MAX_CONTENT_LENGTH remains// a fallback for existing deployments when the new variable is unset.func GetMaxMultipartRequestBodyBytes() int64 {- return getMaxRequestBodyBytes(EnvMaxMultipartContentLength, EnvMaxContentLength, DefaultMaxMultipartRequestBodyBytes)+ return getMaxConfiguredBytes(EnvMaxMultipartContentLength, EnvMaxContentLength, DefaultMaxMultipartRequestBodyBytes)}-func getMaxRequestBodyBytes(envName, fallbackEnvName string, defaultBytes int64) int64 {+// GetMaxDocumentUploadBytes returns the configured per-file limit for dataset document uploads,+// defaulting to 128 MiB.+func GetMaxDocumentUploadBytes() int64 {+ return getMaxConfiguredBytes(EnvMaxDocumentUploadSize, "", DefaultMaxDocumentUploadBytes)+}++func getMaxConfiguredBytes(envName, fallbackEnvName string, defaultBytes int64) int64 {raw := strings.TrimSpace(GetEnv(envName))if raw == "" && fallbackEnvName != "" {raw = strings.TrimSpace(GetEnv(fallbackEnvName))diff --git a/internal/common/http_test.go b/internal/common/http_test.goindex e72c94b527..b2e1fa9280 100644--- a/internal/common/http_test.go+++ b/internal/common/http_test.go@@ -72,6 +72,27 @@ func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {return f(req)}+func TestGetMaxDocumentUploadBytes(t *testing.T) {+ tests := []struct {+ name string+ value string+ want int64+ }{+ {name: "default", want: DefaultMaxDocumentUploadBytes},+ {name: "configured", value: "4096", want: 4096},+ {name: "invalid uses default", value: "invalid", want: DefaultMaxDocumentUploadBytes},+ {name: "nonpositive uses default", value: "0", want: DefaultMaxDocumentUploadBytes},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ t.Setenv(EnvMaxDocumentUploadSize, tt.value)+ if got := GetMaxDocumentUploadBytes(); got != tt.want {+ t.Fatalf("GetMaxDocumentUploadBytes() = %d, want %d", got, tt.want)+ }+ })+ }+}+func TestProviderLoggingRecordsTimingsAndTruncatesVectors(t *testing.T) {core, logs := observer.New(zapcore.InfoLevel)previousLogger := Loggerdiff --git a/internal/service/document/document.go b/internal/service/document/document.goindex 61361adfc5..f23c6f27d8 100644--- a/internal/service/document/document.go+++ b/internal/service/document/document.go@@ -267,10 +267,6 @@ type valueInfo struct {// that holds per-dataset subfolders, mirroring Python KNOWLEDGEBASE_FOLDER_NAME.const knowledgebaseFolderName = ".knowledgebase"-// maxUploadDocSize bounds a single uploaded file held in memory, mirroring the-// Python DOC_MAXIMUM_SIZE default (128 MiB; overridable there via MAX_CONTENT_LENGTH).-const maxUploadDocSize = 128 * 1024 * 1024-// MetadataUpdate is one update item: set key to value.type MetadataUpdate struct {Key string `json:"key"`diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex e72a56222a..ba0e731038 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -628,6 +628,23 @@ func makeTestFileHeader(t *testing.T, field, filename string, content []byte) *mreturn fhs[0]}+func TestReadFileHeaderBytesUsesConfiguredMax(t *testing.T) {+ t.Setenv(common.EnvMaxDocumentUploadSize, "4")++ blob, err := readFileHeaderBytes(makeTestFileHeader(t, "file", "small.txt", []byte("1234")))+ if err != nil {+ t.Fatalf("readFileHeaderBytes() error = %v, want success at configured limit", err)+ }+ if string(blob) != "1234" {+ t.Fatalf("readFileHeaderBytes() = %q, want %q", blob, "1234")+ }++ _, err = readFileHeaderBytes(makeTestFileHeader(t, "file", "large.txt", []byte("12345")))+ if err == nil || !strings.Contains(err.Error(), "4 bytes") {+ t.Fatalf("readFileHeaderBytes() error = %v, want configured 4-byte limit", err)+ }+}+// sptr returns a pointer to the given string.func sptr(s string) *string { return &s }diff --git a/internal/service/document/document_upload.go b/internal/service/document/document_upload.goindex 2eb781cb83..6b9e58a9e4 100644--- a/internal/service/document/document_upload.go+++ b/internal/service/document/document_upload.go@@ -242,7 +242,7 @@ func (s *DocumentService) UploadWebDocument(ctx context.Context, kb *entity.Knowreturn nil, common.CodeServerError, err}- blob, headers, _, err := utility.FetchRemoteFileSafely(ctx, url, maxUploadDocSize)+ blob, headers, _, err := utility.FetchRemoteFileSafely(ctx, url, common.GetMaxDocumentUploadBytes())if err != nil {return nil, common.CodeDataError, err}@@ -385,20 +385,21 @@ func docToRawMap(doc *entity.Document) map[string]interface{} {}func readFileHeaderBytes(fh *multipart.FileHeader) ([]byte, error) {- if fh.Size > maxUploadDocSize {- return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxUploadDocSize)+ maxBytes := common.GetMaxDocumentUploadBytes()+ if fh.Size > maxBytes {+ return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxBytes)}src, err := fh.Open()if err != nil {return nil, err}defer src.Close()- blob, err := io.ReadAll(io.LimitReader(src, maxUploadDocSize+1))+ blob, err := io.ReadAll(io.LimitReader(src, maxBytes+1))if err != nil {return nil, err}- if len(blob) > maxUploadDocSize {- return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxUploadDocSize)+ if int64(len(blob)) > maxBytes {+ return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxBytes)}return blob, nil}