Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat: configure dataset document upload size (#21517)

buua43618ddc7f87faf
diff --git a/docker/.env b/docker/.env
index a98ff61fe9..ace2a8ba3a 100644
--- a/docker/.env
+++ b/docker/.env
@@ -352,10 +352,12 @@ TZ=Asia/Shanghai
# Uncomment the following line if your operating system is MacOS:
# MACOS=1
-# The maximum file size limit (in bytes) for each upload to your dataset or RAGFlow's File system.
-# To change the 1GB file size limit, uncomment the line below and update as needed.
-# MAX_CONTENT_LENGTH=1073741824
-# After updating, ensure `client_max_body_size` in nginx/nginx.conf is updated accordingly.
+# Maximum total request-body size for multipart uploads (default 1 GiB).
+# MAX_MULTIPART_CONTENT_LENGTH=1073741824
+# MAX_CONTENT_LENGTH=1073741824 # fallback name when MAX_MULTIPART_CONTENT_LENGTH is unset
+# Maximum size in bytes for one dataset document upload (default 128 MiB).
+# MAX_DOCUMENT_UPLOAD_SIZE=134217728
+# When raising the multipart limit, ensure `client_max_body_size` in nginx/nginx.conf is updated accordingly.
# Note that neither `MAX_CONTENT_LENGTH` nor `client_max_body_size` sets the maximum size for files uploaded to an agent.
# See https://ragflow.io/docs/dev/begin_component for details.
@@ -557,4 +559,3 @@ OAUTH_AUTO_REGISTER=true
# RAGFLOW_MCP_PORT=9382
# RAGFLOW_MCP_LAUNCH_MODE=self-host
# RAGFLOW_MCP_HOST_API_KEY=
-
diff --git a/docs/faq.mdx b/docs/faq.mdx
index 21d053efe8..cf902a8d09 100644
--- a/docs/faq.mdx
+++ b/docs/faq.mdx
@@ -328,7 +328,7 @@ If your model is not currently supported but has APIs compatible with those of O
The Go API limits non-multipart request bodies to 1 MiB by default. Set `MAX_JSON_CONTENT_LENGTH` to change that limit in bytes. Multipart upload requests share a 1 GiB default total-body limit; set `MAX_MULTIPART_CONTENT_LENGTH` to change it in bytes. Upload endpoints can enforce smaller per-file or per-request limits.
-The file-manager per-file limit follows `MAX_MULTIPART_CONTENT_LENGTH` (1 GiB by default). Dataset documents allow 128 MiB per file, and document-info uploads allow 100 MiB per file; increasing the shared multipart limit does not raise those per-file limits.
+The file-manager per-file limit follows `MAX_MULTIPART_CONTENT_LENGTH` (1 GiB by default). Dataset documents allow 128 MiB per file by default; set `MAX_DOCUMENT_UPLOAD_SIZE` to change that limit in bytes. The total multipart request limit must also be large enough for the file and multipart overhead. Document-info uploads allow 100 MiB per file, and increasing the shared multipart limit does not raise that limit.
If you raise the multipart limit above 1 GiB, also increase `client_max_body_size` in **docker/nginx/nginx.conf**. The Go image contains its own copy of this file, so make the change effective by either rebuilding `RAGFLOW_IMAGE` or enabling the `./nginx/nginx.conf:/etc/nginx/nginx.conf` bind mount for the deployed service in **docker/docker-compose.yml**, then recreate that service.
diff --git a/internal/common/environments.go b/internal/common/environments.go
index 2d5e673656..40120f83c9 100644
--- a/internal/common/environments.go
+++ b/internal/common/environments.go
@@ -89,6 +89,7 @@ const (
EnvMaxContentLength = "MAX_CONTENT_LENGTH"
EnvMaxJSONContentLength = "MAX_JSON_CONTENT_LENGTH"
EnvMaxMultipartContentLength = "MAX_MULTIPART_CONTENT_LENGTH"
+ EnvMaxDocumentUploadSize = "MAX_DOCUMENT_UPLOAD_SIZE"
EnvRAGFlowDictPath = "RAGFLOW_DICT_PATH"
EnvDefaultSuperuserEmail = "DEFAULT_SUPERUSER_EMAIL"
EnvDefaultSuperuserNickname = "DEFAULT_SUPERUSER_NICKNAME"
diff --git a/internal/common/http.go b/internal/common/http.go
index d106e7d58a..d05c291bd2 100644
--- a/internal/common/http.go
+++ b/internal/common/http.go
@@ -43,6 +43,8 @@ const (
DefaultMaxJSONRequestBodyBytes int64 = 1 << 20
// DefaultMaxMultipartRequestBodyBytes is the default total limit for multipart uploads.
DefaultMaxMultipartRequestBodyBytes int64 = 1 << 30
+ // DefaultMaxDocumentUploadBytes is the default per-file limit for dataset document uploads.
+ DefaultMaxDocumentUploadBytes int64 = 128 << 20
// MaxWebhookRequestBodyBytes is the default and maximum request body size for agent webhooks.
MaxWebhookRequestBodyBytes int64 = 10 * 1_000_000
)
@@ -50,17 +52,23 @@ const (
// GetMaxJSONRequestBodyBytes returns the configured non-multipart request-body
// limit in bytes, defaulting to 1 MiB.
func GetMaxJSONRequestBodyBytes() int64 {
- return getMaxRequestBodyBytes(EnvMaxJSONContentLength, "", DefaultMaxJSONRequestBodyBytes)
+ return getMaxConfiguredBytes(EnvMaxJSONContentLength, "", DefaultMaxJSONRequestBodyBytes)
}
// GetMaxMultipartRequestBodyBytes returns the configured multipart upload
// request-body limit in bytes, defaulting to 1 GiB. MAX_CONTENT_LENGTH remains
// a fallback for existing deployments when the new variable is unset.
func GetMaxMultipartRequestBodyBytes() int64 {
- return getMaxRequestBodyBytes(EnvMaxMultipartContentLength, EnvMaxContentLength, DefaultMaxMultipartRequestBodyBytes)
+ return getMaxConfiguredBytes(EnvMaxMultipartContentLength, EnvMaxContentLength, DefaultMaxMultipartRequestBodyBytes)
}
-func getMaxRequestBodyBytes(envName, fallbackEnvName string, defaultBytes int64) int64 {
+// GetMaxDocumentUploadBytes returns the configured per-file limit for dataset document uploads,
+// defaulting to 128 MiB.
+func GetMaxDocumentUploadBytes() int64 {
+ return getMaxConfiguredBytes(EnvMaxDocumentUploadSize, "", DefaultMaxDocumentUploadBytes)
+}
+
+func getMaxConfiguredBytes(envName, fallbackEnvName string, defaultBytes int64) int64 {
raw := strings.TrimSpace(GetEnv(envName))
if raw == "" && fallbackEnvName != "" {
raw = strings.TrimSpace(GetEnv(fallbackEnvName))
diff --git a/internal/common/http_test.go b/internal/common/http_test.go
index e72c94b527..b2e1fa9280 100644
--- a/internal/common/http_test.go
+++ b/internal/common/http_test.go
@@ -72,6 +72,27 @@ func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
+func TestGetMaxDocumentUploadBytes(t *testing.T) {
+ tests := []struct {
+ name string
+ value string
+ want int64
+ }{
+ {name: "default", want: DefaultMaxDocumentUploadBytes},
+ {name: "configured", value: "4096", want: 4096},
+ {name: "invalid uses default", value: "invalid", want: DefaultMaxDocumentUploadBytes},
+ {name: "nonpositive uses default", value: "0", want: DefaultMaxDocumentUploadBytes},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Setenv(EnvMaxDocumentUploadSize, tt.value)
+ if got := GetMaxDocumentUploadBytes(); got != tt.want {
+ t.Fatalf("GetMaxDocumentUploadBytes() = %d, want %d", got, tt.want)
+ }
+ })
+ }
+}
+
func TestProviderLoggingRecordsTimingsAndTruncatesVectors(t *testing.T) {
core, logs := observer.New(zapcore.InfoLevel)
previousLogger := Logger
diff --git a/internal/service/document/document.go b/internal/service/document/document.go
index 61361adfc5..f23c6f27d8 100644
--- a/internal/service/document/document.go
+++ b/internal/service/document/document.go
@@ -267,10 +267,6 @@ type valueInfo struct {
// that holds per-dataset subfolders, mirroring Python KNOWLEDGEBASE_FOLDER_NAME.
const knowledgebaseFolderName = ".knowledgebase"
-// maxUploadDocSize bounds a single uploaded file held in memory, mirroring the
-// Python DOC_MAXIMUM_SIZE default (128 MiB; overridable there via MAX_CONTENT_LENGTH).
-const maxUploadDocSize = 128 * 1024 * 1024
-
// MetadataUpdate is one update item: set key to value.
type MetadataUpdate struct {
Key string `json:"key"`
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index e72a56222a..ba0e731038 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -628,6 +628,23 @@ func makeTestFileHeader(t *testing.T, field, filename string, content []byte) *m
return fhs[0]
}
+func TestReadFileHeaderBytesUsesConfiguredMax(t *testing.T) {
+ t.Setenv(common.EnvMaxDocumentUploadSize, "4")
+
+ blob, err := readFileHeaderBytes(makeTestFileHeader(t, "file", "small.txt", []byte("1234")))
+ if err != nil {
+ t.Fatalf("readFileHeaderBytes() error = %v, want success at configured limit", err)
+ }
+ if string(blob) != "1234" {
+ t.Fatalf("readFileHeaderBytes() = %q, want %q", blob, "1234")
+ }
+
+ _, err = readFileHeaderBytes(makeTestFileHeader(t, "file", "large.txt", []byte("12345")))
+ if err == nil || !strings.Contains(err.Error(), "4 bytes") {
+ t.Fatalf("readFileHeaderBytes() error = %v, want configured 4-byte limit", err)
+ }
+}
+
// sptr returns a pointer to the given string.
func sptr(s string) *string { return &s }
diff --git a/internal/service/document/document_upload.go b/internal/service/document/document_upload.go
index 2eb781cb83..6b9e58a9e4 100644
--- a/internal/service/document/document_upload.go
+++ b/internal/service/document/document_upload.go
@@ -242,7 +242,7 @@ func (s *DocumentService) UploadWebDocument(ctx context.Context, kb *entity.Know
return nil, common.CodeServerError, err
}
- blob, headers, _, err := utility.FetchRemoteFileSafely(ctx, url, maxUploadDocSize)
+ blob, headers, _, err := utility.FetchRemoteFileSafely(ctx, url, common.GetMaxDocumentUploadBytes())
if err != nil {
return nil, common.CodeDataError, err
}
@@ -385,20 +385,21 @@ func docToRawMap(doc *entity.Document) map[string]interface{} {
}
func readFileHeaderBytes(fh *multipart.FileHeader) ([]byte, error) {
- if fh.Size > maxUploadDocSize {
- return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxUploadDocSize)
+ maxBytes := common.GetMaxDocumentUploadBytes()
+ if fh.Size > maxBytes {
+ return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxBytes)
}
src, err := fh.Open()
if err != nil {
return nil, err
}
defer src.Close()
- blob, err := io.ReadAll(io.LimitReader(src, maxUploadDocSize+1))
+ blob, err := io.ReadAll(io.LimitReader(src, maxBytes+1))
if err != nil {
return nil, err
}
- if len(blob) > maxUploadDocSize {
- return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxUploadDocSize)
+ if int64(len(blob)) > maxBytes {
+ return nil, fmt.Errorf("file exceeds the maximum allowed size of %d bytes", maxBytes)
}
return blob, nil
}