SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
filer: configurable TUS max upload size and session expiry (#10638)
Chris Lucab666fca144
diff --git a/weed/command/filer.go b/weed/command/filer.goindex c4f43dff70..a15075abbd 100644--- a/weed/command/filer.go+++ b/weed/command/filer.go@@ -80,6 +80,8 @@ type FilerOptions struct {allowedOrigins *stringexposeDirectoryData *booltusBasePath *string+ tusMaxSizeMB *int+ tusSessionExpiry *time.Durations3ConfigFile *string // optional path to static S3 identity config// shutdownCtx, when non-nil, tells startFiler to gracefully shut down its// HTTP/gRPC servers once the ctx is cancelled. Used by integration tests@@ -123,6 +125,8 @@ func init() {f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI")f.tusBasePath = cmdFiler.Flag.String("tusBasePath", "/.tus", "TUS resumable upload endpoint base path (e.g., /.tus)")+ f.tusMaxSizeMB = cmdFiler.Flag.Int("tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")+ f.tusSessionExpiry = cmdFiler.Flag.Duration("tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"")// start s3 on filerfilerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")@@ -386,6 +390,8 @@ func (fo *FilerOptions) startFiler() {DiskType: *fo.diskType,AllowedOrigins: strings.Split(*fo.allowedOrigins, ","),TusBasePath: *fo.tusBasePath,+ TusMaxSize: int64(*fo.tusMaxSizeMB) * 1024 * 1024,+ TusSessionExpiry: *fo.tusSessionExpiry,CredentialManager: credentialManager,})if nfs_err != nil {diff --git a/weed/command/mini.go b/weed/command/mini.goindex 09c825b61b..a7a0ba01af 100644--- a/weed/command/mini.go+++ b/weed/command/mini.go@@ -458,6 +458,8 @@ func initMiniFilerFlags() {miniFilerOptions.allowedOrigins = cmdMini.Flag.String("filer.allowedOrigins", "*", "comma separated list of allowed origins")miniFilerOptions.exposeDirectoryData = cmdMini.Flag.Bool("filer.exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI")miniFilerOptions.tusBasePath = cmdMini.Flag.String("filer.tusBasePath", "/.tus", "TUS resumable upload endpoint base path")+ miniFilerOptions.tusMaxSizeMB = cmdMini.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")+ miniFilerOptions.tusSessionExpiry = cmdMini.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration")}// initMiniVolumeFlags initializes Volume server flag optionsdiff --git a/weed/command/server.go b/weed/command/server.goindex 19360d6a87..e89d26682d 100644--- a/weed/command/server.go+++ b/weed/command/server.go@@ -130,6 +130,8 @@ func init() {filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")filerOptions.exposeDirectoryData = cmdServer.Flag.Bool("filer.exposeDirectoryData", true, "expose directory data via filer. If false, filer UI will be inaccessible.")filerOptions.tusBasePath = cmdServer.Flag.String("filer.tusBasePath", "/.tus", "TUS resumable upload endpoint base path (e.g., /.tus)")+ filerOptions.tusMaxSizeMB = cmdServer.Flag.Int("filer.tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")+ filerOptions.tusSessionExpiry = cmdServer.Flag.Duration("filer.tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"")serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")diff --git a/weed/server/filer_server.go b/weed/server/filer_server.goindex 9ea143228b..11b1c080bc 100644--- a/weed/server/filer_server.go+++ b/weed/server/filer_server.go@@ -84,6 +84,8 @@ type FilerOption struct {AllowedOrigins []stringExposeDirectoryData boolTusBasePath string+ TusMaxSize int64+ TusSessionExpiry time.DurationS3ConfigFile string // optional path to static S3 identity config fileCredentialManager *credential.CredentialManager}@@ -254,6 +256,12 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)if option.TusBasePath == "" {glog.Warningf("Invalid TUS base path; TUS disabled (must not be root '/')")} else {+ if option.TusMaxSize <= 0 {+ option.TusMaxSize = TusDefaultMaxSize+ }+ if option.TusSessionExpiry <= 0 {+ option.TusSessionExpiry = TusDefaultSessionExpiry+ }handlePath := option.TusBasePath + "/"defaultMux.HandleFunc(handlePath, fs.filerGuard.WhiteList(requestIDMiddleware(fs.tusHandler)))// Start background cleanup of expired TUS sessions (every hour)diff --git a/weed/server/filer_server_tus_handlers.go b/weed/server/filer_server_tus_handlers.goindex 623cbe6ef4..5b72e45a8b 100644--- a/weed/server/filer_server_tus_handlers.go+++ b/weed/server/filer_server_tus_handlers.go@@ -175,7 +175,7 @@ func writeTusCompleteError(w http.ResponseWriter, err error) {func (fs *FilerServer) tusOptionsHandler(w http.ResponseWriter, r *http.Request) {w.Header().Set("Tus-Version", TusVersion)w.Header().Set("Tus-Extension", TusExtensions)- w.Header().Set("Tus-Max-Size", strconv.FormatInt(TusMaxSize, 10))+ w.Header().Set("Tus-Max-Size", strconv.FormatInt(fs.option.TusMaxSize, 10))w.WriteHeader(http.StatusOK)}@@ -197,7 +197,7 @@ func (fs *FilerServer) tusCreateHandler(w http.ResponseWriter, r *http.Request)http.Error(w, "Invalid Upload-Length", http.StatusBadRequest)return}- if uploadLength > TusMaxSize {+ if uploadLength > fs.option.TusMaxSize {http.Error(w, "Upload-Length exceeds maximum", http.StatusRequestEntityTooLarge)return}diff --git a/weed/server/filer_server_tus_idor_test.go b/weed/server/filer_server_tus_idor_test.goindex aa4ae58065..2a66547bd0 100644--- a/weed/server/filer_server_tus_idor_test.go+++ b/weed/server/filer_server_tus_idor_test.go@@ -82,7 +82,7 @@ func TestFilerServer_tusHandler_RejectsAliasesAndInvalidMetadata(t *testing.T) {{"empty stored target", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "", Size: 1}},{"root stored target", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "/", Size: 1}},{"relative stored target", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "buckets/secret/x.bin", Size: 1}},- {"oversize stored size", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "/buckets/secret/victim.bin", Size: TusMaxSize + 1}},+ {"oversize stored size", tusTestUploadID, TusSession{ID: tusTestUploadID, TargetPath: "/buckets/secret/victim.bin", Size: TusDefaultMaxSize + 1}},}for _, tt := range tests {diff --git a/weed/server/filer_server_tus_jwt_test.go b/weed/server/filer_server_tus_jwt_test.goindex af02a202ee..c8291ed2b6 100644--- a/weed/server/filer_server_tus_jwt_test.go+++ b/weed/server/filer_server_tus_jwt_test.go@@ -48,7 +48,7 @@ func newTusTestServer(t *testing.T, sessions map[string]string) (*FilerServer, *fs := &FilerServer{filer: newRenameTestFiler(t, store),filerGuard: security.NewGuard(nil, tusTestWriteKey, 0, tusTestReadKey, 0),- option: &FilerOption{TusBasePath: "/.tus"},+ option: &FilerOption{TusBasePath: "/.tus", TusMaxSize: TusDefaultMaxSize},}for uploadID, targetPath := range sessions {seedTusSession(t, fs, store, TusSession{ID: uploadID, TargetPath: targetPath, Size: 1})diff --git a/weed/server/filer_server_tus_session.go b/weed/server/filer_server_tus_session.goindex 4bc8384663..5c0494d44c 100644--- a/weed/server/filer_server_tus_session.go+++ b/weed/server/filer_server_tus_session.go@@ -20,12 +20,13 @@ import ()const (- TusVersion = "1.0.0"- TusMaxSize = int64(5 * 1024 * 1024 * 1024) // 5GB default max size- TusUploadsFolder = ".uploads.tus"- TusInfoFileName = ".info"- TusChunkExt = ".chunk"- TusExtensions = "creation,creation-with-upload,termination"+ TusVersion = "1.0.0"+ TusDefaultMaxSize = int64(5 * 1024 * 1024 * 1024) // 5GB+ TusDefaultSessionExpiry = 24 * time.Hour+ TusUploadsFolder = ".uploads.tus"+ TusInfoFileName = ".info"+ TusChunkExt = ".chunk"+ TusExtensions = "creation,creation-with-upload,termination")// ErrWormEnforced marks a TUS completion rejected because the target entry is@@ -124,7 +125,7 @@ func (fs *FilerServer) createTusSession(ctx context.Context, uploadID, targetPatOffset: 0,Metadata: metadata,CreatedAt: time.Now(),- ExpiresAt: time.Now().Add(7 * 24 * time.Hour), // 7 days default expiration+ ExpiresAt: time.Now().Add(fs.option.TusSessionExpiry),Chunks: []*TusChunkInfo{},}@@ -209,7 +210,7 @@ func (fs *FilerServer) readTusSessionInfo(ctx context.Context, uploadID string)if target == "" || target == "/" {return nil, fmt.Errorf("invalid TUS target path: %q", session.TargetPath)}- if session.Size < 0 || session.Size > TusMaxSize {+ if session.Size < 0 || session.Size > fs.option.TusMaxSize {return nil, fmt.Errorf("invalid TUS upload size: %d", session.Size)}// Pin authorization and every later operation to the same canonical path.