Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

filer: extract zip members on ?extract=zip upload (#11782)

Chris Lua8affc851d59
diff --git a/weed/filer/zip_extract.go b/weed/filer/zip_extract.go
new file mode 100644
index 0000000000..99ed3a624f
--- /dev/null
+++ b/weed/filer/zip_extract.go
@@ -0,0 +1,117 @@
+package filer
+
+import (
+ "archive/zip"
+ "bytes"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+const (
+ // ZipExtractMaxMembers bounds how many members one archive write fans out to.
+ ZipExtractMaxMembers = 100000
+ // ZipExtractMaxBytes bounds the total uncompressed bytes one archive may declare.
+ ZipExtractMaxBytes = int64(1) << 40 // 1 TiB
+
+ // ZipExtractMaxArchiveBytes bounds the compressed bytes one archive may
+ // spool to local disk when the caller passes no tighter limit. It matches
+ // the TUS default max upload size.
+ ZipExtractMaxArchiveBytes = int64(5) << 30 // 5 GiB
+
+ zipSpoolMemoryLimit = 32 << 20 // 32 MiB; larger archives spool to a temp file
+)
+
+// ErrZipTooLarge marks a spool failure caused by the archive exceeding the
+// caller's size limit, so handlers can answer 4xx instead of 5xx.
+var ErrZipTooLarge = errors.New("zip archive too large")
+
+// SpoolZipReader buffers r into a seekable source for zip.NewReader: memory for
+// small archives, a temp file otherwise. At most maxArchiveBytes are buffered;
+// a non-positive limit falls back to ZipExtractMaxArchiveBytes. The returned
+// cleanup releases the spool.
+func SpoolZipReader(r io.Reader, maxArchiveBytes int64) (ra io.ReaderAt, n int64, cleanup func(), err error) {
+ nop := func() {}
+ if maxArchiveBytes <= 0 {
+ maxArchiveBytes = ZipExtractMaxArchiveBytes
+ }
+ head, err := io.ReadAll(io.LimitReader(r, zipSpoolMemoryLimit+1))
+ if err != nil {
+ return nil, 0, nop, err
+ }
+ if int64(len(head)) > maxArchiveBytes {
+ return nil, 0, nop, fmt.Errorf("%w: exceeds %d bytes", ErrZipTooLarge, maxArchiveBytes)
+ }
+ if int64(len(head)) <= zipSpoolMemoryLimit {
+ return bytes.NewReader(head), int64(len(head)), nop, nil
+ }
+ f, err := os.CreateTemp("", "seaweedfs-extract-*.zip")
+ if err != nil {
+ return nil, 0, nop, err
+ }
+ cleanup = func() {
+ name := f.Name()
+ f.Close()
+ os.Remove(name)
+ }
+ if _, err = f.Write(head); err == nil {
+ var rest int64
+ // One byte past the limit so an oversized archive errors out instead
+ // of being silently truncated.
+ if rest, err = io.Copy(f, io.LimitReader(r, maxArchiveBytes+1-int64(len(head)))); err == nil {
+ n = int64(len(head)) + rest
+ }
+ }
+ if err == nil && n > maxArchiveBytes {
+ err = fmt.Errorf("%w: exceeds %d bytes", ErrZipTooLarge, maxArchiveBytes)
+ }
+ if err != nil {
+ cleanup()
+ return nil, 0, nop, err
+ }
+ return f, n, cleanup, nil
+}
+
+// CheckZipLimits enforces the member-count and total-size caps on an opened archive.
+func CheckZipLimits(zr *zip.Reader) error {
+ if len(zr.File) > ZipExtractMaxMembers {
+ return fmt.Errorf("zip archive has %d members, limit %d", len(zr.File), ZipExtractMaxMembers)
+ }
+ var total uint64
+ for _, f := range zr.File {
+ if f.UncompressedSize64 > uint64(ZipExtractMaxBytes)-total {
+ return fmt.Errorf("zip archive uncompressed size exceeds %d bytes", ZipExtractMaxBytes)
+ }
+ total += f.UncompressedSize64
+ }
+ return nil
+}
+
+// SanitizeZipMemberName converts a zip member name into a relative filer path.
+// ok=false means the member must be skipped: absolute or escaping paths,
+// non-regular files such as symlinks and devices.
+func SanitizeZipMemberName(f *zip.File) (name string, ok bool) {
+ mode := f.FileInfo().Mode()
+ if mode&(os.ModeSymlink|os.ModeDevice|os.ModeNamedPipe|os.ModeSocket) != 0 {
+ return "", false
+ }
+ name = strings.TrimSuffix(strings.ReplaceAll(f.Name, "\\", "/"), "/")
+ if name == "" || strings.HasPrefix(name, "/") {
+ return "", false
+ }
+ for _, seg := range strings.Split(name, "/") {
+ if seg == "" || seg == "." || seg == ".." || strings.ContainsRune(seg, '\x00') {
+ return "", false
+ }
+ }
+ // filepath.IsLocal is the check static analysis recognizes: no element may
+ // be "." or "..", the path may not be absolute or empty, and on Windows it
+ // may not be a reserved name.
+ if !filepath.IsLocal(name) {
+ return "", false
+ }
+ return name, true
+}
diff --git a/weed/filer/zip_extract_test.go b/weed/filer/zip_extract_test.go
new file mode 100644
index 0000000000..e91b3a635d
--- /dev/null
+++ b/weed/filer/zip_extract_test.go
@@ -0,0 +1,154 @@
+package filer
+
+import (
+ "archive/zip"
+ "bytes"
+ "io"
+ "math"
+ "os"
+ "strings"
+ "testing"
+ "time"
+)
+
+func zipFile(t *testing.T, name string, mode os.FileMode) *zip.File {
+ t.Helper()
+ h := &zip.FileHeader{Name: name, Method: zip.Store, Modified: time.Unix(1700000000, 0)}
+ h.SetMode(mode)
+ return &zip.File{FileHeader: *h}
+}
+
+func TestSanitizeZipMemberName(t *testing.T) {
+ tests := []struct {
+ name string
+ mode os.FileMode
+ want string
+ ok bool
+ }{
+ {"a.png", 0644, "a.png", true},
+ {"dir/b.txt", 0644, "dir/b.txt", true},
+ {"dir\\b.txt", 0644, "dir/b.txt", true},
+ {"dir/", os.ModeDir | 0755, "dir", true},
+ {"../escape.txt", 0644, "", false},
+ {"a/../b.txt", 0644, "", false},
+ {"./dot.txt", 0644, "", false},
+ {"/abs.txt", 0644, "", false},
+ {"a//b.txt", 0644, "", false},
+ {"", 0644, "", false},
+ {"link.txt", os.ModeSymlink | 0777, "", false},
+ {"dev", os.ModeDevice | 0666, "", false},
+ }
+ for _, tt := range tests {
+ got, ok := SanitizeZipMemberName(zipFile(t, tt.name, tt.mode))
+ if ok != tt.ok || got != tt.want {
+ t.Errorf("SanitizeZipMemberName(%q, %v) = (%q, %v), want (%q, %v)", tt.name, tt.mode, got, ok, tt.want, tt.ok)
+ }
+ }
+}
+
+func TestSpoolZipReaderMemory(t *testing.T) {
+ data := []byte(strings.Repeat("x", 1024))
+ ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(data), 0)
+ if err != nil {
+ t.Fatalf("spool: %v", err)
+ }
+ defer cleanup()
+ if n != int64(len(data)) {
+ t.Fatalf("size = %d, want %d", n, len(data))
+ }
+ if _, isFile := ra.(*os.File); isFile {
+ t.Fatal("small archive should spool to memory")
+ }
+ buf := make([]byte, 4)
+ if _, err := ra.ReadAt(buf, 0); err != nil || string(buf) != "xxxx" {
+ t.Fatalf("ReadAt = %q, %v", buf, err)
+ }
+}
+
+func TestSpoolZipReaderTempFile(t *testing.T) {
+ data := bytes.Repeat([]byte("y"), zipSpoolMemoryLimit+1)
+ ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(data), 0)
+ if err != nil {
+ t.Fatalf("spool: %v", err)
+ }
+ f, isFile := ra.(*os.File)
+ if !isFile {
+ cleanup()
+ t.Fatal("large archive should spool to a temp file")
+ }
+ name := f.Name()
+ cleanup()
+ if _, err := os.Stat(name); !os.IsNotExist(err) {
+ t.Fatalf("temp file %s still exists after cleanup: %v", name, err)
+ }
+ if n != int64(len(data)) {
+ t.Fatalf("size = %d, want %d", n, len(data))
+ }
+}
+
+func TestSpoolZipReaderLimit(t *testing.T) {
+ data := []byte(strings.Repeat("x", 2048))
+ if _, _, _, err := SpoolZipReader(bytes.NewReader(data), 1024); err == nil {
+ t.Fatal("expected limit error for in-memory archive")
+ }
+ if _, _, cleanup, err := SpoolZipReader(bytes.NewReader(bytes.Repeat([]byte("y"), zipSpoolMemoryLimit+1024)), zipSpoolMemoryLimit+512); err == nil {
+ t.Fatal("expected limit error for spooled archive")
+ } else {
+ cleanup()
+ }
+}
+
+func TestCheckZipLimits(t *testing.T) {
+ over := &zip.Reader{File: make([]*zip.File, ZipExtractMaxMembers+1)}
+ for i := range over.File {
+ over.File[i] = &zip.File{}
+ }
+ if err := CheckZipLimits(over); err == nil {
+ t.Fatal("expected member count limit error")
+ }
+
+ big := &zip.Reader{File: []*zip.File{{}}}
+ big.File[0].UncompressedSize64 = uint64(ZipExtractMaxBytes) + 1
+ if err := CheckZipLimits(big); err == nil {
+ t.Fatal("expected total size limit error")
+ }
+
+ overflow := &zip.Reader{File: []*zip.File{{}, {}}}
+ overflow.File[0].UncompressedSize64 = math.MaxUint64
+ overflow.File[1].UncompressedSize64 = math.MaxUint64
+ if err := CheckZipLimits(overflow); err == nil {
+ t.Fatal("expected total size overflow error")
+ }
+
+ ok := &zip.Reader{File: []*zip.File{{}, {}}}
+ if err := CheckZipLimits(ok); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+}
+
+func TestSpoolZipReaderRoundTrip(t *testing.T) {
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ w, _ := zw.Create("a/b.txt")
+ w.Write([]byte("hello"))
+ zw.Close()
+
+ ra, n, cleanup, err := SpoolZipReader(bytes.NewReader(buf.Bytes()), 0)
+ if err != nil {
+ t.Fatalf("spool: %v", err)
+ }
+ defer cleanup()
+ zr, err := zip.NewReader(ra, n)
+ if err != nil {
+ t.Fatalf("open zip: %v", err)
+ }
+ rc, err := zr.File[0].Open()
+ if err != nil {
+ t.Fatalf("open member: %v", err)
+ }
+ defer rc.Close()
+ data, _ := io.ReadAll(rc)
+ if string(data) != "hello" {
+ t.Fatalf("member content = %q", data)
+ }
+}
diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go
index 7f03adbb35..f88fc351bc 100644
--- a/weed/s3api/s3_constants/header.go
+++ b/weed/s3api/s3_constants/header.go
@@ -58,6 +58,7 @@ const (
AmzDeleteMarker = "X-Amz-Delete-Marker"
SeaweedFSUploadId = "X-Seaweedfs-Upload-Id"
+ SeaweedFSExtract = "X-Seaweedfs-Extract"
SeaweedFSMultipartPartsCount = "X-Seaweedfs-Multipart-Parts-Count"
SeaweedFSMultipartPartBoundaries = "X-Seaweedfs-Multipart-Part-Boundaries" // JSON: [{part:1,start:0,end:2,etag:"abc"},{part:2,start:2,end:3,etag:"def"}]
SeaweedFSExpiresS3 = "X-Seaweedfs-Expires-S3"
diff --git a/weed/server/filer_server_handlers_extract.go b/weed/server/filer_server_handlers_extract.go
new file mode 100644
index 0000000000..c51d7b9c6f
--- /dev/null
+++ b/weed/server/filer_server_handlers_extract.go
@@ -0,0 +1,339 @@
+package weed_server
+
+import (
+ "archive/zip"
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "mime"
+ "net/http"
+ "os"
+ "path"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/seaweedfs/seaweedfs/weed/filer"
+ "github.com/seaweedfs/seaweedfs/weed/glog"
+ "github.com/seaweedfs/seaweedfs/weed/operation"
+ "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+ "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+ "github.com/seaweedfs/seaweedfs/weed/util"
+ "github.com/seaweedfs/seaweedfs/weed/util/constants"
+)
+
+type FilerExtractResult struct {
+ Name string `json:"name,omitempty"`
+ Extracted []*FilerPostResult `json:"extracted,omitempty"`
+ Errors []string `json:"errors,omitempty"`
+}
+
+func isExtractZip(r *http.Request) bool {
+ return r.URL.Query().Get("extract") == "zip" ||
+ r.Header.Get(s3_constants.SeaweedFSExtract) == "zip"
+}
+
+// extractZipArchive writes a posted archive's members as individual entries
+// under the target directory, like AWS Snowball's auto-extract. The archive
+// itself is only stored when ?extract.keep=true.
+func (fs *FilerServer) extractZipArchive(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+ var reader io.Reader = r.Body
+ archiveName := path.Base(r.URL.Path)
+ if r.Method == http.MethodPost {
+ if mediaType, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")); strings.HasPrefix(mediaType, "multipart/") {
+ multipartReader, multipartReaderErr := r.MultipartReader()
+ if multipartReaderErr != nil {
+ writeJsonError(w, r, http.StatusBadRequest, multipartReaderErr)
+ return
+ }
+ part1, part1Err := multipartReader.NextPart()
+ if part1Err != nil {
+ writeJsonError(w, r, http.StatusBadRequest, part1Err)
+ return
+ }
+ defer part1.Close()
+ reader = part1
+ if fileName := part1.FileName(); fileName != "" {
+ archiveName = path.Base(fileName)
+ }
+ }
+ }
+
+ ra, size, cleanup, err := filer.SpoolZipReader(reader, fs.option.TusMaxSize)
+ if err != nil {
+ code := http.StatusInternalServerError
+ if errors.Is(err, filer.ErrZipTooLarge) {
+ code = http.StatusRequestEntityTooLarge
+ }
+ writeJsonError(w, r, code, fmt.Errorf("read input: %w", err))
+ return
+ }
+ defer cleanup()
+ zr, err := zip.NewReader(ra, size)
+ if err != nil {
+ writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("extract zip: %w", err))
+ return
+ }
+ if err = filer.CheckZipLimits(zr); err != nil {
+ writeJsonError(w, r, http.StatusBadRequest, err)
+ return
+ }
+
+ root := r.URL.Path
+ if !strings.HasSuffix(root, "/") {
+ root = path.Dir(root)
+ }
+ root = strings.TrimSuffix(root, "/")
+
+ query := r.URL.Query()
+ chunkSize := fs.requestChunkSize(query)
+
+ // x-seaweedfs-destination picks storage rules for a logical destination
+ // while members land under the request path, mirroring PostHandler. The
+ // actual write path keeps its own read-only enforcement in
+ // saveExtractedEntry and mkdirExtracted.
+ ruleRoot := root
+ if headerDestination := r.Header.Get(s3_constants.SeaweedStorageDestinationHeader); headerDestination != "" {
+ ruleRoot = strings.TrimSuffix(headerDestination, "/")
+ }
+
+ // Members may land under storage rules different from the request path;
+ // resolve options per destination, keeping the request's explicit overrides.
+ memberStorageOption := func(rulePath, writePath string) (*operation.StorageOption, error) {
+ memberSO, detectErr := fs.detectStorageOption0(ctx, rulePath,
+ query.Get("collection"),
+ query.Get("replication"),
+ query.Get("ttl"),
+ query.Get("disk"),
+ query.Get("fsync"),
+ query.Get("dataCenter"),
+ query.Get("rack"),
+ query.Get("dataNode"),
+ query.Get("saveInside"),
+ )
+ if detectErr != nil {
+ return nil, detectErr
+ }
+ if memberSO.DiskType == "" {
+ memberSO.DiskType = fs.option.DiskType
+ }
+ if strings.HasPrefix(writePath, "/etc") {
+ memberSO.SaveInside = true
+ }
+ return memberSO, nil
+ }
+
+ result := &FilerExtractResult{Name: archiveName}
+ writeMember := func(rel string, memberReader io.Reader, memberSize int64, mtime time.Time, mode os.FileMode) {
+ destPath := root + "/" + rel
+ memberSO, soErr := memberStorageOption(ruleRoot+"/"+rel, destPath)
+ if soErr != nil {
+ result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, soErr))
+ return
+ }
+ memberResult, memberErr := fs.saveExtractedEntry(ctx, destPath, memberReader, memberSize, path.Base(rel), chunkSize, memberSO, mtime, mode)
+ if memberErr != nil {
+ result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, memberErr))
+ return
+ }
+ memberResult.Name = rel
+ result.Extracted = append(result.Extracted, memberResult)
+ }
+ mkdirMember := func(rel string, zf *zip.File) {
+ destPath := root + "/" + rel
+ memberSO, soErr := memberStorageOption(ruleRoot+"/"+rel, destPath)
+ if soErr != nil {
+ result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, soErr))
+ return
+ }
+ if dirErr := fs.mkdirExtracted(ctx, destPath, zf, memberSO); dirErr != nil {
+ result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, dirErr))
+ }
+ }
+
+ if query.Get("extract.keep") == "true" && archiveName != "." && archiveName != "/" {
+ mode := os.FileMode(0660)
+ if modeStr := query.Get("mode"); modeStr != "" {
+ if parsed, parseErr := strconv.ParseUint(modeStr, 8, 32); parseErr == nil {
+ mode = os.FileMode(parsed)
+ }
+ }
+ writeMember(archiveName, io.NewSectionReader(ra, 0, size), size, time.Now(), mode)
+ }
+
+ // Directory members first so their stored metadata is not overwritten by
+ // parents auto-created for an earlier file member.
+ for _, dirsOnly := range []bool{true, false} {
+ for _, zf := range zr.File {
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ result.Errors = append(result.Errors, ctxErr.Error())
+ writeJsonQuiet(w, r, http.StatusCreated, result)
+ return
+ }
+ rel, ok := filer.SanitizeZipMemberName(zf)
+ if !ok {
+ if !dirsOnly {
+ result.Errors = append(result.Errors, fmt.Sprintf("%s: unsafe member name", zf.Name))
+ }
+ continue
+ }
+ if zf.FileInfo().IsDir() != dirsOnly {
+ continue
+ }
+ if dirsOnly {
+ mkdirMember(rel, zf)
+ continue
+ }
+ memberReader, openErr := zf.Open()
+ if openErr != nil {
+ result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", rel, openErr))
+ continue
+ }
+ mode := os.FileMode(0660)
+ if perm := zf.Mode().Perm(); perm != 0 {
+ mode = perm
+ }
+ writeMember(rel, memberReader, int64(zf.UncompressedSize64), zf.Modified, mode)
+ memberReader.Close()
+ }
+ }
+
+ writeJsonQuiet(w, r, http.StatusCreated, result)
+}
+
+func (fs *FilerServer) mkdirExtracted(ctx context.Context, destPath string, zf *zip.File, so *operation.StorageOption) error {
+ if fs.filer.FilerConf.MatchStorageRule(destPath).ReadOnly {
+ return ErrReadOnly
+ }
+ existing, findErr := fs.filer.FindEntry(ctx, util.FullPath(destPath))
+ if findErr == nil {
+ if existing.IsDirectory() {
+ return nil
+ }
+ return fmt.Errorf("%s already exists as a file", destPath)
+ }
+ if !errors.Is(findErr, filer_pb.ErrNotFound) {
+ return findErr
+ }
+ mode := os.FileMode(0660)
+ if perm := zf.Mode().Perm(); perm != 0 {
+ mode = perm
+ }
+ mtime := zf.Modified
+ if mtime.IsZero() {
+ mtime = time.Now()
+ }
+ return fs.filer.CreateEntry(context.WithoutCancel(ctx), &filer.Entry{
+ FullPath: util.FullPath(destPath),
+ Attr: filer.Attr{
+ Mtime: mtime,
+ Crtime: time.Now(),
+ Mode: mode | os.ModeDir,
+ Uid: OS_UID,
+ Gid: OS_GID,
+ TtlSec: so.TtlSeconds,
+ },
+ }, nil, false, false, nil, false, so.MaxFileNameLength)
+}
+
+func (fs *FilerServer) saveExtractedEntry(ctx context.Context, destPath string, reader io.Reader, memberSize int64, fileName string, chunkSize int32, so *operation.StorageOption, mtime time.Time, mode os.FileMode) (*FilerPostResult, error) {
+ if fs.filer.FilerConf.MatchStorageRule(destPath).ReadOnly {
+ return nil, ErrReadOnly
+ }
+ wormEnforced, err := fs.wormEnforcedForEntry(ctx, destPath)
+ if err != nil {
+ return nil, err
+ }
+ if wormEnforced {
+ return nil, errors.New(constants.ErrMsgOperationNotPermitted)
+ }
+ if util.FullPath(destPath).IsLongerFileName(so.MaxFileNameLength) {
+ return nil, filer_pb.ErrEntryNameTooLong
+ }
+
+ contentType := mime.TypeByExtension(path.Ext(fileName))
+
+ // saveInside stores members as inline entry content like a normal
+ // multipart upload; the declared size keeps a corrupt archive from
+ // buffering more than it advertised.
+ if so.SaveInside {
+ if memberSize < 0 || memberSize > filer.ZipExtractMaxBytes {
+ return nil, fmt.Errorf("zip member %s size %d out of bounds", fileName, memberSize)
+ }
+ var buf bytes.Buffer
+ if _, err := buf.ReadFrom(io.LimitReader(reader, memberSize+1)); err != nil {
+ return nil, fmt.Errorf("read member: %w", err)
+ }
+ if int64(buf.Len()) > memberSize {
+ return nil, fmt.Errorf("zip member %s is larger than its declared size", fileName)
+ }
+ if mtime.IsZero() {
+ mtime = time.Now()
+ }
+ entry := &filer.Entry{
+ FullPath: util.FullPath(destPath),
+ Attr: filer.Attr{
+ Mtime: mtime,
+ Crtime: time.Now(),
+ Mode: mode,
+ Uid: OS_UID,
+ Gid: OS_GID,
+ TtlSec: so.TtlSeconds,
+ Mime: contentType,
+ Md5: util.Md5(buf.Bytes()),
+ FileSize: uint64(buf.Len()),
+ },
+ Content: buf.Bytes(),
+ }
+ if err := fs.filer.CreateEntry(context.WithoutCancel(ctx), entry, nil, false, false, nil, false, so.MaxFileNameLength); err != nil {
+ return nil, err
+ }
+ return &FilerPostResult{Name: fileName, Size: int64(buf.Len())}, nil
+ }
+
+ fileChunks, md5Hash, chunkOffset, uploadErr, smallContent := fs.uploadReaderToChunks(ctx, nil, reader, 0, chunkSize, fileName, contentType, false, so)
+ if uploadErr != nil {
+ return nil, uploadErr
+ }
+ if chunkOffset == 0 {
+ smallContent = []byte{}
+ }
+
+ mergedChunks, mergeErr := fs.maybeMergeChunks(ctx, so, fileChunks)
+ if mergeErr != nil {
+ glog.V(0).InfofCtx(ctx, "merge chunks %s: %v", destPath, mergeErr)
+ mergedChunks = fileChunks
+ }
+ mergedChunks, manifestErr := filer.MaybeManifestize(fs.saveAsChunk(ctx, so), fs.filer.DeleteChunksNotRecursive, mergedChunks)
+ if manifestErr != nil {
+ fs.filer.DeleteUncommittedChunks(ctx, mergedChunks)
+ return nil, manifestErr
+ }
+
+ if mtime.IsZero() {
+ mtime = time.Now()
+ }
+ entry := &filer.Entry{
+ FullPath: util.FullPath(destPath),
+ Attr: filer.Attr{
+ Mtime: mtime,
+ Crtime: time.Now(),
+ Mode: mode,
+ Uid: OS_UID,
+ Gid: OS_GID,
+ TtlSec: so.TtlSeconds,
+ Mime: contentType,
+ Md5: md5Hash.Sum(nil),
+ FileSize: uint64(chunkOffset),
+ },
+ Content: smallContent,
+ Chunks: mergedChunks,
+ }
+ if err := fs.filer.CreateEntry(context.WithoutCancel(ctx), entry, nil, false, false, nil, false, so.MaxFileNameLength); err != nil {
+ fs.filer.DeleteUncommittedChunks(ctx, mergedChunks)
+ return nil, err
+ }
+ return &FilerPostResult{Name: fileName, Size: chunkOffset}, nil
+}
diff --git a/weed/server/filer_server_handlers_extract_test.go b/weed/server/filer_server_handlers_extract_test.go
new file mode 100644
index 0000000000..9461e79e5a
--- /dev/null
+++ b/weed/server/filer_server_handlers_extract_test.go
@@ -0,0 +1,274 @@
+package weed_server
+
+import (
+ "archive/zip"
+ "bytes"
+ "encoding/json"
+ "mime/multipart"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/seaweedfs/seaweedfs/weed/filer"
+ "github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func buildTestZip(t *testing.T, members map[string]string, dirs []string, unsafe []string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ for _, dir := range dirs {
+ h := &zip.FileHeader{Name: dir, Method: zip.Store, Modified: time.Unix(1700000000, 0)}
+ h.SetMode(0755 | 1<<31)
+ if _, err := zw.CreateHeader(h); err != nil {
+ t.Fatalf("create dir %s: %v", dir, err)
+ }
+ }
+ write := func(name, content string) {
+ w, err := zw.Create(name)
+ if err != nil {
+ t.Fatalf("create %s: %v", name, err)
+ }
+ if _, err := w.Write([]byte(content)); err != nil {
+ t.Fatalf("write %s: %v", name, err)
+ }
+ }
+ for name, content := range members {
+ write(name, content)
+ }
+ for _, name := range unsafe {
+ write(name, "evil")
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatalf("close zip: %v", err)
+ }
+ return buf.Bytes()
+}
+
+func newExtractTestServer(t *testing.T) (*FilerServer, *filer.Filer) {
+ store := newRenameTestStore()
+ f := newRenameTestFiler(t, store)
+ return &FilerServer{
+ filer: f,
+ option: &FilerOption{
+ MaxMB: 1,
+ SaveToFilerLimit: 1 << 20,
+ },
+ }, f
+}
+
+func TestExtractZipArchivePut(t *testing.T) {
+ fs, f := newExtractTestServer(t)
+ var buf bytes.Buffer
+ zw := zip.NewWriter(&buf)
+ dirHeader := &zip.FileHeader{Name: "empty/", Method: zip.Store, Modified: time.Unix(1700000000, 0)}
+ dirHeader.SetMode(0755 | 1<<31)
+ if _, err := zw.CreateHeader(dirHeader); err != nil {
+ t.Fatal(err)
+ }
+ fileHeader := &zip.FileHeader{Name: "a/b.txt", Modified: time.Unix(1600000000, 0)}
+ fileHeader.SetMode(0600)
+ fw, err := zw.CreateHeader(fileHeader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := fw.Write([]byte("hello")); err != nil {
+ t.Fatal(err)
+ }
+ pngW, err := zw.Create("c.png")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := pngW.Write([]byte("\x89png")); err != nil {
+ t.Fatal(err)
+ }
+ evil, err := zw.Create("../evil.txt")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := evil.Write([]byte("evil")); err != nil {
+ t.Fatal(err)
+ }
+ if err := zw.Close(); err != nil {
+ t.Fatal(err)
+ }
+ zipBytes := buf.Bytes()
+
+ r := httptest.NewRequest(http.MethodPut, "/dst/archive.zip?extract=zip", bytes.NewReader(zipBytes))
+ w := httptest.NewRecorder()
+ fs.PostHandler(w, r, int64(len(zipBytes)))
+
+ if w.Code != http.StatusCreated {
+ t.Fatalf("status = %d, want 201, body: %s", w.Code, w.Body.String())
+ }
+ var result FilerExtractResult
+ if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil {
+ t.Fatalf("decode result: %v", err)
+ }
+ if len(result.Extracted) != 2 {
+ t.Fatalf("extracted %d members, want 2: %+v", len(result.Extracted), result)
+ }
+ if len(result.Errors) != 1 {
+ t.Fatalf("errors = %v, want 1 unsafe-name entry", result.Errors)
+ }
+
+ entry, err := f.FindEntry(r.Context(), util.FullPath("/dst/a/b.txt"))
+ if err != nil {
+ t.Fatalf("find extracted member: %v", err)
+ }
+ if string(entry.Content) != "hello" {
+ t.Fatalf("member content = %q", entry.Content)
+ }
+ if entry.Mtime.Unix() != 1600000000 {
+ t.Fatalf("member mtime = %v, want %v", entry.Mtime, time.Unix(1600000000, 0))
+ }
+ if entry.Mode.Perm() != 0600 {
+ t.Fatalf("member mode = %v, want 0600", entry.Mode.Perm())
+ }
+
+ dirEntry, err := f.FindEntry(r.Context(), util.FullPath("/dst/empty"))
+ if err != nil || !dirEntry.IsDirectory() {
+ t.Fatalf("dir member missing: %v %+v", err, dirEntry)
+ }
+ if dirEntry.Mtime.Unix() != 1700000000 {
+ t.Fatalf("dir mtime = %v, want %v", dirEntry.Mtime, time.Unix(1700000000, 0))
+ }
+ if _, err := f.FindEntry(r.Context(), util.FullPath("/dst/archive.zip")); err == nil {
+ t.Fatal("archive should not be stored without extract.keep")
+ }
+ if _, err := f.FindEntry(r.Context(), util.FullPath("/dst/../evil.txt")); err == nil {
+ t.Fatal("unsafe member should not be extracted")
+ }
+ if _, err := f.FindEntry(r.Context(), util.FullPath("/evil.txt")); err == nil {
+ t.Fatal("zip-slip member escaped the extraction root")
+ }
+}
+
+func TestExtractZipArchiveKeep(t *testing.T) {
+ fs, f := newExtractTestServer(t)
+ zipBytes := buildTestZip(t, map[string]string{"x.txt": "data"}, nil, nil)
+
+ r := httptest.NewRequest(http.MethodPut, "/dst/pack.zip?extract=zip&extract.keep=true", bytes.NewReader(zipBytes))
+ w := httptest.NewRecorder()
+ fs.PostHandler(w, r, int64(len(zipBytes)))
+
+ if w.Code != http.StatusCreated {
+ t.Fatalf("status = %d, body: %s", w.Code, w.Body.String())
+ }
+ archive, err := f.FindEntry(r.Context(), util.FullPath("/dst/pack.zip"))
+ if err != nil {
+ t.Fatalf("archive not stored with extract.keep: %v", err)
+ }
+ if int64(len(archive.Content)) != int64(len(zipBytes)) {
+ t.Fatalf("archive size = %d, want %d", len(archive.Content), len(zipBytes))
+ }
+ if _, err := f.FindEntry(r.Context(), util.FullPath("/dst/x.txt")); err != nil {
+ t.Fatalf("member missing: %v", err)
+ }
+}
+
+func TestExtractZipArchiveNotAZip(t *testing.T) {
+ fs, _ := newExtractTestServer(t)
+ r := httptest.NewRequest(http.MethodPut, "/dst/notzip.bin?extract=zip", bytes.NewReader([]byte("not a zip")))
+ w := httptest.NewRecorder()
+ fs.PostHandler(w, r, 9)
+ if w.Code != http.StatusBadRequest {
+ t.Fatalf("status = %d, want 400", w.Code)
+ }
+}
+
+// A member larger than chunkSize must not take the inline fast path when
+// SaveToFilerLimit exceeds chunkSize — the first chunk is not the whole member.
+func TestExtractZipArchiveMemberLargerThanChunk(t *testing.T) {
+ store := newRenameTestStore()
+ f := newRenameTestFiler(t, store)
+ fs, _ := newInlineAppendUploadServer(t, f, 4<<20, 1) // inline limit exceeds the 1 MiB chunkSize
+ member := bytes.Repeat([]byte("z"), (1<<20)+1)
+ zipBytes := buildTestZip(t, map[string]string{"big.bin": string(member)}, nil, nil)
+
+ r := httptest.NewRequest(http.MethodPut, "/dst/pack.zip?extract=zip", bytes.NewReader(zipBytes))
+ w := httptest.NewRecorder()
+ fs.PostHandler(w, r, int64(len(zipBytes)))
+
+ if w.Code != http.StatusCreated {
+ t.Fatalf("status = %d, body: %s", w.Code, w.Body.String())
+ }
+ var result FilerExtractResult
+ if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil {
+ t.Fatalf("decode result: %v", err)
+ }
+ if len(result.Errors) != 0 {
+ t.Fatalf("member errors: %v", result.Errors)
+ }
+ entry, err := f.FindEntry(r.Context(), util.FullPath("/dst/big.bin"))
+ if err != nil {
+ t.Fatalf("member missing: %v", err)
+ }
+ if int64(entry.Attr.FileSize) != int64(len(member)) {
+ t.Fatalf("member stored truncated: %d bytes, want %d", entry.Attr.FileSize, len(member))
+ }
+}
+
+func TestExtractZipArchiveRawPost(t *testing.T) {
+ fs, f := newExtractTestServer(t)
+ zipBytes := buildTestZip(t, map[string]string{"raw.txt": "raw"}, nil, nil)
+
+ r := httptest.NewRequest(http.MethodPost, "/dst/?extract=zip", bytes.NewReader(zipBytes))
+ r.Header.Set("Content-Type", "application/zip")
+ w := httptest.NewRecorder()
+ fs.PostHandler(w, r, int64(len(zipBytes)))
+
+ if w.Code != http.StatusCreated {
+ t.Fatalf("status = %d, body: %s", w.Code, w.Body.String())
+ }
+ entry, err := f.FindEntry(r.Context(), util.FullPath("/dst/raw.txt"))
+ if err != nil {
+ t.Fatalf("member missing: %v", err)
+ }
+ if string(entry.Content) != "raw" {
+ t.Fatalf("member content = %q", entry.Content)
+ }
+}
+
+func TestExtractZipArchiveMultipartPost(t *testing.T) {
+ fs, f := newExtractTestServer(t)
+ zipBytes := buildTestZip(t, map[string]string{"img/1.png": "png-bytes"}, nil, nil)
+
+ var body bytes.Buffer
+ mw := multipartWriter(t, &body, "bundle.zip", zipBytes)
+ r := httptest.NewRequest(http.MethodPost, "/assets/?extract=zip", &body)
+ r.Header.Set("Content-Type", mw)
+ w := httptest.NewRecorder()
+ fs.PostHandler(w, r, int64(body.Len()))
+
+ if w.Code != http.StatusCreated {
+ t.Fatalf("status = %d, body: %s", w.Code, w.Body.String())
+ }
+ entry, err := f.FindEntry(r.Context(), util.FullPath("/assets/img/1.png"))
+ if err != nil {
+ t.Fatalf("member missing: %v", err)
+ }
+ if string(entry.Content) != "png-bytes" {
+ t.Fatalf("member content = %q", entry.Content)
+ }
+ if _, err := f.FindEntry(r.Context(), util.FullPath("/assets/bundle.zip")); err == nil {
+ t.Fatal("archive should not be stored")
+ }
+}
+
+func multipartWriter(t *testing.T, buf *bytes.Buffer, fileName string, data []byte) string {
+ t.Helper()
+ mw := multipart.NewWriter(buf)
+ part, err := mw.CreateFormFile("file", fileName)
+ if err != nil {
+ t.Fatalf("create form file: %v", err)
+ }
+ if _, err := part.Write(data); err != nil {
+ t.Fatalf("write part: %v", err)
+ }
+ if err := mw.Close(); err != nil {
+ t.Fatalf("close multipart: %v", err)
+ }
+ return mw.FormDataContentType()
+}
diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go
index b1a96e07c4..4bb91c9d0d 100644
--- a/weed/server/filer_server_handlers_write.go
+++ b/weed/server/filer_server_handlers_write.go
@@ -141,6 +141,8 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte
fs.move(ctx, w, r, so)
} else if query.Has("cp.from") {
fs.copy(ctx, w, r, so)
+ } else if isExtractZip(r) {
+ fs.extractZipArchive(ctx, w, r)
} else {
fs.autoChunk(ctx, w, r, contentLength, so)
}
diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go
index a2de8e74f1..f6df863eb2 100644
--- a/weed/server/filer_server_handlers_write_autochunk.go
+++ b/weed/server/filer_server_handlers_write_autochunk.go
@@ -6,7 +6,9 @@ import (
"errors"
"fmt"
"io"
+ "math"
"net/http"
+ "net/url"
"os"
"path"
"strconv"
@@ -29,13 +31,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
// autoChunking can be set at the command-line level or as a query param. Query param overrides command-line
query := r.URL.Query()
- parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32)
- maxMB := int32(parsedMaxMB)
- if maxMB <= 0 && fs.option.MaxMB > 0 {
- maxMB = int32(fs.option.MaxMB)
- }
-
- chunkSize := 1024 * 1024 * maxMB
+ chunkSize := fs.requestChunkSize(query)
var reply *FilerPostResult
var err error
@@ -178,6 +174,22 @@ func skipCheckParentDirEntry(r *http.Request) bool {
return r.URL.Query().Get("skipCheckParentDir") == "true"
}
+// requestChunkSize resolves the auto-chunking size from the maxMB query param
+// or the server default, clamped so the int32 chunk size cannot overflow into
+// a non-positive value that would drop the upload's contents.
+func (fs *FilerServer) requestChunkSize(query url.Values) int32 {
+ parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32)
+ maxMB := parsedMaxMB
+ if maxMB <= 0 {
+ maxMB = int64(fs.option.MaxMB)
+ }
+ const maxChunkSizeMB = math.MaxInt32 / (1024 * 1024) // 2047
+ if maxMB > maxChunkSizeMB {
+ maxMB = maxChunkSizeMB
+ }
+ return int32(maxMB) * 1024 * 1024
+}
+
func (fs *FilerServer) checkPermissions(ctx context.Context, r *http.Request, fileName string) error {
fullPath := fs.fixFilePath(ctx, r, fileName)
enforced, err := fs.wormEnforcedForEntry(ctx, fullPath)
diff --git a/weed/server/filer_server_handlers_write_upload.go b/weed/server/filer_server_handlers_write_upload.go
index f41ba61515..5631e3e554 100644
--- a/weed/server/filer_server_handlers_write_upload.go
+++ b/weed/server/filer_server_handlers_write_upload.go
@@ -149,7 +149,9 @@ func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request
break
}
if chunkOffset == 0 && !isAppend {
- if dataSize < fs.option.SaveToFilerLimit {
+ // A short first read proves EOF, so the buffer holds the whole file.
+ // A full chunk may have more data behind it — store it chunked.
+ if dataSize < fs.option.SaveToFilerLimit && dataSize < int64(chunkSize) {
chunkOffset += dataSize
smallContent = make([]byte, dataSize)
bytesBuffer.Read(smallContent)