π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Share upload validation and simplify preview signing
Neeraj Gupta62b5af421a4e
diff --git a/server/ente/filedata/preview_upload.go b/server/ente/filedata/preview_upload.goindex e04d85ac87..d241b7e60c 100644--- a/server/ente/filedata/preview_upload.go+++ b/server/ente/filedata/preview_upload.go@@ -1,18 +1,12 @@package filedataimport (- "crypto/md5"- "encoding/base64""fmt""github.com/ente/museum/ente")-const (- maxPreviewUploadSize = int64(10 << 30)- maxPreviewPartSize = int64(5 << 30)- minPreviewPartSize = int64(5 << 20)-)+const maxPreviewUploadSize = int64(10 << 30)type PreviewUploadRequest struct {FileID int64 `json:"fileID" binding:"required"`@@ -25,10 +19,10 @@ func (r PreviewUploadRequest) Validate() error {if err := validatePreviewUpload(r.FileID, r.Type, r.ContentLength); err != nil {return err}- if r.ContentLength > maxPreviewPartSize {+ if r.ContentLength > ente.MaxMultipartPartSize {return ente.NewBadRequestWithMessage("previews larger than 5 GiB require multipart upload")}- return validatePreviewMD5(r.ContentMD5)+ return ente.ValidateMD5(r.ContentMD5)}type MultipartPreviewUploadRequest struct {@@ -43,18 +37,15 @@ func (r MultipartPreviewUploadRequest) Validate() error {if err := validatePreviewUpload(r.FileID, r.Type, r.ContentLength); err != nil {return err}- if r.PartLength <= 0 || r.PartLength > maxPreviewPartSize {- return ente.NewBadRequestWithMessage("partLength must be between 1 byte and 5 GiB")- }- if r.ContentLength > r.PartLength && r.PartLength < minPreviewPartSize {- return ente.NewBadRequestWithMessage("partLength must be at least 5 MiB when more than one part is required")+ if err := ente.ValidateMultipartPartLength(r.ContentLength, r.PartLength); err != nil {+ return err}partCount := (r.ContentLength-1)/r.PartLength + 1if int64(len(r.PartMD5s)) != partCount {return ente.NewBadRequestWithMessage(fmt.Sprintf("partMd5s must contain exactly %d checksums", partCount))}for _, checksum := range r.PartMD5s {- if err := validatePreviewMD5(checksum); err != nil {+ if err := ente.ValidateMD5(checksum); err != nil {return err}}@@ -73,11 +64,3 @@ func validatePreviewUpload(fileID int64, objectType ente.ObjectType, contentLeng}return nil}--func validatePreviewMD5(checksum string) error {- digest, err := base64.StdEncoding.DecodeString(checksum)- if err != nil || len(digest) != md5.Size || base64.StdEncoding.EncodeToString(digest) != checksum {- return ente.NewBadRequestWithMessage("MD5 checksums must be base64-encoded 16-byte digests")- }- return nil-}diff --git a/server/ente/filedata/preview_upload_test.go b/server/ente/filedata/preview_upload_test.goindex feaba9e70a..31187cb203 100644--- a/server/ente/filedata/preview_upload_test.go+++ b/server/ente/filedata/preview_upload_test.go@@ -9,7 +9,7 @@ import (func TestPreviewUploadSizeBoundaries(t *testing.T) {const checksum = "XUFAKrxLKna5cZ2REBfFkg=="- for _, size := range []int64{1, maxPreviewPartSize} {+ for _, size := range []int64{1, ente.MaxMultipartPartSize} {require.NoError(t, (PreviewUploadRequest{FileID: 1, Type: ente.PreviewImage, ContentLength: size, ContentMD5: checksum,}).Validate())@@ -21,9 +21,9 @@ func TestPreviewUploadSizeBoundaries(t *testing.T) {partCount int}{{"small single part", 1, 1, 1},- {"part length exceeds object", 1, minPreviewPartSize, 1},- {"exact multiple", minPreviewPartSize * 2, minPreviewPartSize, 2},- {"maximum size", maxPreviewUploadSize, maxPreviewPartSize, 2},+ {"part length exceeds object", 1, ente.MinMultipartPartSize, 1},+ {"exact multiple", ente.MinMultipartPartSize * 2, ente.MinMultipartPartSize, 2},+ {"maximum size", maxPreviewUploadSize, ente.MaxMultipartPartSize, 2},} {t.Run(tc.name, func(t *testing.T) {checksums := make([]string, tc.partCount)diff --git a/server/ente/upload.go b/server/ente/upload.gonew file mode 100644index 0000000000..6d2f791421--- /dev/null+++ b/server/ente/upload.go@@ -0,0 +1,47 @@+package ente++import (+ "crypto/md5"+ "encoding/base64"+ "strings"++ "github.com/ente/stacktrace"+)++const (+ MinMultipartPartSize = int64(5 << 20)+ MaxMultipartPartSize = int64(5 << 30)+)++func NormalizeMD5(value string) (string, error) {+ trimmed := strings.TrimSpace(value)+ if trimmed == "" {+ return "", stacktrace.Propagate(ErrBadRequest, "contentMD5 must not be empty")+ }+ decoded, err := base64.StdEncoding.DecodeString(trimmed)+ if err != nil {+ return "", stacktrace.Propagate(ErrBadRequest, "contentMD5 must be base64 encoded")+ }+ if len(decoded) != md5.Size {+ return "", stacktrace.Propagate(ErrBadRequest, "contentMD5 must be exactly 16 bytes")+ }+ return base64.StdEncoding.EncodeToString(decoded), nil+}++func ValidateMD5(value string) error {+ normalized, err := NormalizeMD5(value)+ if err != nil || normalized != value {+ return NewBadRequestWithMessage("MD5 checksums must be base64-encoded 16-byte digests")+ }+ return nil+}++func ValidateMultipartPartLength(contentLength, partLength int64) error {+ if partLength <= 0 || partLength > MaxMultipartPartSize {+ return NewBadRequestWithMessage("partLength must be between 1 byte and 5 GiB")+ }+ if contentLength > partLength && partLength < MinMultipartPartSize {+ return NewBadRequestWithMessage("partLength must be at least 5 MiB when more than one part is required")+ }+ return nil+}diff --git a/server/pkg/controller/file.go b/server/pkg/controller/file.goindex 53aadebdc8..28333d136e 100644--- a/server/pkg/controller/file.go+++ b/server/pkg/controller/file.go@@ -2,10 +2,7 @@ package controllerimport ("context"- "crypto/md5""database/sql"- "encoding/base64"- "encoding/hex""encoding/json""errors""fmt"@@ -70,11 +67,7 @@ const InternalUserMaxFileSize = int64(1024 * 1024 * 1024 * 20)const MaxUploadURLsLimit = 50-const (- minMultipartPartSize = int64(5 * 1024 * 1024)- maxMultipartPartSize = int64(5 * 1024 * 1024 * 1024)- maxMultipartPartCount = 10000-)+const maxMultipartPartCount = 10000const (DeletedObjectQueueLock = "deleted_objects_queue_lock")@@ -381,7 +374,7 @@ func (c *FileController) GetUploadURLWithMetadata(ctx context.Context, userID inif !isFileSizeAllowed {return ente.UploadURL{}, stacktrace.Propagate(ente.ErrBadRequest, "contentLength exceeds max file size %d", MaxFileSize)}- checksum, err := normalizeMD5String(req.ContentMD5)+ checksum, err := ente.NormalizeMD5(req.ContentMD5)if err != nil {return ente.UploadURL{}, err}@@ -1177,8 +1170,9 @@ func (c *FileController) GetMultipartUploadURLWithMetadata(ctx context.Context,if req.PartMD5s != nil && len(req.PartMD5s) == 0 {return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "partMd5s must not be empty")}- if err := validateMultipartPartLength(req.ContentLength, req.PartLength); err != nil {- return ente.MultipartUploadURLs{}, err+ if err := ente.ValidateMultipartPartLength(req.ContentLength, req.PartLength); err != nil {+ // Preserve the regular upload API's generic bad-request response.+ return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "%v", err)}partCount := calculateMultipartPartCount(req.ContentLength, req.PartLength)if partCount > maxMultipartPartCount {@@ -1191,7 +1185,7 @@ func (c *FileController) GetMultipartUploadURLWithMetadata(ctx context.Context,}normalizedChecksums = make([]string, partCount)for i, checksum := range req.PartMD5s {- normalized, err := normalizeMD5String(checksum)+ normalized, err := ente.NormalizeMD5(checksum)if err != nil {return ente.MultipartUploadURLs{}, err}@@ -1276,24 +1270,6 @@ func (c *FileController) getPartURL(s3Client s3.S3, objectKey string, partNumberreturn url, nil}-func normalizeMD5String(value string) (string, error) {- trimmed := strings.TrimSpace(value)- if trimmed == "" {- return "", stacktrace.Propagate(ente.ErrBadRequest, "contentMD5 must not be empty")- }- decoded, err := base64.StdEncoding.DecodeString(trimmed)- if err != nil {- decoded, err = hex.DecodeString(trimmed)- if err != nil {- return "", stacktrace.Propagate(ente.ErrBadRequest, "contentMD5 must be base64 or hex encoded")- }- }- if len(decoded) != md5.Size {- return "", stacktrace.Propagate(ente.ErrBadRequest, "contentMD5 must be exactly 16 bytes")- }- return base64.StdEncoding.EncodeToString(decoded), nil-}-func calculateMultipartPartCount(contentLength int64, partLength int64) int {if partLength <= 0 {return 0@@ -1321,16 +1297,3 @@ func computePartLengths(contentLength int64, partLength int64, partCount int) []}return lengths}--func validateMultipartPartLength(contentLength int64, partLength int64) error {- if partLength <= 0 {- return stacktrace.Propagate(ente.ErrBadRequest, "partLength must be greater than 0")- }- if partLength > maxMultipartPartSize {- return stacktrace.Propagate(ente.ErrBadRequest, "partLength exceeds 5GB limit")- }- if contentLength > partLength && partLength < minMultipartPartSize {- return stacktrace.Propagate(ente.ErrBadRequest, "partLength must be at least 5MB when more than one part is required")- }- return nil-}diff --git a/server/pkg/controller/filedata/preview_files.go b/server/pkg/controller/filedata/preview_files.goindex c6c88c1274..9ca90cebef 100644--- a/server/pkg/controller/filedata/preview_files.go+++ b/server/pkg/controller/filedata/preview_files.go@@ -49,13 +49,13 @@ func (c *Controller) PreviewUploadURL(ctx *gin.Context, request filedata.PreviewCompleteURL: &multiPartUploadURLs.CompleteURL,}, nil}- enteUrl, err := c.getUploadURL(object)+ url, err := c.getUploadURL(object)if err != nil {return nil, stacktrace.Propagate(err, "")}return &filedata.PreviewUploadUrl{ObjectID: id,- Url: &enteUrl.URL,+ Url: &url,}, nil}@@ -69,11 +69,11 @@ func (c *Controller) PreviewUploadURLWithMetadata(ctx *gin.Context, request file}object.ContentLength = &request.ContentLengthobject.ContentMD5 = &request.ContentMD5- upload, err := c.getUploadURL(object)+ url, err := c.getUploadURL(object)if err != nil {return nil, err}- return &filedata.PreviewUploadUrl{ObjectID: id, Url: &upload.URL}, nil+ return &filedata.PreviewUploadUrl{ObjectID: id, Url: &url}, nil}func (c *Controller) MultipartPreviewUploadURLWithMetadata(ctx *gin.Context, request filedata.MultipartPreviewUploadRequest) (*filedata.PreviewUploadUrl, error) {@@ -99,15 +99,11 @@ func (c *Controller) previewUploadObject(ctx *gin.Context, fileID int64, objectTif err := c._checkPreviewWritePerm(ctx, fileID, actorUser); err != nil {return "", ente.TempObject{}, err}- fileOwnerID, err := c.FileRepo.GetOwnerID(fileID)- if err != nil {- return "", ente.TempObject{}, stacktrace.Propagate(err, "")- }id := filedata.NewUploadID(objectType)return id, ente.TempObject{- ObjectKey: filedata.ObjectKey(fileID, fileOwnerID, objectType, &id),+ ObjectKey: filedata.ObjectKey(fileID, actorUser, objectType, &id),BucketId: c.S3Config.GetBucketID(objectType),- UserID: fileOwnerID,+ UserID: actorUser,App: auth.GetApp(ctx),Purpose: string(objectType),Client: network.GetClientInfo(ctx),diff --git a/server/pkg/controller/filedata/s3.go b/server/pkg/controller/filedata/s3.goindex 61f464a279..c7c584d78e 100644--- a/server/pkg/controller/filedata/s3.go+++ b/server/pkg/controller/filedata/s3.go@@ -21,7 +21,7 @@ import (const PreSignedRequestValidityDuration = 7 * 24 * stime.Hour-func (c *Controller) getUploadURL(object ente.TempObject) (*ente.UploadURL, error) {+func (c *Controller) getUploadURL(object ente.TempObject) (string, error) {s3Client := c.S3Config.GetS3Client(object.BucketId)r, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{Bucket: c.S3Config.GetBucket(object.BucketId),@@ -31,16 +31,13 @@ func (c *Controller) getUploadURL(object ente.TempObject) (*ente.UploadURL, erro})url, err := r.Presign(PreSignedRequestValidityDuration)if err != nil {- return nil, stacktrace.Propagate(err, "")+ return "", stacktrace.Propagate(err, "")}err = c.ObjectCleanupController.AddTempObject(object)if err != nil {- return nil, stacktrace.Propagate(err, "")+ return "", stacktrace.Propagate(err, "")}- return &ente.UploadURL{- ObjectKey: object.ObjectKey,- URL: url,- }, nil+ return url, nil}func (c *Controller) getMultiPartUploadURL(object ente.TempObject, count int64, partLength int64, partMD5s []string) (*ente.MultipartUploadURLs, error) {s3Client := c.S3Config.GetS3Client(object.BucketId)