Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Share upload validation and simplify preview signing

Neeraj Gupta62b5af421a4e
diff --git a/server/ente/filedata/preview_upload.go b/server/ente/filedata/preview_upload.go
index e04d85ac87..d241b7e60c 100644
--- a/server/ente/filedata/preview_upload.go
+++ b/server/ente/filedata/preview_upload.go
@@ -1,18 +1,12 @@
package filedata
import (
- "crypto/md5"
- "encoding/base64"
"fmt"
"github.com/ente/museum/ente"
)
-const (
- maxPreviewUploadSize = int64(10 << 30)
- maxPreviewPartSize = int64(5 << 30)
- minPreviewPartSize = int64(5 << 20)
-)
+const maxPreviewUploadSize = int64(10 << 30)
type PreviewUploadRequest struct {
FileID int64 `json:"fileID" binding:"required"`
@@ -25,10 +19,10 @@ func (r PreviewUploadRequest) Validate() error {
if err := validatePreviewUpload(r.FileID, r.Type, r.ContentLength); err != nil {
return err
}
- if r.ContentLength > maxPreviewPartSize {
+ if r.ContentLength > ente.MaxMultipartPartSize {
return ente.NewBadRequestWithMessage("previews larger than 5 GiB require multipart upload")
}
- return validatePreviewMD5(r.ContentMD5)
+ return ente.ValidateMD5(r.ContentMD5)
}
type MultipartPreviewUploadRequest struct {
@@ -43,18 +37,15 @@ func (r MultipartPreviewUploadRequest) Validate() error {
if err := validatePreviewUpload(r.FileID, r.Type, r.ContentLength); err != nil {
return err
}
- if r.PartLength <= 0 || r.PartLength > maxPreviewPartSize {
- return ente.NewBadRequestWithMessage("partLength must be between 1 byte and 5 GiB")
- }
- if r.ContentLength > r.PartLength && r.PartLength < minPreviewPartSize {
- return ente.NewBadRequestWithMessage("partLength must be at least 5 MiB when more than one part is required")
+ if err := ente.ValidateMultipartPartLength(r.ContentLength, r.PartLength); err != nil {
+ return err
}
partCount := (r.ContentLength-1)/r.PartLength + 1
if int64(len(r.PartMD5s)) != partCount {
return ente.NewBadRequestWithMessage(fmt.Sprintf("partMd5s must contain exactly %d checksums", partCount))
}
for _, checksum := range r.PartMD5s {
- if err := validatePreviewMD5(checksum); err != nil {
+ if err := ente.ValidateMD5(checksum); err != nil {
return err
}
}
@@ -73,11 +64,3 @@ func validatePreviewUpload(fileID int64, objectType ente.ObjectType, contentLeng
}
return nil
}
-
-func validatePreviewMD5(checksum string) error {
- digest, err := base64.StdEncoding.DecodeString(checksum)
- if err != nil || len(digest) != md5.Size || base64.StdEncoding.EncodeToString(digest) != checksum {
- return ente.NewBadRequestWithMessage("MD5 checksums must be base64-encoded 16-byte digests")
- }
- return nil
-}
diff --git a/server/ente/filedata/preview_upload_test.go b/server/ente/filedata/preview_upload_test.go
index feaba9e70a..31187cb203 100644
--- a/server/ente/filedata/preview_upload_test.go
+++ b/server/ente/filedata/preview_upload_test.go
@@ -9,7 +9,7 @@ import (
func TestPreviewUploadSizeBoundaries(t *testing.T) {
const checksum = "XUFAKrxLKna5cZ2REBfFkg=="
- for _, size := range []int64{1, maxPreviewPartSize} {
+ for _, size := range []int64{1, ente.MaxMultipartPartSize} {
require.NoError(t, (PreviewUploadRequest{
FileID: 1, Type: ente.PreviewImage, ContentLength: size, ContentMD5: checksum,
}).Validate())
@@ -21,9 +21,9 @@ func TestPreviewUploadSizeBoundaries(t *testing.T) {
partCount int
}{
{"small single part", 1, 1, 1},
- {"part length exceeds object", 1, minPreviewPartSize, 1},
- {"exact multiple", minPreviewPartSize * 2, minPreviewPartSize, 2},
- {"maximum size", maxPreviewUploadSize, maxPreviewPartSize, 2},
+ {"part length exceeds object", 1, ente.MinMultipartPartSize, 1},
+ {"exact multiple", ente.MinMultipartPartSize * 2, ente.MinMultipartPartSize, 2},
+ {"maximum size", maxPreviewUploadSize, ente.MaxMultipartPartSize, 2},
} {
t.Run(tc.name, func(t *testing.T) {
checksums := make([]string, tc.partCount)
diff --git a/server/ente/upload.go b/server/ente/upload.go
new file mode 100644
index 0000000000..6d2f791421
--- /dev/null
+++ b/server/ente/upload.go
@@ -0,0 +1,47 @@
+package ente
+
+import (
+ "crypto/md5"
+ "encoding/base64"
+ "strings"
+
+ "github.com/ente/stacktrace"
+)
+
+const (
+ MinMultipartPartSize = int64(5 << 20)
+ MaxMultipartPartSize = int64(5 << 30)
+)
+
+func NormalizeMD5(value string) (string, error) {
+ trimmed := strings.TrimSpace(value)
+ if trimmed == "" {
+ return "", stacktrace.Propagate(ErrBadRequest, "contentMD5 must not be empty")
+ }
+ decoded, err := base64.StdEncoding.DecodeString(trimmed)
+ if err != nil {
+ return "", stacktrace.Propagate(ErrBadRequest, "contentMD5 must be base64 encoded")
+ }
+ if len(decoded) != md5.Size {
+ return "", stacktrace.Propagate(ErrBadRequest, "contentMD5 must be exactly 16 bytes")
+ }
+ return base64.StdEncoding.EncodeToString(decoded), nil
+}
+
+func ValidateMD5(value string) error {
+ normalized, err := NormalizeMD5(value)
+ if err != nil || normalized != value {
+ return NewBadRequestWithMessage("MD5 checksums must be base64-encoded 16-byte digests")
+ }
+ return nil
+}
+
+func ValidateMultipartPartLength(contentLength, partLength int64) error {
+ if partLength <= 0 || partLength > MaxMultipartPartSize {
+ return NewBadRequestWithMessage("partLength must be between 1 byte and 5 GiB")
+ }
+ if contentLength > partLength && partLength < MinMultipartPartSize {
+ return NewBadRequestWithMessage("partLength must be at least 5 MiB when more than one part is required")
+ }
+ return nil
+}
diff --git a/server/pkg/controller/file.go b/server/pkg/controller/file.go
index 53aadebdc8..28333d136e 100644
--- a/server/pkg/controller/file.go
+++ b/server/pkg/controller/file.go
@@ -2,10 +2,7 @@ package controller
import (
"context"
- "crypto/md5"
"database/sql"
- "encoding/base64"
- "encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -70,11 +67,7 @@ const InternalUserMaxFileSize = int64(1024 * 1024 * 1024 * 20)
const MaxUploadURLsLimit = 50
-const (
- minMultipartPartSize = int64(5 * 1024 * 1024)
- maxMultipartPartSize = int64(5 * 1024 * 1024 * 1024)
- maxMultipartPartCount = 10000
-)
+const maxMultipartPartCount = 10000
const (
DeletedObjectQueueLock = "deleted_objects_queue_lock"
)
@@ -381,7 +374,7 @@ func (c *FileController) GetUploadURLWithMetadata(ctx context.Context, userID in
if !isFileSizeAllowed {
return ente.UploadURL{}, stacktrace.Propagate(ente.ErrBadRequest, "contentLength exceeds max file size %d", MaxFileSize)
}
- checksum, err := normalizeMD5String(req.ContentMD5)
+ checksum, err := ente.NormalizeMD5(req.ContentMD5)
if err != nil {
return ente.UploadURL{}, err
}
@@ -1177,8 +1170,9 @@ func (c *FileController) GetMultipartUploadURLWithMetadata(ctx context.Context,
if req.PartMD5s != nil && len(req.PartMD5s) == 0 {
return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "partMd5s must not be empty")
}
- if err := validateMultipartPartLength(req.ContentLength, req.PartLength); err != nil {
- return ente.MultipartUploadURLs{}, err
+ if err := ente.ValidateMultipartPartLength(req.ContentLength, req.PartLength); err != nil {
+ // Preserve the regular upload API's generic bad-request response.
+ return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "%v", err)
}
partCount := calculateMultipartPartCount(req.ContentLength, req.PartLength)
if partCount > maxMultipartPartCount {
@@ -1191,7 +1185,7 @@ func (c *FileController) GetMultipartUploadURLWithMetadata(ctx context.Context,
}
normalizedChecksums = make([]string, partCount)
for i, checksum := range req.PartMD5s {
- normalized, err := normalizeMD5String(checksum)
+ normalized, err := ente.NormalizeMD5(checksum)
if err != nil {
return ente.MultipartUploadURLs{}, err
}
@@ -1276,24 +1270,6 @@ func (c *FileController) getPartURL(s3Client s3.S3, objectKey string, partNumber
return url, nil
}
-func normalizeMD5String(value string) (string, error) {
- trimmed := strings.TrimSpace(value)
- if trimmed == "" {
- return "", stacktrace.Propagate(ente.ErrBadRequest, "contentMD5 must not be empty")
- }
- decoded, err := base64.StdEncoding.DecodeString(trimmed)
- if err != nil {
- decoded, err = hex.DecodeString(trimmed)
- if err != nil {
- return "", stacktrace.Propagate(ente.ErrBadRequest, "contentMD5 must be base64 or hex encoded")
- }
- }
- if len(decoded) != md5.Size {
- return "", stacktrace.Propagate(ente.ErrBadRequest, "contentMD5 must be exactly 16 bytes")
- }
- return base64.StdEncoding.EncodeToString(decoded), nil
-}
-
func calculateMultipartPartCount(contentLength int64, partLength int64) int {
if partLength <= 0 {
return 0
@@ -1321,16 +1297,3 @@ func computePartLengths(contentLength int64, partLength int64, partCount int) []
}
return lengths
}
-
-func validateMultipartPartLength(contentLength int64, partLength int64) error {
- if partLength <= 0 {
- return stacktrace.Propagate(ente.ErrBadRequest, "partLength must be greater than 0")
- }
- if partLength > maxMultipartPartSize {
- return stacktrace.Propagate(ente.ErrBadRequest, "partLength exceeds 5GB limit")
- }
- if contentLength > partLength && partLength < minMultipartPartSize {
- return stacktrace.Propagate(ente.ErrBadRequest, "partLength must be at least 5MB when more than one part is required")
- }
- return nil
-}
diff --git a/server/pkg/controller/filedata/preview_files.go b/server/pkg/controller/filedata/preview_files.go
index c6c88c1274..9ca90cebef 100644
--- a/server/pkg/controller/filedata/preview_files.go
+++ b/server/pkg/controller/filedata/preview_files.go
@@ -49,13 +49,13 @@ func (c *Controller) PreviewUploadURL(ctx *gin.Context, request filedata.Preview
CompleteURL: &multiPartUploadURLs.CompleteURL,
}, nil
}
- enteUrl, err := c.getUploadURL(object)
+ url, err := c.getUploadURL(object)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
return &filedata.PreviewUploadUrl{
ObjectID: id,
- Url: &enteUrl.URL,
+ Url: &url,
}, nil
}
@@ -69,11 +69,11 @@ func (c *Controller) PreviewUploadURLWithMetadata(ctx *gin.Context, request file
}
object.ContentLength = &request.ContentLength
object.ContentMD5 = &request.ContentMD5
- upload, err := c.getUploadURL(object)
+ url, err := c.getUploadURL(object)
if err != nil {
return nil, err
}
- return &filedata.PreviewUploadUrl{ObjectID: id, Url: &upload.URL}, nil
+ return &filedata.PreviewUploadUrl{ObjectID: id, Url: &url}, nil
}
func (c *Controller) MultipartPreviewUploadURLWithMetadata(ctx *gin.Context, request filedata.MultipartPreviewUploadRequest) (*filedata.PreviewUploadUrl, error) {
@@ -99,15 +99,11 @@ func (c *Controller) previewUploadObject(ctx *gin.Context, fileID int64, objectT
if err := c._checkPreviewWritePerm(ctx, fileID, actorUser); err != nil {
return "", ente.TempObject{}, err
}
- fileOwnerID, err := c.FileRepo.GetOwnerID(fileID)
- if err != nil {
- return "", ente.TempObject{}, stacktrace.Propagate(err, "")
- }
id := filedata.NewUploadID(objectType)
return id, ente.TempObject{
- ObjectKey: filedata.ObjectKey(fileID, fileOwnerID, objectType, &id),
+ ObjectKey: filedata.ObjectKey(fileID, actorUser, objectType, &id),
BucketId: c.S3Config.GetBucketID(objectType),
- UserID: fileOwnerID,
+ UserID: actorUser,
App: auth.GetApp(ctx),
Purpose: string(objectType),
Client: network.GetClientInfo(ctx),
diff --git a/server/pkg/controller/filedata/s3.go b/server/pkg/controller/filedata/s3.go
index 61f464a279..c7c584d78e 100644
--- a/server/pkg/controller/filedata/s3.go
+++ b/server/pkg/controller/filedata/s3.go
@@ -21,7 +21,7 @@ import (
const PreSignedRequestValidityDuration = 7 * 24 * stime.Hour
-func (c *Controller) getUploadURL(object ente.TempObject) (*ente.UploadURL, error) {
+func (c *Controller) getUploadURL(object ente.TempObject) (string, error) {
s3Client := c.S3Config.GetS3Client(object.BucketId)
r, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{
Bucket: c.S3Config.GetBucket(object.BucketId),
@@ -31,16 +31,13 @@ func (c *Controller) getUploadURL(object ente.TempObject) (*ente.UploadURL, erro
})
url, err := r.Presign(PreSignedRequestValidityDuration)
if err != nil {
- return nil, stacktrace.Propagate(err, "")
+ return "", stacktrace.Propagate(err, "")
}
err = c.ObjectCleanupController.AddTempObject(object)
if err != nil {
- return nil, stacktrace.Propagate(err, "")
+ return "", stacktrace.Propagate(err, "")
}
- return &ente.UploadURL{
- ObjectKey: object.ObjectKey,
- URL: url,
- }, nil
+ return url, nil
}
func (c *Controller) getMultiPartUploadURL(object ente.TempObject, count int64, partLength int64, partMD5s []string) (*ente.MultipartUploadURLs, error) {
s3Client := c.S3Config.GetS3Client(object.BucketId)