Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Support deferred multipart checksums

Neeraj Gupta827048098eb2
diff --git a/server/ente/file.go b/server/ente/file.go
index 4f2f9972fe..c48f38c7af 100644
--- a/server/ente/file.go
+++ b/server/ente/file.go
@@ -178,7 +178,7 @@ type MultipartUploadURLs struct {
type MultipartUploadURLRequest struct {
ContentLength int64 `json:"contentLength" binding:"required"`
PartLength int64 `json:"partLength" binding:"required"`
- PartMD5s []string `json:"partMd5s" binding:"required"`
+ PartMD5s []string `json:"partMd5s"`
}
type ObjectType string
diff --git a/server/ente/remotestore.go b/server/ente/remotestore.go
index 23ffeddb16..5d53c06fee 100644
--- a/server/ente/remotestore.go
+++ b/server/ente/remotestore.go
@@ -61,6 +61,8 @@ const (
// only when those clients no longer need to be isolated from a new flag.
// CastSessionsV2 gates cast sessions v2 support.
CastSessionsV2 int64 = 1 << 5
+ // DeferredMultipartChecksums gates multipart URLs without precomputed checksums.
+ DeferredMultipartChecksums int64 = 1 << 6
)
type FlagKey string
diff --git a/server/pkg/controller/file.go b/server/pkg/controller/file.go
index f6a3e54ca8..6f887ee9f7 100644
--- a/server/pkg/controller/file.go
+++ b/server/pkg/controller/file.go
@@ -1172,7 +1172,7 @@ func (c *FileController) GetMultipartUploadURLs(ctx context.Context, userID int6
return multipartUploadURLs, nil
}
-// GetMultipartUploadURLWithMetadata enforces content length & per-part checksum requirements
+// GetMultipartUploadURLWithMetadata enforces content length and optional per-part checksums.
func (c *FileController) GetMultipartUploadURLWithMetadata(ctx context.Context, userID int64, req ente.MultipartUploadURLRequest, app ente.App) (ente.MultipartUploadURLs, error) {
if req.ContentLength <= 0 {
return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "contentLength must be greater than 0")
@@ -1180,7 +1180,7 @@ func (c *FileController) GetMultipartUploadURLWithMetadata(ctx context.Context,
if req.ContentLength > MaxFileSize {
return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "contentLength exceeds max file size %d", MaxFileSize)
}
- if len(req.PartMD5s) == 0 {
+ if req.PartMD5s != nil && len(req.PartMD5s) == 0 {
return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "partMd5s must not be empty")
}
if err := validateMultipartPartLength(req.ContentLength, req.PartLength); err != nil {
@@ -1190,16 +1190,19 @@ func (c *FileController) GetMultipartUploadURLWithMetadata(ctx context.Context,
if partCount > maxMultipartPartCount {
return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "multipart upload cannot exceed %d parts", maxMultipartPartCount)
}
- if len(req.PartMD5s) != partCount {
- return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "partMd5s size (%d) does not match computed part count (%d)", len(req.PartMD5s), partCount)
- }
- normalizedChecksums := make([]string, partCount)
- for i, checksum := range req.PartMD5s {
- normalized, err := normalizeMD5String(checksum)
- if err != nil {
- return ente.MultipartUploadURLs{}, err
+ var normalizedChecksums []string
+ if req.PartMD5s != nil {
+ if len(req.PartMD5s) != partCount {
+ return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "partMd5s size (%d) does not match computed part count (%d)", len(req.PartMD5s), partCount)
+ }
+ normalizedChecksums = make([]string, partCount)
+ for i, checksum := range req.PartMD5s {
+ normalized, err := normalizeMD5String(checksum)
+ if err != nil {
+ return ente.MultipartUploadURLs{}, err
+ }
+ normalizedChecksums[i] = normalized
}
- normalizedChecksums[i] = normalized
}
partLengths := computePartLengths(req.ContentLength, req.PartLength, partCount)
if err := c.UsageCtrl.CanUploadFile(ctx, userID, nil, app); err != nil {
@@ -1224,9 +1227,11 @@ func (c *FileController) GetMultipartUploadURLWithMetadata(ctx context.Context,
for i := 0; i < partCount; i++ {
partNumber := int64(i + 1)
length := partLengths[i]
- lengthCopy := length
- checksumCopy := normalizedChecksums[i]
- url, err := c.getPartURL(*s3Client, objectKey, partNumber, r.UploadId, &lengthCopy, &checksumCopy)
+ var checksum *string
+ if normalizedChecksums != nil {
+ checksum = &normalizedChecksums[i]
+ }
+ url, err := c.getPartURL(*s3Client, objectKey, partNumber, r.UploadId, &length, checksum)
if err != nil {
return multipartUploadURLs, stacktrace.Propagate(err, "")
}
diff --git a/server/pkg/controller/remotestore/controller.go b/server/pkg/controller/remotestore/controller.go
index f6e96c7d5f..6792a0709b 100644
--- a/server/pkg/controller/remotestore/controller.go
+++ b/server/pkg/controller/remotestore/controller.go
@@ -110,7 +110,7 @@ func (c *Controller) GetFeatureFlags(ctx *gin.Context) (*ente.FeatureFlagRespons
// Changing it to false will hide the option and disable multi part upload for everyone
// except internal user.rt
EnableMobMultiPart: true,
- ServerApiFlag: ente.UploadV2 | ente.Comments | ente.BackupOptions | ente.CastSessionsV2,
+ ServerApiFlag: ente.UploadV2 | ente.Comments | ente.BackupOptions | ente.CastSessionsV2 | ente.DeferredMultipartChecksums,
CastUrl: viper.GetString("apps.cast"),
EmbedUrl: viper.GetString("apps.embed-albums"),
CustomDomainCNAME: viper.GetString("apps.custom-domain.cname"),