Skip to content

armgit / ente

Public Imported from GitHub
0

💚 End-to-end encrypted cloud for everything.

Record pending upload metadata

Neeraj Gupta5b499563b92f
diff --git a/server/ente/file.go b/server/ente/file.go
index b84f09d438..e0b27d09e2 100644
--- a/server/ente/file.go
+++ b/server/ente/file.go
@@ -191,10 +191,16 @@ type ObjectState struct {
}
type TempObject struct {
- ObjectKey string
- IsMultipart bool
- UploadID string
- BucketId string
+ ObjectKey string
+ IsMultipart bool
+ UploadID string
+ BucketId string
+ UserID int64
+ App App
+ Purpose string
+ ContentLength *int64
+ ContentMD5 *string
+ Client string
}
type DuplicateFiles struct {
diff --git a/server/migrations/143_pending_upload_metadata.down.sql b/server/migrations/143_pending_upload_metadata.down.sql
new file mode 100644
index 0000000000..e9b931e93e
--- /dev/null
+++ b/server/migrations/143_pending_upload_metadata.down.sql
@@ -0,0 +1,12 @@
+ALTER TABLE space_temp_objects
+ DROP COLUMN content_md5,
+ DROP COLUMN client;
+
+ALTER TABLE temp_objects
+ DROP COLUMN user_id,
+ DROP COLUMN created_at,
+ DROP COLUMN app,
+ DROP COLUMN purpose,
+ DROP COLUMN content_length,
+ DROP COLUMN content_md5,
+ DROP COLUMN client;
diff --git a/server/migrations/143_pending_upload_metadata.up.sql b/server/migrations/143_pending_upload_metadata.up.sql
new file mode 100644
index 0000000000..3e34007112
--- /dev/null
+++ b/server/migrations/143_pending_upload_metadata.up.sql
@@ -0,0 +1,15 @@
+ALTER TABLE temp_objects
+ ADD COLUMN user_id BIGINT,
+ ADD COLUMN created_at BIGINT,
+ ADD COLUMN app TEXT,
+ ADD COLUMN purpose TEXT,
+ ADD COLUMN content_length BIGINT,
+ ADD COLUMN content_md5 TEXT,
+ ADD COLUMN client TEXT;
+
+ALTER TABLE temp_objects
+ ALTER COLUMN created_at SET DEFAULT now_utc_micro_seconds();
+
+ALTER TABLE space_temp_objects
+ ADD COLUMN content_md5 TEXT,
+ ADD COLUMN client TEXT;
diff --git a/server/migrations/144_temp_objects_user_created_idx.down.sql b/server/migrations/144_temp_objects_user_created_idx.down.sql
new file mode 100644
index 0000000000..987f15d22c
--- /dev/null
+++ b/server/migrations/144_temp_objects_user_created_idx.down.sql
@@ -0,0 +1 @@
+DROP INDEX CONCURRENTLY temp_objects_user_created_idx;
diff --git a/server/migrations/144_temp_objects_user_created_idx.up.sql b/server/migrations/144_temp_objects_user_created_idx.up.sql
new file mode 100644
index 0000000000..5797644f0a
--- /dev/null
+++ b/server/migrations/144_temp_objects_user_created_idx.up.sql
@@ -0,0 +1,2 @@
+CREATE INDEX CONCURRENTLY temp_objects_user_created_idx
+ ON temp_objects (user_id, created_at);
diff --git a/server/pkg/api/file.go b/server/pkg/api/file.go
index ec4d4b6b9a..f0a330d692 100644
--- a/server/pkg/api/file.go
+++ b/server/pkg/api/file.go
@@ -22,6 +22,7 @@ import (
"github.com/ente/museum/pkg/controller"
"github.com/ente/museum/pkg/utils/auth"
"github.com/ente/museum/pkg/utils/handler"
+ "github.com/ente/museum/pkg/utils/network"
"github.com/ente/museum/pkg/utils/time"
"github.com/gin-gonic/gin"
)
@@ -135,7 +136,7 @@ func (h *FileHandler) GetUploadURLs(c *gin.Context) {
userID := auth.GetUserID(c.Request.Header)
count, _ := strconv.Atoi(c.Query("count"))
- urls, err := h.Controller.GetUploadURLs(c, userID, count, enteApp, false)
+ urls, err := h.Controller.GetUploadURLs(c, userID, count, enteApp, false, network.GetClientInfoForStorage(c))
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -163,7 +164,7 @@ func (h *FileHandler) GetUploadURLV2(c *gin.Context) {
handler.Error(c, stacktrace.Propagate(err, ""))
return
}
- url, err := h.Controller.GetUploadURLWithMetadata(c, userID, req, enteApp)
+ url, err := h.Controller.GetUploadURLWithMetadata(c, userID, req, enteApp, network.GetClientInfoForStorage(c))
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -176,7 +177,7 @@ func (h *FileHandler) GetMultipartUploadURLs(c *gin.Context) {
userID := auth.GetUserID(c.Request.Header)
count, _ := strconv.Atoi(c.Query("count"))
- urls, err := h.Controller.GetMultipartUploadURLs(c, userID, count, enteApp)
+ urls, err := h.Controller.GetMultipartUploadURLs(c, userID, count, enteApp, network.GetClientInfoForStorage(c))
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -194,7 +195,7 @@ func (h *FileHandler) GetMultipartUploadURLV2(c *gin.Context) {
handler.Error(c, stacktrace.Propagate(err, ""))
return
}
- upload, err := h.Controller.GetMultipartUploadURLWithMetadata(c, userID, req, enteApp)
+ upload, err := h.Controller.GetMultipartUploadURLWithMetadata(c, userID, req, enteApp, network.GetClientInfoForStorage(c))
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
diff --git a/server/pkg/api/public_collection.go b/server/pkg/api/public_collection.go
index b28fe9221d..267f73486e 100644
--- a/server/pkg/api/public_collection.go
+++ b/server/pkg/api/public_collection.go
@@ -16,6 +16,7 @@ import (
"github.com/ente/museum/pkg/controller"
"github.com/ente/museum/pkg/utils/auth"
"github.com/ente/museum/pkg/utils/handler"
+ "github.com/ente/museum/pkg/utils/network"
"github.com/ente/stacktrace"
"github.com/gin-gonic/gin"
)
@@ -140,7 +141,7 @@ func (h *PublicCollectionHandler) GetUploadURLV2(c *gin.Context) {
handler.Error(c, stacktrace.Propagate(err, ""))
return
}
- url, err := h.FileCtrl.GetUploadURLWithMetadata(c, collection.Owner.ID, req, enteApp)
+ url, err := h.FileCtrl.GetUploadURLWithMetadata(c, collection.Owner.ID, req, enteApp, network.GetClientInfoForStorage(c))
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
@@ -165,7 +166,7 @@ func (h *PublicCollectionHandler) GetMultipartUploadURLV2(c *gin.Context) {
handler.Error(c, stacktrace.Propagate(err, ""))
return
}
- upload, err := h.FileCtrl.GetMultipartUploadURLWithMetadata(c, collection.Owner.ID, req, enteApp)
+ upload, err := h.FileCtrl.GetMultipartUploadURLWithMetadata(c, collection.Owner.ID, req, enteApp, network.GetClientInfoForStorage(c))
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
diff --git a/server/pkg/controller/contact/controller.go b/server/pkg/controller/contact/controller.go
index 9de1489a50..1ca1d19938 100644
--- a/server/pkg/controller/contact/controller.go
+++ b/server/pkg/controller/contact/controller.go
@@ -2,6 +2,8 @@ package contact
import (
"context"
+ "crypto/md5"
+ "encoding/base64"
"errors"
"io"
"os"
@@ -16,6 +18,7 @@ import (
contactrepo "github.com/ente/museum/pkg/repo/contact"
"github.com/ente/museum/pkg/utils/auth"
fileutil "github.com/ente/museum/pkg/utils/file"
+ "github.com/ente/museum/pkg/utils/network"
"github.com/ente/museum/pkg/utils/s3config"
enteTime "github.com/ente/museum/pkg/utils/time"
"github.com/ente/stacktrace"
@@ -188,7 +191,20 @@ func (c *Controller) GetAttachmentUploadURL(ctx *gin.Context, attachmentTypeRaw
if err != nil {
return nil, stacktrace.Propagate(err, "failed to presign profile picture upload url")
}
- if err := c.ObjectCleanupController.AddTempObjectKey(objectKey, dc); err != nil {
+ var contentMD5 *string
+ if digest, err := base64.StdEncoding.DecodeString(req.ContentMD5); err == nil && len(digest) == md5.Size {
+ contentMD5 = aws.String(base64.StdEncoding.EncodeToString(digest))
+ }
+ if err := c.ObjectCleanupController.AddTempObject(ente.TempObject{
+ ObjectKey: objectKey,
+ BucketId: dc,
+ UserID: userID,
+ App: auth.GetApp(ctx),
+ Purpose: "attachment",
+ ContentLength: &req.ContentLength,
+ ContentMD5: contentMD5,
+ Client: network.GetClientInfoForStorage(ctx),
+ }); err != nil {
return nil, stacktrace.Propagate(err, "failed to stage temp object for attachment upload")
}
diff --git a/server/pkg/controller/contact/controller_test.go b/server/pkg/controller/contact/controller_test.go
index 3a4d12f195..30785e08a4 100644
--- a/server/pkg/controller/contact/controller_test.go
+++ b/server/pkg/controller/contact/controller_test.go
@@ -765,12 +765,14 @@ func TestUserAttachmentsRejectDuplicateBucketMembership(t *testing.T) {
func TestAttachmentLifecycle(t *testing.T) {
ctrl, db, ctx, s3Cfg := setupContactControllerTest(t)
mustInsertTestUser(t, db, 1)
+ ctx.Request.Header.Set("X-Client-Package", "io.ente.photos")
+ ctx.Request.Header.Set("X-Client-Version", "1.0")
created := createContactForTest(t, db, ctrl, ctx, 41, "wrapped-key-1", "payload-1")
upload1, err := ctrl.GetAttachmentUploadURL(ctx, string(contactmodel.ProfilePicture), contactmodel.AttachmentUploadURLRequest{
ContentLength: 128,
- ContentMD5: "ZmFrZS1tZDU=",
+ ContentMD5: "XUFAKrxLKna5cZ2REBfFkg==",
})
if err != nil {
t.Fatalf("GetAttachmentUploadURL() error = %v", err)
@@ -787,6 +789,12 @@ func TestAttachmentLifecycle(t *testing.T) {
if bucketID != s3Cfg.GetAttachmentBucketID(string(contactmodel.ProfilePicture)) {
t.Fatalf("bucket_id = %q, want %q", bucketID, s3Cfg.GetAttachmentBucketID(string(contactmodel.ProfilePicture)))
}
+ var metadataMatches bool
+ if err := db.QueryRow(`SELECT user_id = 1 AND app = 'photos' AND purpose = 'attachment'
+ AND content_length = 128 AND content_md5 = 'XUFAKrxLKna5cZ2REBfFkg==' AND client = 'io.ente.photos/1.0'
+ FROM temp_objects WHERE object_key = $1`, objectKey1).Scan(&metadataMatches); err != nil || !metadataMatches {
+ t.Fatalf("unexpected upload metadata: matches=%v err=%v", metadataMatches, err)
+ }
withPicture, err := ctrl.AttachContactAttachment(ctx, created.ID, string(contactmodel.ProfilePicture), contactmodel.CommitAttachmentRequest{
AttachmentID: upload1.AttachmentID,
@@ -817,6 +825,10 @@ func TestAttachmentLifecycle(t *testing.T) {
if err != nil {
t.Fatalf("second GetAttachmentUploadURL() error = %v", err)
}
+ if err := db.QueryRow(`SELECT content_md5 IS NULL FROM temp_objects WHERE object_key = $1`,
+ contactmodel.AttachmentObjectKey(1, contactmodel.ProfilePicture, upload2.AttachmentID)).Scan(&metadataMatches); err != nil || !metadataMatches {
+ t.Fatalf("invalid MD5 should not be recorded: matches=%v err=%v", metadataMatches, err)
+ }
replaced, err := ctrl.AttachContactAttachment(ctx, created.ID, string(contactmodel.ProfilePicture), contactmodel.CommitAttachmentRequest{
AttachmentID: upload2.AttachmentID,
Size: 256,
diff --git a/server/pkg/controller/file.go b/server/pkg/controller/file.go
index d8ff9a8b5e..53aadebdc8 100644
--- a/server/pkg/controller/file.go
+++ b/server/pkg/controller/file.go
@@ -332,14 +332,12 @@ func (c *FileController) Update(ctx context.Context, userID int64, file ente.Fil
return response, nil
}
-func (c *FileController) GetUploadURLs(ctx context.Context, userID int64, count int, app ente.App, ignoreLimit bool) ([]ente.UploadURL, error) {
+func (c *FileController) GetUploadURLs(ctx context.Context, userID int64, count int, app ente.App, ignoreLimit bool, client string) ([]ente.UploadURL, error) {
err := c.UsageCtrl.CanUploadFile(ctx, userID, nil, app)
if err != nil {
return []ente.UploadURL{}, stacktrace.Propagate(err, "")
}
- s3Client := c.S3Config.GetHotS3Client()
dc := c.S3Config.GetHotDataCenter()
- bucket := c.S3Config.GetHotBucket()
urls := make([]ente.UploadURL, 0)
objectKeys := make([]string, 0)
if count > MaxUploadURLsLimit && !ignoreLimit {
@@ -348,7 +346,14 @@ func (c *FileController) GetUploadURLs(ctx context.Context, userID int64, count
for i := 0; i < count; i++ {
objectKey := strconv.FormatInt(userID, 10) + "/" + uuid.NewString()
objectKeys = append(objectKeys, objectKey)
- url, err := c.getObjectURL(s3Client, dc, bucket, objectKey, nil, nil)
+ url, err := c.getObjectURL(ente.TempObject{
+ ObjectKey: objectKey,
+ BucketId: dc,
+ UserID: userID,
+ App: app,
+ Purpose: "file_upload",
+ Client: client,
+ })
if err != nil {
return urls, stacktrace.Propagate(err, "")
}
@@ -365,7 +370,7 @@ func (c *FileController) ValidateUploadEligibility(ctx context.Context, userID i
return nil
}
-func (c *FileController) GetUploadURLWithMetadata(ctx context.Context, userID int64, req ente.UploadURLRequest, app ente.App) (ente.UploadURL, error) {
+func (c *FileController) GetUploadURLWithMetadata(ctx context.Context, userID int64, req ente.UploadURLRequest, app ente.App, client string) (ente.UploadURL, error) {
if req.ContentLength <= 0 {
return ente.UploadURL{}, stacktrace.Propagate(ente.ErrBadRequest, "contentLength must be greater than 0")
}
@@ -383,13 +388,18 @@ func (c *FileController) GetUploadURLWithMetadata(ctx context.Context, userID in
if err := c.UsageCtrl.CanUploadFile(ctx, userID, &req.ContentLength, app); err != nil {
return ente.UploadURL{}, stacktrace.Propagate(err, "")
}
- s3Client := c.S3Config.GetHotS3Client()
dc := c.S3Config.GetHotDataCenter()
- bucket := c.S3Config.GetHotBucket()
objectKey := strconv.FormatInt(userID, 10) + "/" + uuid.NewString()
- length := req.ContentLength
- checksumCopy := checksum
- url, err := c.getObjectURL(s3Client, dc, bucket, objectKey, &length, &checksumCopy)
+ url, err := c.getObjectURL(ente.TempObject{
+ ObjectKey: objectKey,
+ BucketId: dc,
+ UserID: userID,
+ App: app,
+ Purpose: "file_upload",
+ ContentLength: &req.ContentLength,
+ ContentMD5: &checksum,
+ Client: client,
+ })
if err != nil {
return ente.UploadURL{}, stacktrace.Propagate(err, "")
}
@@ -1077,30 +1087,27 @@ func (c *FileController) deleteObjectVersionFromHotStorage(objectKey string, ver
return nil
}
-func (c *FileController) getObjectURL(s3Client *s3.S3, dc string, bucket *string, objectKey string, contentLength *int64, contentMD5 *string) (ente.UploadURL, error) {
+func (c *FileController) getObjectURL(object ente.TempObject) (ente.UploadURL, error) {
+ s3Client := c.S3Config.GetS3Client(object.BucketId)
input := &s3.PutObjectInput{
- Bucket: bucket,
- Key: &objectKey,
- }
- if contentLength != nil {
- input.ContentLength = contentLength
- }
- if contentMD5 != nil {
- input.ContentMD5 = contentMD5
+ Bucket: c.S3Config.GetBucket(object.BucketId),
+ Key: &object.ObjectKey,
+ ContentLength: object.ContentLength,
+ ContentMD5: object.ContentMD5,
}
r, _ := s3Client.PutObjectRequest(input)
url, err := r.Presign(PreSignedRequestValidityDuration)
if err != nil {
return ente.UploadURL{}, stacktrace.Propagate(err, "")
}
- err = c.ObjectCleanupCtrl.AddTempObjectKey(objectKey, dc)
+ err = c.ObjectCleanupCtrl.AddTempObject(object)
if err != nil {
return ente.UploadURL{}, stacktrace.Propagate(err, "")
}
- return ente.UploadURL{ObjectKey: objectKey, URL: url}, nil
+ return ente.UploadURL{ObjectKey: object.ObjectKey, URL: url}, nil
}
-func (c *FileController) GetMultipartUploadURLs(ctx context.Context, userID int64, count int, app ente.App) (ente.MultipartUploadURLs, error) {
+func (c *FileController) GetMultipartUploadURLs(ctx context.Context, userID int64, count int, app ente.App, client string) (ente.MultipartUploadURLs, error) {
if count <= 0 || count > maxMultipartPartCount {
return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "multipart upload cannot exceed %d parts", maxMultipartPartCount)
}
@@ -1119,7 +1126,16 @@ func (c *FileController) GetMultipartUploadURLs(ctx context.Context, userID int6
if err != nil {
return ente.MultipartUploadURLs{}, stacktrace.Propagate(err, "")
}
- err = c.ObjectCleanupCtrl.AddMultipartTempObjectKey(objectKey, *r.UploadId, dc)
+ err = c.ObjectCleanupCtrl.AddTempObject(ente.TempObject{
+ ObjectKey: objectKey,
+ IsMultipart: true,
+ UploadID: *r.UploadId,
+ BucketId: dc,
+ UserID: userID,
+ App: app,
+ Purpose: "file_upload",
+ Client: client,
+ })
if err != nil {
return ente.MultipartUploadURLs{}, stacktrace.Propagate(err, "")
}
@@ -1147,7 +1163,7 @@ func (c *FileController) GetMultipartUploadURLs(ctx context.Context, userID int6
return multipartUploadURLs, nil
}
-func (c *FileController) GetMultipartUploadURLWithMetadata(ctx context.Context, userID int64, req ente.MultipartUploadURLRequest, app ente.App) (ente.MultipartUploadURLs, error) {
+func (c *FileController) GetMultipartUploadURLWithMetadata(ctx context.Context, userID int64, req ente.MultipartUploadURLRequest, app ente.App, client string) (ente.MultipartUploadURLs, error) {
if req.ContentLength <= 0 {
return ente.MultipartUploadURLs{}, stacktrace.Propagate(ente.ErrBadRequest, "contentLength must be greater than 0")
}
@@ -1197,7 +1213,17 @@ func (c *FileController) GetMultipartUploadURLWithMetadata(ctx context.Context,
if err != nil {
return ente.MultipartUploadURLs{}, stacktrace.Propagate(err, "")
}
- if err := c.ObjectCleanupCtrl.AddMultipartTempObjectKey(objectKey, *r.UploadId, dc); err != nil {
+ if err := c.ObjectCleanupCtrl.AddTempObject(ente.TempObject{
+ ObjectKey: objectKey,
+ IsMultipart: true,
+ UploadID: *r.UploadId,
+ BucketId: dc,
+ UserID: userID,
+ App: app,
+ Purpose: "file_upload",
+ ContentLength: &req.ContentLength,
+ Client: client,
+ }); err != nil {
return ente.MultipartUploadURLs{}, stacktrace.Propagate(err, "")
}
multipartUploadURLs := ente.MultipartUploadURLs{ObjectKey: objectKey}
diff --git a/server/pkg/controller/file_copy/file_copy.go b/server/pkg/controller/file_copy/file_copy.go
index c1caa8fdb1..bb2bbd5558 100644
--- a/server/pkg/controller/file_copy/file_copy.go
+++ b/server/pkg/controller/file_copy/file_copy.go
@@ -8,6 +8,7 @@ import (
"github.com/ente/museum/pkg/controller/collections"
"github.com/ente/museum/pkg/repo"
"github.com/ente/museum/pkg/utils/auth"
+ "github.com/ente/museum/pkg/utils/network"
"github.com/ente/museum/pkg/utils/s3config"
enteTime "github.com/ente/museum/pkg/utils/time"
"github.com/gin-contrib/requestid"
@@ -94,7 +95,7 @@ func (fc *FileCopyController) CopyFiles(c *gin.Context, req ente.CopyFileSyncReq
// Reuse upload URLs so abandoned copies are cleaned up as orphan objects.
// todo:(neeraj) optimize this method by removing the need for getting a signed url for each object
- uploadUrls, err := fc.FileController.GetUploadURLs(c, userID, len(s3ObjectsToCopy), app, true)
+ uploadUrls, err := fc.FileController.GetUploadURLs(c, userID, len(s3ObjectsToCopy), app, true, network.GetClientInfoForStorage(c))
if err != nil {
return nil, err
}
diff --git a/server/pkg/controller/filedata/preview_files.go b/server/pkg/controller/filedata/preview_files.go
index 1454bc707b..052f22294b 100644
--- a/server/pkg/controller/filedata/preview_files.go
+++ b/server/pkg/controller/filedata/preview_files.go
@@ -4,6 +4,7 @@ import (
"github.com/ente/museum/ente"
"github.com/ente/museum/ente/filedata"
"github.com/ente/museum/pkg/utils/auth"
+ "github.com/ente/museum/pkg/utils/network"
"github.com/ente/stacktrace"
"github.com/gin-gonic/gin"
)
@@ -44,8 +45,16 @@ func (c *Controller) PreviewUploadURL(ctx *gin.Context, request filedata.Preview
id := filedata.NewUploadID(request.Type)
objectKey := filedata.ObjectKey(request.FileID, fileOwnerID, request.Type, &id)
bucketID := c.S3Config.GetBucketID(request.Type)
+ object := ente.TempObject{
+ ObjectKey: objectKey,
+ BucketId: bucketID,
+ UserID: fileOwnerID,
+ App: auth.GetApp(ctx),
+ Purpose: string(request.Type),
+ Client: network.GetClientInfoForStorage(ctx),
+ }
if request.IsMultiPart {
- multiPartUploadURLs, err2 := c.getMultiPartUploadURL(bucketID, objectKey, request.Count)
+ multiPartUploadURLs, err2 := c.getMultiPartUploadURL(object, request.Count)
if err2 != nil {
return nil, stacktrace.Propagate(err2, "")
}
@@ -55,7 +64,7 @@ func (c *Controller) PreviewUploadURL(ctx *gin.Context, request filedata.Preview
CompleteURL: &multiPartUploadURLs.CompleteURL,
}, nil
}
- enteUrl, err := c.getUploadURL(bucketID, objectKey)
+ enteUrl, err := c.getUploadURL(object)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
diff --git a/server/pkg/controller/filedata/s3.go b/server/pkg/controller/filedata/s3.go
index ac25a1c010..1564c192b8 100644
--- a/server/pkg/controller/filedata/s3.go
+++ b/server/pkg/controller/filedata/s3.go
@@ -21,48 +21,47 @@ import (
const PreSignedRequestValidityDuration = 7 * 24 * stime.Hour
-func (c *Controller) getUploadURL(dc string, objectKey string) (*ente.UploadURL, error) {
- s3Client := c.S3Config.GetS3Client(dc)
+func (c *Controller) getUploadURL(object ente.TempObject) (*ente.UploadURL, error) {
+ s3Client := c.S3Config.GetS3Client(object.BucketId)
r, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{
- Bucket: c.S3Config.GetBucket(dc),
- Key: &objectKey,
+ Bucket: c.S3Config.GetBucket(object.BucketId),
+ Key: &object.ObjectKey,
})
url, err := r.Presign(PreSignedRequestValidityDuration)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
- if err != nil {
- return nil, stacktrace.Propagate(err, "")
- }
- err = c.ObjectCleanupController.AddTempObjectKey(objectKey, dc)
+ err = c.ObjectCleanupController.AddTempObject(object)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
return &ente.UploadURL{
- ObjectKey: objectKey,
+ ObjectKey: object.ObjectKey,
URL: url,
}, nil
}
-func (c *Controller) getMultiPartUploadURL(dc string, objectKey string, count *int64) (*ente.MultipartUploadURLs, error) {
- s3Client := c.S3Config.GetS3Client(dc)
- bucket := c.S3Config.GetBucket(dc)
+func (c *Controller) getMultiPartUploadURL(object ente.TempObject, count *int64) (*ente.MultipartUploadURLs, error) {
+ s3Client := c.S3Config.GetS3Client(object.BucketId)
+ bucket := c.S3Config.GetBucket(object.BucketId)
r, err := s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
Bucket: bucket,
- Key: &objectKey,
+ Key: &object.ObjectKey,
})
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
- err = c.ObjectCleanupController.AddMultipartTempObjectKey(objectKey, *r.UploadId, dc)
+ object.IsMultipart = true
+ object.UploadID = *r.UploadId
+ err = c.ObjectCleanupController.AddTempObject(object)
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
- multipartUploadURLs := ente.MultipartUploadURLs{ObjectKey: objectKey}
+ multipartUploadURLs := ente.MultipartUploadURLs{ObjectKey: object.ObjectKey}
urls := make([]string, 0)
for i := int64(1); i <= *count; i++ {
partReq, _ := s3Client.UploadPartRequest(&s3.UploadPartInput{
Bucket: bucket,
- Key: &objectKey,
+ Key: &object.ObjectKey,
UploadId: r.UploadId,
PartNumber: &i,
})
@@ -75,7 +74,7 @@ func (c *Controller) getMultiPartUploadURL(dc string, objectKey string, count *i
multipartUploadURLs.PartURLs = urls
r2, _ := s3Client.CompleteMultipartUploadRequest(&s3.CompleteMultipartUploadInput{
Bucket: bucket,
- Key: &objectKey,
+ Key: &object.ObjectKey,
UploadId: r.UploadId,
})
url, err := r2.Presign(PreSignedRequestValidityDuration)
diff --git a/server/pkg/controller/object_cleanup.go b/server/pkg/controller/object_cleanup.go
index 58378e9e0f..e496d84a4e 100644
--- a/server/pkg/controller/object_cleanup.go
+++ b/server/pkg/controller/object_cleanup.go
@@ -162,28 +162,13 @@ func (c *ObjectCleanupController) removeUnreportedObject(tx *sql.Tx, t ente.Temp
return nil
}
-func (c *ObjectCleanupController) AddTempObjectKey(objectKey string, dc string) error {
- expiry := time.Microseconds() + (2 * PreSignedRequestValidityDuration.Microseconds())
- return c.addCleanupEntryForObjectKey(objectKey, dc, expiry)
-}
-
-func (c *ObjectCleanupController) addCleanupEntryForObjectKey(objectKey string, dc string, expirationTime int64) error {
- err := c.Repo.AddTempObject(ente.TempObject{
- ObjectKey: objectKey,
- IsMultipart: false,
- BucketId: dc,
- }, expirationTime)
- return stacktrace.Propagate(err, "")
-}
-
-func (c *ObjectCleanupController) AddMultipartTempObjectKey(objectKey string, uploadID string, dc string) error {
- expiry := time.Microseconds() + (2 * PreSignedPartUploadRequestDuration.Microseconds())
- err := c.Repo.AddTempObject(ente.TempObject{
- ObjectKey: objectKey,
- IsMultipart: true,
- UploadID: uploadID,
- BucketId: dc,
- }, expiry)
+func (c *ObjectCleanupController) AddTempObject(object ente.TempObject) error {
+ validity := PreSignedRequestValidityDuration
+ if object.IsMultipart {
+ validity = PreSignedPartUploadRequestDuration
+ }
+ expiry := time.Microseconds() + 2*validity.Microseconds()
+ err := c.Repo.AddTempObject(object, expiry)
return stacktrace.Propagate(err, "")
}
diff --git a/server/pkg/controller/upload_metadata_test.go b/server/pkg/controller/upload_metadata_test.go
new file mode 100644
index 0000000000..280bf4ce9e
--- /dev/null
+++ b/server/pkg/controller/upload_metadata_test.go
@@ -0,0 +1,112 @@
+package controller
+
+import (
+ "database/sql"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/ente/museum/ente"
+ "github.com/ente/museum/internal/testutil"
+ "github.com/ente/museum/pkg/repo"
+ "github.com/stretchr/testify/require"
+)
+
+func TestPendingUploadMetadata(t *testing.T) {
+ s3Server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost || !r.URL.Query().Has("uploads") {
+ t.Errorf("unexpected storage request: %s %s", r.Method, r.URL)
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ _, _ = w.Write([]byte(`<InitiateMultipartUploadResult><UploadId>upload-id</UploadId></InitiateMultipartUploadResult>`))
+ }))
+ t.Cleanup(s3Server.Close)
+ cleanup, _, db := setupObjectCleanupRaceTest(t, s3Server.URL)
+ userID := testutil.InsertUser(t, db, testutil.UserFixture{UserID: 1580559962386438, Email: "[email protected]", CreationTime: 1})
+ testutil.InsertSubscription(t, db, testutil.SubscriptionFixture{
+ UserID: userID, Storage: 10 << 30, ExpiryTime: time.Now().Add(time.Hour).UnixMicro(),
+ })
+ users := &repo.UserRepository{DB: db}
+ controller := &FileController{
+ S3Config: cleanup.S3Config, ObjectCleanupCtrl: cleanup,
+ UsageCtrl: &UsageController{
+ UserRepo: users, UsageRepo: &repo.UsageRepository{DB: db},
+ BillingCtrl: &BillingController{UserRepo: users, BillingRepo: &repo.BillingRepository{DB: db}},
+ },
+ }
+ const checksum = "XUFAKrxLKna5cZ2REBfFkg=="
+ const client = "io.ente.locker/1.0"
+ for _, tc := range []struct {
+ name string
+ multipart bool
+ legacy bool
+ }{
+ {name: "single"},
+ {name: "multipart", multipart: true},
+ {name: "legacy single", legacy: true},
+ {name: "legacy multipart", multipart: true, legacy: true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ startedAt := time.Now().UnixMicro()
+ var objectKey string
+ if tc.multipart {
+ var upload ente.MultipartUploadURLs
+ var err error
+ if tc.legacy {
+ upload, err = controller.GetMultipartUploadURLs(t.Context(), userID, 1, ente.Locker, client)
+ } else {
+ upload, err = controller.GetMultipartUploadURLWithMetadata(t.Context(), userID, ente.MultipartUploadURLRequest{
+ ContentLength: 5, PartLength: 5, PartMD5s: []string{checksum},
+ }, ente.Locker, client)
+ }
+ require.NoError(t, err)
+ require.Len(t, upload.PartURLs, 1)
+ objectKey = upload.ObjectKey
+ } else if tc.legacy {
+ uploads, err := controller.GetUploadURLs(t.Context(), userID, 1, ente.Locker, false, client)
+ require.NoError(t, err)
+ objectKey = uploads[0].ObjectKey
+ } else {
+ upload, err := controller.GetUploadURLWithMetadata(t.Context(), userID, ente.UploadURLRequest{
+ ContentLength: 5, ContentMD5: " " + checksum + " ",
+ }, ente.Locker, client)
+ require.NoError(t, err)
+ objectKey = upload.ObjectKey
+ }
+ var owner, createdAt, expiresAt int64
+ var app, purpose, savedClient, bucket string
+ var length sql.NullInt64
+ var md5, uploadID sql.NullString
+ var multipart bool
+ require.NoError(t, db.QueryRow(`SELECT user_id, created_at, expiration_time, app, purpose, client,
+ content_length, content_md5, upload_id, is_multipart, bucket_id FROM temp_objects WHERE object_key = $1`, objectKey).
+ Scan(&owner, &createdAt, &expiresAt, &app, &purpose, &savedClient, &length, &md5, &uploadID, &multipart, &bucket))
+ require.Equal(t, userID, owner)
+ require.Equal(t, "locker", app)
+ require.Equal(t, "file_upload", purpose)
+ require.Equal(t, client, savedClient)
+ require.Equal(t, "b2-eu-cen", bucket)
+ require.Equal(t, tc.multipart, multipart)
+ require.Equal(t, tc.multipart, uploadID.Valid)
+ require.Equal(t, !tc.legacy, length.Valid)
+ if length.Valid {
+ require.Equal(t, int64(5), length.Int64)
+ }
+ require.Equal(t, !tc.legacy && !tc.multipart, md5.Valid)
+ if md5.Valid {
+ require.Equal(t, checksum, md5.String)
+ }
+ require.GreaterOrEqual(t, createdAt, startedAt)
+ require.InDelta(t, (14 * 24 * time.Hour).Microseconds(), expiresAt-createdAt, float64(time.Second.Microseconds()))
+ tx, err := db.Begin()
+ require.NoError(t, err)
+ defer tx.Rollback()
+ require.NoError(t, cleanup.Repo.SetExpiryForTempObject(tx, ente.TempObject{ObjectKey: objectKey}, 1))
+ var afterRetry int64
+ require.NoError(t, tx.QueryRow(`SELECT created_at FROM temp_objects WHERE object_key = $1`, objectKey).Scan(&afterRetry))
+ require.Equal(t, createdAt, afterRetry)
+ })
+ }
+}
diff --git a/server/pkg/repo/object_cleanup.go b/server/pkg/repo/object_cleanup.go
index 3a5628d17a..02a393bd8d 100644
--- a/server/pkg/repo/object_cleanup.go
+++ b/server/pkg/repo/object_cleanup.go
@@ -22,14 +22,13 @@ type ObjectCleanupRepository struct {
}
func (repo *ObjectCleanupRepository) AddTempObject(tempObject ente.TempObject, expirationTime int64) error {
- var err error
- if tempObject.IsMultipart {
- _, err = repo.DB.Exec(`INSERT INTO temp_objects(object_key, expiration_time,upload_id,is_multipart, bucket_id)
- VALUES($1, $2, $3, $4, $5)`, tempObject.ObjectKey, expirationTime, tempObject.UploadID, tempObject.IsMultipart, tempObject.BucketId)
- } else {
- _, err = repo.DB.Exec(`INSERT INTO temp_objects(object_key, expiration_time, bucket_id)
- VALUES($1, $2, $3)`, tempObject.ObjectKey, expirationTime, tempObject.BucketId)
- }
+ _, err := repo.DB.Exec(`
+ INSERT INTO temp_objects (
+ object_key, expiration_time, upload_id, is_multipart, bucket_id,
+ user_id, app, purpose, content_length, content_md5, client
+ ) VALUES ($1, $2, NULLIF($3, ''), $4, $5, NULLIF($6::BIGINT, 0), NULLIF($7, ''), NULLIF($8, ''), $9, $10, NULLIF($11, ''))`,
+ tempObject.ObjectKey, expirationTime, tempObject.UploadID, tempObject.IsMultipart, tempObject.BucketId,
+ tempObject.UserID, tempObject.App, tempObject.Purpose, tempObject.ContentLength, tempObject.ContentMD5, tempObject.Client)
return stacktrace.Propagate(err, "")
}
diff --git a/server/pkg/repo/upload_metadata_test.go b/server/pkg/repo/upload_metadata_test.go
new file mode 100644
index 0000000000..f4e6a9d24c
--- /dev/null
+++ b/server/pkg/repo/upload_metadata_test.go
@@ -0,0 +1,45 @@
+package repo
+
+import (
+ "os"
+ "testing"
+
+ "github.com/ente/museum/internal/testutil"
+ "github.com/stretchr/testify/require"
+)
+
+func TestPendingUploadMetadataMigration(t *testing.T) {
+ testutil.WithServerRoot(t)
+ db := testutil.RequireTestDB(t)
+ tx, err := db.Begin()
+ require.NoError(t, err)
+ defer tx.Rollback()
+ _, err = tx.Exec(`
+ CREATE TEMP TABLE temp_objects (object_key TEXT, expiration_time BIGINT) ON COMMIT DROP;
+ CREATE TEMP TABLE space_temp_objects (object_key TEXT) ON COMMIT DROP;
+ INSERT INTO temp_objects VALUES ('existing', 100);
+ INSERT INTO space_temp_objects VALUES ('existing-space');`)
+ require.NoError(t, err)
+ migration, err := os.ReadFile("migrations/143_pending_upload_metadata.up.sql")
+ require.NoError(t, err)
+ _, err = tx.Exec(string(migration))
+ require.NoError(t, err)
+ var unknown bool
+ require.NoError(t, tx.QueryRow(`SELECT user_id IS NULL AND created_at IS NULL AND app IS NULL
+ AND purpose IS NULL AND content_length IS NULL AND content_md5 IS NULL AND client IS NULL
+ FROM temp_objects WHERE object_key = 'existing'`).Scan(&unknown))
+ require.True(t, unknown)
+ require.NoError(t, tx.QueryRow(`SELECT content_md5 IS NULL AND client IS NULL FROM space_temp_objects`).Scan(&unknown))
+ require.True(t, unknown)
+ var createdAt int64
+ require.NoError(t, tx.QueryRow(`INSERT INTO temp_objects (object_key, expiration_time)
+ VALUES ('old-writer', 200) RETURNING created_at`).Scan(&createdAt))
+ require.Positive(t, createdAt)
+ down, err := os.ReadFile("migrations/143_pending_upload_metadata.down.sql")
+ require.NoError(t, err)
+ _, err = tx.Exec(string(down))
+ require.NoError(t, err)
+ var count int
+ require.NoError(t, tx.QueryRow(`SELECT COUNT(*) FROM temp_objects`).Scan(&count))
+ require.Equal(t, 2, count)
+}
diff --git a/server/pkg/utils/network/network.go b/server/pkg/utils/network/network.go
index ba6ea444f1..af36ddf963 100644
--- a/server/pkg/utils/network/network.go
+++ b/server/pkg/utils/network/network.go
@@ -1,6 +1,9 @@
package network
import (
+ "strings"
+ "unicode/utf8"
+
"github.com/gin-gonic/gin"
"github.com/ua-parser/uap-go/uaparser"
)
@@ -39,3 +42,17 @@ func GetClientInfo(gin *gin.Context) string {
}
return client + "/" + version
}
+
+func GetClientInfoForStorage(c *gin.Context) string {
+ var client strings.Builder
+ for _, r := range GetClientInfo(c) {
+ if r == 0 {
+ continue
+ }
+ if client.Len()+utf8.RuneLen(r) > 256 {
+ break
+ }
+ client.WriteRune(r)
+ }
+ return client.String()
+}
diff --git a/server/pkg/utils/network/network_test.go b/server/pkg/utils/network/network_test.go
new file mode 100644
index 0000000000..a512f0800b
--- /dev/null
+++ b/server/pkg/utils/network/network_test.go
@@ -0,0 +1,27 @@
+package network
+
+import (
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/gin-gonic/gin"
+ "github.com/stretchr/testify/require"
+)
+
+func TestClientInfoForStorage(t *testing.T) {
+ for _, tc := range []struct{ pkg, version, want string }{
+ {"", "", ""},
+ {"io.ente.photos", "", "io.ente.photos"},
+ {"io.ente.photos", "1.0", "io.ente.photos/1.0"},
+ {strings.Repeat("x", 300), "", strings.Repeat("x", 256)},
+ {strings.Repeat("界", 100), "", strings.Repeat("界", 85)},
+ {"client\x00\xff", "", "client�"},
+ } {
+ ctx, _ := gin.CreateTestContext(httptest.NewRecorder())
+ ctx.Request = httptest.NewRequest("GET", "/", nil)
+ ctx.Request.Header.Set("X-Client-Package", tc.pkg)
+ ctx.Request.Header.Set("X-Client-Version", tc.version)
+ require.Equal(t, tc.want, GetClientInfoForStorage(ctx))
+ }
+}
diff --git a/server/space/api/assets.go b/server/space/api/assets.go
index efb9ea0bee..9ff171e2f6 100644
--- a/server/space/api/assets.go
+++ b/server/space/api/assets.go
@@ -1,6 +1,7 @@
package api
import (
+ "github.com/ente/museum/pkg/utils/network"
"github.com/ente/museum/space/models"
spacerepo "github.com/ente/museum/space/repo"
"github.com/gin-gonic/gin"
@@ -11,7 +12,7 @@ func (h *Handlers) PresignUpload(c *gin.Context, space *spacerepo.SpaceRecord) {
if !bindJSON(c, &req) {
return
}
- resp, err := h.Module.Assets.PresignUpload(c, space, req)
+ resp, err := h.Module.Assets.PresignUpload(c, space, req, network.GetClientInfoForStorage(c))
respondJSON(c, resp, err)
}
diff --git a/server/space/controller/assets.go b/server/space/controller/assets.go
index c57a23fd6f..0220724949 100644
--- a/server/space/controller/assets.go
+++ b/server/space/controller/assets.go
@@ -41,7 +41,7 @@ type AssetsController struct {
auth authDeps
}
-func (c *AssetsController) PresignUpload(ctx context.Context, space *spacerepo.SpaceRecord, req models.PresignUploadRequest) (*models.PresignUploadResponse, error) {
+func (c *AssetsController) PresignUpload(ctx context.Context, space *spacerepo.SpaceRecord, req models.PresignUploadRequest, client string) (*models.PresignUploadResponse, error) {
purpose := uploadPurposePost
if req.Purpose != nil && strings.TrimSpace(*req.Purpose) != "" {
purpose = strings.TrimSpace(*req.Purpose)
@@ -79,6 +79,8 @@ func (c *AssetsController) PresignUpload(ctx context.Context, space *spacerepo.S
BucketID: bucketID,
ExpectedSize: req.Size,
ExpiresAt: timeutil.Microseconds() + int64(uploadTempObjectExpiry/time.Microsecond),
+ ContentMD5: sql.NullString{String: contentMD5, Valid: true},
+ Client: sql.NullString{String: client, Valid: client != ""},
})
if errors.Is(err, spacerepo.ErrSpaceUploadLimitReached) {
return nil, newSpaceUploadLimitError()
diff --git a/server/space/controller/assets_test.go b/server/space/controller/assets_test.go
index 0ccc8317b3..df74ba421a 100644
--- a/server/space/controller/assets_test.go
+++ b/server/space/controller/assets_test.go
@@ -7,12 +7,44 @@ import (
"testing"
"github.com/ente/museum/ente"
+ "github.com/ente/museum/pkg/utils/config"
+ "github.com/ente/museum/pkg/utils/s3config"
"github.com/ente/museum/space/models"
spacerepo "github.com/ente/museum/space/repo"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
+func TestPresignUploadStoresMetadata(t *testing.T) {
+ module, repos, _, ctx := setupSpaceAuthControllerTest(t)
+ viper.Reset()
+ t.Cleanup(viper.Reset)
+ require.NoError(t, config.ConfigureViper("local"))
+ viper.Set("s3.b2-eu-cen.key", "test-key")
+ viper.Set("s3.b2-eu-cen.secret", "test-secret")
+ viper.Set("s3.b2-eu-cen.endpoint", "http://localhost:9000")
+ viper.Set("s3.b2-eu-cen.region", "us-east-1")
+ viper.Set("s3.b2-eu-cen.bucket", "test-bucket")
+ viper.Set(spaceAssetsPrimaryBucketConfigKey, "b2-eu-cen")
+ repos.Assets.S3Config = s3config.NewS3Config()
+ ownerID := insertSpaceControllerUser(t, repos, "[email protected]", "public")
+ space, err := testCreateSpace(ctx, repos, ownerID, "upload_metadata", "key", "public", "secret", "nonce", "profile")
+ require.NoError(t, err)
+ const client = "io.ente.space.web/1.0"
+ resp, err := module.Assets.PresignUpload(ctx, space, models.PresignUploadRequest{
+ Size: 5, ContentMD5: "5d41402abc4b2a76b9719d911017c592",
+ }, client)
+ require.NoError(t, err)
+ rec, err := repos.Assets.GetTempObject(ctx, resp.ObjectKey, spacerepo.TempObjectPurposePost, &space.SpaceID)
+ require.NoError(t, err)
+ require.Equal(t, int64(5), rec.ExpectedSize)
+ require.True(t, rec.ContentMD5.Valid)
+ require.Equal(t, resp.Headers["Content-MD5"], rec.ContentMD5.String)
+ require.True(t, rec.Client.Valid)
+ require.Equal(t, client, rec.Client.String)
+ require.Positive(t, rec.CreatedAt)
+}
+
type testSpaceAssetBuckets map[string]bool
func (b testSpaceAssetBuckets) IsBucketActive(bucketID string) bool {
@@ -41,6 +73,7 @@ func TestPresignUploadRejectsOversizedAssets(t *testing.T) {
context.Background(),
&spacerepo.SpaceRecord{SpaceID: "space-1"},
request,
+ "",
)
require.Error(t, err)
require.Contains(t, err.Error(), strconv.FormatInt(test.limit, 10))
@@ -106,7 +139,7 @@ func TestPresignUploadReturnsUnavailableWhenSpaceAssetBucketMissing(t *testing.T
resp, err := module.Assets.PresignUpload(ctx, space, models.PresignUploadRequest{
Size: 1,
ContentMD5: "XUFAKrxLKna5cZ2REBfFkg==",
- })
+ }, "")
require.Nil(t, resp)
requireSpaceAssetsUnavailable(t, err)
diff --git a/server/space/repo/module.go b/server/space/repo/module.go
index 2f8ae19b16..2e9a4c9c63 100644
--- a/server/space/repo/module.go
+++ b/server/space/repo/module.go
@@ -128,6 +128,8 @@ type SpaceTempObjectRecord struct {
ExpiresAt int64
CleanupAfter int64
CreatedAt int64
+ ContentMD5 sql.NullString
+ Client sql.NullString
}
type SpaceMessageRecord struct {
diff --git a/server/space/repo/temp_objects.go b/server/space/repo/temp_objects.go
index 80a8245248..854e3ca97c 100644
--- a/server/space/repo/temp_objects.go
+++ b/server/space/repo/temp_objects.go
@@ -30,9 +30,9 @@ func (r *AssetsRepository) AddTempObject(ctx context.Context, rec SpaceTempObjec
cleanupAfter = rec.ExpiresAt
}
_, err := r.DB.ExecContext(ctx, `
- INSERT INTO space_temp_objects (object_key, space_id, purpose, bucket_id, expected_size, expires_at, cleanup_after)
- VALUES ($1, $2, $3, $4, $5, $6, $7)
- `, rec.ObjectKey, rec.SpaceID, rec.Purpose, rec.BucketID, rec.ExpectedSize, rec.ExpiresAt, cleanupAfter)
+ INSERT INTO space_temp_objects (object_key, space_id, purpose, bucket_id, expected_size, expires_at, cleanup_after, content_md5, client)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+ `, rec.ObjectKey, rec.SpaceID, rec.Purpose, rec.BucketID, rec.ExpectedSize, rec.ExpiresAt, cleanupAfter, rec.ContentMD5, rec.Client)
return stacktrace.Propagate(err, "")
}
@@ -66,9 +66,9 @@ func (r *AssetsRepository) ReserveTempObject(ctx context.Context, rec SpaceTempO
cleanupAfter = rec.ExpiresAt
}
if _, err := tx.ExecContext(ctx, `
- INSERT INTO space_temp_objects (object_key, space_id, purpose, bucket_id, expected_size, expires_at, cleanup_after)
- VALUES ($1, $2, $3, $4, $5, $6, $7)
- `, rec.ObjectKey, rec.SpaceID, rec.Purpose, rec.BucketID, rec.ExpectedSize, rec.ExpiresAt, cleanupAfter); err != nil {
+ INSERT INTO space_temp_objects (object_key, space_id, purpose, bucket_id, expected_size, expires_at, cleanup_after, content_md5, client)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+ `, rec.ObjectKey, rec.SpaceID, rec.Purpose, rec.BucketID, rec.ExpectedSize, rec.ExpiresAt, cleanupAfter, rec.ContentMD5, rec.Client); err != nil {
return stacktrace.Propagate(err, "")
}
return stacktrace.Propagate(tx.Commit(), "")
@@ -96,7 +96,7 @@ func QueueObjectCleanupTx(ctx context.Context, tx *sql.Tx, rec SpaceTempObjectRe
func (r *AssetsRepository) GetTempObject(ctx context.Context, objectKey, purpose string, spaceID *string) (*SpaceTempObjectRecord, error) {
args := []any{objectKey, purpose}
query := `
- SELECT object_key, space_id, purpose, bucket_id, expected_size, expires_at, cleanup_after, created_at
+ SELECT object_key, space_id, purpose, bucket_id, expected_size, expires_at, cleanup_after, created_at, content_md5, client
FROM space_temp_objects
WHERE object_key = $1 AND purpose = $2 AND expires_at > now_utc_micro_seconds()`
if spaceID != nil {
@@ -140,7 +140,7 @@ func (r *AssetsRepository) GetAndLockExpiredTempObjects(ctx context.Context, now
}
}()
rows, err := tx.QueryContext(ctx, `
- SELECT object_key, space_id, purpose, bucket_id, expected_size, expires_at, cleanup_after, created_at
+ SELECT object_key, space_id, purpose, bucket_id, expected_size, expires_at, cleanup_after, created_at, content_md5, client
FROM space_temp_objects
WHERE cleanup_after <= $1
ORDER BY cleanup_after ASC
@@ -220,6 +220,8 @@ func scanSpaceTempObject(scanner interface{ Scan(dest ...any) error }) (*SpaceTe
&rec.ExpiresAt,
&rec.CleanupAfter,
&rec.CreatedAt,
+ &rec.ContentMD5,
+ &rec.Client,
)
if err != nil {
if err == sql.ErrNoRows {