Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Serialize collection moves and restores with Trash

Neeraj Guptaff822e1b91a2
diff --git a/server/pkg/api/trash.go b/server/pkg/api/trash.go
index 82e9e8f974..edf5393f1c 100644
--- a/server/pkg/api/trash.go
+++ b/server/pkg/api/trash.go
@@ -40,7 +40,7 @@ func (t *TrashHandler) Delete(c *gin.Context) {
return
}
request.OwnerID = userID
- err := t.Controller.Delete(c, request)
+ err := t.Controller.Delete(c.Request.Context(), request)
if err != nil {
handler.Error(c, stacktrace.Propagate(err, ""))
return
diff --git a/server/pkg/controller/collections/file_action.go b/server/pkg/controller/collections/file_action.go
index df9bfbd3ec..e70f83b227 100644
--- a/server/pkg/controller/collections/file_action.go
+++ b/server/pkg/controller/collections/file_action.go
@@ -74,7 +74,7 @@ func (c *CollectionController) RestoreFiles(ctx *gin.Context, userID int64, cID
return stacktrace.Propagate(ente.ErrPermissionDenied, "")
}
}
- err = c.CollectionRepo.RestoreFiles(ctx, userID, cID, files)
+ err = c.CollectionRepo.RestoreFiles(ctx.Request.Context(), userID, cID, files)
if err != nil {
return stacktrace.Propagate(err, "")
}
@@ -119,21 +119,7 @@ func (c *CollectionController) MoveFiles(ctx *gin.Context, req ente.MoveFilesReq
return stacktrace.Propagate(err, "Failed to verify fileOwnership")
}
- trashedOrDeletedFileIDs, err := c.TrashRepo.GetFilesInTrashOrDeleted(ctx, userID, fileIDs)
- if err != nil {
- return stacktrace.Propagate(err, "failed to check trash state")
- }
- if len(trashedOrDeletedFileIDs) > 0 {
- log.WithFields(log.Fields{
- "user_id": userID,
- "from_collection_id": req.FromCollectionID,
- "to_collection_id": req.ToCollectionID,
- "trashed_or_deleted_file_ids": trashedOrDeletedFileIDs,
- }).Warn("attempt to move trashed or deleted files between collections")
- return stacktrace.Propagate(&ente.ErrFileInTrash, "")
- }
-
- err = c.CollectionRepo.MoveFiles(ctx, req.ToCollectionID, req.FromCollectionID, req.Files, userID, userID)
+ err = c.CollectionRepo.MoveFiles(ctx.Request.Context(), req.ToCollectionID, req.FromCollectionID, req.Files, userID, userID)
return stacktrace.Propagate(err, "")
}
diff --git a/server/pkg/repo/collection.go b/server/pkg/repo/collection.go
index a1edcaca1f..a50abe66c7 100644
--- a/server/pkg/repo/collection.go
+++ b/server/pkg/repo/collection.go
@@ -769,42 +769,43 @@ func upsertCollectionFiles(
}
func (repo *CollectionRepository) RestoreFiles(ctx context.Context, userID int64, collectionID int64, newCollectionFiles []ente.CollectionFileItem) error {
- fileIDs := make([]int64, 0)
+ fileIDs := make([]int64, 0, len(newCollectionFiles))
for _, newFile := range newCollectionFiles {
fileIDs = append(fileIDs, newFile.ID)
}
- _, canRestoreAllFiles, err := repo.TrashRepo.GetFilesInTrashState(ctx, userID, fileIDs)
+
+ tx, err := repo.DB.BeginTx(ctx, nil)
if err != nil {
return stacktrace.Propagate(err, "")
}
- if !canRestoreAllFiles {
- return stacktrace.Propagate(ente.ErrBadRequest, "some fileIDs are not restorable")
+ defer tx.Rollback()
+ if err := lockFiles(ctx, tx, userID, fileIDs); err != nil {
+ return stacktrace.Propagate(err, "")
}
-
- tx, err := repo.DB.BeginTx(ctx, nil)
updationTime := time.Microseconds()
+ _, canRestoreAllFiles, err := repo.TrashRepo.getFilesInTrashState(ctx, tx, userID, fileIDs)
if err != nil {
return stacktrace.Propagate(err, "")
}
+ if !canRestoreAllFiles {
+ return stacktrace.Propagate(ente.ErrBadRequest, "some fileIDs are not restorable")
+ }
if err := upsertCollectionFiles(ctx, tx, collectionID, userID, newCollectionFiles, userID, updationTime); err != nil {
- tx.Rollback()
return stacktrace.Propagate(err, "")
}
_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1
WHERE collection_id = $2`, updationTime, collectionID)
if err != nil {
- tx.Rollback()
return stacktrace.Propagate(err, "")
}
_, err = tx.ExecContext(ctx, `UPDATE trash SET is_restored = true
WHERE user_id = $1 and file_id = ANY ($2)`, userID, pq.Array(fileIDs))
if err != nil {
- tx.Rollback()
return stacktrace.Propagate(err, "")
}
- return tx.Commit()
+ return stacktrace.Propagate(tx.Commit(), "")
}
func (repo *CollectionRepository) RemoveFilesV3(context context.Context, collectionID int64, collectionOwnerID int64, fileIDs []int64) error {
@@ -872,42 +873,41 @@ func (repo *CollectionRepository) MoveFiles(ctx context.Context,
if collectionOwner != fileOwner {
return fmt.Errorf("move is not supported when collection and file onwer are different")
}
- updationTime := time.Microseconds()
tx, err := repo.DB.BeginTx(ctx, nil)
if err != nil {
return stacktrace.Propagate(err, "")
}
- fileIDs := make([]int64, 0)
+ defer tx.Rollback()
+ fileIDs := make([]int64, 0, len(fileItems))
for _, file := range fileItems {
fileIDs = append(fileIDs, file.ID)
}
+ if err := lockFiles(ctx, tx, fileOwner, fileIDs); err != nil {
+ return stacktrace.Propagate(err, "")
+ }
+ updationTime := time.Microseconds()
+ trashedOrDeletedFileIDs, err := repo.TrashRepo.getFilesInTrashOrDeleted(ctx, tx, fileOwner, fileIDs)
+ if err != nil {
+ return stacktrace.Propagate(err, "failed to check trash state")
+ }
+ if len(trashedOrDeletedFileIDs) > 0 {
+ return stacktrace.Propagate(&ente.ErrFileInTrash, "")
+ }
if err := upsertCollectionFiles(ctx, tx, toCollectionID, collectionOwner, fileItems, fileOwner, updationTime); err != nil {
- if rollbackErr := tx.Rollback(); rollbackErr != nil {
- logrus.WithError(rollbackErr).Error("transaction rollback failed")
- return stacktrace.Propagate(rollbackErr, "")
- }
return stacktrace.Propagate(err, "")
}
_, err = tx.ExecContext(ctx, `UPDATE collection_files
SET is_deleted = $1, updation_time = $2 WHERE collection_id = $3 AND file_id = ANY($4)`,
true, updationTime, fromCollectionID, pq.Array(fileIDs))
if err != nil {
- if rollbackErr := tx.Rollback(); rollbackErr != nil {
- logrus.WithError(rollbackErr).Error("transaction rollback failed")
- return stacktrace.Propagate(rollbackErr, "")
- }
return stacktrace.Propagate(err, "")
}
_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1
WHERE (collection_id = $2 or collection_id = $3 )`, updationTime, toCollectionID, fromCollectionID)
if err != nil {
- if rollbackErr := tx.Rollback(); rollbackErr != nil {
- logrus.WithError(rollbackErr).Error("transaction rollback failed")
- return stacktrace.Propagate(rollbackErr, "")
- }
return stacktrace.Propagate(err, "")
}
- return tx.Commit()
+ return stacktrace.Propagate(tx.Commit(), "")
}
func (repo *CollectionRepository) GetDiff(collectionID int64, sinceTime int64, limit int) ([]ente.File, error) {
diff --git a/server/pkg/repo/collection_memberships_test.go b/server/pkg/repo/collection_memberships_test.go
index 9f956c70ca..dea6ee6dac 100644
--- a/server/pkg/repo/collection_memberships_test.go
+++ b/server/pkg/repo/collection_memberships_test.go
@@ -215,36 +215,22 @@ func TestAddFilesAndTrashFilesSerializeOnFile(t *testing.T) {
linkObjectTestFileToCollection(t, db, sourceCollectionID, fileID, ownerID)
repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)
- blocker, err := db.BeginTx(t.Context(), nil)
- if err != nil {
- t.Fatal(err)
- }
- defer blocker.Rollback()
- if _, err := blocker.ExecContext(t.Context(), `SELECT file_id FROM files
- WHERE file_id = $1 FOR UPDATE`, fileID); err != nil {
- t.Fatal(err)
- }
-
addResult := make(chan error, 1)
- go func() {
- addResult <- repository.AddFiles(t.Context(), destinationCollectionID, ownerID,
- []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}, ownerID)
- }()
trashResult := make(chan error, 1)
- go func() {
- trashResult <- repository.TrashRepo.TrashFiles(t.Context(), ownerID, ente.TrashRequest{
- TrashItems: []ente.TrashItemRequest{{
- FileID: fileID,
- CollectionID: sourceCollectionID,
- }},
- })
- }()
-
- waitForFileLockWaiters(t, db)
- lockReleaseTime := time.Now().UnixMicro()
- if err := blocker.Commit(); err != nil {
- t.Fatal(err)
- }
+ lockReleaseTime := runFileLockRace(t, db, fileID, func() {
+ go func() {
+ addResult <- repository.AddFiles(t.Context(), destinationCollectionID, ownerID,
+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}, ownerID)
+ }()
+ go func() {
+ trashResult <- repository.TrashRepo.TrashFiles(t.Context(), ownerID, ente.TrashRequest{
+ TrashItems: []ente.TrashItemRequest{{
+ FileID: fileID,
+ CollectionID: sourceCollectionID,
+ }},
+ })
+ }()
+ })
addErr := <-addResult
if addErr != nil && !errors.Is(addErr, &ente.ErrFileInTrash) {
@@ -310,6 +296,72 @@ func TestRestoreFilesUpsertsBatchAndMarksTrashRestored(t *testing.T) {
}
}
+func TestRestoreFilesAndDeleteSerializeOnFile(t *testing.T) {
+ repository, db, ownerID := setupCollectionMembershipTest(t)
+ collectionID := insertObjectTestCollection(t, db, ownerID)
+ fileID := insertObjectTestFile(t, db, ownerID)
+ linkObjectTestFileToCollection(t, db, collectionID, fileID, ownerID)
+ if _, err := db.Exec(`UPDATE collection_files SET is_deleted = TRUE
+ WHERE collection_id = $1 AND file_id = $2`, collectionID, fileID); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec(`INSERT INTO trash(file_id, user_id, collection_id, delete_by)
+ VALUES($1, $2, $3, $4)`, fileID, ownerID, collectionID, int64(100)); err != nil {
+ t.Fatal(err)
+ }
+ insertObjectTestKey(t, db, fileID, ente.FILE, "restore-delete-race", 100, []string{"b2-eu-cen"})
+ testutil.InsertUsage(t, db, ownerID, 100)
+ repository.TrashRepo.FileRepo = &FileRepository{
+ DB: db,
+ ObjectRepo: &ObjectRepository{
+ DB: db,
+ QueueRepo: &QueueRepository{DB: db},
+ },
+ }
+
+ restoreResult := make(chan error, 1)
+ deleteResult := make(chan error, 1)
+ lockReleaseTime := runFileLockRace(t, db, fileID, func() {
+ go func() {
+ restoreResult <- repository.RestoreFiles(t.Context(), ownerID, collectionID,
+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)})
+ }()
+ go func() {
+ deleteResult <- repository.TrashRepo.Delete(t.Context(), ownerID, []int64{fileID})
+ }()
+ })
+
+ restoreErr := <-restoreResult
+ if restoreErr != nil && !errors.Is(restoreErr, ente.ErrBadRequest) {
+ t.Fatalf("RestoreFiles() error = %v", restoreErr)
+ }
+ if err := <-deleteResult; err != nil {
+ t.Fatalf("Delete() error = %v", err)
+ }
+
+ var membershipDeleted, trashDeleted, trashRestored, objectDeleted bool
+ if err := db.QueryRow(`SELECT cf.is_deleted, t.is_deleted, t.is_restored, ok.is_deleted
+ FROM collection_files AS cf
+ JOIN trash AS t ON t.file_id = cf.file_id
+ JOIN object_keys AS ok ON ok.file_id = cf.file_id AND ok.o_type = 'file'
+ WHERE cf.collection_id = $1 AND cf.file_id = $2`, collectionID, fileID).Scan(
+ &membershipDeleted, &trashDeleted, &trashRestored, &objectDeleted); err != nil {
+ t.Fatal(err)
+ }
+ if membershipDeleted {
+ if !trashDeleted || trashRestored || !objectDeleted {
+ t.Fatalf("deleted outcome = (trash deleted %t, restored %t, object deleted %t)", trashDeleted, trashRestored, objectDeleted)
+ }
+ } else {
+ if trashDeleted || !trashRestored || objectDeleted {
+ t.Fatalf("restored outcome = (trash deleted %t, restored %t, object deleted %t)", trashDeleted, trashRestored, objectDeleted)
+ }
+ if updationTime := readCollectionMembershipState(t, db, collectionID, fileID).updationTime; updationTime < lockReleaseTime {
+ t.Fatalf("membership updation_time = %d, want >= %d", updationTime, lockReleaseTime)
+ }
+ }
+}
+
func TestMoveFilesUpsertsDestinationAndDeletesSource(t *testing.T) {
repository, db, ownerID := setupCollectionMembershipTest(t)
fromCollectionID := insertObjectTestCollection(t, db, ownerID)
@@ -372,6 +424,54 @@ func TestMoveFilesUpsertsDestinationAndDeletesSource(t *testing.T) {
}
}
+func TestMoveFilesAndTrashFilesSerializeOnFile(t *testing.T) {
+ repository, db, ownerID := setupCollectionMembershipTest(t)
+ sourceCollectionID := insertObjectTestCollection(t, db, ownerID)
+ destinationCollectionID := insertObjectTestCollection(t, db, ownerID)
+ fileID := insertObjectTestFile(t, db, ownerID)
+ linkObjectTestFileToCollection(t, db, sourceCollectionID, fileID, ownerID)
+ repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)
+
+ moveResult := make(chan error, 1)
+ trashResult := make(chan error, 1)
+ lockReleaseTime := runFileLockRace(t, db, fileID, func() {
+ go func() {
+ moveResult <- repository.MoveFiles(t.Context(), destinationCollectionID, sourceCollectionID,
+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}, ownerID, ownerID)
+ }()
+ go func() {
+ trashResult <- repository.TrashRepo.TrashFiles(t.Context(), ownerID, ente.TrashRequest{
+ TrashItems: []ente.TrashItemRequest{{
+ FileID: fileID,
+ CollectionID: sourceCollectionID,
+ }},
+ })
+ }()
+ })
+
+ moveErr := <-moveResult
+ if moveErr != nil && !errors.Is(moveErr, &ente.ErrFileInTrash) {
+ t.Fatalf("MoveFiles() error = %v", moveErr)
+ }
+ if err := <-trashResult; err != nil {
+ t.Fatalf("TrashFiles() error = %v", err)
+ }
+
+ var activeMemberships, activeTrashRows int
+ if err := db.QueryRow(`SELECT
+ (SELECT COUNT(*) FROM collection_files WHERE file_id = $1 AND is_deleted = FALSE),
+ (SELECT COUNT(*) FROM trash WHERE file_id = $1 AND is_deleted = FALSE AND is_restored = FALSE)`,
+ fileID).Scan(&activeMemberships, &activeTrashRows); err != nil {
+ t.Fatal(err)
+ }
+ if activeMemberships != 0 || activeTrashRows != 1 {
+ t.Fatalf("final state = (%d active memberships, %d active Trash rows), want (0, 1)", activeMemberships, activeTrashRows)
+ }
+ if updationTime := readCollectionMembershipState(t, db, sourceCollectionID, fileID).updationTime; updationTime < lockReleaseTime {
+ t.Fatalf("membership updation_time = %d, want >= %d", updationTime, lockReleaseTime)
+ }
+}
+
type collectionMembershipState struct {
encryptedKey string
keyDecryptionNonce string
@@ -473,6 +573,28 @@ func waitForFileLockWaiters(t *testing.T, db *sql.DB) {
t.Fatal("timed out waiting for file-lock waiters")
}
+func runFileLockRace(t *testing.T, db *sql.DB, fileID int64, start func()) int64 {
+ t.Helper()
+
+ blocker, err := db.BeginTx(t.Context(), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer blocker.Rollback()
+ if _, err := blocker.ExecContext(t.Context(), `SELECT file_id FROM files
+ WHERE file_id = $1 FOR UPDATE`, fileID); err != nil {
+ t.Fatal(err)
+ }
+
+ start()
+ waitForFileLockWaiters(t, db)
+ lockReleaseTime := time.Now().UnixMicro()
+ if err := blocker.Commit(); err != nil {
+ t.Fatal(err)
+ }
+ return lockReleaseTime
+}
+
func readCollectionMembershipState(t *testing.T, db *sql.DB, collectionID int64, fileID int64) collectionMembershipState {
t.Helper()
diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.go
index e720766b80..4da10da39d 100644
--- a/server/pkg/repo/trash.go
+++ b/server/pkg/repo/trash.go
@@ -208,11 +208,15 @@ func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []in
if len(fileIDs) > TrashDiffLimit {
return fmt.Errorf("can not delete more than %d in one go", TrashDiffLimit)
}
- fileIDsInTrash, _, err := t.GetFilesInTrashState(ctx, userID, fileIDs)
+ tx, err := t.DB.BeginTx(ctx, nil)
if err != nil {
- return err
+ return stacktrace.Propagate(err, "")
}
- tx, err := t.DB.BeginTx(ctx, nil)
+ defer tx.Rollback()
+ if err := lockFiles(ctx, tx, userID, fileIDs); err != nil {
+ return stacktrace.Propagate(err, "")
+ }
+ fileIDsInTrash, _, err := t.getFilesInTrashState(ctx, tx, userID, fileIDs)
if err != nil {
return stacktrace.Propagate(err, "")
}
@@ -220,26 +224,18 @@ func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []in
logrus.WithField("fileIDs", fileIDsInTrash).Info("deleting files")
_, err = tx.ExecContext(ctx, `UPDATE trash SET is_deleted= true WHERE file_id = ANY ($1)`, pq.Array(fileIDsInTrash))
if err != nil {
- if rollbackErr := tx.Rollback(); rollbackErr != nil {
- logrus.WithError(rollbackErr).Error("transaction rollback failed")
- return stacktrace.Propagate(rollbackErr, "")
- }
return stacktrace.Propagate(err, "")
}
err = t.FileRepo.scheduleDeletion(ctx, tx, fileIDsInTrash, userID)
if err != nil {
- if rollbackErr := tx.Rollback(); rollbackErr != nil {
- logrus.WithError(rollbackErr).Error("transaction rollback failed")
- return stacktrace.Propagate(rollbackErr, "")
- }
return stacktrace.Propagate(err, "")
}
- return tx.Commit()
+ return stacktrace.Propagate(tx.Commit(), "")
}
-func (t *TrashRepository) GetFilesInTrashState(ctx context.Context, userID int64, fileIDs []int64) ([]int64, bool, error) {
- rows, err := t.DB.Query(`SELECT file_id FROM trash
+func (t *TrashRepository) getFilesInTrashState(ctx context.Context, tx *sql.Tx, userID int64, fileIDs []int64) ([]int64, bool, error) {
+ rows, err := tx.QueryContext(ctx, `SELECT file_id FROM trash
WHERE user_id = $1 AND file_id = ANY ($2)
AND is_deleted = FALSE AND is_restored = FALSE`, userID, pq.Array(fileIDs))
if err != nil {
@@ -261,16 +257,8 @@ func (t *TrashRepository) GetFilesInTrashState(ctx context.Context, userID int64
return fileIDsInTrash, canRestoreOrDeleteAllFiles, nil
}
-func (t *TrashRepository) GetFilesInTrashOrDeleted(ctx context.Context, userID int64, fileIDs []int64) ([]int64, error) {
- return t.getFilesInTrashOrDeleted(ctx, t.DB, userID, fileIDs)
-}
-
-type trashQueryer interface {
- QueryContext(context.Context, string, ...any) (*sql.Rows, error)
-}
-
-func (t *TrashRepository) getFilesInTrashOrDeleted(ctx context.Context, queryer trashQueryer, userID int64, fileIDs []int64) ([]int64, error) {
- rows, err := queryer.QueryContext(ctx, `SELECT file_id FROM trash
+func (t *TrashRepository) getFilesInTrashOrDeleted(ctx context.Context, tx *sql.Tx, userID int64, fileIDs []int64) ([]int64, error) {
+ rows, err := tx.QueryContext(ctx, `SELECT file_id FROM trash
WHERE user_id = $1 AND file_id = ANY ($2)
AND is_restored = FALSE`, userID, pq.Array(fileIDs))
if err != nil {