π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Serialize collection moves and restores with Trash
Neeraj Guptaff822e1b91a2
diff --git a/server/pkg/api/trash.go b/server/pkg/api/trash.goindex 82e9e8f974..edf5393f1c 100644--- a/server/pkg/api/trash.go+++ b/server/pkg/api/trash.go@@ -40,7 +40,7 @@ func (t *TrashHandler) Delete(c *gin.Context) {return}request.OwnerID = userID- err := t.Controller.Delete(c, request)+ err := t.Controller.Delete(c.Request.Context(), request)if err != nil {handler.Error(c, stacktrace.Propagate(err, ""))returndiff --git a/server/pkg/controller/collections/file_action.go b/server/pkg/controller/collections/file_action.goindex df9bfbd3ec..e70f83b227 100644--- a/server/pkg/controller/collections/file_action.go+++ b/server/pkg/controller/collections/file_action.go@@ -74,7 +74,7 @@ func (c *CollectionController) RestoreFiles(ctx *gin.Context, userID int64, cIDreturn stacktrace.Propagate(ente.ErrPermissionDenied, "")}}- err = c.CollectionRepo.RestoreFiles(ctx, userID, cID, files)+ err = c.CollectionRepo.RestoreFiles(ctx.Request.Context(), userID, cID, files)if err != nil {return stacktrace.Propagate(err, "")}@@ -119,21 +119,7 @@ func (c *CollectionController) MoveFiles(ctx *gin.Context, req ente.MoveFilesReqreturn stacktrace.Propagate(err, "Failed to verify fileOwnership")}- trashedOrDeletedFileIDs, err := c.TrashRepo.GetFilesInTrashOrDeleted(ctx, userID, fileIDs)- if err != nil {- return stacktrace.Propagate(err, "failed to check trash state")- }- if len(trashedOrDeletedFileIDs) > 0 {- log.WithFields(log.Fields{- "user_id": userID,- "from_collection_id": req.FromCollectionID,- "to_collection_id": req.ToCollectionID,- "trashed_or_deleted_file_ids": trashedOrDeletedFileIDs,- }).Warn("attempt to move trashed or deleted files between collections")- return stacktrace.Propagate(&ente.ErrFileInTrash, "")- }-- err = c.CollectionRepo.MoveFiles(ctx, req.ToCollectionID, req.FromCollectionID, req.Files, userID, userID)+ err = c.CollectionRepo.MoveFiles(ctx.Request.Context(), req.ToCollectionID, req.FromCollectionID, req.Files, userID, userID)return stacktrace.Propagate(err, "")}diff --git a/server/pkg/repo/collection.go b/server/pkg/repo/collection.goindex a1edcaca1f..a50abe66c7 100644--- a/server/pkg/repo/collection.go+++ b/server/pkg/repo/collection.go@@ -769,42 +769,43 @@ func upsertCollectionFiles(}func (repo *CollectionRepository) RestoreFiles(ctx context.Context, userID int64, collectionID int64, newCollectionFiles []ente.CollectionFileItem) error {- fileIDs := make([]int64, 0)+ fileIDs := make([]int64, 0, len(newCollectionFiles))for _, newFile := range newCollectionFiles {fileIDs = append(fileIDs, newFile.ID)}- _, canRestoreAllFiles, err := repo.TrashRepo.GetFilesInTrashState(ctx, userID, fileIDs)++ tx, err := repo.DB.BeginTx(ctx, nil)if err != nil {return stacktrace.Propagate(err, "")}- if !canRestoreAllFiles {- return stacktrace.Propagate(ente.ErrBadRequest, "some fileIDs are not restorable")+ defer tx.Rollback()+ if err := lockFiles(ctx, tx, userID, fileIDs); err != nil {+ return stacktrace.Propagate(err, "")}-- tx, err := repo.DB.BeginTx(ctx, nil)updationTime := time.Microseconds()+ _, canRestoreAllFiles, err := repo.TrashRepo.getFilesInTrashState(ctx, tx, userID, fileIDs)if err != nil {return stacktrace.Propagate(err, "")}+ if !canRestoreAllFiles {+ return stacktrace.Propagate(ente.ErrBadRequest, "some fileIDs are not restorable")+ }if err := upsertCollectionFiles(ctx, tx, collectionID, userID, newCollectionFiles, userID, updationTime); err != nil {- tx.Rollback()return stacktrace.Propagate(err, "")}_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1WHERE collection_id = $2`, updationTime, collectionID)if err != nil {- tx.Rollback()return stacktrace.Propagate(err, "")}_, err = tx.ExecContext(ctx, `UPDATE trash SET is_restored = trueWHERE user_id = $1 and file_id = ANY ($2)`, userID, pq.Array(fileIDs))if err != nil {- tx.Rollback()return stacktrace.Propagate(err, "")}- return tx.Commit()+ return stacktrace.Propagate(tx.Commit(), "")}func (repo *CollectionRepository) RemoveFilesV3(context context.Context, collectionID int64, collectionOwnerID int64, fileIDs []int64) error {@@ -872,42 +873,41 @@ func (repo *CollectionRepository) MoveFiles(ctx context.Context,if collectionOwner != fileOwner {return fmt.Errorf("move is not supported when collection and file onwer are different")}- updationTime := time.Microseconds()tx, err := repo.DB.BeginTx(ctx, nil)if err != nil {return stacktrace.Propagate(err, "")}- fileIDs := make([]int64, 0)+ defer tx.Rollback()+ fileIDs := make([]int64, 0, len(fileItems))for _, file := range fileItems {fileIDs = append(fileIDs, file.ID)}+ if err := lockFiles(ctx, tx, fileOwner, fileIDs); err != nil {+ return stacktrace.Propagate(err, "")+ }+ updationTime := time.Microseconds()+ trashedOrDeletedFileIDs, err := repo.TrashRepo.getFilesInTrashOrDeleted(ctx, tx, fileOwner, fileIDs)+ if err != nil {+ return stacktrace.Propagate(err, "failed to check trash state")+ }+ if len(trashedOrDeletedFileIDs) > 0 {+ return stacktrace.Propagate(&ente.ErrFileInTrash, "")+ }if err := upsertCollectionFiles(ctx, tx, toCollectionID, collectionOwner, fileItems, fileOwner, updationTime); err != nil {- if rollbackErr := tx.Rollback(); rollbackErr != nil {- logrus.WithError(rollbackErr).Error("transaction rollback failed")- return stacktrace.Propagate(rollbackErr, "")- }return stacktrace.Propagate(err, "")}_, err = tx.ExecContext(ctx, `UPDATE collection_filesSET is_deleted = $1, updation_time = $2 WHERE collection_id = $3 AND file_id = ANY($4)`,true, updationTime, fromCollectionID, pq.Array(fileIDs))if err != nil {- if rollbackErr := tx.Rollback(); rollbackErr != nil {- logrus.WithError(rollbackErr).Error("transaction rollback failed")- return stacktrace.Propagate(rollbackErr, "")- }return stacktrace.Propagate(err, "")}_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1WHERE (collection_id = $2 or collection_id = $3 )`, updationTime, toCollectionID, fromCollectionID)if err != nil {- if rollbackErr := tx.Rollback(); rollbackErr != nil {- logrus.WithError(rollbackErr).Error("transaction rollback failed")- return stacktrace.Propagate(rollbackErr, "")- }return stacktrace.Propagate(err, "")}- return tx.Commit()+ return stacktrace.Propagate(tx.Commit(), "")}func (repo *CollectionRepository) GetDiff(collectionID int64, sinceTime int64, limit int) ([]ente.File, error) {diff --git a/server/pkg/repo/collection_memberships_test.go b/server/pkg/repo/collection_memberships_test.goindex 9f956c70ca..dea6ee6dac 100644--- a/server/pkg/repo/collection_memberships_test.go+++ b/server/pkg/repo/collection_memberships_test.go@@ -215,36 +215,22 @@ func TestAddFilesAndTrashFilesSerializeOnFile(t *testing.T) {linkObjectTestFileToCollection(t, db, sourceCollectionID, fileID, ownerID)repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)- blocker, err := db.BeginTx(t.Context(), nil)- if err != nil {- t.Fatal(err)- }- defer blocker.Rollback()- if _, err := blocker.ExecContext(t.Context(), `SELECT file_id FROM files- WHERE file_id = $1 FOR UPDATE`, fileID); err != nil {- t.Fatal(err)- }-addResult := make(chan error, 1)- go func() {- addResult <- repository.AddFiles(t.Context(), destinationCollectionID, ownerID,- []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}, ownerID)- }()trashResult := make(chan error, 1)- go func() {- trashResult <- repository.TrashRepo.TrashFiles(t.Context(), ownerID, ente.TrashRequest{- TrashItems: []ente.TrashItemRequest{{- FileID: fileID,- CollectionID: sourceCollectionID,- }},- })- }()-- waitForFileLockWaiters(t, db)- lockReleaseTime := time.Now().UnixMicro()- if err := blocker.Commit(); err != nil {- t.Fatal(err)- }+ lockReleaseTime := runFileLockRace(t, db, fileID, func() {+ go func() {+ addResult <- repository.AddFiles(t.Context(), destinationCollectionID, ownerID,+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}, ownerID)+ }()+ go func() {+ trashResult <- repository.TrashRepo.TrashFiles(t.Context(), ownerID, ente.TrashRequest{+ TrashItems: []ente.TrashItemRequest{{+ FileID: fileID,+ CollectionID: sourceCollectionID,+ }},+ })+ }()+ })addErr := <-addResultif addErr != nil && !errors.Is(addErr, &ente.ErrFileInTrash) {@@ -310,6 +296,72 @@ func TestRestoreFilesUpsertsBatchAndMarksTrashRestored(t *testing.T) {}}+func TestRestoreFilesAndDeleteSerializeOnFile(t *testing.T) {+ repository, db, ownerID := setupCollectionMembershipTest(t)+ collectionID := insertObjectTestCollection(t, db, ownerID)+ fileID := insertObjectTestFile(t, db, ownerID)+ linkObjectTestFileToCollection(t, db, collectionID, fileID, ownerID)+ if _, err := db.Exec(`UPDATE collection_files SET is_deleted = TRUE+ WHERE collection_id = $1 AND file_id = $2`, collectionID, fileID); err != nil {+ t.Fatal(err)+ }+ if _, err := db.Exec(`INSERT INTO trash(file_id, user_id, collection_id, delete_by)+ VALUES($1, $2, $3, $4)`, fileID, ownerID, collectionID, int64(100)); err != nil {+ t.Fatal(err)+ }+ insertObjectTestKey(t, db, fileID, ente.FILE, "restore-delete-race", 100, []string{"b2-eu-cen"})+ testutil.InsertUsage(t, db, ownerID, 100)+ repository.TrashRepo.FileRepo = &FileRepository{+ DB: db,+ ObjectRepo: &ObjectRepository{+ DB: db,+ QueueRepo: &QueueRepository{DB: db},+ },+ }++ restoreResult := make(chan error, 1)+ deleteResult := make(chan error, 1)+ lockReleaseTime := runFileLockRace(t, db, fileID, func() {+ go func() {+ restoreResult <- repository.RestoreFiles(t.Context(), ownerID, collectionID,+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)})+ }()+ go func() {+ deleteResult <- repository.TrashRepo.Delete(t.Context(), ownerID, []int64{fileID})+ }()+ })++ restoreErr := <-restoreResult+ if restoreErr != nil && !errors.Is(restoreErr, ente.ErrBadRequest) {+ t.Fatalf("RestoreFiles() error = %v", restoreErr)+ }+ if err := <-deleteResult; err != nil {+ t.Fatalf("Delete() error = %v", err)+ }++ var membershipDeleted, trashDeleted, trashRestored, objectDeleted bool+ if err := db.QueryRow(`SELECT cf.is_deleted, t.is_deleted, t.is_restored, ok.is_deleted+ FROM collection_files AS cf+ JOIN trash AS t ON t.file_id = cf.file_id+ JOIN object_keys AS ok ON ok.file_id = cf.file_id AND ok.o_type = 'file'+ WHERE cf.collection_id = $1 AND cf.file_id = $2`, collectionID, fileID).Scan(+ &membershipDeleted, &trashDeleted, &trashRestored, &objectDeleted); err != nil {+ t.Fatal(err)+ }+ if membershipDeleted {+ if !trashDeleted || trashRestored || !objectDeleted {+ t.Fatalf("deleted outcome = (trash deleted %t, restored %t, object deleted %t)", trashDeleted, trashRestored, objectDeleted)+ }+ } else {+ if trashDeleted || !trashRestored || objectDeleted {+ t.Fatalf("restored outcome = (trash deleted %t, restored %t, object deleted %t)", trashDeleted, trashRestored, objectDeleted)+ }+ if updationTime := readCollectionMembershipState(t, db, collectionID, fileID).updationTime; updationTime < lockReleaseTime {+ t.Fatalf("membership updation_time = %d, want >= %d", updationTime, lockReleaseTime)+ }+ }+}+func TestMoveFilesUpsertsDestinationAndDeletesSource(t *testing.T) {repository, db, ownerID := setupCollectionMembershipTest(t)fromCollectionID := insertObjectTestCollection(t, db, ownerID)@@ -372,6 +424,54 @@ func TestMoveFilesUpsertsDestinationAndDeletesSource(t *testing.T) {}}+func TestMoveFilesAndTrashFilesSerializeOnFile(t *testing.T) {+ repository, db, ownerID := setupCollectionMembershipTest(t)+ sourceCollectionID := insertObjectTestCollection(t, db, ownerID)+ destinationCollectionID := insertObjectTestCollection(t, db, ownerID)+ fileID := insertObjectTestFile(t, db, ownerID)+ linkObjectTestFileToCollection(t, db, sourceCollectionID, fileID, ownerID)+ repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)++ moveResult := make(chan error, 1)+ trashResult := make(chan error, 1)+ lockReleaseTime := runFileLockRace(t, db, fileID, func() {+ go func() {+ moveResult <- repository.MoveFiles(t.Context(), destinationCollectionID, sourceCollectionID,+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}, ownerID, ownerID)+ }()+ go func() {+ trashResult <- repository.TrashRepo.TrashFiles(t.Context(), ownerID, ente.TrashRequest{+ TrashItems: []ente.TrashItemRequest{{+ FileID: fileID,+ CollectionID: sourceCollectionID,+ }},+ })+ }()+ })++ moveErr := <-moveResult+ if moveErr != nil && !errors.Is(moveErr, &ente.ErrFileInTrash) {+ t.Fatalf("MoveFiles() error = %v", moveErr)+ }+ if err := <-trashResult; err != nil {+ t.Fatalf("TrashFiles() error = %v", err)+ }++ var activeMemberships, activeTrashRows int+ if err := db.QueryRow(`SELECT+ (SELECT COUNT(*) FROM collection_files WHERE file_id = $1 AND is_deleted = FALSE),+ (SELECT COUNT(*) FROM trash WHERE file_id = $1 AND is_deleted = FALSE AND is_restored = FALSE)`,+ fileID).Scan(&activeMemberships, &activeTrashRows); err != nil {+ t.Fatal(err)+ }+ if activeMemberships != 0 || activeTrashRows != 1 {+ t.Fatalf("final state = (%d active memberships, %d active Trash rows), want (0, 1)", activeMemberships, activeTrashRows)+ }+ if updationTime := readCollectionMembershipState(t, db, sourceCollectionID, fileID).updationTime; updationTime < lockReleaseTime {+ t.Fatalf("membership updation_time = %d, want >= %d", updationTime, lockReleaseTime)+ }+}+type collectionMembershipState struct {encryptedKey stringkeyDecryptionNonce string@@ -473,6 +573,28 @@ func waitForFileLockWaiters(t *testing.T, db *sql.DB) {t.Fatal("timed out waiting for file-lock waiters")}+func runFileLockRace(t *testing.T, db *sql.DB, fileID int64, start func()) int64 {+ t.Helper()++ blocker, err := db.BeginTx(t.Context(), nil)+ if err != nil {+ t.Fatal(err)+ }+ defer blocker.Rollback()+ if _, err := blocker.ExecContext(t.Context(), `SELECT file_id FROM files+ WHERE file_id = $1 FOR UPDATE`, fileID); err != nil {+ t.Fatal(err)+ }++ start()+ waitForFileLockWaiters(t, db)+ lockReleaseTime := time.Now().UnixMicro()+ if err := blocker.Commit(); err != nil {+ t.Fatal(err)+ }+ return lockReleaseTime+}+func readCollectionMembershipState(t *testing.T, db *sql.DB, collectionID int64, fileID int64) collectionMembershipState {t.Helper()diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.goindex e720766b80..4da10da39d 100644--- a/server/pkg/repo/trash.go+++ b/server/pkg/repo/trash.go@@ -208,11 +208,15 @@ func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []inif len(fileIDs) > TrashDiffLimit {return fmt.Errorf("can not delete more than %d in one go", TrashDiffLimit)}- fileIDsInTrash, _, err := t.GetFilesInTrashState(ctx, userID, fileIDs)+ tx, err := t.DB.BeginTx(ctx, nil)if err != nil {- return err+ return stacktrace.Propagate(err, "")}- tx, err := t.DB.BeginTx(ctx, nil)+ defer tx.Rollback()+ if err := lockFiles(ctx, tx, userID, fileIDs); err != nil {+ return stacktrace.Propagate(err, "")+ }+ fileIDsInTrash, _, err := t.getFilesInTrashState(ctx, tx, userID, fileIDs)if err != nil {return stacktrace.Propagate(err, "")}@@ -220,26 +224,18 @@ func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []inlogrus.WithField("fileIDs", fileIDsInTrash).Info("deleting files")_, err = tx.ExecContext(ctx, `UPDATE trash SET is_deleted= true WHERE file_id = ANY ($1)`, pq.Array(fileIDsInTrash))if err != nil {- if rollbackErr := tx.Rollback(); rollbackErr != nil {- logrus.WithError(rollbackErr).Error("transaction rollback failed")- return stacktrace.Propagate(rollbackErr, "")- }return stacktrace.Propagate(err, "")}err = t.FileRepo.scheduleDeletion(ctx, tx, fileIDsInTrash, userID)if err != nil {- if rollbackErr := tx.Rollback(); rollbackErr != nil {- logrus.WithError(rollbackErr).Error("transaction rollback failed")- return stacktrace.Propagate(rollbackErr, "")- }return stacktrace.Propagate(err, "")}- return tx.Commit()+ return stacktrace.Propagate(tx.Commit(), "")}-func (t *TrashRepository) GetFilesInTrashState(ctx context.Context, userID int64, fileIDs []int64) ([]int64, bool, error) {- rows, err := t.DB.Query(`SELECT file_id FROM trash+func (t *TrashRepository) getFilesInTrashState(ctx context.Context, tx *sql.Tx, userID int64, fileIDs []int64) ([]int64, bool, error) {+ rows, err := tx.QueryContext(ctx, `SELECT file_id FROM trashWHERE user_id = $1 AND file_id = ANY ($2)AND is_deleted = FALSE AND is_restored = FALSE`, userID, pq.Array(fileIDs))if err != nil {@@ -261,16 +257,8 @@ func (t *TrashRepository) GetFilesInTrashState(ctx context.Context, userID int64return fileIDsInTrash, canRestoreOrDeleteAllFiles, nil}-func (t *TrashRepository) GetFilesInTrashOrDeleted(ctx context.Context, userID int64, fileIDs []int64) ([]int64, error) {- return t.getFilesInTrashOrDeleted(ctx, t.DB, userID, fileIDs)-}--type trashQueryer interface {- QueryContext(context.Context, string, ...any) (*sql.Rows, error)-}--func (t *TrashRepository) getFilesInTrashOrDeleted(ctx context.Context, queryer trashQueryer, userID int64, fileIDs []int64) ([]int64, error) {- rows, err := queryer.QueryContext(ctx, `SELECT file_id FROM trash+func (t *TrashRepository) getFilesInTrashOrDeleted(ctx context.Context, tx *sql.Tx, userID int64, fileIDs []int64) ([]int64, error) {+ rows, err := tx.QueryContext(ctx, `SELECT file_id FROM trashWHERE user_id = $1 AND file_id = ANY ($2)AND is_restored = FALSE`, userID, pq.Array(fileIDs))if err != nil {