π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Serialize collection additions with Trash
Neeraj Gupta5cccee425d30
diff --git a/server/pkg/controller/collections/file_action.go b/server/pkg/controller/collections/file_action.goindex 05f0c0e017..df9bfbd3ec 100644--- a/server/pkg/controller/collections/file_action.go+++ b/server/pkg/controller/collections/file_action.go@@ -42,19 +42,6 @@ func (c *CollectionController) AddFiles(ctx *gin.Context, userID int64, files []return stacktrace.Propagate(err, "Failed to verify fileOwnership")}- trashedOrDeletedFileIDs, err := c.TrashRepo.GetFilesInTrashOrDeleted(ctx, userID, fileIDs)- if err != nil {- return stacktrace.Propagate(err, "failed to check trash state")- }- if len(trashedOrDeletedFileIDs) > 0 {- log.WithFields(log.Fields{- "user_id": userID,- "collection_id": cID,- "trashed_or_deleted_file_ids": trashedOrDeletedFileIDs,- }).Warn("attempt to add trashed or deleted files to collection")- return stacktrace.Propagate(&ente.ErrFileInTrash, "")- }-err = c.CollectionRepo.AddFiles(ctx.Request.Context(), cID, collectionOwnerID, files, filesOwnerID)if err != nil {return stacktrace.Propagate(err, "")diff --git a/server/pkg/repo/collection.go b/server/pkg/repo/collection.goindex a2b8284fe6..daeeb3c610 100644--- a/server/pkg/repo/collection.go+++ b/server/pkg/repo/collection.go@@ -685,14 +685,28 @@ func (repo *CollectionRepository) AddFiles(if err != nil {return stacktrace.Propagate(err, "")}+ defer tx.Rollback()++ fileIDs := make([]int64, 0, len(files))+ for _, file := range files {+ fileIDs = append(fileIDs, file.ID)+ }+ if err := lockFiles(ctx, tx, fileOwnerID, fileIDs); err != nil {+ return stacktrace.Propagate(err, "")+ }+ trashedOrDeletedFileIDs, err := repo.TrashRepo.getFilesInTrashOrDeleted(ctx, tx, fileOwnerID, fileIDs)+ if err != nil {+ return stacktrace.Propagate(err, "failed to check trash state")+ }+ if len(trashedOrDeletedFileIDs) > 0 {+ return stacktrace.Propagate(&ente.ErrFileInTrash, "")+ }if err := upsertCollectionFiles(ctx, tx, collectionID, collectionOwnerID, files, fileOwnerID, updationTime); err != nil {- tx.Rollback()return stacktrace.Propagate(err, "")}_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1WHERE collection_id = $2`, updationTime, collectionID)if err != nil {- tx.Rollback()return stacktrace.Propagate(err, "")}err = tx.Commit()diff --git a/server/pkg/repo/collection_memberships_test.go b/server/pkg/repo/collection_memberships_test.goindex e546a7f78a..d79c9d5ced 100644--- a/server/pkg/repo/collection_memberships_test.go+++ b/server/pkg/repo/collection_memberships_test.go@@ -3,11 +3,14 @@ package repoimport ("context""database/sql"+ "errors""fmt""testing"+ "time""github.com/ente/museum/ente""github.com/ente/museum/internal/testutil"+ "github.com/ente/museum/pkg/repo/public")func TestAddFilesUpsertsBatchAndPreservesConflictFields(t *testing.T) {@@ -180,6 +183,88 @@ func TestAddFilesHonorsCanceledContext(t *testing.T) {assertCollectionMembershipTestUpdationTime(t, db, collectionID, 1)}+func TestAddFilesRejectsTrashedFileInsideTransaction(t *testing.T) {+ repository, db, ownerID := setupCollectionMembershipTest(t)+ collectionID := insertObjectTestCollection(t, db, ownerID)+ fileID := insertObjectTestFile(t, db, ownerID)+ linkObjectTestFileToCollection(t, db, collectionID, fileID, ownerID)+ if _, err := db.Exec(`UPDATE collection_files SET is_deleted = TRUE+ WHERE collection_id = $1 AND file_id = $2`, collectionID, fileID); err != nil {+ t.Fatal(err)+ }+ if _, err := db.Exec(`INSERT INTO trash(file_id, user_id, collection_id, delete_by)+ VALUES($1, $2, $3, $4)`, fileID, ownerID, collectionID, int64(100)); err != nil {+ t.Fatal(err)+ }++ err := repository.AddFiles(t.Context(), collectionID, ownerID, []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}, ownerID)+ if !errors.Is(err, &ente.ErrFileInTrash) {+ t.Fatalf("AddFiles() error = %v, want ErrFileInTrash", err)+ }++ if state := readCollectionMembershipState(t, db, collectionID, fileID); !state.isDeleted {+ t.Fatal("AddFiles() reactivated trashed membership")+ }+}++func TestAddFilesAndTrashFilesSerializeOnFile(t *testing.T) {+ repository, db, ownerID := setupCollectionMembershipTest(t)+ sourceCollectionID := insertObjectTestCollection(t, db, ownerID)+ destinationCollectionID := insertObjectTestCollection(t, db, ownerID)+ fileID := insertObjectTestFile(t, db, ownerID)+ linkObjectTestFileToCollection(t, db, sourceCollectionID, fileID, ownerID)+ repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)++ blocker, err := db.BeginTx(t.Context(), nil)+ if err != nil {+ t.Fatal(err)+ }+ defer blocker.Rollback()+ if _, err := blocker.ExecContext(t.Context(), `SELECT file_id FROM files+ WHERE file_id = $1 FOR UPDATE`, fileID); err != nil {+ t.Fatal(err)+ }++ addResult := make(chan error, 1)+ go func() {+ addResult <- repository.AddFiles(t.Context(), destinationCollectionID, ownerID,+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}, ownerID)+ }()+ trashResult := make(chan error, 1)+ go func() {+ trashResult <- repository.TrashRepo.TrashFiles(t.Context(), ownerID, ente.TrashRequest{+ TrashItems: []ente.TrashItemRequest{{+ FileID: fileID,+ CollectionID: sourceCollectionID,+ }},+ })+ }()++ waitForFileLockWaiters(t, db)+ if err := blocker.Commit(); err != nil {+ t.Fatal(err)+ }++ addErr := <-addResult+ if addErr != nil && !errors.Is(addErr, &ente.ErrFileInTrash) {+ t.Fatalf("AddFiles() error = %v", addErr)+ }+ if err := <-trashResult; err != nil {+ t.Fatalf("TrashFiles() error = %v", err)+ }++ var activeMemberships, activeTrashRows int+ if err := db.QueryRow(`SELECT+ (SELECT COUNT(*) FROM collection_files WHERE file_id = $1 AND is_deleted = FALSE),+ (SELECT COUNT(*) FROM trash WHERE file_id = $1 AND is_deleted = FALSE AND is_restored = FALSE)`,+ fileID).Scan(&activeMemberships, &activeTrashRows); err != nil {+ t.Fatal(err)+ }+ if activeMemberships != 0 || activeTrashRows != 1 {+ t.Fatalf("final state = (%d active memberships, %d active Trash rows), want (0, 1)", activeMemberships, activeTrashRows)+ }+}+func TestRestoreFilesUpsertsBatchAndMarksTrashRestored(t *testing.T) {repository, db, ownerID := setupCollectionMembershipTest(t)collectionID := insertObjectTestCollection(t, db, ownerID)@@ -362,6 +447,28 @@ func insertCollectionMembershipTestFiles(t *testing.T, db *sql.DB, ownerID int64return fileIDs}+func waitForFileLockWaiters(t *testing.T, db *sql.DB) {+ t.Helper()++ deadline := time.Now().Add(5 * time.Second)+ for time.Now().Before(deadline) {+ var count int+ if err := db.QueryRow(`SELECT COUNT(*) FROM pg_stat_activity+ WHERE datname = current_database()+ AND pid <> pg_backend_pid()+ AND wait_event_type = 'Lock'+ AND query LIKE '%FROM files%'+ AND query LIKE '%FOR UPDATE%'`).Scan(&count); err != nil {+ t.Fatal(err)+ }+ if count >= 2 {+ return+ }+ time.Sleep(10 * time.Millisecond)+ }+ t.Fatal("timed out waiting for file-lock waiters")+}+func readCollectionMembershipState(t *testing.T, db *sql.DB, collectionID int64, fileID int64) collectionMembershipState {t.Helper()diff --git a/server/pkg/repo/file_lock.go b/server/pkg/repo/file_lock.gonew file mode 100644index 0000000000..c3287b12c6--- /dev/null+++ b/server/pkg/repo/file_lock.go@@ -0,0 +1,33 @@+package repo++import (+ "context"+ "database/sql"++ "github.com/ente/stacktrace"+ "github.com/lib/pq"+)++func lockFiles(ctx context.Context, tx *sql.Tx, userID int64, fileIDs []int64) error {+ if len(fileIDs) == 0 {+ return nil+ }++ rows, err := tx.QueryContext(ctx, `SELECT file_id+ FROM files+ WHERE owner_id = $1 AND file_id = ANY($2)+ ORDER BY file_id+ FOR UPDATE`, userID, pq.Array(fileIDs))+ if err != nil {+ return stacktrace.Propagate(err, "")+ }+ defer rows.Close()++ for rows.Next() {+ var fileID int64+ if err := rows.Scan(&fileID); err != nil {+ return stacktrace.Propagate(err, "")+ }+ }+ return stacktrace.Propagate(rows.Err(), "")+}diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.goindex 8c8db3cd72..28ce911898 100644--- a/server/pkg/repo/trash.go+++ b/server/pkg/repo/trash.go@@ -125,6 +125,9 @@ func (t *TrashRepository) TrashFiles(ctx context.Context, userID int64, trash enreturn stacktrace.Propagate(err, "")}defer tx.Rollback()+ if err := lockFiles(ctx, tx, userID, fileIDs); err != nil {+ return stacktrace.Propagate(err, "")+ }rows, err := tx.QueryContext(ctx, `SELECT DISTINCT collection_id FROMcollection_files WHERE file_id = ANY($1) AND is_deleted = $2`, pq.Array(fileIDs), false)if err != nil {@@ -259,7 +262,15 @@ func (t *TrashRepository) GetFilesInTrashState(ctx context.Context, userID int64}func (t *TrashRepository) GetFilesInTrashOrDeleted(ctx context.Context, userID int64, fileIDs []int64) ([]int64, error) {- rows, err := t.DB.QueryContext(ctx, `SELECT file_id FROM trash+ return t.getFilesInTrashOrDeleted(ctx, t.DB, userID, fileIDs)+}++type trashQueryer interface {+ QueryContext(context.Context, string, ...any) (*sql.Rows, error)+}++func (t *TrashRepository) getFilesInTrashOrDeleted(ctx context.Context, queryer trashQueryer, userID int64, fileIDs []int64) ([]int64, error) {+ rows, err := queryer.QueryContext(ctx, `SELECT file_id FROM trashWHERE user_id = $1 AND file_id = ANY ($2)AND is_restored = FALSE`, userID, pq.Array(fileIDs))if err != nil {