Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Serialize collection additions with Trash

Neeraj Gupta5cccee425d30
diff --git a/server/pkg/controller/collections/file_action.go b/server/pkg/controller/collections/file_action.go
index 05f0c0e017..df9bfbd3ec 100644
--- a/server/pkg/controller/collections/file_action.go
+++ b/server/pkg/controller/collections/file_action.go
@@ -42,19 +42,6 @@ func (c *CollectionController) AddFiles(ctx *gin.Context, userID int64, files []
return stacktrace.Propagate(err, "Failed to verify fileOwnership")
}
- trashedOrDeletedFileIDs, err := c.TrashRepo.GetFilesInTrashOrDeleted(ctx, userID, fileIDs)
- if err != nil {
- return stacktrace.Propagate(err, "failed to check trash state")
- }
- if len(trashedOrDeletedFileIDs) > 0 {
- log.WithFields(log.Fields{
- "user_id": userID,
- "collection_id": cID,
- "trashed_or_deleted_file_ids": trashedOrDeletedFileIDs,
- }).Warn("attempt to add trashed or deleted files to collection")
- return stacktrace.Propagate(&ente.ErrFileInTrash, "")
- }
-
err = c.CollectionRepo.AddFiles(ctx.Request.Context(), cID, collectionOwnerID, files, filesOwnerID)
if err != nil {
return stacktrace.Propagate(err, "")
diff --git a/server/pkg/repo/collection.go b/server/pkg/repo/collection.go
index a2b8284fe6..daeeb3c610 100644
--- a/server/pkg/repo/collection.go
+++ b/server/pkg/repo/collection.go
@@ -685,14 +685,28 @@ func (repo *CollectionRepository) AddFiles(
if err != nil {
return stacktrace.Propagate(err, "")
}
+ defer tx.Rollback()
+
+ fileIDs := make([]int64, 0, len(files))
+ for _, file := range files {
+ fileIDs = append(fileIDs, file.ID)
+ }
+ if err := lockFiles(ctx, tx, fileOwnerID, fileIDs); err != nil {
+ return stacktrace.Propagate(err, "")
+ }
+ trashedOrDeletedFileIDs, err := repo.TrashRepo.getFilesInTrashOrDeleted(ctx, tx, fileOwnerID, fileIDs)
+ if err != nil {
+ return stacktrace.Propagate(err, "failed to check trash state")
+ }
+ if len(trashedOrDeletedFileIDs) > 0 {
+ return stacktrace.Propagate(&ente.ErrFileInTrash, "")
+ }
if err := upsertCollectionFiles(ctx, tx, collectionID, collectionOwnerID, files, fileOwnerID, updationTime); err != nil {
- tx.Rollback()
return stacktrace.Propagate(err, "")
}
_, err = tx.ExecContext(ctx, `UPDATE collections SET updation_time = $1
WHERE collection_id = $2`, updationTime, collectionID)
if err != nil {
- tx.Rollback()
return stacktrace.Propagate(err, "")
}
err = tx.Commit()
diff --git a/server/pkg/repo/collection_memberships_test.go b/server/pkg/repo/collection_memberships_test.go
index e546a7f78a..d79c9d5ced 100644
--- a/server/pkg/repo/collection_memberships_test.go
+++ b/server/pkg/repo/collection_memberships_test.go
@@ -3,11 +3,14 @@ package repo
import (
"context"
"database/sql"
+ "errors"
"fmt"
"testing"
+ "time"
"github.com/ente/museum/ente"
"github.com/ente/museum/internal/testutil"
+ "github.com/ente/museum/pkg/repo/public"
)
func TestAddFilesUpsertsBatchAndPreservesConflictFields(t *testing.T) {
@@ -180,6 +183,88 @@ func TestAddFilesHonorsCanceledContext(t *testing.T) {
assertCollectionMembershipTestUpdationTime(t, db, collectionID, 1)
}
+func TestAddFilesRejectsTrashedFileInsideTransaction(t *testing.T) {
+ repository, db, ownerID := setupCollectionMembershipTest(t)
+ collectionID := insertObjectTestCollection(t, db, ownerID)
+ fileID := insertObjectTestFile(t, db, ownerID)
+ linkObjectTestFileToCollection(t, db, collectionID, fileID, ownerID)
+ if _, err := db.Exec(`UPDATE collection_files SET is_deleted = TRUE
+ WHERE collection_id = $1 AND file_id = $2`, collectionID, fileID); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec(`INSERT INTO trash(file_id, user_id, collection_id, delete_by)
+ VALUES($1, $2, $3, $4)`, fileID, ownerID, collectionID, int64(100)); err != nil {
+ t.Fatal(err)
+ }
+
+ err := repository.AddFiles(t.Context(), collectionID, ownerID, []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}, ownerID)
+ if !errors.Is(err, &ente.ErrFileInTrash) {
+ t.Fatalf("AddFiles() error = %v, want ErrFileInTrash", err)
+ }
+
+ if state := readCollectionMembershipState(t, db, collectionID, fileID); !state.isDeleted {
+ t.Fatal("AddFiles() reactivated trashed membership")
+ }
+}
+
+func TestAddFilesAndTrashFilesSerializeOnFile(t *testing.T) {
+ repository, db, ownerID := setupCollectionMembershipTest(t)
+ sourceCollectionID := insertObjectTestCollection(t, db, ownerID)
+ destinationCollectionID := insertObjectTestCollection(t, db, ownerID)
+ fileID := insertObjectTestFile(t, db, ownerID)
+ linkObjectTestFileToCollection(t, db, sourceCollectionID, fileID, ownerID)
+ repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)
+
+ blocker, err := db.BeginTx(t.Context(), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer blocker.Rollback()
+ if _, err := blocker.ExecContext(t.Context(), `SELECT file_id FROM files
+ WHERE file_id = $1 FOR UPDATE`, fileID); err != nil {
+ t.Fatal(err)
+ }
+
+ addResult := make(chan error, 1)
+ go func() {
+ addResult <- repository.AddFiles(t.Context(), destinationCollectionID, ownerID,
+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}, ownerID)
+ }()
+ trashResult := make(chan error, 1)
+ go func() {
+ trashResult <- repository.TrashRepo.TrashFiles(t.Context(), ownerID, ente.TrashRequest{
+ TrashItems: []ente.TrashItemRequest{{
+ FileID: fileID,
+ CollectionID: sourceCollectionID,
+ }},
+ })
+ }()
+
+ waitForFileLockWaiters(t, db)
+ if err := blocker.Commit(); err != nil {
+ t.Fatal(err)
+ }
+
+ addErr := <-addResult
+ if addErr != nil && !errors.Is(addErr, &ente.ErrFileInTrash) {
+ t.Fatalf("AddFiles() error = %v", addErr)
+ }
+ if err := <-trashResult; err != nil {
+ t.Fatalf("TrashFiles() error = %v", err)
+ }
+
+ var activeMemberships, activeTrashRows int
+ if err := db.QueryRow(`SELECT
+ (SELECT COUNT(*) FROM collection_files WHERE file_id = $1 AND is_deleted = FALSE),
+ (SELECT COUNT(*) FROM trash WHERE file_id = $1 AND is_deleted = FALSE AND is_restored = FALSE)`,
+ fileID).Scan(&activeMemberships, &activeTrashRows); err != nil {
+ t.Fatal(err)
+ }
+ if activeMemberships != 0 || activeTrashRows != 1 {
+ t.Fatalf("final state = (%d active memberships, %d active Trash rows), want (0, 1)", activeMemberships, activeTrashRows)
+ }
+}
+
func TestRestoreFilesUpsertsBatchAndMarksTrashRestored(t *testing.T) {
repository, db, ownerID := setupCollectionMembershipTest(t)
collectionID := insertObjectTestCollection(t, db, ownerID)
@@ -362,6 +447,28 @@ func insertCollectionMembershipTestFiles(t *testing.T, db *sql.DB, ownerID int64
return fileIDs
}
+func waitForFileLockWaiters(t *testing.T, db *sql.DB) {
+ t.Helper()
+
+ deadline := time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadline) {
+ var count int
+ if err := db.QueryRow(`SELECT COUNT(*) FROM pg_stat_activity
+ WHERE datname = current_database()
+ AND pid <> pg_backend_pid()
+ AND wait_event_type = 'Lock'
+ AND query LIKE '%FROM files%'
+ AND query LIKE '%FOR UPDATE%'`).Scan(&count); err != nil {
+ t.Fatal(err)
+ }
+ if count >= 2 {
+ return
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ t.Fatal("timed out waiting for file-lock waiters")
+}
+
func readCollectionMembershipState(t *testing.T, db *sql.DB, collectionID int64, fileID int64) collectionMembershipState {
t.Helper()
diff --git a/server/pkg/repo/file_lock.go b/server/pkg/repo/file_lock.go
new file mode 100644
index 0000000000..c3287b12c6
--- /dev/null
+++ b/server/pkg/repo/file_lock.go
@@ -0,0 +1,33 @@
+package repo
+
+import (
+ "context"
+ "database/sql"
+
+ "github.com/ente/stacktrace"
+ "github.com/lib/pq"
+)
+
+func lockFiles(ctx context.Context, tx *sql.Tx, userID int64, fileIDs []int64) error {
+ if len(fileIDs) == 0 {
+ return nil
+ }
+
+ rows, err := tx.QueryContext(ctx, `SELECT file_id
+ FROM files
+ WHERE owner_id = $1 AND file_id = ANY($2)
+ ORDER BY file_id
+ FOR UPDATE`, userID, pq.Array(fileIDs))
+ if err != nil {
+ return stacktrace.Propagate(err, "")
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var fileID int64
+ if err := rows.Scan(&fileID); err != nil {
+ return stacktrace.Propagate(err, "")
+ }
+ }
+ return stacktrace.Propagate(rows.Err(), "")
+}
diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.go
index 8c8db3cd72..28ce911898 100644
--- a/server/pkg/repo/trash.go
+++ b/server/pkg/repo/trash.go
@@ -125,6 +125,9 @@ func (t *TrashRepository) TrashFiles(ctx context.Context, userID int64, trash en
return stacktrace.Propagate(err, "")
}
defer tx.Rollback()
+ if err := lockFiles(ctx, tx, userID, fileIDs); err != nil {
+ return stacktrace.Propagate(err, "")
+ }
rows, err := tx.QueryContext(ctx, `SELECT DISTINCT collection_id FROM
collection_files WHERE file_id = ANY($1) AND is_deleted = $2`, pq.Array(fileIDs), false)
if err != nil {
@@ -259,7 +262,15 @@ func (t *TrashRepository) GetFilesInTrashState(ctx context.Context, userID int64
}
func (t *TrashRepository) GetFilesInTrashOrDeleted(ctx context.Context, userID int64, fileIDs []int64) ([]int64, error) {
- rows, err := t.DB.QueryContext(ctx, `SELECT file_id FROM trash
+ return t.getFilesInTrashOrDeleted(ctx, t.DB, userID, fileIDs)
+}
+
+type trashQueryer interface {
+ QueryContext(context.Context, string, ...any) (*sql.Rows, error)
+}
+
+func (t *TrashRepository) getFilesInTrashOrDeleted(ctx context.Context, queryer trashQueryer, userID int64, fileIDs []int64) ([]int64, error) {
+ rows, err := queryer.QueryContext(ctx, `SELECT file_id FROM trash
WHERE user_id = $1 AND file_id = ANY ($2)
AND is_restored = FALSE`, userID, pq.Array(fileIDs))
if err != nil {