Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Maintain file counts on trash and restore

Neeraj Guptad8ce9ef4fa5d
diff --git a/server/pkg/repo/collection.go b/server/pkg/repo/collection.go
index 394f745944..ad7a358d65 100644
--- a/server/pkg/repo/collection.go
+++ b/server/pkg/repo/collection.go
@@ -786,6 +786,19 @@ func (repo *CollectionRepository) RestoreFiles(ctx context.Context, userID int64
if !canRestoreAllFiles {
return stacktrace.Propagate(ente.ErrBadRequest, "some fileIDs are not restorable")
}
+ var app ente.App
+ if err := tx.QueryRowContext(ctx, `SELECT app FROM collections
+ WHERE collection_id = $1 AND owner_id = $2 AND is_deleted = FALSE
+ FOR UPDATE`, collectionID, userID).Scan(&app); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return stacktrace.Propagate(ente.ErrPermissionDenied, "restore destination is not an active owned collection")
+ }
+ return stacktrace.Propagate(err, "")
+ }
+ filesBecomingActive, err := inactiveOwnedFileCount(ctx, tx, userID, fileIDs)
+ if err != nil {
+ return stacktrace.Propagate(err, "failed to calculate file count transition")
+ }
if err := upsertCollectionFiles(ctx, tx, collectionID, userID, newCollectionFiles, userID, updationTime); err != nil {
return stacktrace.Propagate(err, "")
@@ -801,6 +814,15 @@ func (repo *CollectionRepository) RestoreFiles(ctx context.Context, userID int64
if err != nil {
return stacktrace.Propagate(err, "")
}
+ if filesBecomingActive != 0 {
+ photosFileDelta, lockerFileDelta, ok := fileCountDelta(app, filesBecomingActive)
+ if !ok {
+ return stacktrace.Propagate(ente.ErrInvalidApp, "unsupported collection app %s", app)
+ }
+ if _, err := applyUsageDelta(ctx, tx, userID, 0, photosFileDelta, lockerFileDelta, false); err != nil {
+ return stacktrace.Propagate(err, "failed to update file counts")
+ }
+ }
return stacktrace.Propagate(tx.Commit(), "")
}
diff --git a/server/pkg/repo/collection_memberships_test.go b/server/pkg/repo/collection_memberships_test.go
index dea6ee6dac..b6bd31f448 100644
--- a/server/pkg/repo/collection_memberships_test.go
+++ b/server/pkg/repo/collection_memberships_test.go
@@ -310,7 +310,9 @@ func TestRestoreFilesAndDeleteSerializeOnFile(t *testing.T) {
t.Fatal(err)
}
insertObjectTestKey(t, db, fileID, ente.FILE, "restore-delete-race", 100, []string{"b2-eu-cen"})
- testutil.InsertUsage(t, db, ownerID, 100)
+ if _, err := db.Exec(`UPDATE usage SET storage_consumed = 100 WHERE user_id = $1`, ownerID); err != nil {
+ t.Fatal(err)
+ }
repository.TrashRepo.FileRepo = &FileRepository{
DB: db,
ObjectRepo: &ObjectRepository{
@@ -493,6 +495,7 @@ func setupCollectionMembershipTest(t *testing.T) (*CollectionRepository, *sql.DB
CreationTime: 1,
})
+ testutil.InsertUsage(t, db, ownerID, 0)
return &CollectionRepository{
DB: db,
TrashRepo: &TrashRepository{DB: db},
diff --git a/server/pkg/repo/file.go b/server/pkg/repo/file.go
index f7304546b5..a3af8b1f7f 100644
--- a/server/pkg/repo/file.go
+++ b/server/pkg/repo/file.go
@@ -780,35 +780,13 @@ func (repo *FileRepository) scheduleDeletion(ctx context.Context, tx *sql.Tx, fi
}
func (repo *FileRepository) updateUsageForFileCreation(ctx context.Context, tx *sql.Tx, userID, storageDiff int64, app ente.App) (int64, error) {
- switch app {
- case ente.Photos:
- return repo.updateUsage(ctx, tx, userID, storageDiff, 1, 0)
- case ente.Locker:
- return repo.updateUsage(ctx, tx, userID, storageDiff, 0, 1)
- default:
+ photosFileCountDiff, lockerFileCountDiff, ok := fileCountDelta(app, 1)
+ if !ok {
return -1, stacktrace.Propagate(ente.ErrInvalidApp, "")
}
+ return repo.updateUsage(ctx, tx, userID, storageDiff, photosFileCountDiff, lockerFileCountDiff)
}
func (repo *FileRepository) updateUsage(ctx context.Context, tx *sql.Tx, userID, storageDiff, photosFileCountDiff, lockerFileCountDiff int64) (int64, error) {
- fileCountSourceVersionDiff := int64(0)
- if photosFileCountDiff != 0 || lockerFileCountDiff != 0 {
- fileCountSourceVersionDiff = 1
- }
- var usage int64
- err := tx.QueryRowContext(ctx, `UPDATE usage SET
- storage_consumed = storage_consumed + $2,
- photos_file_count = photos_file_count + $3,
- locker_file_count = locker_file_count + $4,
- file_count_source_version = file_count_source_version + $5
- WHERE user_id = $1
- RETURNING storage_consumed`,
- userID, storageDiff, photosFileCountDiff, lockerFileCountDiff, fileCountSourceVersionDiff).Scan(&usage)
- if err != nil {
- if errors.Is(err, sql.ErrNoRows) {
- return -1, stacktrace.Propagate(err, "missing usage row for user %d", userID)
- }
- return -1, stacktrace.Propagate(err, "")
- }
- return usage, nil
+ return applyUsageDelta(ctx, tx, userID, storageDiff, photosFileCountDiff, lockerFileCountDiff, false)
}
diff --git a/server/pkg/repo/file_usage_test.go b/server/pkg/repo/file_usage_test.go
index 52a8fd2fc3..863ab4054a 100644
--- a/server/pkg/repo/file_usage_test.go
+++ b/server/pkg/repo/file_usage_test.go
@@ -2,6 +2,7 @@ package repo
import (
"database/sql"
+ "errors"
"testing"
"github.com/ente/museum/ente"
@@ -83,9 +84,13 @@ func TestUpdateUsageRequiresUsageRow(t *testing.T) {
}
defer tx.Rollback()
- if _, err := (&FileRepository{}).updateUsage(t.Context(), tx, userID, 5, 1, 0); err == nil {
+ _, err = (&FileRepository{}).updateUsage(t.Context(), tx, userID, 5, 1, 0)
+ if err == nil {
t.Fatal("updateUsage() created a missing usage row")
}
+ if errors.Is(err, sql.ErrNoRows) {
+ t.Fatal("updateUsage() exposed missing usage row as not found")
+ }
var count int
if err := db.QueryRow(`SELECT COUNT(*) FROM usage WHERE user_id = $1`, userID).Scan(&count); err != nil {
diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.go
index 54b5d42cec..b33e1604d1 100644
--- a/server/pkg/repo/trash.go
+++ b/server/pkg/repo/trash.go
@@ -127,6 +127,16 @@ func (t *TrashRepository) TrashFiles(ctx context.Context, userID int64, trash en
if err := lockFiles(ctx, tx, userID, fileIDs); err != nil {
return stacktrace.Propagate(err, "")
}
+ photosFileDelta, lockerFileDelta, ambiguousFileApp, err := activeOwnedFileCountDeltas(ctx, tx, userID, fileIDs)
+ if err != nil {
+ return stacktrace.Propagate(err, "failed to calculate file count transition")
+ }
+ if ambiguousFileApp {
+ logrus.WithFields(logrus.Fields{
+ "user_id": userID,
+ "file_count": len(fileIDs),
+ }).Error("found cross-app or invalid app memberships while trashing files")
+ }
updationTime := time.Microseconds()
rows, err := tx.QueryContext(ctx, `SELECT DISTINCT collection_id FROM
collection_files WHERE file_id = ANY($1) AND is_deleted = $2`, pq.Array(fileIDs), false)
@@ -160,6 +170,11 @@ func (t *TrashRepository) TrashFiles(ctx context.Context, userID int64, trash en
if err = t.FileLinkRepo.DisableLinkForFilesTx(ctx, tx, fileIDs); err != nil {
return stacktrace.Propagate(err, "failed to disable file links for files being trashed")
}
+ if photosFileDelta != 0 || lockerFileDelta != 0 || ambiguousFileApp {
+ if _, err := applyUsageDelta(ctx, tx, userID, 0, photosFileDelta, lockerFileDelta, ambiguousFileApp); err != nil {
+ return stacktrace.Propagate(err, "failed to update file counts")
+ }
+ }
return stacktrace.Propagate(tx.Commit(), "")
}
diff --git a/server/pkg/repo/trash_test.go b/server/pkg/repo/trash_test.go
index 05c8a319db..b2b301b4c5 100644
--- a/server/pkg/repo/trash_test.go
+++ b/server/pkg/repo/trash_test.go
@@ -16,6 +16,7 @@ func TestTrashFilesUsesRequestItemsAsItsScope(t *testing.T) {
CreationTime: 1,
})
+ testutil.InsertUsage(t, db, ownerID, 0)
collectionID := insertObjectTestCollection(t, db, ownerID)
requestedFileID := insertObjectTestFile(t, db, ownerID)
untouchedFileID := insertObjectTestFile(t, db, ownerID)
@@ -86,6 +87,7 @@ func TestTrashFilesRollsBackWhenFileLinkCleanupFails(t *testing.T) {
CreationTime: 1,
})
+ testutil.InsertUsage(t, db, ownerID, 0)
collectionID := insertObjectTestCollection(t, db, ownerID)
fileID := insertObjectTestFile(t, db, ownerID)
linkObjectTestFileToCollection(t, db, collectionID, fileID, ownerID)
diff --git a/server/pkg/repo/usage_file_count.go b/server/pkg/repo/usage_file_count.go
new file mode 100644
index 0000000000..0a9e07d2c1
--- /dev/null
+++ b/server/pkg/repo/usage_file_count.go
@@ -0,0 +1,127 @@
+package repo
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+
+ "github.com/ente/museum/ente"
+ "github.com/ente/stacktrace"
+ "github.com/lib/pq"
+ "github.com/sirupsen/logrus"
+)
+
+func applyUsageDelta(
+ ctx context.Context,
+ tx *sql.Tx,
+ userID int64,
+ storageDelta int64,
+ photosFileDelta int64,
+ lockerFileDelta int64,
+ invalidateFileCounts bool,
+) (int64, error) {
+ row := tx.QueryRowContext(ctx, `WITH current_usage AS (
+ SELECT user_id, storage_consumed, photos_file_count, locker_file_count, file_count_source_version
+ FROM usage
+ WHERE user_id = $1
+ FOR UPDATE
+ )
+ UPDATE usage AS target
+ SET storage_consumed = current_usage.storage_consumed + $2,
+ photos_file_count = CASE
+ WHEN $5
+ OR current_usage.photos_file_count IS NULL
+ OR current_usage.locker_file_count IS NULL
+ OR current_usage.photos_file_count + $3 < 0
+ OR current_usage.locker_file_count + $4 < 0
+ THEN NULL
+ ELSE current_usage.photos_file_count + $3
+ END,
+ locker_file_count = CASE
+ WHEN $5
+ OR current_usage.photos_file_count IS NULL
+ OR current_usage.locker_file_count IS NULL
+ OR current_usage.photos_file_count + $3 < 0
+ OR current_usage.locker_file_count + $4 < 0
+ THEN NULL
+ ELSE current_usage.locker_file_count + $4
+ END,
+ file_count_source_version = current_usage.file_count_source_version
+ + CASE WHEN $3 <> 0 OR $4 <> 0 OR $5 THEN 1 ELSE 0 END
+ FROM current_usage
+ WHERE target.user_id = current_usage.user_id
+ RETURNING target.storage_consumed,
+ current_usage.photos_file_count IS NOT NULL
+ AND current_usage.locker_file_count IS NOT NULL
+ AND ($5 OR current_usage.photos_file_count + $3 < 0 OR current_usage.locker_file_count + $4 < 0)`,
+ userID,
+ storageDelta,
+ photosFileDelta,
+ lockerFileDelta,
+ invalidateFileCounts,
+ )
+ var storageConsumed int64
+ var fileCountsInvalidated bool
+ if err := row.Scan(&storageConsumed, &fileCountsInvalidated); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return -1, stacktrace.NewError("missing usage row for user %d", userID)
+ }
+ return -1, stacktrace.Propagate(err, "")
+ }
+ if fileCountsInvalidated {
+ logrus.WithFields(logrus.Fields{
+ "user_id": userID,
+ "photos_file_delta": photosFileDelta,
+ "locker_file_delta": lockerFileDelta,
+ }).Error("invalidated file counts")
+ }
+ return storageConsumed, nil
+}
+
+func fileCountDelta(app ente.App, delta int64) (photos int64, locker int64, ok bool) {
+ switch app {
+ case ente.Photos:
+ return delta, 0, true
+ case ente.Locker:
+ return 0, delta, true
+ default:
+ return 0, 0, false
+ }
+}
+
+func activeOwnedFileCountDeltas(ctx context.Context, tx *sql.Tx, userID int64, fileIDs []int64) (photos int64, locker int64, ambiguous bool, err error) {
+ err = tx.QueryRowContext(ctx, `WITH active_apps AS (
+ SELECT DISTINCT cf.file_id, c.app
+ FROM collection_files AS cf
+ JOIN files AS f ON f.file_id = cf.file_id AND f.owner_id = $1
+ JOIN collections AS c ON c.collection_id = cf.collection_id AND c.owner_id = $1
+ WHERE cf.file_id = ANY($2) AND cf.is_deleted = FALSE
+ )
+ SELECT
+ -COUNT(*) FILTER (WHERE app = $3),
+ -COUNT(*) FILTER (WHERE app = $4),
+ COUNT(*) <> COUNT(DISTINCT file_id)
+ OR COUNT(*) FILTER (WHERE app IS NULL OR app NOT IN ($3, $4)) > 0
+ FROM active_apps`, userID, pq.Array(fileIDs), ente.Photos, ente.Locker).Scan(&photos, &locker, &ambiguous)
+ if err != nil {
+ return 0, 0, false, stacktrace.Propagate(err, "")
+ }
+ return photos, locker, ambiguous, nil
+}
+
+func inactiveOwnedFileCount(ctx context.Context, tx *sql.Tx, userID int64, fileIDs []int64) (int64, error) {
+ row := tx.QueryRowContext(ctx, `SELECT COUNT(*)
+ FROM files AS f
+ WHERE f.owner_id = $1 AND f.file_id = ANY($2)
+ AND NOT EXISTS (
+ SELECT 1
+ FROM collection_files AS cf
+ JOIN collections AS c ON c.collection_id = cf.collection_id AND c.owner_id = $1
+ WHERE cf.file_id = f.file_id AND cf.is_deleted = FALSE
+ )`, userID, pq.Array(fileIDs))
+ var count int64
+ if err := row.Scan(&count); err != nil {
+ return 0, stacktrace.Propagate(err, "")
+ }
+ return count, nil
+}
diff --git a/server/pkg/repo/usage_file_count_test.go b/server/pkg/repo/usage_file_count_test.go
new file mode 100644
index 0000000000..45ff224c34
--- /dev/null
+++ b/server/pkg/repo/usage_file_count_test.go
@@ -0,0 +1,90 @@
+package repo
+
+import (
+ "database/sql"
+ "testing"
+
+ "github.com/ente/museum/ente"
+ "github.com/ente/museum/pkg/repo/public"
+)
+
+func TestTrashAndRestoreMaintainReadyLockerCount(t *testing.T) {
+ repository, db, userID := setupCollectionMembershipTest(t)
+ setReadyFileCounts(t, db, userID, 0, 1)
+ firstCollectionID := insertObjectTestCollection(t, db, userID)
+ secondCollectionID := insertObjectTestCollection(t, db, userID)
+ if _, err := db.Exec(`UPDATE collections SET app = 'locker'
+ WHERE collection_id IN ($1, $2)`, firstCollectionID, secondCollectionID); err != nil {
+ t.Fatal(err)
+ }
+ fileID := insertObjectTestFile(t, db, userID)
+ linkObjectTestFileToCollection(t, db, firstCollectionID, fileID, userID)
+ linkObjectTestFileToCollection(t, db, secondCollectionID, fileID, userID)
+ repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)
+
+ if err := repository.TrashRepo.TrashFiles(t.Context(), userID, ente.TrashRequest{
+ TrashItems: []ente.TrashItemRequest{{FileID: fileID, CollectionID: firstCollectionID}},
+ }); err != nil {
+ t.Fatalf("TrashFiles() error = %v", err)
+ }
+ assertReadyFileCounts(t, db, userID, 0, 0, 1)
+
+ if err := repository.RestoreFiles(t.Context(), userID, firstCollectionID,
+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}); err != nil {
+ t.Fatalf("RestoreFiles() error = %v", err)
+ }
+ assertReadyFileCounts(t, db, userID, 0, 1, 2)
+}
+
+func TestTrashInvalidatesCrossAppFileCounts(t *testing.T) {
+ repository, db, userID := setupCollectionMembershipTest(t)
+ setReadyFileCounts(t, db, userID, 1, 1)
+ photosCollectionID := insertObjectTestCollection(t, db, userID)
+ lockerCollectionID := insertObjectTestCollection(t, db, userID)
+ if _, err := db.Exec(`UPDATE collections SET app = 'locker' WHERE collection_id = $1`, lockerCollectionID); err != nil {
+ t.Fatal(err)
+ }
+ fileID := insertObjectTestFile(t, db, userID)
+ linkObjectTestFileToCollection(t, db, photosCollectionID, fileID, userID)
+ linkObjectTestFileToCollection(t, db, lockerCollectionID, fileID, userID)
+ repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)
+
+ if err := repository.TrashRepo.TrashFiles(t.Context(), userID, ente.TrashRequest{
+ TrashItems: []ente.TrashItemRequest{{FileID: fileID, CollectionID: photosCollectionID}},
+ }); err != nil {
+ t.Fatalf("TrashFiles() error = %v", err)
+ }
+
+ photos, locker, version := readFileCountState(t, db, userID)
+ if photos.Valid || locker.Valid || version != 1 {
+ t.Fatalf("file count state = (%v, %v, %d), want (NULL, NULL, 1)", photos, locker, version)
+ }
+}
+
+func setReadyFileCounts(t *testing.T, db *sql.DB, userID, photos, locker int64) {
+ t.Helper()
+ if _, err := db.Exec(`UPDATE usage
+ SET photos_file_count = $2, locker_file_count = $3
+ WHERE user_id = $1`, userID, photos, locker); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func assertReadyFileCounts(t *testing.T, db *sql.DB, userID, wantPhotos, wantLocker, wantVersion int64) {
+ t.Helper()
+ photos, locker, version := readFileCountState(t, db, userID)
+ if !photos.Valid || photos.Int64 != wantPhotos || !locker.Valid || locker.Int64 != wantLocker || version != wantVersion {
+ t.Fatalf("file count state = (%v, %v, %d), want (%d, %d, %d)", photos, locker, version, wantPhotos, wantLocker, wantVersion)
+ }
+}
+
+func readFileCountState(t *testing.T, db *sql.DB, userID int64) (sql.NullInt64, sql.NullInt64, int64) {
+ t.Helper()
+ var photos, locker sql.NullInt64
+ var version int64
+ if err := db.QueryRow(`SELECT photos_file_count, locker_file_count, file_count_source_version
+ FROM usage WHERE user_id = $1`, userID).Scan(&photos, &locker, &version); err != nil {
+ t.Fatal(err)
+ }
+ return photos, locker, version
+}