π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Maintain file counts on trash and restore
Neeraj Guptad8ce9ef4fa5d
diff --git a/server/pkg/repo/collection.go b/server/pkg/repo/collection.goindex 394f745944..ad7a358d65 100644--- a/server/pkg/repo/collection.go+++ b/server/pkg/repo/collection.go@@ -786,6 +786,19 @@ func (repo *CollectionRepository) RestoreFiles(ctx context.Context, userID int64if !canRestoreAllFiles {return stacktrace.Propagate(ente.ErrBadRequest, "some fileIDs are not restorable")}+ var app ente.App+ if err := tx.QueryRowContext(ctx, `SELECT app FROM collections+ WHERE collection_id = $1 AND owner_id = $2 AND is_deleted = FALSE+ FOR UPDATE`, collectionID, userID).Scan(&app); err != nil {+ if errors.Is(err, sql.ErrNoRows) {+ return stacktrace.Propagate(ente.ErrPermissionDenied, "restore destination is not an active owned collection")+ }+ return stacktrace.Propagate(err, "")+ }+ filesBecomingActive, err := inactiveOwnedFileCount(ctx, tx, userID, fileIDs)+ if err != nil {+ return stacktrace.Propagate(err, "failed to calculate file count transition")+ }if err := upsertCollectionFiles(ctx, tx, collectionID, userID, newCollectionFiles, userID, updationTime); err != nil {return stacktrace.Propagate(err, "")@@ -801,6 +814,15 @@ func (repo *CollectionRepository) RestoreFiles(ctx context.Context, userID int64if err != nil {return stacktrace.Propagate(err, "")}+ if filesBecomingActive != 0 {+ photosFileDelta, lockerFileDelta, ok := fileCountDelta(app, filesBecomingActive)+ if !ok {+ return stacktrace.Propagate(ente.ErrInvalidApp, "unsupported collection app %s", app)+ }+ if _, err := applyUsageDelta(ctx, tx, userID, 0, photosFileDelta, lockerFileDelta, false); err != nil {+ return stacktrace.Propagate(err, "failed to update file counts")+ }+ }return stacktrace.Propagate(tx.Commit(), "")}diff --git a/server/pkg/repo/collection_memberships_test.go b/server/pkg/repo/collection_memberships_test.goindex dea6ee6dac..b6bd31f448 100644--- a/server/pkg/repo/collection_memberships_test.go+++ b/server/pkg/repo/collection_memberships_test.go@@ -310,7 +310,9 @@ func TestRestoreFilesAndDeleteSerializeOnFile(t *testing.T) {t.Fatal(err)}insertObjectTestKey(t, db, fileID, ente.FILE, "restore-delete-race", 100, []string{"b2-eu-cen"})- testutil.InsertUsage(t, db, ownerID, 100)+ if _, err := db.Exec(`UPDATE usage SET storage_consumed = 100 WHERE user_id = $1`, ownerID); err != nil {+ t.Fatal(err)+ }repository.TrashRepo.FileRepo = &FileRepository{DB: db,ObjectRepo: &ObjectRepository{@@ -493,6 +495,7 @@ func setupCollectionMembershipTest(t *testing.T) (*CollectionRepository, *sql.DBEmail: "[email protected]",CreationTime: 1,})+ testutil.InsertUsage(t, db, ownerID, 0)return &CollectionRepository{DB: db,TrashRepo: &TrashRepository{DB: db},diff --git a/server/pkg/repo/file.go b/server/pkg/repo/file.goindex f7304546b5..a3af8b1f7f 100644--- a/server/pkg/repo/file.go+++ b/server/pkg/repo/file.go@@ -780,35 +780,13 @@ func (repo *FileRepository) scheduleDeletion(ctx context.Context, tx *sql.Tx, fi}func (repo *FileRepository) updateUsageForFileCreation(ctx context.Context, tx *sql.Tx, userID, storageDiff int64, app ente.App) (int64, error) {- switch app {- case ente.Photos:- return repo.updateUsage(ctx, tx, userID, storageDiff, 1, 0)- case ente.Locker:- return repo.updateUsage(ctx, tx, userID, storageDiff, 0, 1)- default:+ photosFileCountDiff, lockerFileCountDiff, ok := fileCountDelta(app, 1)+ if !ok {return -1, stacktrace.Propagate(ente.ErrInvalidApp, "")}+ return repo.updateUsage(ctx, tx, userID, storageDiff, photosFileCountDiff, lockerFileCountDiff)}func (repo *FileRepository) updateUsage(ctx context.Context, tx *sql.Tx, userID, storageDiff, photosFileCountDiff, lockerFileCountDiff int64) (int64, error) {- fileCountSourceVersionDiff := int64(0)- if photosFileCountDiff != 0 || lockerFileCountDiff != 0 {- fileCountSourceVersionDiff = 1- }- var usage int64- err := tx.QueryRowContext(ctx, `UPDATE usage SET- storage_consumed = storage_consumed + $2,- photos_file_count = photos_file_count + $3,- locker_file_count = locker_file_count + $4,- file_count_source_version = file_count_source_version + $5- WHERE user_id = $1- RETURNING storage_consumed`,- userID, storageDiff, photosFileCountDiff, lockerFileCountDiff, fileCountSourceVersionDiff).Scan(&usage)- if err != nil {- if errors.Is(err, sql.ErrNoRows) {- return -1, stacktrace.Propagate(err, "missing usage row for user %d", userID)- }- return -1, stacktrace.Propagate(err, "")- }- return usage, nil+ return applyUsageDelta(ctx, tx, userID, storageDiff, photosFileCountDiff, lockerFileCountDiff, false)}diff --git a/server/pkg/repo/file_usage_test.go b/server/pkg/repo/file_usage_test.goindex 52a8fd2fc3..863ab4054a 100644--- a/server/pkg/repo/file_usage_test.go+++ b/server/pkg/repo/file_usage_test.go@@ -2,6 +2,7 @@ package repoimport ("database/sql"+ "errors""testing""github.com/ente/museum/ente"@@ -83,9 +84,13 @@ func TestUpdateUsageRequiresUsageRow(t *testing.T) {}defer tx.Rollback()- if _, err := (&FileRepository{}).updateUsage(t.Context(), tx, userID, 5, 1, 0); err == nil {+ _, err = (&FileRepository{}).updateUsage(t.Context(), tx, userID, 5, 1, 0)+ if err == nil {t.Fatal("updateUsage() created a missing usage row")}+ if errors.Is(err, sql.ErrNoRows) {+ t.Fatal("updateUsage() exposed missing usage row as not found")+ }var count intif err := db.QueryRow(`SELECT COUNT(*) FROM usage WHERE user_id = $1`, userID).Scan(&count); err != nil {diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.goindex 54b5d42cec..b33e1604d1 100644--- a/server/pkg/repo/trash.go+++ b/server/pkg/repo/trash.go@@ -127,6 +127,16 @@ func (t *TrashRepository) TrashFiles(ctx context.Context, userID int64, trash enif err := lockFiles(ctx, tx, userID, fileIDs); err != nil {return stacktrace.Propagate(err, "")}+ photosFileDelta, lockerFileDelta, ambiguousFileApp, err := activeOwnedFileCountDeltas(ctx, tx, userID, fileIDs)+ if err != nil {+ return stacktrace.Propagate(err, "failed to calculate file count transition")+ }+ if ambiguousFileApp {+ logrus.WithFields(logrus.Fields{+ "user_id": userID,+ "file_count": len(fileIDs),+ }).Error("found cross-app or invalid app memberships while trashing files")+ }updationTime := time.Microseconds()rows, err := tx.QueryContext(ctx, `SELECT DISTINCT collection_id FROMcollection_files WHERE file_id = ANY($1) AND is_deleted = $2`, pq.Array(fileIDs), false)@@ -160,6 +170,11 @@ func (t *TrashRepository) TrashFiles(ctx context.Context, userID int64, trash enif err = t.FileLinkRepo.DisableLinkForFilesTx(ctx, tx, fileIDs); err != nil {return stacktrace.Propagate(err, "failed to disable file links for files being trashed")}+ if photosFileDelta != 0 || lockerFileDelta != 0 || ambiguousFileApp {+ if _, err := applyUsageDelta(ctx, tx, userID, 0, photosFileDelta, lockerFileDelta, ambiguousFileApp); err != nil {+ return stacktrace.Propagate(err, "failed to update file counts")+ }+ }return stacktrace.Propagate(tx.Commit(), "")}diff --git a/server/pkg/repo/trash_test.go b/server/pkg/repo/trash_test.goindex 05c8a319db..b2b301b4c5 100644--- a/server/pkg/repo/trash_test.go+++ b/server/pkg/repo/trash_test.go@@ -16,6 +16,7 @@ func TestTrashFilesUsesRequestItemsAsItsScope(t *testing.T) {Email: "[email protected]",CreationTime: 1,})+ testutil.InsertUsage(t, db, ownerID, 0)collectionID := insertObjectTestCollection(t, db, ownerID)requestedFileID := insertObjectTestFile(t, db, ownerID)untouchedFileID := insertObjectTestFile(t, db, ownerID)@@ -86,6 +87,7 @@ func TestTrashFilesRollsBackWhenFileLinkCleanupFails(t *testing.T) {Email: "[email protected]",CreationTime: 1,})+ testutil.InsertUsage(t, db, ownerID, 0)collectionID := insertObjectTestCollection(t, db, ownerID)fileID := insertObjectTestFile(t, db, ownerID)linkObjectTestFileToCollection(t, db, collectionID, fileID, ownerID)diff --git a/server/pkg/repo/usage_file_count.go b/server/pkg/repo/usage_file_count.gonew file mode 100644index 0000000000..0a9e07d2c1--- /dev/null+++ b/server/pkg/repo/usage_file_count.go@@ -0,0 +1,127 @@+package repo++import (+ "context"+ "database/sql"+ "errors"++ "github.com/ente/museum/ente"+ "github.com/ente/stacktrace"+ "github.com/lib/pq"+ "github.com/sirupsen/logrus"+)++func applyUsageDelta(+ ctx context.Context,+ tx *sql.Tx,+ userID int64,+ storageDelta int64,+ photosFileDelta int64,+ lockerFileDelta int64,+ invalidateFileCounts bool,+) (int64, error) {+ row := tx.QueryRowContext(ctx, `WITH current_usage AS (+ SELECT user_id, storage_consumed, photos_file_count, locker_file_count, file_count_source_version+ FROM usage+ WHERE user_id = $1+ FOR UPDATE+ )+ UPDATE usage AS target+ SET storage_consumed = current_usage.storage_consumed + $2,+ photos_file_count = CASE+ WHEN $5+ OR current_usage.photos_file_count IS NULL+ OR current_usage.locker_file_count IS NULL+ OR current_usage.photos_file_count + $3 < 0+ OR current_usage.locker_file_count + $4 < 0+ THEN NULL+ ELSE current_usage.photos_file_count + $3+ END,+ locker_file_count = CASE+ WHEN $5+ OR current_usage.photos_file_count IS NULL+ OR current_usage.locker_file_count IS NULL+ OR current_usage.photos_file_count + $3 < 0+ OR current_usage.locker_file_count + $4 < 0+ THEN NULL+ ELSE current_usage.locker_file_count + $4+ END,+ file_count_source_version = current_usage.file_count_source_version+ + CASE WHEN $3 <> 0 OR $4 <> 0 OR $5 THEN 1 ELSE 0 END+ FROM current_usage+ WHERE target.user_id = current_usage.user_id+ RETURNING target.storage_consumed,+ current_usage.photos_file_count IS NOT NULL+ AND current_usage.locker_file_count IS NOT NULL+ AND ($5 OR current_usage.photos_file_count + $3 < 0 OR current_usage.locker_file_count + $4 < 0)`,+ userID,+ storageDelta,+ photosFileDelta,+ lockerFileDelta,+ invalidateFileCounts,+ )+ var storageConsumed int64+ var fileCountsInvalidated bool+ if err := row.Scan(&storageConsumed, &fileCountsInvalidated); err != nil {+ if errors.Is(err, sql.ErrNoRows) {+ return -1, stacktrace.NewError("missing usage row for user %d", userID)+ }+ return -1, stacktrace.Propagate(err, "")+ }+ if fileCountsInvalidated {+ logrus.WithFields(logrus.Fields{+ "user_id": userID,+ "photos_file_delta": photosFileDelta,+ "locker_file_delta": lockerFileDelta,+ }).Error("invalidated file counts")+ }+ return storageConsumed, nil+}++func fileCountDelta(app ente.App, delta int64) (photos int64, locker int64, ok bool) {+ switch app {+ case ente.Photos:+ return delta, 0, true+ case ente.Locker:+ return 0, delta, true+ default:+ return 0, 0, false+ }+}++func activeOwnedFileCountDeltas(ctx context.Context, tx *sql.Tx, userID int64, fileIDs []int64) (photos int64, locker int64, ambiguous bool, err error) {+ err = tx.QueryRowContext(ctx, `WITH active_apps AS (+ SELECT DISTINCT cf.file_id, c.app+ FROM collection_files AS cf+ JOIN files AS f ON f.file_id = cf.file_id AND f.owner_id = $1+ JOIN collections AS c ON c.collection_id = cf.collection_id AND c.owner_id = $1+ WHERE cf.file_id = ANY($2) AND cf.is_deleted = FALSE+ )+ SELECT+ -COUNT(*) FILTER (WHERE app = $3),+ -COUNT(*) FILTER (WHERE app = $4),+ COUNT(*) <> COUNT(DISTINCT file_id)+ OR COUNT(*) FILTER (WHERE app IS NULL OR app NOT IN ($3, $4)) > 0+ FROM active_apps`, userID, pq.Array(fileIDs), ente.Photos, ente.Locker).Scan(&photos, &locker, &ambiguous)+ if err != nil {+ return 0, 0, false, stacktrace.Propagate(err, "")+ }+ return photos, locker, ambiguous, nil+}++func inactiveOwnedFileCount(ctx context.Context, tx *sql.Tx, userID int64, fileIDs []int64) (int64, error) {+ row := tx.QueryRowContext(ctx, `SELECT COUNT(*)+ FROM files AS f+ WHERE f.owner_id = $1 AND f.file_id = ANY($2)+ AND NOT EXISTS (+ SELECT 1+ FROM collection_files AS cf+ JOIN collections AS c ON c.collection_id = cf.collection_id AND c.owner_id = $1+ WHERE cf.file_id = f.file_id AND cf.is_deleted = FALSE+ )`, userID, pq.Array(fileIDs))+ var count int64+ if err := row.Scan(&count); err != nil {+ return 0, stacktrace.Propagate(err, "")+ }+ return count, nil+}diff --git a/server/pkg/repo/usage_file_count_test.go b/server/pkg/repo/usage_file_count_test.gonew file mode 100644index 0000000000..45ff224c34--- /dev/null+++ b/server/pkg/repo/usage_file_count_test.go@@ -0,0 +1,90 @@+package repo++import (+ "database/sql"+ "testing"++ "github.com/ente/museum/ente"+ "github.com/ente/museum/pkg/repo/public"+)++func TestTrashAndRestoreMaintainReadyLockerCount(t *testing.T) {+ repository, db, userID := setupCollectionMembershipTest(t)+ setReadyFileCounts(t, db, userID, 0, 1)+ firstCollectionID := insertObjectTestCollection(t, db, userID)+ secondCollectionID := insertObjectTestCollection(t, db, userID)+ if _, err := db.Exec(`UPDATE collections SET app = 'locker'+ WHERE collection_id IN ($1, $2)`, firstCollectionID, secondCollectionID); err != nil {+ t.Fatal(err)+ }+ fileID := insertObjectTestFile(t, db, userID)+ linkObjectTestFileToCollection(t, db, firstCollectionID, fileID, userID)+ linkObjectTestFileToCollection(t, db, secondCollectionID, fileID, userID)+ repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)++ if err := repository.TrashRepo.TrashFiles(t.Context(), userID, ente.TrashRequest{+ TrashItems: []ente.TrashItemRequest{{FileID: fileID, CollectionID: firstCollectionID}},+ }); err != nil {+ t.Fatalf("TrashFiles() error = %v", err)+ }+ assertReadyFileCounts(t, db, userID, 0, 0, 1)++ if err := repository.RestoreFiles(t.Context(), userID, firstCollectionID,+ []ente.CollectionFileItem{collectionMembershipTestItem(fileID)}); err != nil {+ t.Fatalf("RestoreFiles() error = %v", err)+ }+ assertReadyFileCounts(t, db, userID, 0, 1, 2)+}++func TestTrashInvalidatesCrossAppFileCounts(t *testing.T) {+ repository, db, userID := setupCollectionMembershipTest(t)+ setReadyFileCounts(t, db, userID, 1, 1)+ photosCollectionID := insertObjectTestCollection(t, db, userID)+ lockerCollectionID := insertObjectTestCollection(t, db, userID)+ if _, err := db.Exec(`UPDATE collections SET app = 'locker' WHERE collection_id = $1`, lockerCollectionID); err != nil {+ t.Fatal(err)+ }+ fileID := insertObjectTestFile(t, db, userID)+ linkObjectTestFileToCollection(t, db, photosCollectionID, fileID, userID)+ linkObjectTestFileToCollection(t, db, lockerCollectionID, fileID, userID)+ repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)++ if err := repository.TrashRepo.TrashFiles(t.Context(), userID, ente.TrashRequest{+ TrashItems: []ente.TrashItemRequest{{FileID: fileID, CollectionID: photosCollectionID}},+ }); err != nil {+ t.Fatalf("TrashFiles() error = %v", err)+ }++ photos, locker, version := readFileCountState(t, db, userID)+ if photos.Valid || locker.Valid || version != 1 {+ t.Fatalf("file count state = (%v, %v, %d), want (NULL, NULL, 1)", photos, locker, version)+ }+}++func setReadyFileCounts(t *testing.T, db *sql.DB, userID, photos, locker int64) {+ t.Helper()+ if _, err := db.Exec(`UPDATE usage+ SET photos_file_count = $2, locker_file_count = $3+ WHERE user_id = $1`, userID, photos, locker); err != nil {+ t.Fatal(err)+ }+}++func assertReadyFileCounts(t *testing.T, db *sql.DB, userID, wantPhotos, wantLocker, wantVersion int64) {+ t.Helper()+ photos, locker, version := readFileCountState(t, db, userID)+ if !photos.Valid || photos.Int64 != wantPhotos || !locker.Valid || locker.Int64 != wantLocker || version != wantVersion {+ t.Fatalf("file count state = (%v, %v, %d), want (%d, %d, %d)", photos, locker, version, wantPhotos, wantLocker, wantVersion)+ }+}++func readFileCountState(t *testing.T, db *sql.DB, userID int64) (sql.NullInt64, sql.NullInt64, int64) {+ t.Helper()+ var photos, locker sql.NullInt64+ var version int64+ if err := db.QueryRow(`SELECT photos_file_count, locker_file_count, file_count_source_version+ FROM usage WHERE user_id = $1`, userID).Scan(&photos, &locker, &version); err != nil {+ t.Fatal(err)+ }+ return photos, locker, version+}