π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Reuse file count initialization scan for repair
Neeraj Gupta50314aa42d2d
diff --git a/server/pkg/controller/usage_file_count_init.go b/server/pkg/controller/usage_file_count_init.goindex 1159c32b6b..0d030e3254 100644--- a/server/pkg/controller/usage_file_count_init.go+++ b/server/pkg/controller/usage_file_count_init.go@@ -64,13 +64,14 @@ func (c *FileCountInitializer) ProcessBatch() {updated, err := func() (bool, error) {defer c.LockController.ReleaseLock(lockID)updated, err := c.UsageRepo.InitializeFileCounts(ctx, userID)- if errors.Is(err, repo.ErrFileCountIneligible) {- fileIDs, cleanupErr := c.TrashRepo.GetStaleDeletedFileIDs(ctx, userID, repo.TrashBatchSize)- if cleanupErr == nil && len(fileIDs) > 0 {- cleanupErr = c.TrashRepo.CleanUpDeletedFilesFromCollection(ctx, fileIDs, userID)- }+ var ineligibleErr *repo.FileCountIneligibleError+ if errors.As(err, &ineligibleErr) && ineligibleErr.StaleDeletedFileID != 0 {+ cleanupErr := c.TrashRepo.CleanUpDeletedFilesFromCollection(ctx, []int64{ineligibleErr.StaleDeletedFileID}, userID)if cleanupErr != nil {- log.WithError(cleanupErr).WithField("user_id", userID).Error("Failed to clean stale deleted file memberships")+ log.WithError(cleanupErr).WithFields(log.Fields{+ "user_id": userID,+ "file_id": ineligibleErr.StaleDeletedFileID,+ }).Error("Failed to clean stale deleted file membership")}}return updated, errdiff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.goindex aff6ab9df2..15d1dc27ef 100644--- a/server/pkg/repo/trash.go+++ b/server/pkg/repo/trash.go@@ -239,25 +239,6 @@ func (t *TrashRepository) CleanUpDeletedFilesFromCollection(ctx context.Context,return nil}-func (t *TrashRepository) GetStaleDeletedFileIDs(ctx context.Context, userID int64, limit int) ([]int64, error) {- rows, err := t.DB.QueryContext(ctx, `SELECT DISTINCT cf.file_id- FROM collections c- JOIN collection_files cf ON cf.collection_id = c.collection_id AND cf.is_deleted = FALSE- JOIN files f ON f.file_id = cf.file_id AND f.owner_id = $1- JOIN trash t ON t.file_id = cf.file_id AND t.user_id = $1- AND t.is_deleted = TRUE AND t.is_restored = FALSE- WHERE c.owner_id = $1- AND NOT EXISTS (- SELECT 1 FROM object_keys ok- WHERE ok.file_id = cf.file_id AND ok.is_deleted = FALSE- )- LIMIT $2`, userID, limit)- if err != nil {- return nil, stacktrace.Propagate(err, "")- }- return convertRowsToFileId(rows)-}-func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []int64) error {if len(fileIDs) > TrashDiffLimit {return fmt.Errorf("can not delete more than %d in one go", TrashDiffLimit)@@ -347,23 +328,19 @@ func (t *TrashRepository) verifyFilesAreDeleted(ctx context.Context, userID int6return stacktrace.NewError("all file ids are not deleted from trash")}- row := t.DB.QueryRowContext(ctx, `SELECT coalesce(sum(size),0) FROM object_keys WHERE file_id = ANY($1) and is_deleted = FALSE`,- pq.Array(fileIDs))- var totalUsage int64- err = row.Scan(&totalUsage)+ row := t.DB.QueryRowContext(ctx, `SELECT EXISTS (+ SELECT 1 FROM object_keys WHERE file_id = ANY($1) AND is_deleted = FALSE+ )`, pq.Array(fileIDs))+ var hasLiveObjects bool+ err = row.Scan(&hasLiveObjects)if err != nil {- if errors.Is(err, sql.ErrNoRows) {- totalUsage = 0- } else {- return stacktrace.Propagate(err, "failed to get total usage for fileIDs")- }+ return stacktrace.Propagate(err, "failed to find live objects for fileIDs")}- if totalUsage != 0 {+ if hasLiveObjects {logrus.WithFields(logrus.Fields{"user_id": userID,"input_fileIds": fileIDs,"trash_fileIds": filesDeleted,- "total_usage": totalUsage,}).Error("object_keys table still has entries for deleted files")return stacktrace.NewError("object_keys table still has entries for deleted files")}diff --git a/server/pkg/repo/trash_test.go b/server/pkg/repo/trash_test.goindex b5d422a1c0..e22d3cb081 100644--- a/server/pkg/repo/trash_test.go+++ b/server/pkg/repo/trash_test.go@@ -204,33 +204,26 @@ func TestStaleCleanupInvalidatesOnlyRemovedOwnedMemberships(t *testing.T) {}}-func TestGetStaleDeletedFileIDs(t *testing.T) {+func TestStaleCleanupRejectsLiveZeroByteObject(t *testing.T) {repository, db, userID := setupCollectionMembershipTest(t)- otherUserID := testutil.InsertUser(t, db, testutil.UserFixture{UserID: 2, Email: "[email protected]", CreationTime: 1})collectionID := insertObjectTestCollection(t, db, userID)- eligibleFileID := insertObjectTestFile(t, db, userID)- liveObjectFileID := insertObjectTestFile(t, db, userID)- activeTrashFileID := insertObjectTestFile(t, db, userID)- wrongTrashOwnerFileID := insertObjectTestFile(t, db, userID)- noTrashFileID := insertObjectTestFile(t, db, userID)- for _, fileID := range []int64{eligibleFileID, liveObjectFileID, activeTrashFileID, wrongTrashOwnerFileID, noTrashFileID} {- linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)- }- insertObjectTestKey(t, db, liveObjectFileID, ente.FILE, "live-zero-byte-object", 0, []string{"b2-eu-cen"})+ fileID := insertObjectTestFile(t, db, userID)+ linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)+ insertObjectTestKey(t, db, fileID, ente.FILE, "live-zero-byte-object", 0, []string{"b2-eu-cen"})if _, err := db.Exec(`INSERT INTO trash(file_id, collection_id, user_id, delete_by, updated_at, is_deleted)- VALUES ($1, $5, $6, 1, 1, TRUE), ($2, $5, $6, 1, 1, TRUE),- ($3, $5, $6, 1, 1, FALSE), ($4, $5, $7, 1, 1, TRUE)`,- eligibleFileID, liveObjectFileID, activeTrashFileID, wrongTrashOwnerFileID,- collectionID, userID, otherUserID); err != nil {+ VALUES ($1, $2, $3, 1, 1, TRUE)`, fileID, collectionID, userID); err != nil {t.Fatal(err)}- fileIDs, err := repository.TrashRepo.GetStaleDeletedFileIDs(t.Context(), userID, TrashBatchSize)- if err != nil {+ if err := repository.TrashRepo.CleanUpDeletedFilesFromCollection(t.Context(), []int64{fileID}, userID); err == nil {+ t.Fatal("CleanUpDeletedFilesFromCollection() succeeded with a live object")+ }+ var deleted bool+ if err := db.QueryRow(`SELECT is_deleted FROM collection_files WHERE file_id = $1`, fileID).Scan(&deleted); err != nil {t.Fatal(err)}- if len(fileIDs) != 1 || fileIDs[0] != eligibleFileID {- t.Fatalf("GetStaleDeletedFileIDs() = %v, want [%d]", fileIDs, eligibleFileID)+ if deleted {+ t.Fatal("collection membership was deleted")}}diff --git a/server/pkg/repo/usage_file_count_init.go b/server/pkg/repo/usage_file_count_init.goindex 0a841e5221..04d08b2fee 100644--- a/server/pkg/repo/usage_file_count_init.go+++ b/server/pkg/repo/usage_file_count_init.go@@ -8,6 +8,17 @@ import (var ErrFileCountIneligible = errors.New("file counts are ineligible")+type FileCountIneligibleError struct {+ reason string+ StaleDeletedFileID int64+}++func (e *FileCountIneligibleError) Error() string {+ return fmt.Sprintf("%s: %s", ErrFileCountIneligible, e.reason)+}++func (e *FileCountIneligibleError) Unwrap() error { return ErrFileCountIneligible }+func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID int64) (bool, error) {_, photos, _, err := repo.GetStoredFileCounts(ctx, userID)if err != nil || photos != -1 {@@ -21,7 +32,7 @@ func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID inreturn false, nil}if counts.ineligibilityReason != "" {- return false, fmt.Errorf("%w: %s", ErrFileCountIneligible, counts.ineligibilityReason)+ return false, &FileCountIneligibleError{counts.ineligibilityReason, counts.staleDeletedFileID}}return repo.publishInitialFileCounts(ctx, userID, counts)}@@ -50,6 +61,7 @@ func (repo *UsageRepository) GetFileCountInitializationCandidates(ctx context.Cotype fileCountInitSnapshot struct {photos, locker, version int64+ staleDeletedFileID int64uninitialized boolineligibilityReason string}@@ -71,6 +83,11 @@ func (repo *UsageRepository) readFileCountInitSnapshot(ctx context.Context, userCOUNT(DISTINCT file_id) FILTER (WHERE owner_id = $1 AND app = 'locker') AS locker,BOOL_OR(app = 'locker' AND (f_owner_id IS NOT DISTINCT FROM $1) <> (owner_id = $1)) AS locker_mismatchFROM memberships+ ), trash_conflict AS (+ SELECT f.file_id, t.user_id, t.is_deleted+ FROM owned_files AS f+ JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE+ LIMIT 1)SELECT u.file_count_source_version,u.photos_file_count IS NULL AND u.locker_file_count IS NULL,@@ -84,16 +101,17 @@ func (repo *UsageRepository) readFileCountInitSnapshot(ctx context.Context, userGROUP BY f.file_idHAVING COUNT(DISTINCT c.app) > 1 OR BOOL_OR(c.app NOT IN ('photos', 'locker'))) THEN 'cross-app or unsupported app memberships'- WHEN EXISTS (- SELECT 1 FROM owned_files AS f- JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE- ) THEN 'active membership or wrong owner in Trash'+ WHEN trash_conflict.file_id IS NOT NULL THEN 'active membership or wrong owner in Trash'WHEN source.locker_mismatch THEN 'Locker ownership predicates disagree'ELSE ''- END+ END,+ COALESCE(CASE WHEN trash_conflict.user_id = $1 AND trash_conflict.is_deleted+ THEN trash_conflict.file_id END, 0)FROM usage AS u CROSS JOIN source+ LEFT JOIN trash_conflict ON TRUEWHERE u.user_id = $1`, userID).Scan(- &counts.version, &counts.uninitialized, &counts.photos, &counts.locker, &counts.ineligibilityReason)+ &counts.version, &counts.uninitialized, &counts.photos, &counts.locker,+ &counts.ineligibilityReason, &counts.staleDeletedFileID)return counts, err}diff --git a/server/pkg/repo/usage_file_count_init_test.go b/server/pkg/repo/usage_file_count_init_test.goindex b362daa561..e1b7e357dc 100644--- a/server/pkg/repo/usage_file_count_init_test.go+++ b/server/pkg/repo/usage_file_count_init_test.go@@ -52,19 +52,23 @@ func TestInitializeFileCountsIncludesLegacyMembershipsInDeletedCollections(t *tefunc TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {for _, tt := range []struct {- name string- sql string+ name string+ sql string+ staleDeletedFileID int64}{{"cross_app", `INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)- VALUES (102, 201, 'key', 'nonce', 1)`},+ VALUES (102, 201, 'key', 'nonce', 1)`, 0},{"cross_app_shared", `UPDATE collections SET owner_id = 2 WHERE collection_id = 102;INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)- VALUES (102, 201, 'key', 'nonce', 1)`},- {"active_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 1, 101, 1)`},- {"wrong_trash_owner", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 2, 101, 1)`},- {"locker_null_owner", `UPDATE collection_files SET f_owner_id = NULL WHERE file_id = 201`},+ VALUES (102, 201, 'key', 'nonce', 1)`, 0},+ {"active_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 1, 101, 1)`, 0},+ {"deleted_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)+ VALUES (201, 1, 101, 1, TRUE)`, 201},+ {"wrong_trash_owner", `INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)+ VALUES (201, 2, 101, 1, TRUE)`, 0},+ {"locker_null_owner", `UPDATE collection_files SET f_owner_id = NULL WHERE file_id = 201`, 0},{"locker_equal_counts_different_files", `UPDATE collection_files- SET f_owner_id = CASE WHEN file_id = 201 THEN NULL ELSE 1 END WHERE collection_id = 101`},+ SET f_owner_id = CASE WHEN file_id = 201 THEN NULL ELSE 1 END WHERE collection_id = 101`, 0},} {t.Run(tt.name, func(t *testing.T) {_, db, userID := setupCollectionMembershipTest(t)@@ -91,6 +95,13 @@ func TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {if initialized || !errors.Is(err, ErrFileCountIneligible) {t.Fatalf("InitializeFileCounts() = (%t, %v), want ineligible", initialized, err)}+ var ineligibleErr *FileCountIneligibleError+ if !errors.As(err, &ineligibleErr) {+ t.Fatalf("error = %v, want FileCountIneligibleError", err)+ }+ if ineligibleErr.StaleDeletedFileID != tt.staleDeletedFileID {+ t.Fatalf("stale deleted file = %d, want %d", ineligibleErr.StaleDeletedFileID, tt.staleDeletedFileID)+ }photos, locker, version := readFileCountState(t, db, userID)if photos.Valid || locker.Valid || version != 0 {t.Fatalf("ineligible user changed: (%v, %v, %d)", photos, locker, version)