π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Simplify stale file membership repair
Neeraj Guptab5ff18dc8e1b
diff --git a/server/pkg/controller/usage_file_count_init.go b/server/pkg/controller/usage_file_count_init.goindex 0d030e3254..c42023f7ce 100644--- a/server/pkg/controller/usage_file_count_init.go+++ b/server/pkg/controller/usage_file_count_init.go@@ -64,14 +64,18 @@ func (c *FileCountInitializer) ProcessBatch() {updated, err := func() (bool, error) {defer c.LockController.ReleaseLock(lockID)updated, err := c.UsageRepo.InitializeFileCounts(ctx, userID)- var ineligibleErr *repo.FileCountIneligibleError- if errors.As(err, &ineligibleErr) && ineligibleErr.StaleDeletedFileID != 0 {- cleanupErr := c.TrashRepo.CleanUpDeletedFilesFromCollection(ctx, []int64{ineligibleErr.StaleDeletedFileID}, userID)+ if errors.Is(err, repo.ErrFileCountIneligible) {+ fileIDs, cleanupErr := c.TrashRepo.GetStaleDeletedFileIDs(ctx, userID)if cleanupErr != nil {- log.WithError(cleanupErr).WithFields(log.Fields{- "user_id": userID,- "file_id": ineligibleErr.StaleDeletedFileID,- }).Error("Failed to clean stale deleted file membership")+ log.WithError(cleanupErr).WithField("user_id", userID).Error("Failed to find stale deleted file memberships")+ }+ for _, fileID := range fileIDs {+ if cleanupErr := c.TrashRepo.CleanUpDeletedFilesFromCollection(ctx, []int64{fileID}, userID); cleanupErr != nil {+ log.WithError(cleanupErr).WithFields(log.Fields{+ "user_id": userID,+ "file_id": fileID,+ }).Error("Failed to clean stale deleted file membership")+ }}}return updated, errdiff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.goindex 15d1dc27ef..9284db8b29 100644--- a/server/pkg/repo/trash.go+++ b/server/pkg/repo/trash.go@@ -19,7 +19,8 @@ const (TrashDurationInDays = 30TrashDiffLimit = 2500- TrashBatchSize = 1000+ TrashBatchSize = 1000+ staleDeletedFileLimit = 10EmptyTrashQueueItemSeparator = "::")@@ -239,6 +240,21 @@ func (t *TrashRepository) CleanUpDeletedFilesFromCollection(ctx context.Context,return nil}+func (t *TrashRepository) GetStaleDeletedFileIDs(ctx context.Context, userID int64) ([]int64, error) {+ rows, err := t.DB.QueryContext(ctx, `SELECT DISTINCT f.file_id+ FROM collections c+ JOIN collection_files cf ON cf.collection_id = c.collection_id AND cf.is_deleted = FALSE+ JOIN files f ON f.file_id = cf.file_id+ JOIN trash t ON t.file_id = f.file_id+ WHERE c.owner_id = $1 AND f.owner_id = $1 AND t.user_id = $1+ AND t.is_deleted = TRUE AND t.is_restored = FALSE+ LIMIT $2`, userID, staleDeletedFileLimit)+ if err != nil {+ return nil, stacktrace.Propagate(err, "")+ }+ return convertRowsToFileId(rows)+}+func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []int64) error {if len(fileIDs) > TrashDiffLimit {return fmt.Errorf("can not delete more than %d in one go", TrashDiffLimit)diff --git a/server/pkg/repo/trash_test.go b/server/pkg/repo/trash_test.goindex e22d3cb081..9a32813193 100644--- a/server/pkg/repo/trash_test.go+++ b/server/pkg/repo/trash_test.go@@ -204,7 +204,7 @@ func TestStaleCleanupInvalidatesOnlyRemovedOwnedMemberships(t *testing.T) {}}-func TestStaleCleanupRejectsLiveZeroByteObject(t *testing.T) {+func TestStaleDeletedFileCleanupRejectsLiveObject(t *testing.T) {repository, db, userID := setupCollectionMembershipTest(t)collectionID := insertObjectTestCollection(t, db, userID)fileID := insertObjectTestFile(t, db, userID)@@ -215,7 +215,14 @@ func TestStaleCleanupRejectsLiveZeroByteObject(t *testing.T) {t.Fatal(err)}- if err := repository.TrashRepo.CleanUpDeletedFilesFromCollection(t.Context(), []int64{fileID}, userID); err == nil {+ fileIDs, err := repository.TrashRepo.GetStaleDeletedFileIDs(t.Context(), userID)+ if err != nil {+ t.Fatal(err)+ }+ if len(fileIDs) != 1 || fileIDs[0] != fileID {+ t.Fatalf("GetStaleDeletedFileIDs() = %v, want [%d]", fileIDs, fileID)+ }+ if err := repository.TrashRepo.CleanUpDeletedFilesFromCollection(t.Context(), fileIDs, userID); err == nil {t.Fatal("CleanUpDeletedFilesFromCollection() succeeded with a live object")}var deleted booldiff --git a/server/pkg/repo/usage_file_count_init.go b/server/pkg/repo/usage_file_count_init.goindex 04d08b2fee..0a841e5221 100644--- a/server/pkg/repo/usage_file_count_init.go+++ b/server/pkg/repo/usage_file_count_init.go@@ -8,17 +8,6 @@ import (var ErrFileCountIneligible = errors.New("file counts are ineligible")-type FileCountIneligibleError struct {- reason string- StaleDeletedFileID int64-}--func (e *FileCountIneligibleError) Error() string {- return fmt.Sprintf("%s: %s", ErrFileCountIneligible, e.reason)-}--func (e *FileCountIneligibleError) Unwrap() error { return ErrFileCountIneligible }-func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID int64) (bool, error) {_, photos, _, err := repo.GetStoredFileCounts(ctx, userID)if err != nil || photos != -1 {@@ -32,7 +21,7 @@ func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID inreturn false, nil}if counts.ineligibilityReason != "" {- return false, &FileCountIneligibleError{counts.ineligibilityReason, counts.staleDeletedFileID}+ return false, fmt.Errorf("%w: %s", ErrFileCountIneligible, counts.ineligibilityReason)}return repo.publishInitialFileCounts(ctx, userID, counts)}@@ -61,7 +50,6 @@ func (repo *UsageRepository) GetFileCountInitializationCandidates(ctx context.Cotype fileCountInitSnapshot struct {photos, locker, version int64- staleDeletedFileID int64uninitialized boolineligibilityReason string}@@ -83,11 +71,6 @@ func (repo *UsageRepository) readFileCountInitSnapshot(ctx context.Context, userCOUNT(DISTINCT file_id) FILTER (WHERE owner_id = $1 AND app = 'locker') AS locker,BOOL_OR(app = 'locker' AND (f_owner_id IS NOT DISTINCT FROM $1) <> (owner_id = $1)) AS locker_mismatchFROM memberships- ), trash_conflict AS (- SELECT f.file_id, t.user_id, t.is_deleted- FROM owned_files AS f- JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE- LIMIT 1)SELECT u.file_count_source_version,u.photos_file_count IS NULL AND u.locker_file_count IS NULL,@@ -101,17 +84,16 @@ func (repo *UsageRepository) readFileCountInitSnapshot(ctx context.Context, userGROUP BY f.file_idHAVING COUNT(DISTINCT c.app) > 1 OR BOOL_OR(c.app NOT IN ('photos', 'locker'))) THEN 'cross-app or unsupported app memberships'- WHEN trash_conflict.file_id IS NOT NULL THEN 'active membership or wrong owner in Trash'+ WHEN EXISTS (+ SELECT 1 FROM owned_files AS f+ JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE+ ) THEN 'active membership or wrong owner in Trash'WHEN source.locker_mismatch THEN 'Locker ownership predicates disagree'ELSE ''- END,- COALESCE(CASE WHEN trash_conflict.user_id = $1 AND trash_conflict.is_deleted- THEN trash_conflict.file_id END, 0)+ ENDFROM usage AS u CROSS JOIN source- LEFT JOIN trash_conflict ON TRUEWHERE u.user_id = $1`, userID).Scan(- &counts.version, &counts.uninitialized, &counts.photos, &counts.locker,- &counts.ineligibilityReason, &counts.staleDeletedFileID)+ &counts.version, &counts.uninitialized, &counts.photos, &counts.locker, &counts.ineligibilityReason)return counts, err}diff --git a/server/pkg/repo/usage_file_count_init_test.go b/server/pkg/repo/usage_file_count_init_test.goindex e1b7e357dc..b362daa561 100644--- a/server/pkg/repo/usage_file_count_init_test.go+++ b/server/pkg/repo/usage_file_count_init_test.go@@ -52,23 +52,19 @@ func TestInitializeFileCountsIncludesLegacyMembershipsInDeletedCollections(t *tefunc TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {for _, tt := range []struct {- name string- sql string- staleDeletedFileID int64+ name string+ sql string}{{"cross_app", `INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)- VALUES (102, 201, 'key', 'nonce', 1)`, 0},+ VALUES (102, 201, 'key', 'nonce', 1)`},{"cross_app_shared", `UPDATE collections SET owner_id = 2 WHERE collection_id = 102;INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)- VALUES (102, 201, 'key', 'nonce', 1)`, 0},- {"active_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 1, 101, 1)`, 0},- {"deleted_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)- VALUES (201, 1, 101, 1, TRUE)`, 201},- {"wrong_trash_owner", `INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)- VALUES (201, 2, 101, 1, TRUE)`, 0},- {"locker_null_owner", `UPDATE collection_files SET f_owner_id = NULL WHERE file_id = 201`, 0},+ VALUES (102, 201, 'key', 'nonce', 1)`},+ {"active_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 1, 101, 1)`},+ {"wrong_trash_owner", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 2, 101, 1)`},+ {"locker_null_owner", `UPDATE collection_files SET f_owner_id = NULL WHERE file_id = 201`},{"locker_equal_counts_different_files", `UPDATE collection_files- SET f_owner_id = CASE WHEN file_id = 201 THEN NULL ELSE 1 END WHERE collection_id = 101`, 0},+ SET f_owner_id = CASE WHEN file_id = 201 THEN NULL ELSE 1 END WHERE collection_id = 101`},} {t.Run(tt.name, func(t *testing.T) {_, db, userID := setupCollectionMembershipTest(t)@@ -95,13 +91,6 @@ func TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {if initialized || !errors.Is(err, ErrFileCountIneligible) {t.Fatalf("InitializeFileCounts() = (%t, %v), want ineligible", initialized, err)}- var ineligibleErr *FileCountIneligibleError- if !errors.As(err, &ineligibleErr) {- t.Fatalf("error = %v, want FileCountIneligibleError", err)- }- if ineligibleErr.StaleDeletedFileID != tt.staleDeletedFileID {- t.Fatalf("stale deleted file = %d, want %d", ineligibleErr.StaleDeletedFileID, tt.staleDeletedFileID)- }photos, locker, version := readFileCountState(t, db, userID)if photos.Valid || locker.Valid || version != 0 {t.Fatalf("ineligible user changed: (%v, %v, %d)", photos, locker, version)