Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Simplify stale file membership repair

Neeraj Guptab5ff18dc8e1b
diff --git a/server/pkg/controller/usage_file_count_init.go b/server/pkg/controller/usage_file_count_init.go
index 0d030e3254..c42023f7ce 100644
--- a/server/pkg/controller/usage_file_count_init.go
+++ b/server/pkg/controller/usage_file_count_init.go
@@ -64,14 +64,18 @@ func (c *FileCountInitializer) ProcessBatch() {
updated, err := func() (bool, error) {
defer c.LockController.ReleaseLock(lockID)
updated, err := c.UsageRepo.InitializeFileCounts(ctx, userID)
- var ineligibleErr *repo.FileCountIneligibleError
- if errors.As(err, &ineligibleErr) && ineligibleErr.StaleDeletedFileID != 0 {
- cleanupErr := c.TrashRepo.CleanUpDeletedFilesFromCollection(ctx, []int64{ineligibleErr.StaleDeletedFileID}, userID)
+ if errors.Is(err, repo.ErrFileCountIneligible) {
+ fileIDs, cleanupErr := c.TrashRepo.GetStaleDeletedFileIDs(ctx, userID)
if cleanupErr != nil {
- log.WithError(cleanupErr).WithFields(log.Fields{
- "user_id": userID,
- "file_id": ineligibleErr.StaleDeletedFileID,
- }).Error("Failed to clean stale deleted file membership")
+ log.WithError(cleanupErr).WithField("user_id", userID).Error("Failed to find stale deleted file memberships")
+ }
+ for _, fileID := range fileIDs {
+ if cleanupErr := c.TrashRepo.CleanUpDeletedFilesFromCollection(ctx, []int64{fileID}, userID); cleanupErr != nil {
+ log.WithError(cleanupErr).WithFields(log.Fields{
+ "user_id": userID,
+ "file_id": fileID,
+ }).Error("Failed to clean stale deleted file membership")
+ }
}
}
return updated, err
diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.go
index 15d1dc27ef..9284db8b29 100644
--- a/server/pkg/repo/trash.go
+++ b/server/pkg/repo/trash.go
@@ -19,7 +19,8 @@ const (
TrashDurationInDays = 30
TrashDiffLimit = 2500
- TrashBatchSize = 1000
+ TrashBatchSize = 1000
+ staleDeletedFileLimit = 10
EmptyTrashQueueItemSeparator = "::"
)
@@ -239,6 +240,21 @@ func (t *TrashRepository) CleanUpDeletedFilesFromCollection(ctx context.Context,
return nil
}
+func (t *TrashRepository) GetStaleDeletedFileIDs(ctx context.Context, userID int64) ([]int64, error) {
+ rows, err := t.DB.QueryContext(ctx, `SELECT DISTINCT f.file_id
+ FROM collections c
+ JOIN collection_files cf ON cf.collection_id = c.collection_id AND cf.is_deleted = FALSE
+ JOIN files f ON f.file_id = cf.file_id
+ JOIN trash t ON t.file_id = f.file_id
+ WHERE c.owner_id = $1 AND f.owner_id = $1 AND t.user_id = $1
+ AND t.is_deleted = TRUE AND t.is_restored = FALSE
+ LIMIT $2`, userID, staleDeletedFileLimit)
+ if err != nil {
+ return nil, stacktrace.Propagate(err, "")
+ }
+ return convertRowsToFileId(rows)
+}
+
func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []int64) error {
if len(fileIDs) > TrashDiffLimit {
return fmt.Errorf("can not delete more than %d in one go", TrashDiffLimit)
diff --git a/server/pkg/repo/trash_test.go b/server/pkg/repo/trash_test.go
index e22d3cb081..9a32813193 100644
--- a/server/pkg/repo/trash_test.go
+++ b/server/pkg/repo/trash_test.go
@@ -204,7 +204,7 @@ func TestStaleCleanupInvalidatesOnlyRemovedOwnedMemberships(t *testing.T) {
}
}
-func TestStaleCleanupRejectsLiveZeroByteObject(t *testing.T) {
+func TestStaleDeletedFileCleanupRejectsLiveObject(t *testing.T) {
repository, db, userID := setupCollectionMembershipTest(t)
collectionID := insertObjectTestCollection(t, db, userID)
fileID := insertObjectTestFile(t, db, userID)
@@ -215,7 +215,14 @@ func TestStaleCleanupRejectsLiveZeroByteObject(t *testing.T) {
t.Fatal(err)
}
- if err := repository.TrashRepo.CleanUpDeletedFilesFromCollection(t.Context(), []int64{fileID}, userID); err == nil {
+ fileIDs, err := repository.TrashRepo.GetStaleDeletedFileIDs(t.Context(), userID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(fileIDs) != 1 || fileIDs[0] != fileID {
+ t.Fatalf("GetStaleDeletedFileIDs() = %v, want [%d]", fileIDs, fileID)
+ }
+ if err := repository.TrashRepo.CleanUpDeletedFilesFromCollection(t.Context(), fileIDs, userID); err == nil {
t.Fatal("CleanUpDeletedFilesFromCollection() succeeded with a live object")
}
var deleted bool
diff --git a/server/pkg/repo/usage_file_count_init.go b/server/pkg/repo/usage_file_count_init.go
index 04d08b2fee..0a841e5221 100644
--- a/server/pkg/repo/usage_file_count_init.go
+++ b/server/pkg/repo/usage_file_count_init.go
@@ -8,17 +8,6 @@ import (
var ErrFileCountIneligible = errors.New("file counts are ineligible")
-type FileCountIneligibleError struct {
- reason string
- StaleDeletedFileID int64
-}
-
-func (e *FileCountIneligibleError) Error() string {
- return fmt.Sprintf("%s: %s", ErrFileCountIneligible, e.reason)
-}
-
-func (e *FileCountIneligibleError) Unwrap() error { return ErrFileCountIneligible }
-
func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID int64) (bool, error) {
_, photos, _, err := repo.GetStoredFileCounts(ctx, userID)
if err != nil || photos != -1 {
@@ -32,7 +21,7 @@ func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID in
return false, nil
}
if counts.ineligibilityReason != "" {
- return false, &FileCountIneligibleError{counts.ineligibilityReason, counts.staleDeletedFileID}
+ return false, fmt.Errorf("%w: %s", ErrFileCountIneligible, counts.ineligibilityReason)
}
return repo.publishInitialFileCounts(ctx, userID, counts)
}
@@ -61,7 +50,6 @@ func (repo *UsageRepository) GetFileCountInitializationCandidates(ctx context.Co
type fileCountInitSnapshot struct {
photos, locker, version int64
- staleDeletedFileID int64
uninitialized bool
ineligibilityReason string
}
@@ -83,11 +71,6 @@ func (repo *UsageRepository) readFileCountInitSnapshot(ctx context.Context, user
COUNT(DISTINCT file_id) FILTER (WHERE owner_id = $1 AND app = 'locker') AS locker,
BOOL_OR(app = 'locker' AND (f_owner_id IS NOT DISTINCT FROM $1) <> (owner_id = $1)) AS locker_mismatch
FROM memberships
- ), trash_conflict AS (
- SELECT f.file_id, t.user_id, t.is_deleted
- FROM owned_files AS f
- JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE
- LIMIT 1
)
SELECT u.file_count_source_version,
u.photos_file_count IS NULL AND u.locker_file_count IS NULL,
@@ -101,17 +84,16 @@ func (repo *UsageRepository) readFileCountInitSnapshot(ctx context.Context, user
GROUP BY f.file_id
HAVING COUNT(DISTINCT c.app) > 1 OR BOOL_OR(c.app NOT IN ('photos', 'locker'))
) THEN 'cross-app or unsupported app memberships'
- WHEN trash_conflict.file_id IS NOT NULL THEN 'active membership or wrong owner in Trash'
+ WHEN EXISTS (
+ SELECT 1 FROM owned_files AS f
+ JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE
+ ) THEN 'active membership or wrong owner in Trash'
WHEN source.locker_mismatch THEN 'Locker ownership predicates disagree'
ELSE ''
- END,
- COALESCE(CASE WHEN trash_conflict.user_id = $1 AND trash_conflict.is_deleted
- THEN trash_conflict.file_id END, 0)
+ END
FROM usage AS u CROSS JOIN source
- LEFT JOIN trash_conflict ON TRUE
WHERE u.user_id = $1`, userID).Scan(
- &counts.version, &counts.uninitialized, &counts.photos, &counts.locker,
- &counts.ineligibilityReason, &counts.staleDeletedFileID)
+ &counts.version, &counts.uninitialized, &counts.photos, &counts.locker, &counts.ineligibilityReason)
return counts, err
}
diff --git a/server/pkg/repo/usage_file_count_init_test.go b/server/pkg/repo/usage_file_count_init_test.go
index e1b7e357dc..b362daa561 100644
--- a/server/pkg/repo/usage_file_count_init_test.go
+++ b/server/pkg/repo/usage_file_count_init_test.go
@@ -52,23 +52,19 @@ func TestInitializeFileCountsIncludesLegacyMembershipsInDeletedCollections(t *te
func TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {
for _, tt := range []struct {
- name string
- sql string
- staleDeletedFileID int64
+ name string
+ sql string
}{
{"cross_app", `INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)
- VALUES (102, 201, 'key', 'nonce', 1)`, 0},
+ VALUES (102, 201, 'key', 'nonce', 1)`},
{"cross_app_shared", `UPDATE collections SET owner_id = 2 WHERE collection_id = 102;
INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)
- VALUES (102, 201, 'key', 'nonce', 1)`, 0},
- {"active_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 1, 101, 1)`, 0},
- {"deleted_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)
- VALUES (201, 1, 101, 1, TRUE)`, 201},
- {"wrong_trash_owner", `INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)
- VALUES (201, 2, 101, 1, TRUE)`, 0},
- {"locker_null_owner", `UPDATE collection_files SET f_owner_id = NULL WHERE file_id = 201`, 0},
+ VALUES (102, 201, 'key', 'nonce', 1)`},
+ {"active_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 1, 101, 1)`},
+ {"wrong_trash_owner", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 2, 101, 1)`},
+ {"locker_null_owner", `UPDATE collection_files SET f_owner_id = NULL WHERE file_id = 201`},
{"locker_equal_counts_different_files", `UPDATE collection_files
- SET f_owner_id = CASE WHEN file_id = 201 THEN NULL ELSE 1 END WHERE collection_id = 101`, 0},
+ SET f_owner_id = CASE WHEN file_id = 201 THEN NULL ELSE 1 END WHERE collection_id = 101`},
} {
t.Run(tt.name, func(t *testing.T) {
_, db, userID := setupCollectionMembershipTest(t)
@@ -95,13 +91,6 @@ func TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {
if initialized || !errors.Is(err, ErrFileCountIneligible) {
t.Fatalf("InitializeFileCounts() = (%t, %v), want ineligible", initialized, err)
}
- var ineligibleErr *FileCountIneligibleError
- if !errors.As(err, &ineligibleErr) {
- t.Fatalf("error = %v, want FileCountIneligibleError", err)
- }
- if ineligibleErr.StaleDeletedFileID != tt.staleDeletedFileID {
- t.Fatalf("stale deleted file = %d, want %d", ineligibleErr.StaleDeletedFileID, tt.staleDeletedFileID)
- }
photos, locker, version := readFileCountState(t, db, userID)
if photos.Valid || locker.Valid || version != 0 {
t.Fatalf("ineligible user changed: (%v, %v, %d)", photos, locker, version)