π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Repair stale deleted file memberships
Neeraj Gupta139098e3b264
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.goindex f5cbe2dbb0..0b914cc19a 100644--- a/server/cmd/museum/main.go+++ b/server/cmd/museum/main.go@@ -298,7 +298,7 @@ func main() {FileRepo: fileRepo,UploadResultCache: make(map[int64]bool),}- fileCountInitializer := &controller.FileCountInitializer{UsageRepo: usageRepo, LockController: lockController}+ fileCountInitializer := &controller.FileCountInitializer{UsageRepo: usageRepo, TrashRepo: trashRepo, LockController: lockController}accessCtrl := access.NewAccessController(accessCollectionRepo, accessFileRepo)commentsRepo := &socialrepo.CommentsRepository{DB: db}diff --git a/server/pkg/controller/usage_file_count_init.go b/server/pkg/controller/usage_file_count_init.goindex 3bc444d58c..1159c32b6b 100644--- a/server/pkg/controller/usage_file_count_init.go+++ b/server/pkg/controller/usage_file_count_init.go@@ -15,6 +15,7 @@ import (type FileCountInitializer struct {UsageRepo *repo.UsageRepository+ TrashRepo *repo.TrashRepositoryLockController *lock.LockControllerafterUserID int64resumeAt time.Time@@ -35,7 +36,8 @@ func (c *FileCountInitializer) ProcessBatch() {return}- userIDs, err := c.UsageRepo.GetFileCountInitializationCandidates(context.Background(), c.afterUserID, fileCountInitializationBatchSize)+ ctx := context.Background()+ userIDs, err := c.UsageRepo.GetFileCountInitializationCandidates(ctx, c.afterUserID, fileCountInitializationBatchSize)if err != nil {log.WithError(err).Error("Failed to fetch file count initialization candidates")return@@ -61,7 +63,17 @@ func (c *FileCountInitializer) ProcessBatch() {}updated, err := func() (bool, error) {defer c.LockController.ReleaseLock(lockID)- return c.UsageRepo.InitializeFileCounts(context.Background(), userID)+ updated, err := c.UsageRepo.InitializeFileCounts(ctx, userID)+ if errors.Is(err, repo.ErrFileCountIneligible) {+ fileIDs, cleanupErr := c.TrashRepo.GetStaleDeletedFileIDs(ctx, userID, repo.TrashBatchSize)+ if cleanupErr == nil && len(fileIDs) > 0 {+ cleanupErr = c.TrashRepo.CleanUpDeletedFilesFromCollection(ctx, fileIDs, userID)+ }+ if cleanupErr != nil {+ log.WithError(cleanupErr).WithField("user_id", userID).Error("Failed to clean stale deleted file memberships")+ }+ }+ return updated, err}()if errors.Is(err, repo.ErrFileCountIneligible) {ineligible++diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.goindex 13d51c0a3c..aff6ab9df2 100644--- a/server/pkg/repo/trash.go+++ b/server/pkg/repo/trash.go@@ -239,6 +239,25 @@ func (t *TrashRepository) CleanUpDeletedFilesFromCollection(ctx context.Context,return nil}+func (t *TrashRepository) GetStaleDeletedFileIDs(ctx context.Context, userID int64, limit int) ([]int64, error) {+ rows, err := t.DB.QueryContext(ctx, `SELECT DISTINCT cf.file_id+ FROM collections c+ JOIN collection_files cf ON cf.collection_id = c.collection_id AND cf.is_deleted = FALSE+ JOIN files f ON f.file_id = cf.file_id AND f.owner_id = $1+ JOIN trash t ON t.file_id = cf.file_id AND t.user_id = $1+ AND t.is_deleted = TRUE AND t.is_restored = FALSE+ WHERE c.owner_id = $1+ AND NOT EXISTS (+ SELECT 1 FROM object_keys ok+ WHERE ok.file_id = cf.file_id AND ok.is_deleted = FALSE+ )+ LIMIT $2`, userID, limit)+ if err != nil {+ return nil, stacktrace.Propagate(err, "")+ }+ return convertRowsToFileId(rows)+}+func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []int64) error {if len(fileIDs) > TrashDiffLimit {return fmt.Errorf("can not delete more than %d in one go", TrashDiffLimit)diff --git a/server/pkg/repo/trash_test.go b/server/pkg/repo/trash_test.goindex a39d33f250..b5d422a1c0 100644--- a/server/pkg/repo/trash_test.go+++ b/server/pkg/repo/trash_test.go@@ -204,6 +204,36 @@ func TestStaleCleanupInvalidatesOnlyRemovedOwnedMemberships(t *testing.T) {}}+func TestGetStaleDeletedFileIDs(t *testing.T) {+ repository, db, userID := setupCollectionMembershipTest(t)+ otherUserID := testutil.InsertUser(t, db, testutil.UserFixture{UserID: 2, Email: "[email protected]", CreationTime: 1})+ collectionID := insertObjectTestCollection(t, db, userID)+ eligibleFileID := insertObjectTestFile(t, db, userID)+ liveObjectFileID := insertObjectTestFile(t, db, userID)+ activeTrashFileID := insertObjectTestFile(t, db, userID)+ wrongTrashOwnerFileID := insertObjectTestFile(t, db, userID)+ noTrashFileID := insertObjectTestFile(t, db, userID)+ for _, fileID := range []int64{eligibleFileID, liveObjectFileID, activeTrashFileID, wrongTrashOwnerFileID, noTrashFileID} {+ linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)+ }+ insertObjectTestKey(t, db, liveObjectFileID, ente.FILE, "live-zero-byte-object", 0, []string{"b2-eu-cen"})+ if _, err := db.Exec(`INSERT INTO trash(file_id, collection_id, user_id, delete_by, updated_at, is_deleted)+ VALUES ($1, $5, $6, 1, 1, TRUE), ($2, $5, $6, 1, 1, TRUE),+ ($3, $5, $6, 1, 1, FALSE), ($4, $5, $7, 1, 1, TRUE)`,+ eligibleFileID, liveObjectFileID, activeTrashFileID, wrongTrashOwnerFileID,+ collectionID, userID, otherUserID); err != nil {+ t.Fatal(err)+ }++ fileIDs, err := repository.TrashRepo.GetStaleDeletedFileIDs(t.Context(), userID, TrashBatchSize)+ if err != nil {+ t.Fatal(err)+ }+ if len(fileIDs) != 1 || fileIDs[0] != eligibleFileID {+ t.Fatalf("GetStaleDeletedFileIDs() = %v, want [%d]", fileIDs, eligibleFileID)+ }+}+func setupTrashTest(t *testing.T) (*TrashRepository, *sql.DB) {t.Helper()