Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Repair stale deleted file memberships

Neeraj Gupta139098e3b264
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go
index f5cbe2dbb0..0b914cc19a 100644
--- a/server/cmd/museum/main.go
+++ b/server/cmd/museum/main.go
@@ -298,7 +298,7 @@ func main() {
FileRepo: fileRepo,
UploadResultCache: make(map[int64]bool),
}
- fileCountInitializer := &controller.FileCountInitializer{UsageRepo: usageRepo, LockController: lockController}
+ fileCountInitializer := &controller.FileCountInitializer{UsageRepo: usageRepo, TrashRepo: trashRepo, LockController: lockController}
accessCtrl := access.NewAccessController(accessCollectionRepo, accessFileRepo)
commentsRepo := &socialrepo.CommentsRepository{DB: db}
diff --git a/server/pkg/controller/usage_file_count_init.go b/server/pkg/controller/usage_file_count_init.go
index 3bc444d58c..1159c32b6b 100644
--- a/server/pkg/controller/usage_file_count_init.go
+++ b/server/pkg/controller/usage_file_count_init.go
@@ -15,6 +15,7 @@ import (
type FileCountInitializer struct {
UsageRepo *repo.UsageRepository
+ TrashRepo *repo.TrashRepository
LockController *lock.LockController
afterUserID int64
resumeAt time.Time
@@ -35,7 +36,8 @@ func (c *FileCountInitializer) ProcessBatch() {
return
}
- userIDs, err := c.UsageRepo.GetFileCountInitializationCandidates(context.Background(), c.afterUserID, fileCountInitializationBatchSize)
+ ctx := context.Background()
+ userIDs, err := c.UsageRepo.GetFileCountInitializationCandidates(ctx, c.afterUserID, fileCountInitializationBatchSize)
if err != nil {
log.WithError(err).Error("Failed to fetch file count initialization candidates")
return
@@ -61,7 +63,17 @@ func (c *FileCountInitializer) ProcessBatch() {
}
updated, err := func() (bool, error) {
defer c.LockController.ReleaseLock(lockID)
- return c.UsageRepo.InitializeFileCounts(context.Background(), userID)
+ updated, err := c.UsageRepo.InitializeFileCounts(ctx, userID)
+ if errors.Is(err, repo.ErrFileCountIneligible) {
+ fileIDs, cleanupErr := c.TrashRepo.GetStaleDeletedFileIDs(ctx, userID, repo.TrashBatchSize)
+ if cleanupErr == nil && len(fileIDs) > 0 {
+ cleanupErr = c.TrashRepo.CleanUpDeletedFilesFromCollection(ctx, fileIDs, userID)
+ }
+ if cleanupErr != nil {
+ log.WithError(cleanupErr).WithField("user_id", userID).Error("Failed to clean stale deleted file memberships")
+ }
+ }
+ return updated, err
}()
if errors.Is(err, repo.ErrFileCountIneligible) {
ineligible++
diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.go
index 13d51c0a3c..aff6ab9df2 100644
--- a/server/pkg/repo/trash.go
+++ b/server/pkg/repo/trash.go
@@ -239,6 +239,25 @@ func (t *TrashRepository) CleanUpDeletedFilesFromCollection(ctx context.Context,
return nil
}
+func (t *TrashRepository) GetStaleDeletedFileIDs(ctx context.Context, userID int64, limit int) ([]int64, error) {
+ rows, err := t.DB.QueryContext(ctx, `SELECT DISTINCT cf.file_id
+ FROM collections c
+ JOIN collection_files cf ON cf.collection_id = c.collection_id AND cf.is_deleted = FALSE
+ JOIN files f ON f.file_id = cf.file_id AND f.owner_id = $1
+ JOIN trash t ON t.file_id = cf.file_id AND t.user_id = $1
+ AND t.is_deleted = TRUE AND t.is_restored = FALSE
+ WHERE c.owner_id = $1
+ AND NOT EXISTS (
+ SELECT 1 FROM object_keys ok
+ WHERE ok.file_id = cf.file_id AND ok.is_deleted = FALSE
+ )
+ LIMIT $2`, userID, limit)
+ if err != nil {
+ return nil, stacktrace.Propagate(err, "")
+ }
+ return convertRowsToFileId(rows)
+}
+
func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []int64) error {
if len(fileIDs) > TrashDiffLimit {
return fmt.Errorf("can not delete more than %d in one go", TrashDiffLimit)
diff --git a/server/pkg/repo/trash_test.go b/server/pkg/repo/trash_test.go
index a39d33f250..b5d422a1c0 100644
--- a/server/pkg/repo/trash_test.go
+++ b/server/pkg/repo/trash_test.go
@@ -204,6 +204,36 @@ func TestStaleCleanupInvalidatesOnlyRemovedOwnedMemberships(t *testing.T) {
}
}
+func TestGetStaleDeletedFileIDs(t *testing.T) {
+ repository, db, userID := setupCollectionMembershipTest(t)
+ otherUserID := testutil.InsertUser(t, db, testutil.UserFixture{UserID: 2, Email: "[email protected]", CreationTime: 1})
+ collectionID := insertObjectTestCollection(t, db, userID)
+ eligibleFileID := insertObjectTestFile(t, db, userID)
+ liveObjectFileID := insertObjectTestFile(t, db, userID)
+ activeTrashFileID := insertObjectTestFile(t, db, userID)
+ wrongTrashOwnerFileID := insertObjectTestFile(t, db, userID)
+ noTrashFileID := insertObjectTestFile(t, db, userID)
+ for _, fileID := range []int64{eligibleFileID, liveObjectFileID, activeTrashFileID, wrongTrashOwnerFileID, noTrashFileID} {
+ linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)
+ }
+ insertObjectTestKey(t, db, liveObjectFileID, ente.FILE, "live-zero-byte-object", 0, []string{"b2-eu-cen"})
+ if _, err := db.Exec(`INSERT INTO trash(file_id, collection_id, user_id, delete_by, updated_at, is_deleted)
+ VALUES ($1, $5, $6, 1, 1, TRUE), ($2, $5, $6, 1, 1, TRUE),
+ ($3, $5, $6, 1, 1, FALSE), ($4, $5, $7, 1, 1, TRUE)`,
+ eligibleFileID, liveObjectFileID, activeTrashFileID, wrongTrashOwnerFileID,
+ collectionID, userID, otherUserID); err != nil {
+ t.Fatal(err)
+ }
+
+ fileIDs, err := repository.TrashRepo.GetStaleDeletedFileIDs(t.Context(), userID, TrashBatchSize)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(fileIDs) != 1 || fileIDs[0] != eligibleFileID {
+ t.Fatalf("GetStaleDeletedFileIDs() = %v, want [%d]", fileIDs, eligibleFileID)
+ }
+}
+
func setupTrashTest(t *testing.T) (*TrashRepository, *sql.DB) {
t.Helper()