Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Reuse file count initialization scan for repair

Neeraj Gupta50314aa42d2d
diff --git a/server/pkg/controller/usage_file_count_init.go b/server/pkg/controller/usage_file_count_init.go
index 1159c32b6b..0d030e3254 100644
--- a/server/pkg/controller/usage_file_count_init.go
+++ b/server/pkg/controller/usage_file_count_init.go
@@ -64,13 +64,14 @@ func (c *FileCountInitializer) ProcessBatch() {
updated, err := func() (bool, error) {
defer c.LockController.ReleaseLock(lockID)
updated, err := c.UsageRepo.InitializeFileCounts(ctx, userID)
- if errors.Is(err, repo.ErrFileCountIneligible) {
- fileIDs, cleanupErr := c.TrashRepo.GetStaleDeletedFileIDs(ctx, userID, repo.TrashBatchSize)
- if cleanupErr == nil && len(fileIDs) > 0 {
- cleanupErr = c.TrashRepo.CleanUpDeletedFilesFromCollection(ctx, fileIDs, userID)
- }
+ var ineligibleErr *repo.FileCountIneligibleError
+ if errors.As(err, &ineligibleErr) && ineligibleErr.StaleDeletedFileID != 0 {
+ cleanupErr := c.TrashRepo.CleanUpDeletedFilesFromCollection(ctx, []int64{ineligibleErr.StaleDeletedFileID}, userID)
if cleanupErr != nil {
- log.WithError(cleanupErr).WithField("user_id", userID).Error("Failed to clean stale deleted file memberships")
+ log.WithError(cleanupErr).WithFields(log.Fields{
+ "user_id": userID,
+ "file_id": ineligibleErr.StaleDeletedFileID,
+ }).Error("Failed to clean stale deleted file membership")
}
}
return updated, err
diff --git a/server/pkg/repo/trash.go b/server/pkg/repo/trash.go
index aff6ab9df2..15d1dc27ef 100644
--- a/server/pkg/repo/trash.go
+++ b/server/pkg/repo/trash.go
@@ -239,25 +239,6 @@ func (t *TrashRepository) CleanUpDeletedFilesFromCollection(ctx context.Context,
return nil
}
-func (t *TrashRepository) GetStaleDeletedFileIDs(ctx context.Context, userID int64, limit int) ([]int64, error) {
- rows, err := t.DB.QueryContext(ctx, `SELECT DISTINCT cf.file_id
- FROM collections c
- JOIN collection_files cf ON cf.collection_id = c.collection_id AND cf.is_deleted = FALSE
- JOIN files f ON f.file_id = cf.file_id AND f.owner_id = $1
- JOIN trash t ON t.file_id = cf.file_id AND t.user_id = $1
- AND t.is_deleted = TRUE AND t.is_restored = FALSE
- WHERE c.owner_id = $1
- AND NOT EXISTS (
- SELECT 1 FROM object_keys ok
- WHERE ok.file_id = cf.file_id AND ok.is_deleted = FALSE
- )
- LIMIT $2`, userID, limit)
- if err != nil {
- return nil, stacktrace.Propagate(err, "")
- }
- return convertRowsToFileId(rows)
-}
-
func (t *TrashRepository) Delete(ctx context.Context, userID int64, fileIDs []int64) error {
if len(fileIDs) > TrashDiffLimit {
return fmt.Errorf("can not delete more than %d in one go", TrashDiffLimit)
@@ -347,23 +328,19 @@ func (t *TrashRepository) verifyFilesAreDeleted(ctx context.Context, userID int6
return stacktrace.NewError("all file ids are not deleted from trash")
}
- row := t.DB.QueryRowContext(ctx, `SELECT coalesce(sum(size),0) FROM object_keys WHERE file_id = ANY($1) and is_deleted = FALSE`,
- pq.Array(fileIDs))
- var totalUsage int64
- err = row.Scan(&totalUsage)
+ row := t.DB.QueryRowContext(ctx, `SELECT EXISTS (
+ SELECT 1 FROM object_keys WHERE file_id = ANY($1) AND is_deleted = FALSE
+ )`, pq.Array(fileIDs))
+ var hasLiveObjects bool
+ err = row.Scan(&hasLiveObjects)
if err != nil {
- if errors.Is(err, sql.ErrNoRows) {
- totalUsage = 0
- } else {
- return stacktrace.Propagate(err, "failed to get total usage for fileIDs")
- }
+ return stacktrace.Propagate(err, "failed to find live objects for fileIDs")
}
- if totalUsage != 0 {
+ if hasLiveObjects {
logrus.WithFields(logrus.Fields{
"user_id": userID,
"input_fileIds": fileIDs,
"trash_fileIds": filesDeleted,
- "total_usage": totalUsage,
}).Error("object_keys table still has entries for deleted files")
return stacktrace.NewError("object_keys table still has entries for deleted files")
}
diff --git a/server/pkg/repo/trash_test.go b/server/pkg/repo/trash_test.go
index b5d422a1c0..e22d3cb081 100644
--- a/server/pkg/repo/trash_test.go
+++ b/server/pkg/repo/trash_test.go
@@ -204,33 +204,26 @@ func TestStaleCleanupInvalidatesOnlyRemovedOwnedMemberships(t *testing.T) {
}
}
-func TestGetStaleDeletedFileIDs(t *testing.T) {
+func TestStaleCleanupRejectsLiveZeroByteObject(t *testing.T) {
repository, db, userID := setupCollectionMembershipTest(t)
- otherUserID := testutil.InsertUser(t, db, testutil.UserFixture{UserID: 2, Email: "[email protected]", CreationTime: 1})
collectionID := insertObjectTestCollection(t, db, userID)
- eligibleFileID := insertObjectTestFile(t, db, userID)
- liveObjectFileID := insertObjectTestFile(t, db, userID)
- activeTrashFileID := insertObjectTestFile(t, db, userID)
- wrongTrashOwnerFileID := insertObjectTestFile(t, db, userID)
- noTrashFileID := insertObjectTestFile(t, db, userID)
- for _, fileID := range []int64{eligibleFileID, liveObjectFileID, activeTrashFileID, wrongTrashOwnerFileID, noTrashFileID} {
- linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)
- }
- insertObjectTestKey(t, db, liveObjectFileID, ente.FILE, "live-zero-byte-object", 0, []string{"b2-eu-cen"})
+ fileID := insertObjectTestFile(t, db, userID)
+ linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)
+ insertObjectTestKey(t, db, fileID, ente.FILE, "live-zero-byte-object", 0, []string{"b2-eu-cen"})
if _, err := db.Exec(`INSERT INTO trash(file_id, collection_id, user_id, delete_by, updated_at, is_deleted)
- VALUES ($1, $5, $6, 1, 1, TRUE), ($2, $5, $6, 1, 1, TRUE),
- ($3, $5, $6, 1, 1, FALSE), ($4, $5, $7, 1, 1, TRUE)`,
- eligibleFileID, liveObjectFileID, activeTrashFileID, wrongTrashOwnerFileID,
- collectionID, userID, otherUserID); err != nil {
+ VALUES ($1, $2, $3, 1, 1, TRUE)`, fileID, collectionID, userID); err != nil {
t.Fatal(err)
}
- fileIDs, err := repository.TrashRepo.GetStaleDeletedFileIDs(t.Context(), userID, TrashBatchSize)
- if err != nil {
+ if err := repository.TrashRepo.CleanUpDeletedFilesFromCollection(t.Context(), []int64{fileID}, userID); err == nil {
+ t.Fatal("CleanUpDeletedFilesFromCollection() succeeded with a live object")
+ }
+ var deleted bool
+ if err := db.QueryRow(`SELECT is_deleted FROM collection_files WHERE file_id = $1`, fileID).Scan(&deleted); err != nil {
t.Fatal(err)
}
- if len(fileIDs) != 1 || fileIDs[0] != eligibleFileID {
- t.Fatalf("GetStaleDeletedFileIDs() = %v, want [%d]", fileIDs, eligibleFileID)
+ if deleted {
+ t.Fatal("collection membership was deleted")
}
}
diff --git a/server/pkg/repo/usage_file_count_init.go b/server/pkg/repo/usage_file_count_init.go
index 0a841e5221..04d08b2fee 100644
--- a/server/pkg/repo/usage_file_count_init.go
+++ b/server/pkg/repo/usage_file_count_init.go
@@ -8,6 +8,17 @@ import (
var ErrFileCountIneligible = errors.New("file counts are ineligible")
+type FileCountIneligibleError struct {
+ reason string
+ StaleDeletedFileID int64
+}
+
+func (e *FileCountIneligibleError) Error() string {
+ return fmt.Sprintf("%s: %s", ErrFileCountIneligible, e.reason)
+}
+
+func (e *FileCountIneligibleError) Unwrap() error { return ErrFileCountIneligible }
+
func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID int64) (bool, error) {
_, photos, _, err := repo.GetStoredFileCounts(ctx, userID)
if err != nil || photos != -1 {
@@ -21,7 +32,7 @@ func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID in
return false, nil
}
if counts.ineligibilityReason != "" {
- return false, fmt.Errorf("%w: %s", ErrFileCountIneligible, counts.ineligibilityReason)
+ return false, &FileCountIneligibleError{counts.ineligibilityReason, counts.staleDeletedFileID}
}
return repo.publishInitialFileCounts(ctx, userID, counts)
}
@@ -50,6 +61,7 @@ func (repo *UsageRepository) GetFileCountInitializationCandidates(ctx context.Co
type fileCountInitSnapshot struct {
photos, locker, version int64
+ staleDeletedFileID int64
uninitialized bool
ineligibilityReason string
}
@@ -71,6 +83,11 @@ func (repo *UsageRepository) readFileCountInitSnapshot(ctx context.Context, user
COUNT(DISTINCT file_id) FILTER (WHERE owner_id = $1 AND app = 'locker') AS locker,
BOOL_OR(app = 'locker' AND (f_owner_id IS NOT DISTINCT FROM $1) <> (owner_id = $1)) AS locker_mismatch
FROM memberships
+ ), trash_conflict AS (
+ SELECT f.file_id, t.user_id, t.is_deleted
+ FROM owned_files AS f
+ JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE
+ LIMIT 1
)
SELECT u.file_count_source_version,
u.photos_file_count IS NULL AND u.locker_file_count IS NULL,
@@ -84,16 +101,17 @@ func (repo *UsageRepository) readFileCountInitSnapshot(ctx context.Context, user
GROUP BY f.file_id
HAVING COUNT(DISTINCT c.app) > 1 OR BOOL_OR(c.app NOT IN ('photos', 'locker'))
) THEN 'cross-app or unsupported app memberships'
- WHEN EXISTS (
- SELECT 1 FROM owned_files AS f
- JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE
- ) THEN 'active membership or wrong owner in Trash'
+ WHEN trash_conflict.file_id IS NOT NULL THEN 'active membership or wrong owner in Trash'
WHEN source.locker_mismatch THEN 'Locker ownership predicates disagree'
ELSE ''
- END
+ END,
+ COALESCE(CASE WHEN trash_conflict.user_id = $1 AND trash_conflict.is_deleted
+ THEN trash_conflict.file_id END, 0)
FROM usage AS u CROSS JOIN source
+ LEFT JOIN trash_conflict ON TRUE
WHERE u.user_id = $1`, userID).Scan(
- &counts.version, &counts.uninitialized, &counts.photos, &counts.locker, &counts.ineligibilityReason)
+ &counts.version, &counts.uninitialized, &counts.photos, &counts.locker,
+ &counts.ineligibilityReason, &counts.staleDeletedFileID)
return counts, err
}
diff --git a/server/pkg/repo/usage_file_count_init_test.go b/server/pkg/repo/usage_file_count_init_test.go
index b362daa561..e1b7e357dc 100644
--- a/server/pkg/repo/usage_file_count_init_test.go
+++ b/server/pkg/repo/usage_file_count_init_test.go
@@ -52,19 +52,23 @@ func TestInitializeFileCountsIncludesLegacyMembershipsInDeletedCollections(t *te
func TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {
for _, tt := range []struct {
- name string
- sql string
+ name string
+ sql string
+ staleDeletedFileID int64
}{
{"cross_app", `INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)
- VALUES (102, 201, 'key', 'nonce', 1)`},
+ VALUES (102, 201, 'key', 'nonce', 1)`, 0},
{"cross_app_shared", `UPDATE collections SET owner_id = 2 WHERE collection_id = 102;
INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)
- VALUES (102, 201, 'key', 'nonce', 1)`},
- {"active_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 1, 101, 1)`},
- {"wrong_trash_owner", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 2, 101, 1)`},
- {"locker_null_owner", `UPDATE collection_files SET f_owner_id = NULL WHERE file_id = 201`},
+ VALUES (102, 201, 'key', 'nonce', 1)`, 0},
+ {"active_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 1, 101, 1)`, 0},
+ {"deleted_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)
+ VALUES (201, 1, 101, 1, TRUE)`, 201},
+ {"wrong_trash_owner", `INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)
+ VALUES (201, 2, 101, 1, TRUE)`, 0},
+ {"locker_null_owner", `UPDATE collection_files SET f_owner_id = NULL WHERE file_id = 201`, 0},
{"locker_equal_counts_different_files", `UPDATE collection_files
- SET f_owner_id = CASE WHEN file_id = 201 THEN NULL ELSE 1 END WHERE collection_id = 101`},
+ SET f_owner_id = CASE WHEN file_id = 201 THEN NULL ELSE 1 END WHERE collection_id = 101`, 0},
} {
t.Run(tt.name, func(t *testing.T) {
_, db, userID := setupCollectionMembershipTest(t)
@@ -91,6 +95,13 @@ func TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {
if initialized || !errors.Is(err, ErrFileCountIneligible) {
t.Fatalf("InitializeFileCounts() = (%t, %v), want ineligible", initialized, err)
}
+ var ineligibleErr *FileCountIneligibleError
+ if !errors.As(err, &ineligibleErr) {
+ t.Fatalf("error = %v, want FileCountIneligibleError", err)
+ }
+ if ineligibleErr.StaleDeletedFileID != tt.staleDeletedFileID {
+ t.Fatalf("stale deleted file = %d, want %d", ineligibleErr.StaleDeletedFileID, tt.staleDeletedFileID)
+ }
photos, locker, version := readFileCountState(t, db, userID)
if photos.Valid || locker.Valid || version != 0 {
t.Fatalf("ineligible user changed: (%v, %v, %d)", photos, locker, version)