π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Initialize file counts from active memberships
Neeraj Guptaddca7d3bf873
diff --git a/server/pkg/api/admin.go b/server/pkg/api/admin.goindex 33ec95fb9c..ad8112787d 100644--- a/server/pkg/api/admin.go+++ b/server/pkg/api/admin.go@@ -710,7 +710,7 @@ func (h *AdminHandler) attachSubscription(ctx *gin.Context, userID int64, responif err == nil {response["details"] = details}- photos, locker, err := h.UsageRepo.GetFileCounts(ctx.Request.Context(), userID)+ photos, locker, err := h.UsageRepo.GetStoredFileCounts(ctx.Request.Context(), userID)if err == nil {response["photosFileCount"] = photosresponse["lockerFileCount"] = lockerdiff --git a/server/pkg/api/admin_file_count_test.go b/server/pkg/api/admin_file_count_test.goindex 9c2fc495b3..51a178d712 100644--- a/server/pkg/api/admin_file_count_test.go+++ b/server/pkg/api/admin_file_count_test.go@@ -24,10 +24,14 @@ func TestInitializeFileCountsHandler(t *testing.T) {{"initialize", "", http.StatusOK, `{"initialized":true}`},{"already_ready", `UPDATE usage SET photos_file_count = 0, locker_file_count = 0`,http.StatusOK, `{"initialized":false}`},- {"ineligible", `INSERT INTO files(owner_id, file_decryption_header, thumbnail_decryption_header,+ {"ineligible", `INSERT INTO collections(owner_id, encrypted_key, key_decryption_nonce, name, type, attributes, updation_time, app)+ VALUES (1, 'key', 'nonce', 'name', 'album', '{}', 1, 'auth');+ INSERT INTO files(owner_id, file_decryption_header, thumbnail_decryption_header,metadata_decryption_header, encrypted_metadata, updation_time)- VALUES (1, 'header', 'header', 'header', 'metadata', 1)`,- http.StatusOK, `{"initialized":false,"reason":"file counts are ineligible: untrashed file without an owned membership"}`},+ VALUES (1, 'header', 'header', 'header', 'metadata', 1);+ INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)+ SELECT collection_id, file_id, 'key', 'nonce', 1 FROM collections CROSS JOIN files`,+ http.StatusOK, `{"initialized":false,"reason":"file counts are ineligible: cross-app or unsupported app memberships"}`},{"missing_usage", `DELETE FROM usage`, http.StatusNotFound, `{}`},} {t.Run(tt.name, func(t *testing.T) {diff --git a/server/pkg/repo/usage.go b/server/pkg/repo/usage.goindex 64e3bcc139..293de6567b 100644--- a/server/pkg/repo/usage.go+++ b/server/pkg/repo/usage.go@@ -43,8 +43,7 @@ func (repo *UsageRepository) GetUsage(userID int64) (int64, error) {return usage, stacktrace.Propagate(err, "")}-// GetFileCounts returns the stored counts, or -1 for both until initialized.-func (repo *UsageRepository) GetFileCounts(ctx context.Context, userID int64) (int64, int64, error) {+func (repo *UsageRepository) GetStoredFileCounts(ctx context.Context, userID int64) (int64, int64, error) {var photos, locker int64err := repo.DB.QueryRowContext(ctx, `SELECT photos_file_count, locker_file_countFROM usage WHERE user_id = $1diff --git a/server/pkg/repo/usage_file_count_init.go b/server/pkg/repo/usage_file_count_init.goindex 82d726eb1a..1ff588313a 100644--- a/server/pkg/repo/usage_file_count_init.go+++ b/server/pkg/repo/usage_file_count_init.go@@ -8,79 +8,73 @@ import (var ErrFileCountIneligible = errors.New("file counts are ineligible")-// InitializeFileCounts publishes both counters without holding a usage-row lock-// while counting. All count-changing writers must be deployed before calling it.func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID int64) (bool, error) {- counts, err := repo.readInitialFileCounts(ctx, userID)+ photos, _, err := repo.GetStoredFileCounts(ctx, userID)+ if err != nil || photos != -1 {+ return false, err+ }+ counts, err := repo.readFileCountInitSnapshot(ctx, userID)if err != nil {return false, err}- if !counts.legacy {+ if !counts.uninitialized {return false, nil}- if counts.ineligible != "" {- return false, fmt.Errorf("%w: %s", ErrFileCountIneligible, counts.ineligible)+ if counts.ineligibilityReason != "" {+ return false, fmt.Errorf("%w: %s", ErrFileCountIneligible, counts.ineligibilityReason)}return repo.publishInitialFileCounts(ctx, userID, counts)}-type initialFileCounts struct {+type fileCountInitSnapshot struct {photos, locker, version int64- legacy bool- ineligible string+ uninitialized bool+ ineligibilityReason string}-func (repo *UsageRepository) readInitialFileCounts(ctx context.Context, userID int64) (initialFileCounts, error) {- var counts initialFileCounts- // One statement gives the source version, counts and checks the same snapshot.- // The per-file aggregate keeps membership lookups bounded to this owner's files.- err := repo.DB.QueryRowContext(ctx, `WITH source AS (+func (repo *UsageRepository) readFileCountInitSnapshot(ctx context.Context, userID int64) (fileCountInitSnapshot, error) {+ var counts fileCountInitSnapshot+ err := repo.DB.QueryRowContext(ctx, `WITH memberships AS MATERIALIZED (+ SELECT cf.file_id, c.app, f.owner_id, cf.f_owner_id+ FROM collections AS c+ JOIN collection_files AS cf ON cf.collection_id = c.collection_id AND cf.is_deleted = FALSE+ JOIN files AS f ON f.file_id = cf.file_id+ WHERE c.owner_id = $1+ ), owned_files AS MATERIALIZED (+ SELECT DISTINCT file_id FROM memberships WHERE owner_id = $1+ ), source AS (SELECT- COUNT(*) FILTER (WHERE memberships.photos) AS photos,- COUNT(*) FILTER (WHERE memberships.locker) AS locker,- BOOL_OR(memberships.app_count > 1 OR memberships.unsupported_app) AS ambiguous_app,- BOOL_OR(memberships.photos IS NOT TRUE AND memberships.locker IS NOT TRUE- AND t.file_id IS NULL) AS orphan,- BOOL_OR(t.file_id IS NOT NULL- AND (memberships.photos OR memberships.locker OR t.user_id <> $1)) AS inconsistent_trash- FROM files AS f- CROSS JOIN LATERAL (- SELECT- BOOL_OR(c.owner_id = $1 AND c.app = 'photos') AS photos,- BOOL_OR(c.owner_id = $1 AND c.app = 'locker') AS locker,- COUNT(DISTINCT c.app) AS app_count,- BOOL_OR(c.app NOT IN ('photos', 'locker')) AS unsupported_app- FROM collection_files AS cf- JOIN collections AS c ON c.collection_id = cf.collection_id- WHERE cf.file_id = f.file_id AND cf.is_deleted = FALSE- ) AS memberships- LEFT JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE- WHERE f.owner_id = $1+ COUNT(DISTINCT file_id) FILTER (WHERE owner_id = $1 AND app = 'photos') AS photos,+ COUNT(DISTINCT file_id) FILTER (WHERE owner_id = $1 AND app = 'locker') AS locker,+ BOOL_OR(app = 'locker' AND (f_owner_id IS NOT DISTINCT FROM $1) <> (owner_id = $1)) AS locker_mismatch+ FROM memberships)SELECT u.file_count_source_version,u.photos_file_count IS NULL AND u.locker_file_count IS NULL,source.photos, source.locker,CASE- WHEN source.ambiguous_app THEN 'cross-app or unsupported app memberships'- WHEN source.orphan THEN 'untrashed file without an owned membership'- WHEN source.inconsistent_trash THEN 'active membership or wrong owner in Trash'WHEN EXISTS (SELECT 1- FROM collections AS c- JOIN collection_files AS cf ON cf.collection_id = c.collection_id- JOIN files AS f ON f.file_id = cf.file_id- WHERE c.owner_id = $1 AND c.app = 'locker' AND cf.is_deleted = FALSE- AND (cf.f_owner_id IS NOT DISTINCT FROM $1) <> (f.owner_id = $1)- ) THEN 'Locker ownership predicates disagree'+ FROM owned_files AS f+ JOIN collection_files AS cf ON cf.file_id = f.file_id AND cf.is_deleted = FALSE+ JOIN collections AS c ON c.collection_id = cf.collection_id+ GROUP BY f.file_id+ HAVING COUNT(DISTINCT c.app) > 1 OR BOOL_OR(c.app NOT IN ('photos', 'locker'))+ ) THEN 'cross-app or unsupported app memberships'+ WHEN EXISTS (+ SELECT 1 FROM owned_files AS f+ JOIN trash AS t ON t.file_id = f.file_id AND t.is_restored = FALSE+ ) THEN 'active membership or wrong owner in Trash'+ WHEN source.locker_mismatch THEN 'Locker ownership predicates disagree'ELSE ''ENDFROM usage AS u CROSS JOIN sourceWHERE u.user_id = $1`, userID).Scan(- &counts.version, &counts.legacy, &counts.photos, &counts.locker, &counts.ineligible)+ &counts.version, &counts.uninitialized, &counts.photos, &counts.locker, &counts.ineligibilityReason)return counts, err}-func (repo *UsageRepository) publishInitialFileCounts(ctx context.Context, userID int64, counts initialFileCounts) (bool, error) {+func (repo *UsageRepository) publishInitialFileCounts(ctx context.Context, userID int64, counts fileCountInitSnapshot) (bool, error) {result, err := repo.DB.ExecContext(ctx, `UPDATE usageSET photos_file_count = $2, locker_file_count = $3WHERE user_id = $1 AND file_count_source_version = $4diff --git a/server/pkg/repo/usage_file_count_init_test.go b/server/pkg/repo/usage_file_count_init_test.goindex 72e61816f5..b362daa561 100644--- a/server/pkg/repo/usage_file_count_init_test.go+++ b/server/pkg/repo/usage_file_count_init_test.go@@ -9,7 +9,7 @@ import ("github.com/ente/museum/pkg/repo/public")-func TestInitializeFileCountsUsesActiveOwnedMemberships(t *testing.T) {+func TestInitializeFileCountsIncludesLegacyMembershipsInDeletedCollections(t *testing.T) {_, db, userID := setupCollectionMembershipTest(t)usageRepo := &UsageRepository{DB: db}otherUserID := testutil.InsertUser(t, db, testutil.UserFixture{UserID: 2, Email: "[email protected]", CreationTime: 1})@@ -20,8 +20,6 @@ func TestInitializeFileCountsUsesActiveOwnedMemberships(t *testing.T) {collectionID := insertObjectTestCollection(t, db, userID)linkObjectTestFileToCollection(t, db, collectionID, photosFile, userID)linkObjectTestFileToCollection(t, db, collectionID, foreignFile, otherUserID)- // Pre-60 Photos memberships have NULL denormalized owners. A collection- // scheduled for deletion remains countable until its memberships are trashed.if _, err := db.Exec(`UPDATE collection_files SET c_owner_id = NULL, f_owner_id = NULLWHERE collection_id = $1`, collectionID); err != nil {t.Fatal(err)@@ -57,16 +55,15 @@ func TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {name stringsql string}{- {"orphan", `UPDATE collection_files SET is_deleted = TRUE WHERE file_id = 201`},- {"shared_only", `UPDATE collections SET owner_id = 2 WHERE collection_id = 101`},{"cross_app", `INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)VALUES (102, 201, 'key', 'nonce', 1)`},+ {"cross_app_shared", `UPDATE collections SET owner_id = 2 WHERE collection_id = 102;+ INSERT INTO collection_files(collection_id, file_id, encrypted_key, key_decryption_nonce, updation_time)+ VALUES (102, 201, 'key', 'nonce', 1)`},{"active_trash", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 1, 101, 1)`},- {"wrong_trash_owner", `UPDATE collection_files SET is_deleted = TRUE WHERE file_id = 201;- INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 2, 101, 1)`},+ {"wrong_trash_owner", `INSERT INTO trash(file_id, user_id, collection_id, delete_by) VALUES (201, 2, 101, 1)`},{"locker_null_owner", `UPDATE collection_files SET f_owner_id = NULL WHERE file_id = 201`},- // The legacy and canonical totals both equal one; different files qualify.- {"locker_canceling_errors", `UPDATE collection_files+ {"locker_equal_counts_different_files", `UPDATE collection_filesSET f_owner_id = CASE WHEN file_id = 201 THEN NULL ELSE 1 END WHERE collection_id = 101`},} {t.Run(tt.name, func(t *testing.T) {@@ -102,20 +99,22 @@ func TestInitializeFileCountsRejectsInconsistentHistory(t *testing.T) {}}-func TestInitializeFileCountsAllowsEmptyAndPermanentlyDeletedLibraries(t *testing.T) {- for _, name := range []string{"empty", "permanently_deleted"} {+func TestInitializeFileCountsExcludesInactiveFiles(t *testing.T) {+ for _, name := range []string{"empty", "orphan", "permanently_deleted"} {t.Run(name, func(t *testing.T) {_, db, userID := setupCollectionMembershipTest(t)- if name == "permanently_deleted" {+ if name != "empty" {collectionID := insertObjectTestCollection(t, db, userID)fileID := insertObjectTestFile(t, db, userID)linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)if _, err := db.Exec(`UPDATE collection_files SET is_deleted = TRUE WHERE file_id = $1`, fileID); err != nil {t.Fatal(err)}- if _, err := db.Exec(`INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)- VALUES ($1, $2, $3, 1, TRUE)`, fileID, userID, collectionID); err != nil {- t.Fatal(err)+ if name == "permanently_deleted" {+ if _, err := db.Exec(`INSERT INTO trash(file_id, user_id, collection_id, delete_by, is_deleted)+ VALUES ($1, $2, $3, 1, TRUE)`, fileID, userID, collectionID); err != nil {+ t.Fatal(err)+ }}}if initialized, err := (&UsageRepository{DB: db}).InitializeFileCounts(t.Context(), userID); err != nil || !initialized {@@ -126,7 +125,7 @@ func TestInitializeFileCountsAllowsEmptyAndPermanentlyDeletedLibraries(t *testin}}-func TestInitializeFileCountsSerializesWithWriters(t *testing.T) {+func TestInitializeFileCountsRejectsStaleSnapshots(t *testing.T) {repository, db, userID := setupCollectionMembershipTest(t)usageRepo := &UsageRepository{DB: db}collectionID := insertObjectTestCollection(t, db, userID)@@ -134,7 +133,7 @@ func TestInitializeFileCountsSerializesWithWriters(t *testing.T) {linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)repository.TrashRepo.FileLinkRepo = public.NewFileLinkRepo(db)- counts, err := usageRepo.readInitialFileCounts(t.Context(), userID)+ counts, err := usageRepo.readFileCountInitSnapshot(t.Context(), userID)if err != nil {t.Fatal(err)}diff --git a/server/pkg/repo/usage_file_count_test.go b/server/pkg/repo/usage_file_count_test.goindex c07d6a117a..b2bdb01c69 100644--- a/server/pkg/repo/usage_file_count_test.go+++ b/server/pkg/repo/usage_file_count_test.go@@ -8,7 +8,7 @@ import ("github.com/ente/museum/pkg/repo/public")-func TestGetFileCountsReturnsStoredCountsOrUninitialized(t *testing.T) {+func TestGetStoredFileCountsReturnsMinusOneUntilInitialized(t *testing.T) {_, db, userID := setupCollectionMembershipTest(t)usageRepo := &UsageRepository{DB: db}for _, tt := range []struct {@@ -17,8 +17,7 @@ func TestGetFileCountsReturnsStoredCountsOrUninitialized(t *testing.T) {photos, locker int64}{{"uninitialized", "", -1, -1},- {"empty", `UPDATE usage SET photos_file_count = 0, locker_file_count = 0 WHERE user_id = $1`, 0, 0},- {"ready", `UPDATE usage SET photos_file_count = 23, locker_file_count = 7 WHERE user_id = $1`, 23, 7},+ {"ready", `UPDATE usage SET photos_file_count = 0, locker_file_count = 7 WHERE user_id = $1`, 0, 7},{"missing", `DELETE FROM usage WHERE user_id = $1`, -1, -1},} {t.Run(tt.name, func(t *testing.T) {@@ -27,9 +26,9 @@ func TestGetFileCountsReturnsStoredCountsOrUninitialized(t *testing.T) {t.Fatal(err)}}- photos, locker, err := usageRepo.GetFileCounts(t.Context(), userID)+ photos, locker, err := usageRepo.GetStoredFileCounts(t.Context(), userID)if err != nil || photos != tt.photos || locker != tt.locker {- t.Fatalf("GetFileCounts() = (%d, %d, %v), want (%d, %d, nil)", photos, locker, err, tt.photos, tt.locker)+ t.Fatalf("GetStoredFileCounts() = (%d, %d, %v), want (%d, %d, nil)", photos, locker, err, tt.photos, tt.locker)}})}