π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Simplify file count cutover
Neeraj Gupta78ce1f831491
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.goindex 48bf877c1e..f8b34cd43f 100644--- a/server/cmd/museum/main.go+++ b/server/cmd/museum/main.go@@ -245,7 +245,7 @@ func main() {userCache := cache2.NewUserCache()userCacheCtrl := &usercache.Controller{UserCache: userCache, FileRepo: fileRepo,UsageRepo: usageRepo, TrashRepo: trashRepo,- StoreBonusRepo: storagBonusRepo, QueueFileCountInitialization: fileCountInitializer.Enqueue}+ StoreBonusRepo: storagBonusRepo}offerController := offer.NewOfferController(*userRepo, discordController, storagBonusRepo, userCacheCtrl)plans := billing.GetPlans()defaultPlan := billing.GetDefaultPlans(plans)diff --git a/server/pkg/controller/user/account_deletion_summary.go b/server/pkg/controller/user/account_deletion_summary.goindex 5bc7461daa..9b81a899fe 100644--- a/server/pkg/controller/user/account_deletion_summary.go+++ b/server/pkg/controller/user/account_deletion_summary.go@@ -16,7 +16,7 @@ func (c *UserController) GetAccountDeletionSummary(ctx context.Context, userID ig := new(errgroup.Group)g.Go(func() error {- count, err := c.UserCacheController.GetUserFileCountWithCache(ctx, userID, ente.Photos)+ count, err := c.UserCacheController.GetUserFileCountWithCache(userID, ente.Photos)if err != nil {return stacktrace.Propagate(err, "failed to get photos file count")}@@ -24,7 +24,7 @@ func (c *UserController) GetAccountDeletionSummary(ctx context.Context, userID ireturn nil})g.Go(func() error {- count, err := c.UserCacheController.GetUserFileCountWithCache(ctx, userID, ente.Locker)+ count, err := c.UserCacheController.GetUserFileCountWithCache(userID, ente.Locker)if err != nil {return stacktrace.Propagate(err, "failed to get locker file count")}diff --git a/server/pkg/controller/user/user_details.go b/server/pkg/controller/user/user_details.goindex 957cc8fa54..5938ea3b6d 100644--- a/server/pkg/controller/user/user_details.go+++ b/server/pkg/controller/user/user_details.go@@ -97,7 +97,7 @@ func (c *UserController) GetDetailsV2(ctx *gin.Context, userID int64, fetchMemorif fetchMemoryCount {g.Go(func() error {- fCount, err := c.UserCacheController.GetUserFileCountWithCache(ctx, userID, app)+ fCount, err := c.UserCacheController.GetUserFileCountWithCache(userID, app)if err == nil {fileCount = fCount}diff --git a/server/pkg/controller/usercache/controller.go b/server/pkg/controller/usercache/controller.goindex 325010c417..9c0ca3d230 100644--- a/server/pkg/controller/usercache/controller.go+++ b/server/pkg/controller/usercache/controller.go@@ -11,12 +11,11 @@ import (// Avoid adding any direct dependencies to the other controller.type Controller struct {- FileRepo *repo.FileRepository- UsageRepo *repo.UsageRepository- TrashRepo *repo.TrashRepository- StoreBonusRepo *storagebonus.Repository- UserCache *cache.UserCache- QueueFileCountInitialization func(int64)+ FileRepo *repo.FileRepository+ UsageRepo *repo.UsageRepository+ TrashRepo *repo.TrashRepository+ StoreBonusRepo *storagebonus.Repository+ UserCache *cache.UserCache}func (c *Controller) GetActiveStorageBonus(ctx context.Context, userID int64) (*bonus.ActiveStorageBonus, error) {diff --git a/server/pkg/controller/usercache/count.go b/server/pkg/controller/usercache/count.goindex da625cdc46..caefe72c40 100644--- a/server/pkg/controller/usercache/count.go+++ b/server/pkg/controller/usercache/count.go@@ -9,8 +9,8 @@ import ("github.com/sirupsen/logrus")-func (c *Controller) GetUserFileCountWithCache(ctx context.Context, userID int64, app ente.App) (int64, error) {- _, photos, locker, err := c.UsageRepo.GetStoredFileCounts(ctx, userID)+func (c *Controller) GetUserFileCountWithCache(userID int64, app ente.App) (int64, error) {+ _, photos, locker, err := c.UsageRepo.GetStoredFileCounts(context.Background(), userID)if err != nil {return 0, stacktrace.Propagate(err, "")}@@ -24,8 +24,8 @@ func (c *Controller) GetUserFileCountWithCache(ctx context.Context, userID int64if count >= 0 {return count, nil}- if (app == ente.Photos || app == ente.Locker) && c.QueueFileCountInitialization != nil {- c.QueueFileCountInitialization(userID)+ if (app == ente.Photos || app == ente.Locker) && c.UsageRepo.QueueFileCountInitialization != nil {+ c.UsageRepo.QueueFileCountInitialization(userID)}if count, ok := c.UserCache.GetFileCount(userID, app); ok {// Cache hit, update the cache asynchronouslydiff --git a/server/pkg/controller/usercache/count_test.go b/server/pkg/controller/usercache/count_test.goindex c77ade2234..853daa614e 100644--- a/server/pkg/controller/usercache/count_test.go+++ b/server/pkg/controller/usercache/count_test.go@@ -23,13 +23,12 @@ func TestGetUserFileCountUsesStoredCountOrQueuesInitialization(t *testing.T) {userCache := cache.NewUserCache()userCache.SetFileCount(userID, &cache.FileCountCache{Count: 41}, ente.Photos)controller := &Controller{- FileRepo: &repo.FileRepository{DB: db},- UsageRepo: &repo.UsageRepository{DB: db},- TrashRepo: &repo.TrashRepository{DB: db},- UserCache: userCache,- QueueFileCountInitialization: func(int64) { queued++ },+ FileRepo: &repo.FileRepository{DB: db},+ UsageRepo: &repo.UsageRepository{DB: db, QueueFileCountInitialization: func(int64) { queued++ }},+ TrashRepo: &repo.TrashRepository{DB: db},+ UserCache: userCache,}- count, err := controller.GetUserFileCountWithCache(t.Context(), userID, ente.Photos)+ count, err := controller.GetUserFileCountWithCache(userID, ente.Photos)if err != nil || count != 9 || queued != 0 {t.Fatalf("ready count = (%d, %v), queued %d; want (9, nil), queued 0", count, err, queued)}@@ -38,7 +37,7 @@ func TestGetUserFileCountUsesStoredCountOrQueuesInitialization(t *testing.T) {t.Fatal(err)}controller.UserCache = cache.NewUserCache()- count, err = controller.GetUserFileCountWithCache(t.Context(), userID, ente.Photos)+ count, err = controller.GetUserFileCountWithCache(userID, ente.Photos)if err != nil || count != 0 || queued != 1 {t.Fatalf("legacy count = (%d, %v), queued %d; want (0, nil), queued 1", count, err, queued)}diff --git a/server/pkg/repo/file_count_read_test.go b/server/pkg/repo/file_count_read_test.goindex 0167e67e78..a07e48de5b 100644--- a/server/pkg/repo/file_count_read_test.go+++ b/server/pkg/repo/file_count_read_test.go@@ -1,10 +1,8 @@package repoimport (- "fmt""testing"- "github.com/ente/museum/ente""github.com/ente/museum/internal/testutil")@@ -16,15 +14,12 @@ func TestGetLockerUsageUsesStoredAndLegacyCountsPerUser(t *testing.T) {testutil.InsertUsage(t, db, legacyUserID, 0)setReadyFileCounts(t, db, readyUserID, 0, 9)- for _, userID := range []int64{readyUserID, legacyUserID} {- fileID := insertObjectTestFile(t, db, userID)- collectionID := insertObjectTestCollection(t, db, userID)- if _, err := db.Exec(`UPDATE collections SET app = 'locker' WHERE collection_id = $1`, collectionID); err != nil {- t.Fatal(err)- }- linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)- insertObjectTestKey(t, db, fileID, ente.FILE, fmt.Sprintf("object-%d", userID), 100, []string{"b2-eu-cen"})+ fileID := insertObjectTestFile(t, db, legacyUserID)+ collectionID := insertObjectTestCollection(t, db, legacyUserID)+ if _, err := db.Exec(`UPDATE collections SET app = 'locker' WHERE collection_id = $1`, collectionID); err != nil {+ t.Fatal(err)}+ linkObjectTestFileToCollection(t, db, collectionID, fileID, legacyUserID)var queued []int64usage, err := (&UsageRepository{@@ -34,8 +29,8 @@ func TestGetLockerUsageUsesStoredAndLegacyCountsPerUser(t *testing.T) {if err != nil {t.Fatal(err)}- if usage.TotalFileCount != 10 || usage.TotalUsage != 200 {- t.Fatalf("family usage = %+v, want 10 files and 200 bytes", usage)+ if usage.TotalFileCount != 10 {+ t.Fatalf("family usage = %+v, want 10 files", usage)}if len(usage.Users) != 2 || usage.Users[0].FileCount != 9 || usage.Users[1].FileCount != 1 {t.Fatalf("member usage = %+v, want stored then legacy count", usage.Users)diff --git a/server/pkg/repo/usage.go b/server/pkg/repo/usage.goindex f383e92d5e..864daeeb36 100644--- a/server/pkg/repo/usage.go+++ b/server/pkg/repo/usage.go@@ -126,11 +126,7 @@ func (repo *UsageRepository) GetLockerUsage(ctx context.Context, userIDs []int64userMap := make(map[int64]*UserLockerUsage)for _, userID := range userIDs {- userMap[userID] = &UserLockerUsage{- UserID: userID,- FileCount: 0,- Usage: 0,- }+ userMap[userID] = &UserLockerUsage{UserID: userID}}rows, err := repo.DB.QueryContext(ctx, `SELECT requested.user_id, u.locker_file_count@@ -139,12 +135,12 @@ func (repo *UsageRepository) GetLockerUsage(ctx context.Context, userIDs []int64if err != nil {return nil, stacktrace.Propagate(err, "")}+ defer rows.Close()var uninitializedUserIDs []int64for rows.Next() {var userID int64var fileCount sql.NullInt64if err := rows.Scan(&userID, &fileCount); err != nil {- rows.Close()return nil, stacktrace.Propagate(err, "")}if fileCount.Valid {@@ -153,9 +149,6 @@ func (repo *UsageRepository) GetLockerUsage(ctx context.Context, userIDs []int64uninitializedUserIDs = append(uninitializedUserIDs, userID)}}- if err := rows.Close(); err != nil {- return nil, stacktrace.Propagate(err, "")- }if err := rows.Err(); err != nil {return nil, stacktrace.Propagate(err, "")}@@ -175,17 +168,14 @@ func (repo *UsageRepository) GetLockerUsage(ctx context.Context, userIDs []int64if err != nil {return nil, stacktrace.Propagate(err, "")}+ defer rows.Close()for rows.Next() {var userID, fileCount int64if err := rows.Scan(&userID, &fileCount); err != nil {- rows.Close()return nil, stacktrace.Propagate(err, "")}userMap[userID].FileCount = fileCount}- if err := rows.Close(); err != nil {- return nil, stacktrace.Propagate(err, "")- }if err := rows.Err(); err != nil {return nil, stacktrace.Propagate(err, "")}