Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Simplify file count cutover

Neeraj Gupta78ce1f831491
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go
index 48bf877c1e..f8b34cd43f 100644
--- a/server/cmd/museum/main.go
+++ b/server/cmd/museum/main.go
@@ -245,7 +245,7 @@ func main() {
userCache := cache2.NewUserCache()
userCacheCtrl := &usercache.Controller{UserCache: userCache, FileRepo: fileRepo,
UsageRepo: usageRepo, TrashRepo: trashRepo,
- StoreBonusRepo: storagBonusRepo, QueueFileCountInitialization: fileCountInitializer.Enqueue}
+ StoreBonusRepo: storagBonusRepo}
offerController := offer.NewOfferController(*userRepo, discordController, storagBonusRepo, userCacheCtrl)
plans := billing.GetPlans()
defaultPlan := billing.GetDefaultPlans(plans)
diff --git a/server/pkg/controller/user/account_deletion_summary.go b/server/pkg/controller/user/account_deletion_summary.go
index 5bc7461daa..9b81a899fe 100644
--- a/server/pkg/controller/user/account_deletion_summary.go
+++ b/server/pkg/controller/user/account_deletion_summary.go
@@ -16,7 +16,7 @@ func (c *UserController) GetAccountDeletionSummary(ctx context.Context, userID i
g := new(errgroup.Group)
g.Go(func() error {
- count, err := c.UserCacheController.GetUserFileCountWithCache(ctx, userID, ente.Photos)
+ count, err := c.UserCacheController.GetUserFileCountWithCache(userID, ente.Photos)
if err != nil {
return stacktrace.Propagate(err, "failed to get photos file count")
}
@@ -24,7 +24,7 @@ func (c *UserController) GetAccountDeletionSummary(ctx context.Context, userID i
return nil
})
g.Go(func() error {
- count, err := c.UserCacheController.GetUserFileCountWithCache(ctx, userID, ente.Locker)
+ count, err := c.UserCacheController.GetUserFileCountWithCache(userID, ente.Locker)
if err != nil {
return stacktrace.Propagate(err, "failed to get locker file count")
}
diff --git a/server/pkg/controller/user/user_details.go b/server/pkg/controller/user/user_details.go
index 957cc8fa54..5938ea3b6d 100644
--- a/server/pkg/controller/user/user_details.go
+++ b/server/pkg/controller/user/user_details.go
@@ -97,7 +97,7 @@ func (c *UserController) GetDetailsV2(ctx *gin.Context, userID int64, fetchMemor
if fetchMemoryCount {
g.Go(func() error {
- fCount, err := c.UserCacheController.GetUserFileCountWithCache(ctx, userID, app)
+ fCount, err := c.UserCacheController.GetUserFileCountWithCache(userID, app)
if err == nil {
fileCount = fCount
}
diff --git a/server/pkg/controller/usercache/controller.go b/server/pkg/controller/usercache/controller.go
index 325010c417..9c0ca3d230 100644
--- a/server/pkg/controller/usercache/controller.go
+++ b/server/pkg/controller/usercache/controller.go
@@ -11,12 +11,11 @@ import (
// Avoid adding any direct dependencies to the other controller.
type Controller struct {
- FileRepo *repo.FileRepository
- UsageRepo *repo.UsageRepository
- TrashRepo *repo.TrashRepository
- StoreBonusRepo *storagebonus.Repository
- UserCache *cache.UserCache
- QueueFileCountInitialization func(int64)
+ FileRepo *repo.FileRepository
+ UsageRepo *repo.UsageRepository
+ TrashRepo *repo.TrashRepository
+ StoreBonusRepo *storagebonus.Repository
+ UserCache *cache.UserCache
}
func (c *Controller) GetActiveStorageBonus(ctx context.Context, userID int64) (*bonus.ActiveStorageBonus, error) {
diff --git a/server/pkg/controller/usercache/count.go b/server/pkg/controller/usercache/count.go
index da625cdc46..caefe72c40 100644
--- a/server/pkg/controller/usercache/count.go
+++ b/server/pkg/controller/usercache/count.go
@@ -9,8 +9,8 @@ import (
"github.com/sirupsen/logrus"
)
-func (c *Controller) GetUserFileCountWithCache(ctx context.Context, userID int64, app ente.App) (int64, error) {
- _, photos, locker, err := c.UsageRepo.GetStoredFileCounts(ctx, userID)
+func (c *Controller) GetUserFileCountWithCache(userID int64, app ente.App) (int64, error) {
+ _, photos, locker, err := c.UsageRepo.GetStoredFileCounts(context.Background(), userID)
if err != nil {
return 0, stacktrace.Propagate(err, "")
}
@@ -24,8 +24,8 @@ func (c *Controller) GetUserFileCountWithCache(ctx context.Context, userID int64
if count >= 0 {
return count, nil
}
- if (app == ente.Photos || app == ente.Locker) && c.QueueFileCountInitialization != nil {
- c.QueueFileCountInitialization(userID)
+ if (app == ente.Photos || app == ente.Locker) && c.UsageRepo.QueueFileCountInitialization != nil {
+ c.UsageRepo.QueueFileCountInitialization(userID)
}
if count, ok := c.UserCache.GetFileCount(userID, app); ok {
// Cache hit, update the cache asynchronously
diff --git a/server/pkg/controller/usercache/count_test.go b/server/pkg/controller/usercache/count_test.go
index c77ade2234..853daa614e 100644
--- a/server/pkg/controller/usercache/count_test.go
+++ b/server/pkg/controller/usercache/count_test.go
@@ -23,13 +23,12 @@ func TestGetUserFileCountUsesStoredCountOrQueuesInitialization(t *testing.T) {
userCache := cache.NewUserCache()
userCache.SetFileCount(userID, &cache.FileCountCache{Count: 41}, ente.Photos)
controller := &Controller{
- FileRepo: &repo.FileRepository{DB: db},
- UsageRepo: &repo.UsageRepository{DB: db},
- TrashRepo: &repo.TrashRepository{DB: db},
- UserCache: userCache,
- QueueFileCountInitialization: func(int64) { queued++ },
+ FileRepo: &repo.FileRepository{DB: db},
+ UsageRepo: &repo.UsageRepository{DB: db, QueueFileCountInitialization: func(int64) { queued++ }},
+ TrashRepo: &repo.TrashRepository{DB: db},
+ UserCache: userCache,
}
- count, err := controller.GetUserFileCountWithCache(t.Context(), userID, ente.Photos)
+ count, err := controller.GetUserFileCountWithCache(userID, ente.Photos)
if err != nil || count != 9 || queued != 0 {
t.Fatalf("ready count = (%d, %v), queued %d; want (9, nil), queued 0", count, err, queued)
}
@@ -38,7 +37,7 @@ func TestGetUserFileCountUsesStoredCountOrQueuesInitialization(t *testing.T) {
t.Fatal(err)
}
controller.UserCache = cache.NewUserCache()
- count, err = controller.GetUserFileCountWithCache(t.Context(), userID, ente.Photos)
+ count, err = controller.GetUserFileCountWithCache(userID, ente.Photos)
if err != nil || count != 0 || queued != 1 {
t.Fatalf("legacy count = (%d, %v), queued %d; want (0, nil), queued 1", count, err, queued)
}
diff --git a/server/pkg/repo/file_count_read_test.go b/server/pkg/repo/file_count_read_test.go
index 0167e67e78..a07e48de5b 100644
--- a/server/pkg/repo/file_count_read_test.go
+++ b/server/pkg/repo/file_count_read_test.go
@@ -1,10 +1,8 @@
package repo
import (
- "fmt"
"testing"
- "github.com/ente/museum/ente"
"github.com/ente/museum/internal/testutil"
)
@@ -16,15 +14,12 @@ func TestGetLockerUsageUsesStoredAndLegacyCountsPerUser(t *testing.T) {
testutil.InsertUsage(t, db, legacyUserID, 0)
setReadyFileCounts(t, db, readyUserID, 0, 9)
- for _, userID := range []int64{readyUserID, legacyUserID} {
- fileID := insertObjectTestFile(t, db, userID)
- collectionID := insertObjectTestCollection(t, db, userID)
- if _, err := db.Exec(`UPDATE collections SET app = 'locker' WHERE collection_id = $1`, collectionID); err != nil {
- t.Fatal(err)
- }
- linkObjectTestFileToCollection(t, db, collectionID, fileID, userID)
- insertObjectTestKey(t, db, fileID, ente.FILE, fmt.Sprintf("object-%d", userID), 100, []string{"b2-eu-cen"})
+ fileID := insertObjectTestFile(t, db, legacyUserID)
+ collectionID := insertObjectTestCollection(t, db, legacyUserID)
+ if _, err := db.Exec(`UPDATE collections SET app = 'locker' WHERE collection_id = $1`, collectionID); err != nil {
+ t.Fatal(err)
}
+ linkObjectTestFileToCollection(t, db, collectionID, fileID, legacyUserID)
var queued []int64
usage, err := (&UsageRepository{
@@ -34,8 +29,8 @@ func TestGetLockerUsageUsesStoredAndLegacyCountsPerUser(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if usage.TotalFileCount != 10 || usage.TotalUsage != 200 {
- t.Fatalf("family usage = %+v, want 10 files and 200 bytes", usage)
+ if usage.TotalFileCount != 10 {
+ t.Fatalf("family usage = %+v, want 10 files", usage)
}
if len(usage.Users) != 2 || usage.Users[0].FileCount != 9 || usage.Users[1].FileCount != 1 {
t.Fatalf("member usage = %+v, want stored then legacy count", usage.Users)
diff --git a/server/pkg/repo/usage.go b/server/pkg/repo/usage.go
index f383e92d5e..864daeeb36 100644
--- a/server/pkg/repo/usage.go
+++ b/server/pkg/repo/usage.go
@@ -126,11 +126,7 @@ func (repo *UsageRepository) GetLockerUsage(ctx context.Context, userIDs []int64
userMap := make(map[int64]*UserLockerUsage)
for _, userID := range userIDs {
- userMap[userID] = &UserLockerUsage{
- UserID: userID,
- FileCount: 0,
- Usage: 0,
- }
+ userMap[userID] = &UserLockerUsage{UserID: userID}
}
rows, err := repo.DB.QueryContext(ctx, `SELECT requested.user_id, u.locker_file_count
@@ -139,12 +135,12 @@ func (repo *UsageRepository) GetLockerUsage(ctx context.Context, userIDs []int64
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
+ defer rows.Close()
var uninitializedUserIDs []int64
for rows.Next() {
var userID int64
var fileCount sql.NullInt64
if err := rows.Scan(&userID, &fileCount); err != nil {
- rows.Close()
return nil, stacktrace.Propagate(err, "")
}
if fileCount.Valid {
@@ -153,9 +149,6 @@ func (repo *UsageRepository) GetLockerUsage(ctx context.Context, userIDs []int64
uninitializedUserIDs = append(uninitializedUserIDs, userID)
}
}
- if err := rows.Close(); err != nil {
- return nil, stacktrace.Propagate(err, "")
- }
if err := rows.Err(); err != nil {
return nil, stacktrace.Propagate(err, "")
}
@@ -175,17 +168,14 @@ func (repo *UsageRepository) GetLockerUsage(ctx context.Context, userIDs []int64
if err != nil {
return nil, stacktrace.Propagate(err, "")
}
+ defer rows.Close()
for rows.Next() {
var userID, fileCount int64
if err := rows.Scan(&userID, &fileCount); err != nil {
- rows.Close()
return nil, stacktrace.Propagate(err, "")
}
userMap[userID].FileCount = fileCount
}
- if err := rows.Close(); err != nil {
- return nil, stacktrace.Propagate(err, "")
- }
if err := rows.Err(); err != nil {
return nil, stacktrace.Propagate(err, "")
}