π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Move file count backfill out of main
Neeraj Guptaa4d01cb289d5
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.goindex 65c85a2f48..f5cbe2dbb0 100644--- a/server/cmd/museum/main.go+++ b/server/cmd/museum/main.go@@ -4,7 +4,6 @@ import ("context""database/sql"b64 "encoding/base64"- "errors""fmt""net/http""os"@@ -299,6 +298,7 @@ func main() {FileRepo: fileRepo,UploadResultCache: make(map[int64]bool),}+ fileCountInitializer := &controller.FileCountInitializer{UsageRepo: usageRepo, LockController: lockController}accessCtrl := access.NewAccessController(accessCollectionRepo, accessFileRepo)commentsRepo := &socialrepo.CommentsRepository{DB: db}@@ -1108,7 +1108,7 @@ func main() {setupAndStartCrons(userAuthRepo, collectionLinkRepo, fileLinkRepo, pasteRepo, twoFactorRepo, passkeysRepo, fileController, taskLockingRepo, emailNotificationCtrl,trashController, pushController, objectController, dataCleanupController, storageBonusCtrl, emergencyCtrl,- embeddingController, healthCheckHandler, castDb, inactiveUserOrchestrator, spaceDripController, usageRepo, lockController)+ embeddingController, healthCheckHandler, castDb, inactiveUserOrchestrator, spaceDripController, fileCountInitializer)primaryDBCollector := sqlstats.NewStatsCollector("prod_db", db)latencySensitiveDBCollector := sqlstats.NewStatsCollector("latency_sensitive_db", latencySensitiveDB)@@ -1299,8 +1299,7 @@ func setupAndStartCrons(userAuthRepo *repo.UserAuthRepository, collectionLinkRepcastDb castRepo.Repository,inactiveUserOrchestrator *user.InactiveUserOrchestrator,spaceDripController *spacecontroller.SpaceDripController,- usageRepo *repo.UsageRepository,- lockController *lock.LockController) {+ fileCountInitializer *controller.FileCountInitializer) {if viper.GetBool("jobs.cron.skip") {log.Info("Skipping cron jobs")return@@ -1426,62 +1425,7 @@ func setupAndStartCrons(userAuthRepo *repo.UserAuthRepository, collectionLinkReppushController.ClearExpiredTokens()})- const (- fileCountInitializationBatchSize = 10- fileCountInitializationLock = "file_count_initialization"- )- var afterFileCountUserID int64- var resumeFileCountInitializationAt time.Time- schedule(c, "@every 1m", func() {- if !lockController.TryLock(fileCountInitializationLock, timeUtil.MicrosecondsAfterHours(3)) {- return- }- defer lockController.ReleaseLock(fileCountInitializationLock)- if time.Now().Before(resumeFileCountInitializationAt) {- return- }-- userIDs, err := usageRepo.GetFileCountInitializationCandidates(context.Background(), afterFileCountUserID, fileCountInitializationBatchSize)- if err != nil {- log.WithError(err).Error("Failed to fetch file count initialization candidates")- return- }- if len(userIDs) == 0 {- afterFileCountUserID = 0- resumeFileCountInitializationAt = time.Now().Add(24 * time.Hour)- return- }- initialized, deferred, ineligible := 0, 0, 0- for _, userID := range userIDs {- afterFileCountUserID = userID- updated, err := usageRepo.InitializeFileCounts(context.Background(), userID)- if errors.Is(err, repo.ErrFileCountIneligible) {- ineligible++- log.WithError(err).WithField("user_id", userID).Warn("File count initialization ineligible")- continue- }- if err != nil {- log.WithError(err).WithField("user_id", userID).Error("Failed to initialize file counts")- return- }- if updated {- initialized++- } else {- deferred++- }- }- if len(userIDs) < fileCountInitializationBatchSize {- afterFileCountUserID = 0- resumeFileCountInitializationAt = time.Now().Add(24 * time.Hour)- }- log.WithFields(log.Fields{- "attempted": len(userIDs),- "initialized": initialized,- "deferred": deferred,- "ineligible": ineligible,- "last_user_id": userIDs[len(userIDs)-1],- }).Info("Processed file count initialization batch")- })+ schedule(c, "@every 1m", fileCountInitializer.ProcessBatch)c.Start()}diff --git a/server/pkg/controller/usage_file_count_init.go b/server/pkg/controller/usage_file_count_init.gonew file mode 100644index 0000000000..098f58c209--- /dev/null+++ b/server/pkg/controller/usage_file_count_init.go@@ -0,0 +1,76 @@+package controller++import (+ "context"+ "errors"+ "time"++ "github.com/ente/museum/pkg/controller/lock"+ "github.com/ente/museum/pkg/repo"+ timeUtil "github.com/ente/museum/pkg/utils/time"+ log "github.com/sirupsen/logrus"+)++type FileCountInitializer struct {+ UsageRepo *repo.UsageRepository+ LockController *lock.LockController+ afterUserID int64+ resumeAt time.Time+}++const (+ fileCountInitializationBatchSize = 10+ fileCountInitializationLock = "file_count_initialization"+)++func (c *FileCountInitializer) ProcessBatch() {+ if !c.LockController.TryLock(fileCountInitializationLock, timeUtil.MicrosecondsAfterHours(3)) {+ return+ }+ defer c.LockController.ReleaseLock(fileCountInitializationLock)+ if time.Now().Before(c.resumeAt) {+ return+ }++ userIDs, err := c.UsageRepo.GetFileCountInitializationCandidates(context.Background(), c.afterUserID, fileCountInitializationBatchSize)+ if err != nil {+ log.WithError(err).Error("Failed to fetch file count initialization candidates")+ return+ }+ if len(userIDs) == 0 {+ c.afterUserID = 0+ c.resumeAt = time.Now().Add(24 * time.Hour)+ return+ }++ initialized, deferred, ineligible := 0, 0, 0+ for _, userID := range userIDs {+ c.afterUserID = userID+ updated, err := c.UsageRepo.InitializeFileCounts(context.Background(), userID)+ if errors.Is(err, repo.ErrFileCountIneligible) {+ ineligible+++ log.WithError(err).WithField("user_id", userID).Warn("File count initialization ineligible")+ continue+ }+ if err != nil {+ log.WithError(err).WithField("user_id", userID).Error("Failed to initialize file counts")+ return+ }+ if updated {+ initialized+++ } else {+ deferred+++ }+ }+ if len(userIDs) < fileCountInitializationBatchSize {+ c.afterUserID = 0+ c.resumeAt = time.Now().Add(24 * time.Hour)+ }+ log.WithFields(log.Fields{+ "attempted": len(userIDs),+ "initialized": initialized,+ "deferred": deferred,+ "ineligible": ineligible,+ "last_user_id": userIDs[len(userIDs)-1],+ }).Info("Processed file count initialization batch")+}