Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Move file count backfill out of main

Neeraj Guptaa4d01cb289d5
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go
index 65c85a2f48..f5cbe2dbb0 100644
--- a/server/cmd/museum/main.go
+++ b/server/cmd/museum/main.go
@@ -4,7 +4,6 @@ import (
"context"
"database/sql"
b64 "encoding/base64"
- "errors"
"fmt"
"net/http"
"os"
@@ -299,6 +298,7 @@ func main() {
FileRepo: fileRepo,
UploadResultCache: make(map[int64]bool),
}
+ fileCountInitializer := &controller.FileCountInitializer{UsageRepo: usageRepo, LockController: lockController}
accessCtrl := access.NewAccessController(accessCollectionRepo, accessFileRepo)
commentsRepo := &socialrepo.CommentsRepository{DB: db}
@@ -1108,7 +1108,7 @@ func main() {
setupAndStartCrons(
userAuthRepo, collectionLinkRepo, fileLinkRepo, pasteRepo, twoFactorRepo, passkeysRepo, fileController, taskLockingRepo, emailNotificationCtrl,
trashController, pushController, objectController, dataCleanupController, storageBonusCtrl, emergencyCtrl,
- embeddingController, healthCheckHandler, castDb, inactiveUserOrchestrator, spaceDripController, usageRepo, lockController)
+ embeddingController, healthCheckHandler, castDb, inactiveUserOrchestrator, spaceDripController, fileCountInitializer)
primaryDBCollector := sqlstats.NewStatsCollector("prod_db", db)
latencySensitiveDBCollector := sqlstats.NewStatsCollector("latency_sensitive_db", latencySensitiveDB)
@@ -1299,8 +1299,7 @@ func setupAndStartCrons(userAuthRepo *repo.UserAuthRepository, collectionLinkRep
castDb castRepo.Repository,
inactiveUserOrchestrator *user.InactiveUserOrchestrator,
spaceDripController *spacecontroller.SpaceDripController,
- usageRepo *repo.UsageRepository,
- lockController *lock.LockController) {
+ fileCountInitializer *controller.FileCountInitializer) {
if viper.GetBool("jobs.cron.skip") {
log.Info("Skipping cron jobs")
return
@@ -1426,62 +1425,7 @@ func setupAndStartCrons(userAuthRepo *repo.UserAuthRepository, collectionLinkRep
pushController.ClearExpiredTokens()
})
- const (
- fileCountInitializationBatchSize = 10
- fileCountInitializationLock = "file_count_initialization"
- )
- var afterFileCountUserID int64
- var resumeFileCountInitializationAt time.Time
- schedule(c, "@every 1m", func() {
- if !lockController.TryLock(fileCountInitializationLock, timeUtil.MicrosecondsAfterHours(3)) {
- return
- }
- defer lockController.ReleaseLock(fileCountInitializationLock)
- if time.Now().Before(resumeFileCountInitializationAt) {
- return
- }
-
- userIDs, err := usageRepo.GetFileCountInitializationCandidates(context.Background(), afterFileCountUserID, fileCountInitializationBatchSize)
- if err != nil {
- log.WithError(err).Error("Failed to fetch file count initialization candidates")
- return
- }
- if len(userIDs) == 0 {
- afterFileCountUserID = 0
- resumeFileCountInitializationAt = time.Now().Add(24 * time.Hour)
- return
- }
- initialized, deferred, ineligible := 0, 0, 0
- for _, userID := range userIDs {
- afterFileCountUserID = userID
- updated, err := usageRepo.InitializeFileCounts(context.Background(), userID)
- if errors.Is(err, repo.ErrFileCountIneligible) {
- ineligible++
- log.WithError(err).WithField("user_id", userID).Warn("File count initialization ineligible")
- continue
- }
- if err != nil {
- log.WithError(err).WithField("user_id", userID).Error("Failed to initialize file counts")
- return
- }
- if updated {
- initialized++
- } else {
- deferred++
- }
- }
- if len(userIDs) < fileCountInitializationBatchSize {
- afterFileCountUserID = 0
- resumeFileCountInitializationAt = time.Now().Add(24 * time.Hour)
- }
- log.WithFields(log.Fields{
- "attempted": len(userIDs),
- "initialized": initialized,
- "deferred": deferred,
- "ineligible": ineligible,
- "last_user_id": userIDs[len(userIDs)-1],
- }).Info("Processed file count initialization batch")
- })
+ schedule(c, "@every 1m", fileCountInitializer.ProcessBatch)
c.Start()
}
diff --git a/server/pkg/controller/usage_file_count_init.go b/server/pkg/controller/usage_file_count_init.go
new file mode 100644
index 0000000000..098f58c209
--- /dev/null
+++ b/server/pkg/controller/usage_file_count_init.go
@@ -0,0 +1,76 @@
+package controller
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/ente/museum/pkg/controller/lock"
+ "github.com/ente/museum/pkg/repo"
+ timeUtil "github.com/ente/museum/pkg/utils/time"
+ log "github.com/sirupsen/logrus"
+)
+
+type FileCountInitializer struct {
+ UsageRepo *repo.UsageRepository
+ LockController *lock.LockController
+ afterUserID int64
+ resumeAt time.Time
+}
+
+const (
+ fileCountInitializationBatchSize = 10
+ fileCountInitializationLock = "file_count_initialization"
+)
+
+func (c *FileCountInitializer) ProcessBatch() {
+ if !c.LockController.TryLock(fileCountInitializationLock, timeUtil.MicrosecondsAfterHours(3)) {
+ return
+ }
+ defer c.LockController.ReleaseLock(fileCountInitializationLock)
+ if time.Now().Before(c.resumeAt) {
+ return
+ }
+
+ userIDs, err := c.UsageRepo.GetFileCountInitializationCandidates(context.Background(), c.afterUserID, fileCountInitializationBatchSize)
+ if err != nil {
+ log.WithError(err).Error("Failed to fetch file count initialization candidates")
+ return
+ }
+ if len(userIDs) == 0 {
+ c.afterUserID = 0
+ c.resumeAt = time.Now().Add(24 * time.Hour)
+ return
+ }
+
+ initialized, deferred, ineligible := 0, 0, 0
+ for _, userID := range userIDs {
+ c.afterUserID = userID
+ updated, err := c.UsageRepo.InitializeFileCounts(context.Background(), userID)
+ if errors.Is(err, repo.ErrFileCountIneligible) {
+ ineligible++
+ log.WithError(err).WithField("user_id", userID).Warn("File count initialization ineligible")
+ continue
+ }
+ if err != nil {
+ log.WithError(err).WithField("user_id", userID).Error("Failed to initialize file counts")
+ return
+ }
+ if updated {
+ initialized++
+ } else {
+ deferred++
+ }
+ }
+ if len(userIDs) < fileCountInitializationBatchSize {
+ c.afterUserID = 0
+ c.resumeAt = time.Now().Add(24 * time.Hour)
+ }
+ log.WithFields(log.Fields{
+ "attempted": len(userIDs),
+ "initialized": initialized,
+ "deferred": deferred,
+ "ineligible": ineligible,
+ "last_user_id": userIDs[len(userIDs)-1],
+ }).Info("Processed file count initialization batch")
+}