π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Backfill file counts incrementally
Neeraj Guptaedec1ba4b7af
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.goindex f346ba5d49..65c85a2f48 100644--- a/server/cmd/museum/main.go+++ b/server/cmd/museum/main.go@@ -4,6 +4,7 @@ import ("context""database/sql"b64 "encoding/base64"+ "errors""fmt""net/http""os"@@ -1107,7 +1108,7 @@ func main() {setupAndStartCrons(userAuthRepo, collectionLinkRepo, fileLinkRepo, pasteRepo, twoFactorRepo, passkeysRepo, fileController, taskLockingRepo, emailNotificationCtrl,trashController, pushController, objectController, dataCleanupController, storageBonusCtrl, emergencyCtrl,- embeddingController, healthCheckHandler, castDb, inactiveUserOrchestrator, spaceDripController)+ embeddingController, healthCheckHandler, castDb, inactiveUserOrchestrator, spaceDripController, usageRepo, lockController)primaryDBCollector := sqlstats.NewStatsCollector("prod_db", db)latencySensitiveDBCollector := sqlstats.NewStatsCollector("latency_sensitive_db", latencySensitiveDB)@@ -1297,7 +1298,9 @@ func setupAndStartCrons(userAuthRepo *repo.UserAuthRepository, collectionLinkRephealthCheckHandler *api.HealthCheckHandler,castDb castRepo.Repository,inactiveUserOrchestrator *user.InactiveUserOrchestrator,- spaceDripController *spacecontroller.SpaceDripController) {+ spaceDripController *spacecontroller.SpaceDripController,+ usageRepo *repo.UsageRepository,+ lockController *lock.LockController) {if viper.GetBool("jobs.cron.skip") {log.Info("Skipping cron jobs")return@@ -1423,6 +1426,63 @@ func setupAndStartCrons(userAuthRepo *repo.UserAuthRepository, collectionLinkReppushController.ClearExpiredTokens()})+ const (+ fileCountInitializationBatchSize = 10+ fileCountInitializationLock = "file_count_initialization"+ )+ var afterFileCountUserID int64+ var resumeFileCountInitializationAt time.Time+ schedule(c, "@every 1m", func() {+ if !lockController.TryLock(fileCountInitializationLock, timeUtil.MicrosecondsAfterHours(3)) {+ return+ }+ defer lockController.ReleaseLock(fileCountInitializationLock)+ if time.Now().Before(resumeFileCountInitializationAt) {+ return+ }++ userIDs, err := usageRepo.GetFileCountInitializationCandidates(context.Background(), afterFileCountUserID, fileCountInitializationBatchSize)+ if err != nil {+ log.WithError(err).Error("Failed to fetch file count initialization candidates")+ return+ }+ if len(userIDs) == 0 {+ afterFileCountUserID = 0+ resumeFileCountInitializationAt = time.Now().Add(24 * time.Hour)+ return+ }+ initialized, deferred, ineligible := 0, 0, 0+ for _, userID := range userIDs {+ afterFileCountUserID = userID+ updated, err := usageRepo.InitializeFileCounts(context.Background(), userID)+ if errors.Is(err, repo.ErrFileCountIneligible) {+ ineligible+++ log.WithError(err).WithField("user_id", userID).Warn("File count initialization ineligible")+ continue+ }+ if err != nil {+ log.WithError(err).WithField("user_id", userID).Error("Failed to initialize file counts")+ return+ }+ if updated {+ initialized+++ } else {+ deferred+++ }+ }+ if len(userIDs) < fileCountInitializationBatchSize {+ afterFileCountUserID = 0+ resumeFileCountInitializationAt = time.Now().Add(24 * time.Hour)+ }+ log.WithFields(log.Fields{+ "attempted": len(userIDs),+ "initialized": initialized,+ "deferred": deferred,+ "ineligible": ineligible,+ "last_user_id": userIDs[len(userIDs)-1],+ }).Info("Processed file count initialization batch")+ })+c.Start()}diff --git a/server/pkg/repo/usage_file_count_init.go b/server/pkg/repo/usage_file_count_init.goindex 1457cc8c61..da2324246c 100644--- a/server/pkg/repo/usage_file_count_init.go+++ b/server/pkg/repo/usage_file_count_init.go@@ -26,6 +26,28 @@ func (repo *UsageRepository) InitializeFileCounts(ctx context.Context, userID inreturn repo.publishInitialFileCounts(ctx, userID, counts)}+func (repo *UsageRepository) GetFileCountInitializationCandidates(ctx context.Context, afterUserID int64, limit int) ([]int64, error) {+ rows, err := repo.DB.QueryContext(ctx, `SELECT usage.user_id FROM usage+ JOIN users ON users.user_id = usage.user_id AND users.encrypted_email IS NOT NULL+ WHERE usage.user_id > $1 AND usage.storage_consumed > 0+ AND photos_file_count IS NULL AND locker_file_count IS NULL+ ORDER BY usage.user_id LIMIT $2`, afterUserID, limit)+ if err != nil {+ return nil, err+ }+ defer rows.Close()++ var userIDs []int64+ for rows.Next() {+ var userID int64+ if err := rows.Scan(&userID); err != nil {+ return nil, err+ }+ userIDs = append(userIDs, userID)+ }+ return userIDs, rows.Err()+}+type fileCountInitSnapshot struct {photos, locker, version int64uninitialized bool