π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Parallelize file count backfill across hosts
Neeraj Guptaaf3111ad718d
diff --git a/server/pkg/controller/lock/lock.go b/server/pkg/controller/lock/lock.goindex 82ecf495a8..cee160b3cf 100644--- a/server/pkg/controller/lock/lock.go+++ b/server/pkg/controller/lock/lock.go@@ -35,7 +35,7 @@ func (c *LockController) ExtendLock(lockID string, lockUntil int64) error {}func (c *LockController) ReleaseLock(lockID string) {- err := c.TaskLockingRepo.ReleaseLock(lockID)+ err := c.TaskLockingRepo.ReleaseLockBy(lockID, c.HostName)if err != nil {log.Errorf("Error while releasing lock %v: %s", lockID, err)}diff --git a/server/pkg/controller/usage_file_count_init.go b/server/pkg/controller/usage_file_count_init.goindex 098f58c209..626612212f 100644--- a/server/pkg/controller/usage_file_count_init.go+++ b/server/pkg/controller/usage_file_count_init.go@@ -3,6 +3,8 @@ package controllerimport ("context""errors"+ "strconv"+ "sync/atomic""time""github.com/ente/museum/pkg/controller/lock"@@ -16,6 +18,7 @@ type FileCountInitializer struct {LockController *lock.LockControllerafterUserID int64resumeAt time.Time+ running atomic.Bool}const (@@ -24,10 +27,10 @@ const ()func (c *FileCountInitializer) ProcessBatch() {- if !c.LockController.TryLock(fileCountInitializationLock, timeUtil.MicrosecondsAfterHours(3)) {+ if !c.running.CompareAndSwap(false, true) {return}- defer c.LockController.ReleaseLock(fileCountInitializationLock)+ defer c.running.Store(false)if time.Now().Before(c.resumeAt) {return}@@ -43,10 +46,18 @@ func (c *FileCountInitializer) ProcessBatch() {return}- initialized, deferred, ineligible := 0, 0, 0+ initialized, deferred, ineligible, contended := 0, 0, 0, 0for _, userID := range userIDs {c.afterUserID = userID- updated, err := c.UsageRepo.InitializeFileCounts(context.Background(), userID)+ lockID := fileCountInitializationLock + ":" + strconv.FormatInt(userID, 10)+ if !c.LockController.TryLock(lockID, timeUtil.MicrosecondsAfterHours(3)) {+ contended+++ continue+ }+ updated, err := func() (bool, error) {+ defer c.LockController.ReleaseLock(lockID)+ return c.UsageRepo.InitializeFileCounts(context.Background(), userID)+ }()if errors.Is(err, repo.ErrFileCountIneligible) {ineligible++log.WithError(err).WithField("user_id", userID).Warn("File count initialization ineligible")@@ -71,6 +82,7 @@ func (c *FileCountInitializer) ProcessBatch() {"initialized": initialized,"deferred": deferred,"ineligible": ineligible,+ "contended": contended,"last_user_id": userIDs[len(userIDs)-1],}).Info("Processed file count initialization batch")}diff --git a/server/pkg/repo/tasklock.go b/server/pkg/repo/tasklock.goindex a7cb00f8bb..c4986a0830 100644--- a/server/pkg/repo/tasklock.go+++ b/server/pkg/repo/tasklock.go@@ -64,6 +64,11 @@ func (repo *TaskLockRepository) ReleaseLock(name string) error {return stacktrace.Propagate(err, "")}+func (repo *TaskLockRepository) ReleaseLockBy(name string, lockedBy string) error {+ _, err := repo.DB.Exec(`DELETE FROM task_lock WHERE task_name = $1 AND locked_by = $2`, name, lockedBy)+ return stacktrace.Propagate(err, "")+}+func (repo *TaskLockRepository) ReleaseLocksBy(lockedBy string) (*int64, error) {result, err := repo.DB.Exec(`DELETE FROM task_lock WHERE locked_by = $1`, lockedBy)if err != nil {