Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Parallelize file count backfill across hosts

Neeraj Guptaaf3111ad718d
diff --git a/server/pkg/controller/lock/lock.go b/server/pkg/controller/lock/lock.go
index 82ecf495a8..cee160b3cf 100644
--- a/server/pkg/controller/lock/lock.go
+++ b/server/pkg/controller/lock/lock.go
@@ -35,7 +35,7 @@ func (c *LockController) ExtendLock(lockID string, lockUntil int64) error {
}
func (c *LockController) ReleaseLock(lockID string) {
- err := c.TaskLockingRepo.ReleaseLock(lockID)
+ err := c.TaskLockingRepo.ReleaseLockBy(lockID, c.HostName)
if err != nil {
log.Errorf("Error while releasing lock %v: %s", lockID, err)
}
diff --git a/server/pkg/controller/usage_file_count_init.go b/server/pkg/controller/usage_file_count_init.go
index 098f58c209..626612212f 100644
--- a/server/pkg/controller/usage_file_count_init.go
+++ b/server/pkg/controller/usage_file_count_init.go
@@ -3,6 +3,8 @@ package controller
import (
"context"
"errors"
+ "strconv"
+ "sync/atomic"
"time"
"github.com/ente/museum/pkg/controller/lock"
@@ -16,6 +18,7 @@ type FileCountInitializer struct {
LockController *lock.LockController
afterUserID int64
resumeAt time.Time
+ running atomic.Bool
}
const (
@@ -24,10 +27,10 @@ const (
)
func (c *FileCountInitializer) ProcessBatch() {
- if !c.LockController.TryLock(fileCountInitializationLock, timeUtil.MicrosecondsAfterHours(3)) {
+ if !c.running.CompareAndSwap(false, true) {
return
}
- defer c.LockController.ReleaseLock(fileCountInitializationLock)
+ defer c.running.Store(false)
if time.Now().Before(c.resumeAt) {
return
}
@@ -43,10 +46,18 @@ func (c *FileCountInitializer) ProcessBatch() {
return
}
- initialized, deferred, ineligible := 0, 0, 0
+ initialized, deferred, ineligible, contended := 0, 0, 0, 0
for _, userID := range userIDs {
c.afterUserID = userID
- updated, err := c.UsageRepo.InitializeFileCounts(context.Background(), userID)
+ lockID := fileCountInitializationLock + ":" + strconv.FormatInt(userID, 10)
+ if !c.LockController.TryLock(lockID, timeUtil.MicrosecondsAfterHours(3)) {
+ contended++
+ continue
+ }
+ updated, err := func() (bool, error) {
+ defer c.LockController.ReleaseLock(lockID)
+ return c.UsageRepo.InitializeFileCounts(context.Background(), userID)
+ }()
if errors.Is(err, repo.ErrFileCountIneligible) {
ineligible++
log.WithError(err).WithField("user_id", userID).Warn("File count initialization ineligible")
@@ -71,6 +82,7 @@ func (c *FileCountInitializer) ProcessBatch() {
"initialized": initialized,
"deferred": deferred,
"ineligible": ineligible,
+ "contended": contended,
"last_user_id": userIDs[len(userIDs)-1],
}).Info("Processed file count initialization batch")
}
diff --git a/server/pkg/repo/tasklock.go b/server/pkg/repo/tasklock.go
index a7cb00f8bb..c4986a0830 100644
--- a/server/pkg/repo/tasklock.go
+++ b/server/pkg/repo/tasklock.go
@@ -64,6 +64,11 @@ func (repo *TaskLockRepository) ReleaseLock(name string) error {
return stacktrace.Propagate(err, "")
}
+func (repo *TaskLockRepository) ReleaseLockBy(name string, lockedBy string) error {
+ _, err := repo.DB.Exec(`DELETE FROM task_lock WHERE task_name = $1 AND locked_by = $2`, name, lockedBy)
+ return stacktrace.Propagate(err, "")
+}
+
func (repo *TaskLockRepository) ReleaseLocksBy(lockedBy string) (*int64, error) {
result, err := repo.DB.Exec(`DELETE FROM task_lock WHERE locked_by = $1`, lockedBy)
if err != nil {