Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(taskrun): simplify scheduler with atomic claim and remove orphan restart (#18703)

p0ny6c811392d390
diff --git a/backend/component/state/state.go b/backend/component/state/state.go
index d11dd6e9e3..ed3c2634f9 100644
--- a/backend/component/state/state.go
+++ b/backend/component/state/state.go
@@ -15,8 +15,6 @@ type State struct {
// TaskRunConnectionID is the map from task run ID to the connection id of the connection to the database.
TaskRunConnectionID sync.Map // map[taskRunID]string
- // RunningTaskRuns is the set of running taskruns.
- RunningTaskRuns sync.Map // map[taskRunID]bool
// RunningTaskRunsCancelFunc is the cancelFunc of running taskruns.
RunningTaskRunsCancelFunc sync.Map // map[taskRunID]context.CancelFunc
diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.go
index a8f3354f3d..3f5d0de37e 100644
--- a/backend/runner/taskrun/running_scheduler.go
+++ b/backend/runner/taskrun/running_scheduler.go
@@ -45,73 +45,29 @@ func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.Wa
}
func (s *Scheduler) scheduleRunningTaskRuns(ctx context.Context) error {
- // Query AVAILABLE tasks (ready for execution)
- availableTaskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{
- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_AVAILABLE},
- })
+ // Atomically claim all AVAILABLE task runs
+ claimed, err := s.store.ClaimAvailableTaskRuns(ctx)
if err != nil {
- return errors.Wrapf(err, "failed to list available task runs")
+ return errors.Wrapf(err, "failed to claim available task runs")
}
- for _, taskRun := range availableTaskRuns {
- if err := s.claimAndExecuteTaskRun(ctx, taskRun); err != nil {
- slog.Error("failed to claim and execute task run", log.BBError(err))
- }
- }
-
- // Also re-execute orphaned RUNNING tasks (for restart recovery)
- runningTaskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{
- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_RUNNING},
- })
- if err != nil {
- return errors.Wrapf(err, "failed to list running task runs")
- }
-
- for _, taskRun := range runningTaskRuns {
- // Skip if already executing in this instance
- if _, ok := s.stateCfg.RunningTaskRuns.Load(taskRun.ID); ok {
- continue
- }
- // Re-execute orphaned RUNNING task
- if err := s.executeTaskRun(ctx, taskRun); err != nil {
- slog.Error("failed to re-execute orphaned task run", log.BBError(err))
+ for _, c := range claimed {
+ if err := s.executeTaskRun(ctx, c.TaskRunUID, c.TaskUID); err != nil {
+ slog.Error("failed to execute task run", slog.Int("id", c.TaskRunUID), log.BBError(err))
}
}
return nil
}
-// claimAndExecuteTaskRun attempts to atomically claim an AVAILABLE task and execute it.
-func (s *Scheduler) claimAndExecuteTaskRun(ctx context.Context, taskRun *store.TaskRunMessage) error {
- // Optimistic locking: attempt to claim by updating AVAILABLE -> RUNNING
- claimed, err := s.store.ClaimAvailableTaskRun(ctx, taskRun.ID)
- if err != nil {
- return errors.Wrapf(err, "failed to claim task run")
- }
- if !claimed {
- // Another instance claimed it
- return nil
- }
-
- return s.executeTaskRun(ctx, taskRun)
-}
-
// executeTaskRun executes a task run that is already in RUNNING status.
-func (s *Scheduler) executeTaskRun(ctx context.Context, taskRun *store.TaskRunMessage) error {
- task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)
+func (s *Scheduler) executeTaskRun(ctx context.Context, taskRunUID, taskUID int) error {
+ task, err := s.store.GetTaskByID(ctx, taskUID)
if err != nil {
return errors.Wrapf(err, "failed to get task")
}
-
- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})
- if err != nil {
- return errors.Wrapf(err, "failed to get instance")
- }
- if instance == nil {
- return errors.Errorf("instance %v not found", task.InstanceID)
- }
- if instance.Deleted {
- return errors.Errorf("instance %v is deleted", task.InstanceID)
+ if task == nil {
+ return errors.Errorf("task %v not found", taskUID)
}
executor, ok := s.executorMap[task.Type]
@@ -120,25 +76,22 @@ func (s *Scheduler) executeTaskRun(ctx context.Context, taskRun *store.TaskRunMe
}
// Update started_at
- if err := s.store.UpdateTaskRunStartAt(ctx, taskRun.ID); err != nil {
+ if err := s.store.UpdateTaskRunStartAt(ctx, taskRunUID); err != nil {
return errors.Wrapf(err, "failed to update task run start at")
}
- // Register as running
- s.stateCfg.RunningTaskRuns.Store(taskRun.ID, true)
-
- s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{
+ s.store.CreateTaskRunLogS(ctx, taskRunUID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{
Type: storepb.TaskRunLog_TASK_RUN_STATUS_UPDATE,
TaskRunStatusUpdate: &storepb.TaskRunLog_TaskRunStatusUpdate{
Status: storepb.TaskRunLog_TaskRunStatusUpdate_RUNNING_RUNNING,
},
})
- go s.runTaskRunOnce(ctx, taskRun, task, executor)
+ go s.runTaskRunOnce(ctx, taskRunUID, task, executor)
return nil
}
-func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMessage, task *store.TaskMessage, executor Executor) {
+func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRunUID int, task *store.TaskMessage, executor Executor) {
defer func() {
if r := recover(); r != nil {
err, ok := r.(error)
@@ -149,14 +102,14 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMe
}
}()
defer func() {
- s.stateCfg.RunningTaskRunsCancelFunc.Delete(taskRun.ID)
+ s.stateCfg.RunningTaskRunsCancelFunc.Delete(taskRunUID)
}()
driverCtx, cancel := context.WithCancel(ctx)
defer cancel()
- s.stateCfg.RunningTaskRunsCancelFunc.Store(taskRun.ID, cancel)
+ s.stateCfg.RunningTaskRunsCancelFunc.Store(taskRunUID, cancel)
- done, result, err := RunExecutorOnce(ctx, driverCtx, executor, task, taskRun.ID)
+ done, result, err := RunExecutorOnce(ctx, driverCtx, executor, task, taskRunUID)
switch {
case !done && err != nil:
@@ -189,7 +142,7 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMe
code := common.Ok
result := string(resultBytes)
taskRunStatusPatch := &store.TaskRunStatusPatch{
- ID: taskRun.ID,
+ ID: taskRunUID,
Updater: common.SystemBotEmail,
Status: storepb.TaskRun_CANCELED,
Code: &code,
@@ -232,7 +185,7 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMe
code := common.ErrorCode(err)
result := string(resultBytes)
taskRunStatusPatch := &store.TaskRunStatusPatch{
- ID: taskRun.ID,
+ ID: taskRunUID,
Updater: common.SystemBotEmail,
Status: storepb.TaskRun_FAILED,
Code: &code,
@@ -262,7 +215,7 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMe
code := common.Ok
result := string(resultBytes)
taskRunStatusPatch := &store.TaskRunStatusPatch{
- ID: taskRun.ID,
+ ID: taskRunUID,
Updater: common.SystemBotEmail,
Status: storepb.TaskRun_DONE,
Code: &code,
diff --git a/backend/store/task_run.go b/backend/store/task_run.go
index 1967da416e..d057fac1e5 100644
--- a/backend/store/task_run.go
+++ b/backend/store/task_run.go
@@ -221,31 +221,50 @@ func (s *Store) UpdateTaskRunStatus(ctx context.Context, patch *TaskRunStatusPat
return taskRun, nil
}
-// ClaimAvailableTaskRun attempts to atomically claim an AVAILABLE task run by updating it to RUNNING.
-// Returns true if the claim succeeded, false if another process claimed it first.
-func (s *Store) ClaimAvailableTaskRun(ctx context.Context, taskRunID int) (bool, error) {
+// ClaimedTaskRun represents a claimed task run with its task UID.
+type ClaimedTaskRun struct {
+ TaskRunUID int
+ TaskUID int
+}
+
+// ClaimAvailableTaskRuns atomically claims all AVAILABLE task runs by updating them to RUNNING
+// and returns the claimed task run and task UIDs. This combines list + claim into a single atomic operation.
+// Uses FOR UPDATE SKIP LOCKED to allow concurrent schedulers to claim different tasks.
+func (s *Store) ClaimAvailableTaskRuns(ctx context.Context) ([]*ClaimedTaskRun, error) {
q := qb.Q().Space(`
UPDATE task_run
SET status = ?, updated_at = now()
- WHERE id = ? AND status = ?
- `, storepb.TaskRun_RUNNING.String(), taskRunID, storepb.TaskRun_AVAILABLE.String())
+ WHERE id IN (
+ SELECT task_run.id FROM task_run
+ WHERE task_run.status = ?
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING id, task_id
+ `, storepb.TaskRun_RUNNING.String(), storepb.TaskRun_AVAILABLE.String())
query, args, err := q.ToSQL()
if err != nil {
- return false, errors.Wrapf(err, "failed to build sql")
+ return nil, errors.Wrapf(err, "failed to build sql")
}
- result, err := s.GetDB().ExecContext(ctx, query, args...)
+ rows, err := s.GetDB().QueryContext(ctx, query, args...)
if err != nil {
- return false, errors.Wrapf(err, "failed to claim task run")
+ return nil, errors.Wrapf(err, "failed to claim task runs")
}
+ defer rows.Close()
- rowsAffected, err := result.RowsAffected()
- if err != nil {
- return false, errors.Wrapf(err, "failed to get rows affected")
+ var claimed []*ClaimedTaskRun
+ for rows.Next() {
+ var c ClaimedTaskRun
+ if err := rows.Scan(&c.TaskRunUID, &c.TaskUID); err != nil {
+ return nil, err
+ }
+ claimed = append(claimed, &c)
}
-
- return rowsAffected == 1, nil
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return claimed, nil
}
func (s *Store) UpdateTaskRunStartAt(ctx context.Context, taskRunID int) error {