Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(taskrun): simplify scheduler with atomic claim and remove orphan restart (#18703)
p0ny6c811392d390
diff --git a/backend/component/state/state.go b/backend/component/state/state.goindex d11dd6e9e3..ed3c2634f9 100644--- a/backend/component/state/state.go+++ b/backend/component/state/state.go@@ -15,8 +15,6 @@ type State struct {// TaskRunConnectionID is the map from task run ID to the connection id of the connection to the database.TaskRunConnectionID sync.Map // map[taskRunID]string- // RunningTaskRuns is the set of running taskruns.- RunningTaskRuns sync.Map // map[taskRunID]bool// RunningTaskRunsCancelFunc is the cancelFunc of running taskruns.RunningTaskRunsCancelFunc sync.Map // map[taskRunID]context.CancelFuncdiff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.goindex a8f3354f3d..3f5d0de37e 100644--- a/backend/runner/taskrun/running_scheduler.go+++ b/backend/runner/taskrun/running_scheduler.go@@ -45,73 +45,29 @@ func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.Wa}func (s *Scheduler) scheduleRunningTaskRuns(ctx context.Context) error {- // Query AVAILABLE tasks (ready for execution)- availableTaskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_AVAILABLE},- })+ // Atomically claim all AVAILABLE task runs+ claimed, err := s.store.ClaimAvailableTaskRuns(ctx)if err != nil {- return errors.Wrapf(err, "failed to list available task runs")+ return errors.Wrapf(err, "failed to claim available task runs")}- for _, taskRun := range availableTaskRuns {- if err := s.claimAndExecuteTaskRun(ctx, taskRun); err != nil {- slog.Error("failed to claim and execute task run", log.BBError(err))- }- }-- // Also re-execute orphaned RUNNING tasks (for restart recovery)- runningTaskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_RUNNING},- })- if err != nil {- return errors.Wrapf(err, "failed to list running task runs")- }-- for _, taskRun := range runningTaskRuns {- // Skip if already executing in this instance- if _, ok := s.stateCfg.RunningTaskRuns.Load(taskRun.ID); ok {- continue- }- // Re-execute orphaned RUNNING task- if err := s.executeTaskRun(ctx, taskRun); err != nil {- slog.Error("failed to re-execute orphaned task run", log.BBError(err))+ for _, c := range claimed {+ if err := s.executeTaskRun(ctx, c.TaskRunUID, c.TaskUID); err != nil {+ slog.Error("failed to execute task run", slog.Int("id", c.TaskRunUID), log.BBError(err))}}return nil}-// claimAndExecuteTaskRun attempts to atomically claim an AVAILABLE task and execute it.-func (s *Scheduler) claimAndExecuteTaskRun(ctx context.Context, taskRun *store.TaskRunMessage) error {- // Optimistic locking: attempt to claim by updating AVAILABLE -> RUNNING- claimed, err := s.store.ClaimAvailableTaskRun(ctx, taskRun.ID)- if err != nil {- return errors.Wrapf(err, "failed to claim task run")- }- if !claimed {- // Another instance claimed it- return nil- }-- return s.executeTaskRun(ctx, taskRun)-}-// executeTaskRun executes a task run that is already in RUNNING status.-func (s *Scheduler) executeTaskRun(ctx context.Context, taskRun *store.TaskRunMessage) error {- task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)+func (s *Scheduler) executeTaskRun(ctx context.Context, taskRunUID, taskUID int) error {+ task, err := s.store.GetTaskByID(ctx, taskUID)if err != nil {return errors.Wrapf(err, "failed to get task")}-- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})- if err != nil {- return errors.Wrapf(err, "failed to get instance")- }- if instance == nil {- return errors.Errorf("instance %v not found", task.InstanceID)- }- if instance.Deleted {- return errors.Errorf("instance %v is deleted", task.InstanceID)+ if task == nil {+ return errors.Errorf("task %v not found", taskUID)}executor, ok := s.executorMap[task.Type]@@ -120,25 +76,22 @@ func (s *Scheduler) executeTaskRun(ctx context.Context, taskRun *store.TaskRunMe}// Update started_at- if err := s.store.UpdateTaskRunStartAt(ctx, taskRun.ID); err != nil {+ if err := s.store.UpdateTaskRunStartAt(ctx, taskRunUID); err != nil {return errors.Wrapf(err, "failed to update task run start at")}- // Register as running- s.stateCfg.RunningTaskRuns.Store(taskRun.ID, true)-- s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{+ s.store.CreateTaskRunLogS(ctx, taskRunUID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{Type: storepb.TaskRunLog_TASK_RUN_STATUS_UPDATE,TaskRunStatusUpdate: &storepb.TaskRunLog_TaskRunStatusUpdate{Status: storepb.TaskRunLog_TaskRunStatusUpdate_RUNNING_RUNNING,},})- go s.runTaskRunOnce(ctx, taskRun, task, executor)+ go s.runTaskRunOnce(ctx, taskRunUID, task, executor)return nil}-func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMessage, task *store.TaskMessage, executor Executor) {+func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRunUID int, task *store.TaskMessage, executor Executor) {defer func() {if r := recover(); r != nil {err, ok := r.(error)@@ -149,14 +102,14 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMe}}()defer func() {- s.stateCfg.RunningTaskRunsCancelFunc.Delete(taskRun.ID)+ s.stateCfg.RunningTaskRunsCancelFunc.Delete(taskRunUID)}()driverCtx, cancel := context.WithCancel(ctx)defer cancel()- s.stateCfg.RunningTaskRunsCancelFunc.Store(taskRun.ID, cancel)+ s.stateCfg.RunningTaskRunsCancelFunc.Store(taskRunUID, cancel)- done, result, err := RunExecutorOnce(ctx, driverCtx, executor, task, taskRun.ID)+ done, result, err := RunExecutorOnce(ctx, driverCtx, executor, task, taskRunUID)switch {case !done && err != nil:@@ -189,7 +142,7 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMecode := common.Okresult := string(resultBytes)taskRunStatusPatch := &store.TaskRunStatusPatch{- ID: taskRun.ID,+ ID: taskRunUID,Updater: common.SystemBotEmail,Status: storepb.TaskRun_CANCELED,Code: &code,@@ -232,7 +185,7 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMecode := common.ErrorCode(err)result := string(resultBytes)taskRunStatusPatch := &store.TaskRunStatusPatch{- ID: taskRun.ID,+ ID: taskRunUID,Updater: common.SystemBotEmail,Status: storepb.TaskRun_FAILED,Code: &code,@@ -262,7 +215,7 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRun *store.TaskRunMecode := common.Okresult := string(resultBytes)taskRunStatusPatch := &store.TaskRunStatusPatch{- ID: taskRun.ID,+ ID: taskRunUID,Updater: common.SystemBotEmail,Status: storepb.TaskRun_DONE,Code: &code,diff --git a/backend/store/task_run.go b/backend/store/task_run.goindex 1967da416e..d057fac1e5 100644--- a/backend/store/task_run.go+++ b/backend/store/task_run.go@@ -221,31 +221,50 @@ func (s *Store) UpdateTaskRunStatus(ctx context.Context, patch *TaskRunStatusPatreturn taskRun, nil}-// ClaimAvailableTaskRun attempts to atomically claim an AVAILABLE task run by updating it to RUNNING.-// Returns true if the claim succeeded, false if another process claimed it first.-func (s *Store) ClaimAvailableTaskRun(ctx context.Context, taskRunID int) (bool, error) {+// ClaimedTaskRun represents a claimed task run with its task UID.+type ClaimedTaskRun struct {+ TaskRunUID int+ TaskUID int+}++// ClaimAvailableTaskRuns atomically claims all AVAILABLE task runs by updating them to RUNNING+// and returns the claimed task run and task UIDs. This combines list + claim into a single atomic operation.+// Uses FOR UPDATE SKIP LOCKED to allow concurrent schedulers to claim different tasks.+func (s *Store) ClaimAvailableTaskRuns(ctx context.Context) ([]*ClaimedTaskRun, error) {q := qb.Q().Space(`UPDATE task_runSET status = ?, updated_at = now()- WHERE id = ? AND status = ?- `, storepb.TaskRun_RUNNING.String(), taskRunID, storepb.TaskRun_AVAILABLE.String())+ WHERE id IN (+ SELECT task_run.id FROM task_run+ WHERE task_run.status = ?+ FOR UPDATE SKIP LOCKED+ )+ RETURNING id, task_id+ `, storepb.TaskRun_RUNNING.String(), storepb.TaskRun_AVAILABLE.String())query, args, err := q.ToSQL()if err != nil {- return false, errors.Wrapf(err, "failed to build sql")+ return nil, errors.Wrapf(err, "failed to build sql")}- result, err := s.GetDB().ExecContext(ctx, query, args...)+ rows, err := s.GetDB().QueryContext(ctx, query, args...)if err != nil {- return false, errors.Wrapf(err, "failed to claim task run")+ return nil, errors.Wrapf(err, "failed to claim task runs")}+ defer rows.Close()- rowsAffected, err := result.RowsAffected()- if err != nil {- return false, errors.Wrapf(err, "failed to get rows affected")+ var claimed []*ClaimedTaskRun+ for rows.Next() {+ var c ClaimedTaskRun+ if err := rows.Scan(&c.TaskRunUID, &c.TaskUID); err != nil {+ return nil, err+ }+ claimed = append(claimed, &c)}-- return rowsAffected == 1, nil+ if err := rows.Err(); err != nil {+ return nil, err+ }+ return claimed, nil}func (s *Store) UpdateTaskRunStartAt(ctx context.Context, taskRunID int) error {