Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(taskrun): optimize pending scheduler with pre-fetched context (#18705)

p0ny21a2da04c071
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.go
index 53f2c2aab6..90c3ad6e5e 100644
--- a/backend/runner/taskrun/pending_scheduler.go
+++ b/backend/runner/taskrun/pending_scheduler.go
@@ -16,88 +16,6 @@ import (
"github.com/bytebase/bytebase/backend/store"
)
-// checkDatabaseMutualExclusion checks if there's already an AVAILABLE or RUNNING task on the database.
-// Returns true if the task can proceed (no conflict), false otherwise.
-func (s *Scheduler) checkDatabaseMutualExclusion(ctx context.Context, task *store.TaskMessage, availableDBs map[string]bool) (bool, *int, error) {
- if task.DatabaseName == nil {
- return true, nil, nil
- }
- if !isSequentialTask(task) {
- return true, nil, nil
- }
-
- databaseKey := getDatabaseKey(task.InstanceID, *task.DatabaseName)
-
- // Check in-memory tracking first (tasks promoted this round)
- if availableDBs[databaseKey] {
- return false, nil, nil
- }
-
- // Check database for AVAILABLE or RUNNING tasks on the same database
- activeTaskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{
- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_AVAILABLE, storepb.TaskRun_RUNNING},
- })
- if err != nil {
- return false, nil, errors.Wrapf(err, "failed to list active task runs")
- }
-
- for _, tr := range activeTaskRuns {
- activeTask, err := s.store.GetTaskByID(ctx, tr.TaskUID)
- if err != nil {
- return false, nil, errors.Wrapf(err, "failed to get task")
- }
- if activeTask.DatabaseName == nil {
- continue
- }
- if !isSequentialTask(activeTask) {
- continue
- }
- if getDatabaseKey(activeTask.InstanceID, *activeTask.DatabaseName) == databaseKey {
- return false, &activeTask.ID, nil
- }
- }
-
- return true, nil, nil
-}
-
-// checkParallelLimit checks if promoting this task would exceed the parallel task limit.
-// Returns true if the task can proceed, false otherwise.
-func (s *Scheduler) checkParallelLimit(ctx context.Context, task *store.TaskMessage, rolloutCounts map[int64]int) (bool, error) {
- plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})
- if err != nil {
- return false, errors.Wrapf(err, "failed to get plan")
- }
- if plan == nil {
- return false, errors.Errorf("plan %v not found", task.PlanID)
- }
-
- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})
- if err != nil {
- return false, errors.Wrapf(err, "failed to get project")
- }
- if project == nil {
- return false, errors.Errorf("project %v not found", plan.ProjectID)
- }
-
- maxParallel := int(project.Setting.GetParallelTasksPerRollout())
- if maxParallel <= 0 {
- // No limit
- return true, nil
- }
-
- // Count current AVAILABLE + RUNNING for this rollout
- activeTaskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{
- PlanUID: &task.PlanID,
- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_AVAILABLE, storepb.TaskRun_RUNNING},
- })
- if err != nil {
- return false, errors.Wrapf(err, "failed to list active task runs")
- }
-
- currentCount := len(activeTaskRuns) + rolloutCounts[task.PlanID]
- return currentCount < maxParallel, nil
-}
-
// runPendingTaskRunsScheduler runs in a separate goroutine to schedule pending task runs.
func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.WaitGroup) {
ticker := time.NewTicker(taskSchedulerInterval)
@@ -125,112 +43,128 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {
Status: &[]storepb.TaskRun_Status{storepb.TaskRun_PENDING},
})
if err != nil {
- return errors.Wrapf(err, "failed to list pending tasks")
+ return errors.Wrapf(err, "failed to list pending task runs")
}
- // Track what we've promoted this round to avoid over-committing
- availableDBs := map[string]bool{} // database key -> has AVAILABLE this round
- rolloutCounts := map[int64]int{} // plan_id -> count promoted this round
+ // Build context once per cycle
+ sc, err := newSchedulingContext(ctx, s.store)
+ if err != nil {
+ return errors.Wrapf(err, "failed to create scheduling context")
+ }
for _, taskRun := range taskRuns {
- promoted, err := s.schedulePendingTaskRun(ctx, taskRun, availableDBs, rolloutCounts)
- if err != nil {
+ if err := s.schedulePendingTaskRun(ctx, taskRun, sc); err != nil {
slog.Error("failed to schedule pending task run", log.BBError(err))
- continue
- }
- if promoted {
- task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)
- if err != nil {
- slog.Error("failed to get task after promotion", log.BBError(err))
- continue
- }
- if task.DatabaseName != nil && isSequentialTask(task) {
- availableDBs[getDatabaseKey(task.InstanceID, *task.DatabaseName)] = true
- }
- rolloutCounts[task.PlanID]++
}
}
return nil
}
-func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.TaskRunMessage, availableDBs map[string]bool, rolloutCounts map[int64]int) (bool, error) {
+func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.TaskRunMessage, sc *schedulingContext) error {
task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)
if err != nil {
- return false, errors.Wrapf(err, "failed to get task")
+ return errors.Wrapf(err, "failed to get task")
}
// Check 1: RunAt time
if taskRun.RunAt != nil && time.Now().Before(*taskRun.RunAt) {
- return false, nil
+ return nil
}
// Check 2: Version ordering (blocking tasks with smaller versions)
if task.DatabaseName != nil {
- schemaVersion := task.Payload.GetSchemaVersion()
- if schemaVersion != "" {
- maybeTaskID, err := s.store.FindBlockingTaskByVersion(ctx, task.PlanID, task.InstanceID, *task.DatabaseName, schemaVersion)
+ if schemaVersion := task.Payload.GetSchemaVersion(); schemaVersion != "" {
+ blockingTaskID, err := s.store.FindBlockingTaskByVersion(ctx, task.PlanID, task.InstanceID, *task.DatabaseName, schemaVersion)
if err != nil {
- return false, errors.Wrapf(err, "failed to find blocking versioned tasks")
+ return errors.Wrapf(err, "failed to find blocking versioned tasks")
}
- if maybeTaskID != nil {
- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
- ReportTime: timestamppb.Now(),
- WaitingCause: &storepb.SchedulerInfo_WaitingCause{
- Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{
- TaskUid: int32(*maybeTaskID),
- },
- },
- })
- return false, nil
+ if blockingTaskID != nil {
+ s.storeWaitingCause(taskRun.ID, *blockingTaskID)
+ return nil
}
}
}
// Check 3: Database mutual exclusion (for sequential tasks)
- canProceed, blockingTaskID, err := s.checkDatabaseMutualExclusion(ctx, task, availableDBs)
- if err != nil {
- return false, errors.Wrapf(err, "failed to check database mutual exclusion")
- }
+ canProceed, blockingTaskID := sc.checkDatabaseMutualExclusion(task)
if !canProceed {
if blockingTaskID != nil {
- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
- ReportTime: timestamppb.Now(),
- WaitingCause: &storepb.SchedulerInfo_WaitingCause{
- Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{
- TaskUid: int32(*blockingTaskID),
- },
- },
- })
+ s.storeWaitingCause(taskRun.ID, *blockingTaskID)
}
- return false, nil
+ return nil
}
// Check 4: Parallel task limit per rollout
- withinLimit, err := s.checkParallelLimit(ctx, task, rolloutCounts)
+ maxParallel, err := s.getMaxParallelForTask(ctx, task)
if err != nil {
- return false, errors.Wrapf(err, "failed to check parallel limit")
- }
- if !withinLimit {
- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{
- ReportTime: timestamppb.Now(),
- WaitingCause: &storepb.SchedulerInfo_WaitingCause{
- Cause: &storepb.SchedulerInfo_WaitingCause_ParallelTasksLimit{
- ParallelTasksLimit: true,
- },
- },
- })
- return false, nil
+ return errors.Wrapf(err, "failed to get max parallel limit")
+ }
+ if !sc.checkParallelLimit(task.PlanID, maxParallel) {
+ s.storeParallelLimitCause(taskRun.ID)
+ return nil
}
// All checks passed - promote to AVAILABLE
+ if err := s.promoteTaskRun(ctx, taskRun); err != nil {
+ return err
+ }
+ sc.markPromoted(task)
+
+ return nil
+}
+
+func (s *Scheduler) storeWaitingCause(taskRunID int, blockingTaskID int) {
+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRunID, &storepb.SchedulerInfo{
+ ReportTime: timestamppb.Now(),
+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{
+ Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{
+ TaskUid: int32(blockingTaskID),
+ },
+ },
+ })
+}
+
+func (s *Scheduler) storeParallelLimitCause(taskRunID int) {
+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRunID, &storepb.SchedulerInfo{
+ ReportTime: timestamppb.Now(),
+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{
+ Cause: &storepb.SchedulerInfo_WaitingCause_ParallelTasksLimit{
+ ParallelTasksLimit: true,
+ },
+ },
+ })
+}
+
+func (s *Scheduler) getMaxParallelForTask(ctx context.Context, task *store.TaskMessage) (int, error) {
+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to get plan")
+ }
+ if plan == nil {
+ return 0, errors.Errorf("plan %v not found", task.PlanID)
+ }
+
+ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to get project")
+ }
+ if project == nil {
+ return 0, errors.Errorf("project %v not found", plan.ProjectID)
+ }
+
+ return int(project.Setting.GetParallelTasksPerRollout()), nil
+}
+
+func (s *Scheduler) promoteTaskRun(ctx context.Context, taskRun *store.TaskRunMessage) error {
s.stateCfg.TaskRunSchedulerInfo.Delete(taskRun.ID)
+
if _, err := s.store.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{
ID: taskRun.ID,
Updater: common.SystemBotEmail,
Status: storepb.TaskRun_AVAILABLE,
}); err != nil {
- return false, errors.Wrapf(err, "failed to update task run status to available")
+ return errors.Wrapf(err, "failed to update task run status to available")
}
s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{
@@ -240,11 +174,110 @@ func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.T
},
})
- // Tickle the running scheduler
select {
case s.stateCfg.TaskRunTickleChan <- 0:
default:
}
+ return nil
+}
+
+// schedulingContext holds pre-fetched and indexed active task run data for a scheduling cycle.
+type schedulingContext struct {
+ // Pre-indexed active task runs (AVAILABLE + RUNNING)
+ activeByDatabase map[string]int // dbKey -> blocking taskID (first found)
+ activeCountByPlan map[int64]int // planID -> active count
+
+ // Tracks promotions this round
+ promotedDBs map[string]bool
+ promotedCounts map[int64]int
+}
+
+func newSchedulingContext(ctx context.Context, s *store.Store) (*schedulingContext, error) {
+ activeTaskRuns, err := s.ListTaskRuns(ctx, &store.FindTaskRunMessage{
+ Status: &[]storepb.TaskRun_Status{storepb.TaskRun_AVAILABLE, storepb.TaskRun_RUNNING},
+ })
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to list active task runs")
+ }
+
+ sc := &schedulingContext{
+ activeByDatabase: make(map[string]int),
+ activeCountByPlan: make(map[int64]int),
+ promotedDBs: make(map[string]bool),
+ promotedCounts: make(map[int64]int),
+ }
+
+ if len(activeTaskRuns) == 0 {
+ return sc, nil
+ }
+
+ // Batch fetch all tasks
+ taskIDs := make([]int, len(activeTaskRuns))
+ for i, tr := range activeTaskRuns {
+ taskIDs[i] = tr.TaskUID
+ }
+
+ tasks, err := s.ListTasks(ctx, &store.TaskFind{IDs: &taskIDs})
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to list tasks")
+ }
+
+ taskByID := make(map[int]*store.TaskMessage, len(tasks))
+ for _, t := range tasks {
+ taskByID[t.ID] = t
+ }
+
+ // Build indexes
+ for _, tr := range activeTaskRuns {
+ task := taskByID[tr.TaskUID]
+ if task == nil {
+ continue
+ }
+ sc.activeCountByPlan[task.PlanID]++
+
+ if task.DatabaseName != nil && isSequentialTask(task) {
+ dbKey := getDatabaseKey(task.InstanceID, *task.DatabaseName)
+ if _, exists := sc.activeByDatabase[dbKey]; !exists {
+ sc.activeByDatabase[dbKey] = task.ID
+ }
+ }
+ }
+
+ return sc, nil
+}
+
+func (sc *schedulingContext) checkDatabaseMutualExclusion(task *store.TaskMessage) (canProceed bool, blockingTaskID *int) {
+ if task.DatabaseName == nil || !isSequentialTask(task) {
+ return true, nil
+ }
+
+ dbKey := getDatabaseKey(task.InstanceID, *task.DatabaseName)
+
+ // Check in-memory tracking (promoted this round)
+ if sc.promotedDBs[dbKey] {
+ return false, nil
+ }
+
+ // Check pre-fetched active tasks
+ if blockingID, exists := sc.activeByDatabase[dbKey]; exists {
+ return false, &blockingID
+ }
+
return true, nil
}
+
+func (sc *schedulingContext) checkParallelLimit(planID int64, maxParallel int) bool {
+ if maxParallel <= 0 {
+ return true // no limit
+ }
+ currentCount := sc.activeCountByPlan[planID] + sc.promotedCounts[planID]
+ return currentCount < maxParallel
+}
+
+func (sc *schedulingContext) markPromoted(task *store.TaskMessage) {
+ if task.DatabaseName != nil && isSequentialTask(task) {
+ sc.promotedDBs[getDatabaseKey(task.InstanceID, *task.DatabaseName)] = true
+ }
+ sc.promotedCounts[task.PlanID]++
+}