Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(taskrun): optimize pending scheduler with pre-fetched context (#18705)
p0ny21a2da04c071
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.goindex 53f2c2aab6..90c3ad6e5e 100644--- a/backend/runner/taskrun/pending_scheduler.go+++ b/backend/runner/taskrun/pending_scheduler.go@@ -16,88 +16,6 @@ import ("github.com/bytebase/bytebase/backend/store")-// checkDatabaseMutualExclusion checks if there's already an AVAILABLE or RUNNING task on the database.-// Returns true if the task can proceed (no conflict), false otherwise.-func (s *Scheduler) checkDatabaseMutualExclusion(ctx context.Context, task *store.TaskMessage, availableDBs map[string]bool) (bool, *int, error) {- if task.DatabaseName == nil {- return true, nil, nil- }- if !isSequentialTask(task) {- return true, nil, nil- }-- databaseKey := getDatabaseKey(task.InstanceID, *task.DatabaseName)-- // Check in-memory tracking first (tasks promoted this round)- if availableDBs[databaseKey] {- return false, nil, nil- }-- // Check database for AVAILABLE or RUNNING tasks on the same database- activeTaskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_AVAILABLE, storepb.TaskRun_RUNNING},- })- if err != nil {- return false, nil, errors.Wrapf(err, "failed to list active task runs")- }-- for _, tr := range activeTaskRuns {- activeTask, err := s.store.GetTaskByID(ctx, tr.TaskUID)- if err != nil {- return false, nil, errors.Wrapf(err, "failed to get task")- }- if activeTask.DatabaseName == nil {- continue- }- if !isSequentialTask(activeTask) {- continue- }- if getDatabaseKey(activeTask.InstanceID, *activeTask.DatabaseName) == databaseKey {- return false, &activeTask.ID, nil- }- }-- return true, nil, nil-}--// checkParallelLimit checks if promoting this task would exceed the parallel task limit.-// Returns true if the task can proceed, false otherwise.-func (s *Scheduler) checkParallelLimit(ctx context.Context, task *store.TaskMessage, rolloutCounts map[int64]int) (bool, error) {- plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})- if err != nil {- return false, errors.Wrapf(err, "failed to get plan")- }- if plan == nil {- return false, errors.Errorf("plan %v not found", task.PlanID)- }-- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})- if err != nil {- return false, errors.Wrapf(err, "failed to get project")- }- if project == nil {- return false, errors.Errorf("project %v not found", plan.ProjectID)- }-- maxParallel := int(project.Setting.GetParallelTasksPerRollout())- if maxParallel <= 0 {- // No limit- return true, nil- }-- // Count current AVAILABLE + RUNNING for this rollout- activeTaskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{- PlanUID: &task.PlanID,- Status: &[]storepb.TaskRun_Status{storepb.TaskRun_AVAILABLE, storepb.TaskRun_RUNNING},- })- if err != nil {- return false, errors.Wrapf(err, "failed to list active task runs")- }-- currentCount := len(activeTaskRuns) + rolloutCounts[task.PlanID]- return currentCount < maxParallel, nil-}-// runPendingTaskRunsScheduler runs in a separate goroutine to schedule pending task runs.func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.WaitGroup) {ticker := time.NewTicker(taskSchedulerInterval)@@ -125,112 +43,128 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {Status: &[]storepb.TaskRun_Status{storepb.TaskRun_PENDING},})if err != nil {- return errors.Wrapf(err, "failed to list pending tasks")+ return errors.Wrapf(err, "failed to list pending task runs")}- // Track what we've promoted this round to avoid over-committing- availableDBs := map[string]bool{} // database key -> has AVAILABLE this round- rolloutCounts := map[int64]int{} // plan_id -> count promoted this round+ // Build context once per cycle+ sc, err := newSchedulingContext(ctx, s.store)+ if err != nil {+ return errors.Wrapf(err, "failed to create scheduling context")+ }for _, taskRun := range taskRuns {- promoted, err := s.schedulePendingTaskRun(ctx, taskRun, availableDBs, rolloutCounts)- if err != nil {+ if err := s.schedulePendingTaskRun(ctx, taskRun, sc); err != nil {slog.Error("failed to schedule pending task run", log.BBError(err))- continue- }- if promoted {- task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)- if err != nil {- slog.Error("failed to get task after promotion", log.BBError(err))- continue- }- if task.DatabaseName != nil && isSequentialTask(task) {- availableDBs[getDatabaseKey(task.InstanceID, *task.DatabaseName)] = true- }- rolloutCounts[task.PlanID]++}}return nil}-func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.TaskRunMessage, availableDBs map[string]bool, rolloutCounts map[int64]int) (bool, error) {+func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.TaskRunMessage, sc *schedulingContext) error {task, err := s.store.GetTaskByID(ctx, taskRun.TaskUID)if err != nil {- return false, errors.Wrapf(err, "failed to get task")+ return errors.Wrapf(err, "failed to get task")}// Check 1: RunAt timeif taskRun.RunAt != nil && time.Now().Before(*taskRun.RunAt) {- return false, nil+ return nil}// Check 2: Version ordering (blocking tasks with smaller versions)if task.DatabaseName != nil {- schemaVersion := task.Payload.GetSchemaVersion()- if schemaVersion != "" {- maybeTaskID, err := s.store.FindBlockingTaskByVersion(ctx, task.PlanID, task.InstanceID, *task.DatabaseName, schemaVersion)+ if schemaVersion := task.Payload.GetSchemaVersion(); schemaVersion != "" {+ blockingTaskID, err := s.store.FindBlockingTaskByVersion(ctx, task.PlanID, task.InstanceID, *task.DatabaseName, schemaVersion)if err != nil {- return false, errors.Wrapf(err, "failed to find blocking versioned tasks")+ return errors.Wrapf(err, "failed to find blocking versioned tasks")}- if maybeTaskID != nil {- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{- ReportTime: timestamppb.Now(),- WaitingCause: &storepb.SchedulerInfo_WaitingCause{- Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{- TaskUid: int32(*maybeTaskID),- },- },- })- return false, nil+ if blockingTaskID != nil {+ s.storeWaitingCause(taskRun.ID, *blockingTaskID)+ return nil}}}// Check 3: Database mutual exclusion (for sequential tasks)- canProceed, blockingTaskID, err := s.checkDatabaseMutualExclusion(ctx, task, availableDBs)- if err != nil {- return false, errors.Wrapf(err, "failed to check database mutual exclusion")- }+ canProceed, blockingTaskID := sc.checkDatabaseMutualExclusion(task)if !canProceed {if blockingTaskID != nil {- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{- ReportTime: timestamppb.Now(),- WaitingCause: &storepb.SchedulerInfo_WaitingCause{- Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{- TaskUid: int32(*blockingTaskID),- },- },- })+ s.storeWaitingCause(taskRun.ID, *blockingTaskID)}- return false, nil+ return nil}// Check 4: Parallel task limit per rollout- withinLimit, err := s.checkParallelLimit(ctx, task, rolloutCounts)+ maxParallel, err := s.getMaxParallelForTask(ctx, task)if err != nil {- return false, errors.Wrapf(err, "failed to check parallel limit")- }- if !withinLimit {- s.stateCfg.TaskRunSchedulerInfo.Store(taskRun.ID, &storepb.SchedulerInfo{- ReportTime: timestamppb.Now(),- WaitingCause: &storepb.SchedulerInfo_WaitingCause{- Cause: &storepb.SchedulerInfo_WaitingCause_ParallelTasksLimit{- ParallelTasksLimit: true,- },- },- })- return false, nil+ return errors.Wrapf(err, "failed to get max parallel limit")+ }+ if !sc.checkParallelLimit(task.PlanID, maxParallel) {+ s.storeParallelLimitCause(taskRun.ID)+ return nil}// All checks passed - promote to AVAILABLE+ if err := s.promoteTaskRun(ctx, taskRun); err != nil {+ return err+ }+ sc.markPromoted(task)++ return nil+}++func (s *Scheduler) storeWaitingCause(taskRunID int, blockingTaskID int) {+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRunID, &storepb.SchedulerInfo{+ ReportTime: timestamppb.Now(),+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{+ Cause: &storepb.SchedulerInfo_WaitingCause_TaskUid{+ TaskUid: int32(blockingTaskID),+ },+ },+ })+}++func (s *Scheduler) storeParallelLimitCause(taskRunID int) {+ s.stateCfg.TaskRunSchedulerInfo.Store(taskRunID, &storepb.SchedulerInfo{+ ReportTime: timestamppb.Now(),+ WaitingCause: &storepb.SchedulerInfo_WaitingCause{+ Cause: &storepb.SchedulerInfo_WaitingCause_ParallelTasksLimit{+ ParallelTasksLimit: true,+ },+ },+ })+}++func (s *Scheduler) getMaxParallelForTask(ctx context.Context, task *store.TaskMessage) (int, error) {+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})+ if err != nil {+ return 0, errors.Wrapf(err, "failed to get plan")+ }+ if plan == nil {+ return 0, errors.Errorf("plan %v not found", task.PlanID)+ }++ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})+ if err != nil {+ return 0, errors.Wrapf(err, "failed to get project")+ }+ if project == nil {+ return 0, errors.Errorf("project %v not found", plan.ProjectID)+ }++ return int(project.Setting.GetParallelTasksPerRollout()), nil+}++func (s *Scheduler) promoteTaskRun(ctx context.Context, taskRun *store.TaskRunMessage) error {s.stateCfg.TaskRunSchedulerInfo.Delete(taskRun.ID)+if _, err := s.store.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{ID: taskRun.ID,Updater: common.SystemBotEmail,Status: storepb.TaskRun_AVAILABLE,}); err != nil {- return false, errors.Wrapf(err, "failed to update task run status to available")+ return errors.Wrapf(err, "failed to update task run status to available")}s.store.CreateTaskRunLogS(ctx, taskRun.ID, time.Now(), s.profile.DeployID, &storepb.TaskRunLog{@@ -240,11 +174,110 @@ func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.T},})- // Tickle the running schedulerselect {case s.stateCfg.TaskRunTickleChan <- 0:default:}+ return nil+}++// schedulingContext holds pre-fetched and indexed active task run data for a scheduling cycle.+type schedulingContext struct {+ // Pre-indexed active task runs (AVAILABLE + RUNNING)+ activeByDatabase map[string]int // dbKey -> blocking taskID (first found)+ activeCountByPlan map[int64]int // planID -> active count++ // Tracks promotions this round+ promotedDBs map[string]bool+ promotedCounts map[int64]int+}++func newSchedulingContext(ctx context.Context, s *store.Store) (*schedulingContext, error) {+ activeTaskRuns, err := s.ListTaskRuns(ctx, &store.FindTaskRunMessage{+ Status: &[]storepb.TaskRun_Status{storepb.TaskRun_AVAILABLE, storepb.TaskRun_RUNNING},+ })+ if err != nil {+ return nil, errors.Wrapf(err, "failed to list active task runs")+ }++ sc := &schedulingContext{+ activeByDatabase: make(map[string]int),+ activeCountByPlan: make(map[int64]int),+ promotedDBs: make(map[string]bool),+ promotedCounts: make(map[int64]int),+ }++ if len(activeTaskRuns) == 0 {+ return sc, nil+ }++ // Batch fetch all tasks+ taskIDs := make([]int, len(activeTaskRuns))+ for i, tr := range activeTaskRuns {+ taskIDs[i] = tr.TaskUID+ }++ tasks, err := s.ListTasks(ctx, &store.TaskFind{IDs: &taskIDs})+ if err != nil {+ return nil, errors.Wrapf(err, "failed to list tasks")+ }++ taskByID := make(map[int]*store.TaskMessage, len(tasks))+ for _, t := range tasks {+ taskByID[t.ID] = t+ }++ // Build indexes+ for _, tr := range activeTaskRuns {+ task := taskByID[tr.TaskUID]+ if task == nil {+ continue+ }+ sc.activeCountByPlan[task.PlanID]++++ if task.DatabaseName != nil && isSequentialTask(task) {+ dbKey := getDatabaseKey(task.InstanceID, *task.DatabaseName)+ if _, exists := sc.activeByDatabase[dbKey]; !exists {+ sc.activeByDatabase[dbKey] = task.ID+ }+ }+ }++ return sc, nil+}++func (sc *schedulingContext) checkDatabaseMutualExclusion(task *store.TaskMessage) (canProceed bool, blockingTaskID *int) {+ if task.DatabaseName == nil || !isSequentialTask(task) {+ return true, nil+ }++ dbKey := getDatabaseKey(task.InstanceID, *task.DatabaseName)++ // Check in-memory tracking (promoted this round)+ if sc.promotedDBs[dbKey] {+ return false, nil+ }++ // Check pre-fetched active tasks+ if blockingID, exists := sc.activeByDatabase[dbKey]; exists {+ return false, &blockingID+ }+return true, nil}++func (sc *schedulingContext) checkParallelLimit(planID int64, maxParallel int) bool {+ if maxParallel <= 0 {+ return true // no limit+ }+ currentCount := sc.activeCountByPlan[planID] + sc.promotedCounts[planID]+ return currentCount < maxParallel+}++func (sc *schedulingContext) markPromoted(task *store.TaskMessage) {+ if task.DatabaseName != nil && isSequentialTask(task) {+ sc.promotedDBs[getDatabaseKey(task.InstanceID, *task.DatabaseName)] = true+ }+ sc.promotedCounts[task.PlanID]+++}