Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: make plan check run scheduler HA compatible (#18765)
p0nyc6daba9dd695
diff --git a/backend/api/v1/plan_service.go b/backend/api/v1/plan_service.goindex 185b444886..9eb7193b8b 100644--- a/backend/api/v1/plan_service.go+++ b/backend/api/v1/plan_service.go@@ -490,11 +490,11 @@ func (s *PlanService) CancelPlanCheckRun(ctx context.Context, request *connect.Rreturn nil, connect.NewError(connect.CodeNotFound, errors.Errorf("plan check run not found for plan %d", planUID))}- if planCheckRun.Status != store.PlanCheckRunStatusRunning {- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("plan check run is not running"))+ if planCheckRun.Status != store.PlanCheckRunStatusRunning && planCheckRun.Status != store.PlanCheckRunStatusAvailable {+ return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("plan check run is not running or available"))}- // Cancel the plan check run.+ // Cancel in-flight plan check run if running.if cancelFunc, ok := s.bus.RunningPlanCheckRunsCancelFunc.Load(planCheckRun.UID); ok {cancelFunc.(context.CancelFunc)()}diff --git a/backend/component/bus/bus.go b/backend/component/bus/bus.goindex afa2ae7323..2d0cb4a6c5 100644--- a/backend/component/bus/bus.go+++ b/backend/component/bus/bus.go@@ -16,8 +16,6 @@ type Bus struct {// RunningTaskRunsCancelFunc is the cancelFunc of running taskruns.RunningTaskRunsCancelFunc sync.Map // map[taskRunID]context.CancelFunc- // RunningPlanChecks is the set of running plan checks.- RunningPlanChecks sync.Map// RunningPlanCheckRunsCancelFunc is the cancelFunc of running plan checks.RunningPlanCheckRunsCancelFunc sync.Map // map[planCheckRunUID]context.CancelFuncdiff --git a/backend/migrator/migration/3.14/0022##plan_check_run_ha.sql b/backend/migrator/migration/3.14/0022##plan_check_run_ha.sqlnew file mode 100644index 0000000000..715b7cfca5--- /dev/null+++ b/backend/migrator/migration/3.14/0022##plan_check_run_ha.sql@@ -0,0 +1,15 @@+-- Add AVAILABLE status for HA-compatible plan check scheduling.+-- Uses FOR UPDATE SKIP LOCKED pattern for atomic claiming.++-- Update status constraint to include AVAILABLE+ALTER TABLE plan_check_run+ DROP CONSTRAINT plan_check_run_status_check,+ ADD CONSTRAINT plan_check_run_status_check+ CHECK (status IN ('AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED'));++-- Convert existing RUNNING to AVAILABLE (will be re-claimed after deployment)+UPDATE plan_check_run SET status = 'AVAILABLE' WHERE status = 'RUNNING';++-- Update index to include AVAILABLE for efficient claiming+DROP INDEX IF EXISTS idx_plan_check_run_active_status;+CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status IN ('AVAILABLE', 'RUNNING');diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sqlindex 3a8abdd9e4..fca73f1a77 100644--- a/backend/migrator/migration/LATEST.sql+++ b/backend/migrator/migration/LATEST.sql@@ -215,14 +215,14 @@ CREATE TABLE plan_check_run (created_at timestamptz NOT NULL DEFAULT now(),updated_at timestamptz NOT NULL DEFAULT now(),plan_id bigint NOT NULL REFERENCES plan(id),- status text NOT NULL CHECK (status IN ('RUNNING', 'DONE', 'FAILED', 'CANCELED')),+ status text NOT NULL CHECK (status IN ('AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED')),-- Stored as PlanCheckRunResult (proto/store/store/plan_check_run.proto)result jsonb NOT NULL DEFAULT '{}');CREATE UNIQUE INDEX idx_plan_check_run_unique_plan_id ON plan_check_run(plan_id);-CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status = 'RUNNING';+CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status IN ('AVAILABLE', 'RUNNING');ALTER SEQUENCE plan_check_run_id_seq RESTART WITH 101;diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.goindex 636c0d226a..7028e63940 100644--- a/backend/migrator/migrator_test.go+++ b/backend/migrator/migrator_test.go@@ -12,8 +12,8 @@ import (func TestLatestVersion(t *testing.T) {files, err := getSortedVersionedFiles()require.NoError(t, err)- require.Equal(t, semver.MustParse("3.14.21"), *files[len(files)-1].version)- require.Equal(t, "migration/3.14/0021##remove_plan_check_run_config_payload.sql", files[len(files)-1].path)+ require.Equal(t, semver.MustParse("3.14.22"), *files[len(files)-1].version)+ require.Equal(t, "migration/3.14/0022##plan_check_run_ha.sql", files[len(files)-1].path)}func TestVersionUnique(t *testing.T) {diff --git a/backend/runner/plancheck/scheduler.go b/backend/runner/plancheck/scheduler.goindex 7957dcf732..14ce31ca28 100644--- a/backend/runner/plancheck/scheduler.go+++ b/backend/runner/plancheck/scheduler.go@@ -68,112 +68,96 @@ func (s *Scheduler) runOnce(ctx context.Context) {}}()- planCheckRuns, err := s.store.ListPlanCheckRuns(ctx, &store.FindPlanCheckRunMessage{- Status: &[]store.PlanCheckRunStatus{- store.PlanCheckRunStatusRunning,- },- })+ claimed, err := s.store.ClaimAvailablePlanCheckRuns(ctx)if err != nil {- slog.Error("failed to list running plan check runs", log.BBError(err))+ slog.Error("failed to claim available plan check runs", log.BBError(err))return}- for _, planCheckRun := range planCheckRuns {- s.runPlanCheckRun(ctx, planCheckRun)+ for _, c := range claimed {+ go s.runPlanCheckRun(ctx, c.UID, c.PlanUID)}}-func (s *Scheduler) runPlanCheckRun(ctx context.Context, planCheckRun *store.PlanCheckRunMessage) {- // Skip the plan check run if it is already running.- if _, ok := s.bus.RunningPlanChecks.Load(planCheckRun.UID); ok {+func (s *Scheduler) runPlanCheckRun(ctx context.Context, uid int, planUID int64) {+ ctxWithCancel, cancel := context.WithCancel(ctx)+ defer cancel()+ s.bus.RunningPlanCheckRunsCancelFunc.Store(uid, cancel)+ defer s.bus.RunningPlanCheckRunsCancelFunc.Delete(uid)++ // Fetch plan to derive check targets at runtime+ plan, err := s.store.GetPlan(ctxWithCancel, &store.FindPlanMessage{UID: &planUID})+ if err != nil {+ s.markPlanCheckRunFailed(ctxWithCancel, uid, err.Error())+ return+ }+ if plan == nil {+ s.markPlanCheckRunFailed(ctxWithCancel, uid, "plan not found")return}- s.bus.RunningPlanChecks.Store(planCheckRun.UID, true)- go func() {- defer func() {- s.bus.RunningPlanChecks.Delete(planCheckRun.UID)- s.bus.RunningPlanCheckRunsCancelFunc.Delete(planCheckRun.UID)- }()-- ctxWithCancel, cancel := context.WithCancel(ctx)- defer cancel()- s.bus.RunningPlanCheckRunsCancelFunc.Store(planCheckRun.UID, cancel)-- // Fetch plan to derive check targets at runtime- plan, err := s.store.GetPlan(ctxWithCancel, &store.FindPlanMessage{UID: &planCheckRun.PlanUID})- if err != nil {- s.markPlanCheckRunFailed(ctx, planCheckRun, err.Error())- return- }- if plan == nil {- s.markPlanCheckRunFailed(ctx, planCheckRun, "plan not found")- return- }-- project, err := s.store.GetProject(ctxWithCancel, &store.FindProjectMessage{ResourceID: &plan.ProjectID})- if err != nil {- s.markPlanCheckRunFailed(ctx, planCheckRun, err.Error())- return- }- if project == nil {- s.markPlanCheckRunFailed(ctx, planCheckRun, "project not found")- return- }+ project, err := s.store.GetProject(ctxWithCancel, &store.FindProjectMessage{ResourceID: &plan.ProjectID})+ if err != nil {+ s.markPlanCheckRunFailed(ctxWithCancel, uid, err.Error())+ return+ }+ if project == nil {+ s.markPlanCheckRunFailed(ctxWithCancel, uid, "project not found")+ return+ }- // Get database group if needed (for spec expansion)- databaseGroup, err := s.getDatabaseGroupForPlan(ctxWithCancel, plan)- if err != nil {- s.markPlanCheckRunFailed(ctx, planCheckRun, err.Error())- return- }+ // Get database group if needed (for spec expansion)+ databaseGroup, err := s.getDatabaseGroupForPlan(ctxWithCancel, plan)+ if err != nil {+ s.markPlanCheckRunFailed(ctxWithCancel, uid, err.Error())+ return+ }- // Derive check targets from plan- targets, err := DeriveCheckTargets(project, plan, databaseGroup)- if err != nil {- s.markPlanCheckRunFailed(ctx, planCheckRun, err.Error())- return- }+ // Derive check targets from plan+ targets, err := DeriveCheckTargets(project, plan, databaseGroup)+ if err != nil {+ s.markPlanCheckRunFailed(ctxWithCancel, uid, err.Error())+ return+ }- var results []*storepb.PlanCheckRunResult_Result- for _, target := range targets {- targetResults, targetErr := s.executor.RunForTarget(ctxWithCancel, target)- if targetErr != nil {- err = targetErr- break- }- results = append(results, targetResults...)+ var results []*storepb.PlanCheckRunResult_Result+ for _, target := range targets {+ targetResults, targetErr := s.executor.RunForTarget(ctxWithCancel, target)+ if targetErr != nil {+ err = targetErr+ break}- if err != nil {- if errors.Is(err, context.Canceled) {- s.markPlanCheckRunCanceled(ctx, planCheckRun, err.Error())- } else {- s.markPlanCheckRunFailed(ctx, planCheckRun, err.Error())- }+ results = append(results, targetResults...)+ }+ if err != nil {+ if errors.Is(err, context.Canceled) {+ s.markPlanCheckRunCanceled(ctxWithCancel, uid, err.Error())} else {- s.markPlanCheckRunDone(ctx, planCheckRun, results)+ s.markPlanCheckRunFailed(ctxWithCancel, uid, err.Error())}- }()+ } else {+ s.markPlanCheckRunDone(ctxWithCancel, uid, planUID, results)+ }}-func (s *Scheduler) markPlanCheckRunDone(ctx context.Context, planCheckRun *store.PlanCheckRunMessage, results []*storepb.PlanCheckRunResult_Result) {+func (s *Scheduler) markPlanCheckRunDone(ctx context.Context, uid int, planUID int64, results []*storepb.PlanCheckRunResult_Result) {result := &storepb.PlanCheckRunResult{Results: results,}if err := s.store.UpdatePlanCheckRun(ctx,store.PlanCheckRunStatusDone,result,- planCheckRun.UID,+ uid,); err != nil {slog.Error("failed to mark plan check run done", log.BBError(err))return}// Auto-create rollout if plan checks pass- issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PlanUID: &planCheckRun.PlanUID})+ issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PlanUID: &planUID})if err != nil {slog.Error("failed to get issue for approval check after plan check",- slog.Int("plan_id", int(planCheckRun.PlanUID)),+ slog.Int("plan_id", int(planUID)),log.BBError(err))return}@@ -181,31 +165,31 @@ func (s *Scheduler) markPlanCheckRunDone(ctx context.Context, planCheckRun *stor// Trigger approval findings.bus.ApprovalCheckChan <- int64(issue.UID)// Trigger rollout creation (existing behavior)- s.bus.RolloutCreationChan <- planCheckRun.PlanUID+ s.bus.RolloutCreationChan <- planUID}}-func (s *Scheduler) markPlanCheckRunFailed(ctx context.Context, planCheckRun *store.PlanCheckRunMessage, reason string) {+func (s *Scheduler) markPlanCheckRunFailed(ctx context.Context, uid int, reason string) {result := &storepb.PlanCheckRunResult{Error: reason,}if err := s.store.UpdatePlanCheckRun(ctx,store.PlanCheckRunStatusFailed,result,- planCheckRun.UID,+ uid,); err != nil {slog.Error("failed to mark plan check run failed", log.BBError(err))}}-func (s *Scheduler) markPlanCheckRunCanceled(ctx context.Context, planCheckRun *store.PlanCheckRunMessage, reason string) {+func (s *Scheduler) markPlanCheckRunCanceled(ctx context.Context, uid int, reason string) {result := &storepb.PlanCheckRunResult{Error: reason,}if err := s.store.UpdatePlanCheckRun(ctx,store.PlanCheckRunStatusCanceled,result,- planCheckRun.UID,+ uid,); err != nil {slog.Error("failed to mark plan check run canceled", log.BBError(err))}diff --git a/backend/store/plan_check_run.go b/backend/store/plan_check_run.goindex c099e55e01..3174d125b5 100644--- a/backend/store/plan_check_run.go+++ b/backend/store/plan_check_run.go@@ -16,6 +16,8 @@ import (type PlanCheckRunStatus stringconst (+ // PlanCheckRunStatusAvailable is the plan check status for AVAILABLE.+ PlanCheckRunStatusAvailable PlanCheckRunStatus = "AVAILABLE"// PlanCheckRunStatusRunning is the plan check status for RUNNING.PlanCheckRunStatusRunning PlanCheckRunStatus = "RUNNING"// PlanCheckRunStatusDone is the plan check status for DONE.@@ -47,6 +49,7 @@ type FindPlanCheckRunMessage struct {}// CreatePlanCheckRun creates or replaces the plan check run for a plan.+// Always creates with AVAILABLE status for HA-safe scheduling.func (s *Store) CreatePlanCheckRun(ctx context.Context, create *PlanCheckRunMessage) error {result, err := protojson.Marshal(create.Result)if err != nil {@@ -61,7 +64,7 @@ func (s *Store) CreatePlanCheckRun(ctx context.Context, create *PlanCheckRunMessresult = EXCLUDED.result,updated_at = now()`- if _, err := s.GetDB().ExecContext(ctx, query, create.PlanUID, create.Status, result); err != nil {+ if _, err := s.GetDB().ExecContext(ctx, query, create.PlanUID, PlanCheckRunStatusAvailable, result); err != nil {return errors.Wrapf(err, "failed to upsert plan check run")}return nil@@ -187,3 +190,50 @@ func (s *Store) BatchCancelPlanCheckRuns(ctx context.Context, planCheckRunUIDs [}return nil}++// ClaimedPlanCheckRun represents a plan check run that was atomically claimed.+type ClaimedPlanCheckRun struct {+ UID int+ PlanUID int64+}++// ClaimAvailablePlanCheckRuns atomically claims all AVAILABLE plan check runs by updating them to RUNNING+// and returns the claimed UIDs. Uses FOR UPDATE SKIP LOCKED to allow concurrent schedulers to claim different runs.+func (s *Store) ClaimAvailablePlanCheckRuns(ctx context.Context) ([]*ClaimedPlanCheckRun, error) {+ q := qb.Q().Space(`+ UPDATE plan_check_run+ SET status = ?, updated_at = now()+ WHERE id IN (+ SELECT id FROM plan_check_run+ WHERE status = ?+ FOR UPDATE SKIP LOCKED+ )+ RETURNING id, plan_id+ `, PlanCheckRunStatusRunning, PlanCheckRunStatusAvailable)++ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrapf(err, "failed to build sql")+ }++ rows, err := s.GetDB().QueryContext(ctx, query, args...)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to claim plan check runs")+ }+ defer rows.Close()++ var claimed []*ClaimedPlanCheckRun+ for rows.Next() {+ var c ClaimedPlanCheckRun+ if err := rows.Scan(&c.UID, &c.PlanUID); err != nil {+ return nil, err+ }+ claimed = append(claimed, &c)+ }++ if err := rows.Err(); err != nil {+ return nil, err+ }++ return claimed, nil+}diff --git a/docs/plans/2026-01-05-plan-check-run-ha-design.md b/docs/plans/2026-01-05-plan-check-run-ha-design.mdnew file mode 100644index 0000000000..7ed5ac07b9--- /dev/null+++ b/docs/plans/2026-01-05-plan-check-run-ha-design.md@@ -0,0 +1,105 @@+# Plan Check Run Scheduler HA Compatibility++## Overview++Make the plan check run scheduler HA (High Availability) compatible by following patterns established in the task run scheduler.++## Current State++The plan check scheduler is **not HA-compatible**:+- Uses in-memory `sync.Map` to track running checks+- Simple polling loop every 5 seconds+- No database locking - relies on single-instance assumption+- Status: `RUNNING → DONE/FAILED/CANCELED`++## Target State++Follow the task run scheduler HA pattern:+- Use `FOR UPDATE SKIP LOCKED` to atomically claim work+- Status state machine: `AVAILABLE → RUNNING → DONE/FAILED/CANCELED`+- Multiple instances compete fairly via database queries+- No leader election - peer-to-peer work queue pattern++## Design++### Schema Changes++**Migration file:** `backend/migrator/migration/3.X/XXXX##plan_check_run_ha.sql`++```sql+-- Add AVAILABLE to status check constraint+ALTER TABLE plan_check_run+ DROP CONSTRAINT plan_check_run_status_check,+ ADD CONSTRAINT plan_check_run_status_check+ CHECK (status IN ('AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED'));++-- Convert existing RUNNING to AVAILABLE (will be re-claimed)+UPDATE plan_check_run SET status = 'AVAILABLE' WHERE status = 'RUNNING';++-- Update index to include AVAILABLE for efficient claiming+DROP INDEX IF EXISTS idx_plan_check_run_active_status;+CREATE INDEX idx_plan_check_run_active_status+ ON plan_check_run(status, id)+ WHERE status IN ('AVAILABLE', 'RUNNING');+```++### Store Layer Changes++**File:** `backend/store/plan_check_run.go`++Add claiming function:++```go+// ClaimAvailablePlanCheckRuns atomically claims all AVAILABLE plan check runs.+// Uses FOR UPDATE SKIP LOCKED for HA-safe concurrent claiming.+func (s *Store) ClaimAvailablePlanCheckRuns(ctx context.Context) ([]*PlanCheckRunMessage, error) {+ query := `+ UPDATE plan_check_run+ SET status = 'RUNNING', updated_at = now()+ WHERE id IN (+ SELECT id FROM plan_check_run+ WHERE status = 'AVAILABLE'+ FOR UPDATE SKIP LOCKED+ )+ RETURNING id, plan_id+ `+ // Execute and return claimed runs+}+```++Modify `CreatePlanCheckRun`: Change default status from `RUNNING` to `AVAILABLE`.++### Scheduler Changes++**File:** `backend/runner/plancheck/scheduler.go`++1. **Remove in-memory tracking** - Delete `RunningPlanChecks` and `RunningPlanCheckRunsCancelFunc` sync.Maps++2. **Update `runOnce()`** - Use atomic claiming instead of querying RUNNING status:+ ```go+ func (s *Scheduler) runOnce(ctx context.Context) {+ claimed, err := s.store.ClaimAvailablePlanCheckRuns(ctx)+ for _, planCheckRun := range claimed {+ go s.runPlanCheckRun(ctx, planCheckRun)+ }+ }+ ```++3. **Simplify cancellation** - Update database status to `CANCELED` directly++### API Changes++No changes needed - store layer handles status transition.++## Files to Modify++| File | Change |+|------|--------|+| `backend/migrator/migration/3.X/XXXX##plan_check_run_ha.sql` | New migration |+| `backend/migrator/migration/LATEST.sql` | Update schema |+| `backend/store/plan_check_run.go` | Add claiming, change default status |+| `backend/runner/plancheck/scheduler.go` | Remove sync.Map, use claiming |++## Migration Strategy++Existing `RUNNING` plan check runs are converted to `AVAILABLE` during migration. They will be re-claimed and re-executed by the scheduler after deployment.diff --git a/docs/plans/2026-01-05-plan-check-run-ha.md b/docs/plans/2026-01-05-plan-check-run-ha.mdnew file mode 100644index 0000000000..b2cd80435d--- /dev/null+++ b/docs/plans/2026-01-05-plan-check-run-ha.md@@ -0,0 +1,551 @@+# Plan Check Run HA Implementation Plan++> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.++**Goal:** Make the plan check run scheduler HA-compatible using database-level atomic claiming.++**Architecture:** Add `AVAILABLE` status to plan check runs. Scheduler atomically claims `AVAILABLE` runs using `FOR UPDATE SKIP LOCKED`, transitioning them to `RUNNING`. Remove in-memory tracking from bus.++**Tech Stack:** Go, PostgreSQL, protobuf++---++## Task 1: Add AVAILABLE status constant to store++**Files:**+- Modify: `backend/store/plan_check_run.go:18-27`++**Step 1: Add the new status constant**++In `backend/store/plan_check_run.go`, add `PlanCheckRunStatusAvailable` after line 18:++```go+const (+ // PlanCheckRunStatusAvailable is the plan check status for AVAILABLE.+ PlanCheckRunStatusAvailable PlanCheckRunStatus = "AVAILABLE"+ // PlanCheckRunStatusRunning is the plan check status for RUNNING.+ PlanCheckRunStatusRunning PlanCheckRunStatus = "RUNNING"+ // PlanCheckRunStatusDone is the plan check status for DONE.+ PlanCheckRunStatusDone PlanCheckRunStatus = "DONE"+ // PlanCheckRunStatusFailed is the plan check status for FAILED.+ PlanCheckRunStatusFailed PlanCheckRunStatus = "FAILED"+ // PlanCheckRunStatusCanceled is the plan check status for CANCELED.+ PlanCheckRunStatusCanceled PlanCheckRunStatus = "CANCELED"+)+```++**Step 2: Verify build**++Run: `go build ./backend/store/...`+Expected: Build succeeds++**Step 3: Commit**++```bash+but commit plan-check-ha -m "feat(store): add AVAILABLE status for plan check runs"+```++---++## Task 2: Add ClaimAvailablePlanCheckRuns function to store++**Files:**+- Modify: `backend/store/plan_check_run.go`++**Step 1: Add ClaimedPlanCheckRun struct and claiming function**++Add after `BatchCancelPlanCheckRuns` function (after line 189):++```go+// ClaimedPlanCheckRun represents a plan check run that was atomically claimed.+type ClaimedPlanCheckRun struct {+ UID int+ PlanUID int64+}++// ClaimAvailablePlanCheckRuns atomically claims all AVAILABLE plan check runs by updating them to RUNNING+// and returns the claimed UIDs. Uses FOR UPDATE SKIP LOCKED to allow concurrent schedulers to claim different runs.+func (s *Store) ClaimAvailablePlanCheckRuns(ctx context.Context) ([]*ClaimedPlanCheckRun, error) {+ q := qb.Q().Space(`+ UPDATE plan_check_run+ SET status = ?, updated_at = now()+ WHERE id IN (+ SELECT id FROM plan_check_run+ WHERE status = ?+ FOR UPDATE SKIP LOCKED+ )+ RETURNING id, plan_id+ `, PlanCheckRunStatusRunning, PlanCheckRunStatusAvailable)++ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrapf(err, "failed to build sql")+ }++ rows, err := s.GetDB().QueryContext(ctx, query, args...)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to claim plan check runs")+ }+ defer rows.Close()++ var claimed []*ClaimedPlanCheckRun+ for rows.Next() {+ var c ClaimedPlanCheckRun+ if err := rows.Scan(&c.UID, &c.PlanUID); err != nil {+ return nil, err+ }+ claimed = append(claimed, &c)+ }++ if err := rows.Err(); err != nil {+ return nil, err+ }++ return claimed, nil+}+```++**Step 2: Verify build**++Run: `go build ./backend/store/...`+Expected: Build succeeds++**Step 3: Commit**++```bash+but commit plan-check-ha -m "feat(store): add ClaimAvailablePlanCheckRuns for HA scheduling"+```++---++## Task 3: Update CreatePlanCheckRun to use AVAILABLE status++**Files:**+- Modify: `backend/store/plan_check_run.go:50-68`++**Step 1: Modify CreatePlanCheckRun to always use AVAILABLE**++Replace the `CreatePlanCheckRun` function to ignore the passed status and always use AVAILABLE:++```go+// CreatePlanCheckRun creates or replaces the plan check run for a plan.+// Always creates with AVAILABLE status for HA-safe scheduling.+func (s *Store) CreatePlanCheckRun(ctx context.Context, create *PlanCheckRunMessage) error {+ result, err := protojson.Marshal(create.Result)+ if err != nil {+ return errors.Wrapf(err, "failed to marshal result")+ }++ query := `+ INSERT INTO plan_check_run (plan_id, status, result)+ VALUES ($1, $2, $3)+ ON CONFLICT (plan_id) DO UPDATE SET+ status = EXCLUDED.status,+ result = EXCLUDED.result,+ updated_at = now()+ `+ if _, err := s.GetDB().ExecContext(ctx, query, create.PlanUID, PlanCheckRunStatusAvailable, result); err != nil {+ return errors.Wrapf(err, "failed to upsert plan check run")+ }+ return nil+}+```++**Step 2: Verify build**++Run: `go build ./backend/store/...`+Expected: Build succeeds++**Step 3: Commit**++```bash+but commit plan-check-ha -m "feat(store): CreatePlanCheckRun always uses AVAILABLE status"+```++---++## Task 4: Remove in-memory plan check tracking from bus++**Files:**+- Modify: `backend/component/bus/bus.go:19-22`++**Step 1: Remove RunningPlanChecks and RunningPlanCheckRunsCancelFunc**++Remove these two lines from the Bus struct:++```go+// RunningPlanChecks is the set of running plan checks.+RunningPlanChecks sync.Map+// RunningPlanCheckRunsCancelFunc is the cancelFunc of running plan checks.+RunningPlanCheckRunsCancelFunc sync.Map // map[planCheckRunUID]context.CancelFunc+```++The Bus struct should now look like:++```go+// Bus is the message bus for all in-memory communication within the server.+type Bus struct {+ // ApprovalCheckChan signals when an issue needs approval template finding.+ // Triggered by plan check completion, issue creation (if checks already done).+ ApprovalCheckChan chan int64 // issue UID++ TaskRunSchedulerInfo sync.Map // map[taskRunID]*storepb.SchedulerInfo++ // RunningTaskRunsCancelFunc is the cancelFunc of running taskruns.+ RunningTaskRunsCancelFunc sync.Map // map[taskRunID]context.CancelFunc++ // PlanCheckTickleChan is the tickler for plan check scheduler.+ PlanCheckTickleChan chan int+ // TaskRunTickleChan is the tickler for task run scheduler.+ TaskRunTickleChan chan int++ // RolloutCreationChan is the channel for automatic rollout creation.+ RolloutCreationChan chan int64++ // PlanCompletionCheckChan signals when a plan might be complete (for PIPELINE_COMPLETED webhook).+ PlanCompletionCheckChan chan int64+}+```++**Step 2: Verify build fails (expected - scheduler still references these)**++Run: `go build ./backend/...`+Expected: Build fails with references to deleted fields++**Step 3: Commit (partial - will fix in next task)**++```bash+but commit plan-check-ha -m "refactor(bus): remove in-memory plan check tracking"+```++---++## Task 5: Update scheduler to use atomic claiming++**Files:**+- Modify: `backend/runner/plancheck/scheduler.go`++**Step 1: Update runOnce to use claiming**++Replace the `runOnce` function:++```go+func (s *Scheduler) runOnce(ctx context.Context) {+ defer func() {+ if r := recover(); r != nil {+ err, ok := r.(error)+ if !ok {+ err = errors.Errorf("%v", r)+ }+ slog.Error("Plan check scheduler PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))+ }+ }()++ claimed, err := s.store.ClaimAvailablePlanCheckRuns(ctx)+ if err != nil {+ slog.Error("failed to claim available plan check runs", log.BBError(err))+ return+ }++ for _, c := range claimed {+ go s.runPlanCheckRun(ctx, c.UID, c.PlanUID)+ }+}+```++**Step 2: Update runPlanCheckRun signature and remove in-memory tracking**++Replace the `runPlanCheckRun` function:++```go+func (s *Scheduler) runPlanCheckRun(ctx context.Context, uid int, planUID int64) {+ // Fetch plan to derive check targets at runtime+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &planUID})+ if err != nil {+ s.markPlanCheckRunFailed(ctx, uid, planUID, err.Error())+ return+ }+ if plan == nil {+ s.markPlanCheckRunFailed(ctx, uid, planUID, "plan not found")+ return+ }++ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})+ if err != nil {+ s.markPlanCheckRunFailed(ctx, uid, planUID, err.Error())+ return+ }+ if project == nil {+ s.markPlanCheckRunFailed(ctx, uid, planUID, "project not found")+ return+ }++ // Get database group if needed (for spec expansion)+ databaseGroup, err := s.getDatabaseGroupForPlan(ctx, plan)+ if err != nil {+ s.markPlanCheckRunFailed(ctx, uid, planUID, err.Error())+ return+ }++ // Derive check targets from plan+ targets, err := DeriveCheckTargets(project, plan, databaseGroup)+ if err != nil {+ s.markPlanCheckRunFailed(ctx, uid, planUID, err.Error())+ return+ }++ var results []*storepb.PlanCheckRunResult_Result+ for _, target := range targets {+ targetResults, targetErr := s.executor.RunForTarget(ctx, target)+ if targetErr != nil {+ err = targetErr+ break+ }+ results = append(results, targetResults...)+ }+ if err != nil {+ if errors.Is(err, context.Canceled) {+ s.markPlanCheckRunCanceled(ctx, uid, planUID, err.Error())+ } else {+ s.markPlanCheckRunFailed(ctx, uid, planUID, err.Error())+ }+ } else {+ s.markPlanCheckRunDone(ctx, uid, planUID, results)+ }+}+```++**Step 3: Update helper functions to use uid and planUID parameters**++Replace the three mark functions:++```go+func (s *Scheduler) markPlanCheckRunDone(ctx context.Context, uid int, planUID int64, results []*storepb.PlanCheckRunResult_Result) {+ result := &storepb.PlanCheckRunResult{+ Results: results,+ }+ if err := s.store.UpdatePlanCheckRun(ctx,+ store.PlanCheckRunStatusDone,+ result,+ uid,+ ); err != nil {+ slog.Error("failed to mark plan check run done", log.BBError(err))+ return+ }++ // Auto-create rollout if plan checks pass+ issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PlanUID: &planUID})+ if err != nil {+ slog.Error("failed to get issue for approval check after plan check",+ slog.Int("plan_id", int(planUID)),+ log.BBError(err))+ return+ }+ if issue != nil && issue.PlanUID != nil {+ // Trigger approval finding+ s.bus.ApprovalCheckChan <- int64(issue.UID)+ // Trigger rollout creation (existing behavior)+ s.bus.RolloutCreationChan <- planUID+ }+}++func (s *Scheduler) markPlanCheckRunFailed(ctx context.Context, uid int, planUID int64, reason string) {+ result := &storepb.PlanCheckRunResult{+ Error: reason,+ }+ if err := s.store.UpdatePlanCheckRun(ctx,+ store.PlanCheckRunStatusFailed,+ result,+ uid,+ ); err != nil {+ slog.Error("failed to mark plan check run failed", log.BBError(err))+ }+}++func (s *Scheduler) markPlanCheckRunCanceled(ctx context.Context, uid int, planUID int64, reason string) {+ result := &storepb.PlanCheckRunResult{+ Error: reason,+ }+ if err := s.store.UpdatePlanCheckRun(ctx,+ store.PlanCheckRunStatusCanceled,+ result,+ uid,+ ); err != nil {+ slog.Error("failed to mark plan check run canceled", log.BBError(err))+ }+}+```++**Step 4: Verify build**++Run: `go build ./backend/...`+Expected: Build succeeds++**Step 5: Commit**++```bash+but commit plan-check-ha -m "refactor(plancheck): use atomic claiming instead of in-memory tracking"+```++---++## Task 6: Add database migration++**Files:**+- Create: `backend/migrator/migration/3.14/0022##plan_check_run_ha.sql`+- Modify: `backend/migrator/migration/LATEST.sql:218,225`++**Step 1: Create migration file**++Create `backend/migrator/migration/3.14/0022##plan_check_run_ha.sql`:++```sql+-- Add AVAILABLE status for HA-compatible plan check scheduling.+-- Uses FOR UPDATE SKIP LOCKED pattern for atomic claiming.++-- Update status constraint to include AVAILABLE+ALTER TABLE plan_check_run+ DROP CONSTRAINT plan_check_run_status_check,+ ADD CONSTRAINT plan_check_run_status_check+ CHECK (status IN ('AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED'));++-- Convert existing RUNNING to AVAILABLE (will be re-claimed after deployment)+UPDATE plan_check_run SET status = 'AVAILABLE' WHERE status = 'RUNNING';++-- Update index to include AVAILABLE for efficient claiming+DROP INDEX IF EXISTS idx_plan_check_run_active_status;+CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status IN ('AVAILABLE', 'RUNNING');+```++**Step 2: Update LATEST.sql constraint**++In `backend/migrator/migration/LATEST.sql`, change line 218 from:++```sql+ status text NOT NULL CHECK (status IN ('RUNNING', 'DONE', 'FAILED', 'CANCELED')),+```++to:++```sql+ status text NOT NULL CHECK (status IN ('AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED')),+```++**Step 3: Update LATEST.sql index**++In `backend/migrator/migration/LATEST.sql`, change line 225 from:++```sql+CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status = 'RUNNING';+```++to:++```sql+CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status IN ('AVAILABLE', 'RUNNING');+```++**Step 4: Commit**++```bash+but commit plan-check-ha -m "chore(migration): add AVAILABLE status for plan check runs"+```++---++## Task 7: Update migrator test version++**Files:**+- Modify: `backend/migrator/migrator_test.go`++**Step 1: Find and update TestLatestVersion**++Search for `TestLatestVersion` and update the migration count to include the new migration file.++Run: `grep -n "TestLatestVersion\|3.14" backend/migrator/migrator_test.go`++Update the `3.14` entry to include the new migration count (should be 22 now).++**Step 2: Verify test**++Run: `go test -v -count=1 github.com/bytebase/bytebase/backend/migrator -run ^TestLatestVersion$`+Expected: Test passes++**Step 3: Commit**++```bash+but commit plan-check-ha -m "test(migrator): update version for plan check run HA migration"+```++---++## Task 8: Run linter and fix issues++**Step 1: Run golangci-lint**++Run: `golangci-lint run --allow-parallel-runners`++**Step 2: Fix any issues reported**++Common issues to watch for:+- Unused parameters (prefix with `_`)+- Import ordering++**Step 3: Run lint again until clean**++Run: `golangci-lint run --allow-parallel-runners`+Expected: No issues++**Step 4: Commit if any fixes**++```bash+but commit plan-check-ha -m "fix: address linter issues"+```++---++## Task 9: Build and verify++**Step 1: Full backend build**++Run: `go build -ldflags "-w -s" -p=16 -o ./bytebase-build/bytebase ./backend/bin/server/main.go`+Expected: Build succeeds++**Step 2: Run related tests**++Run: `go test -v -count=1 github.com/bytebase/bytebase/backend/store -run PlanCheck`+Run: `go test -v -count=1 github.com/bytebase/bytebase/backend/runner/plancheck/...`++**Step 3: Commit if any fixes needed**++---++## Task 10: Create PR++**Step 1: Push branch**++```bash+but push plan-check-ha+```++**Step 2: Create PR**++```bash+gh pr create --base main --head plan-check-ha \+ --title "feat: make plan check run scheduler HA compatible" \+ --body "$(cat <<'EOF'+## Summary+- Add `AVAILABLE` status to plan check runs+- Implement atomic claiming with `FOR UPDATE SKIP LOCKED`+- Remove in-memory tracking from bus component+- Follows the same HA pattern as task run scheduler++## Test plan+- [ ] Verify plan checks still execute correctly in single-instance mode+- [ ] Verify migration applies cleanly+- [ ] Verify existing RUNNING plan checks are re-executed after deployment++🤖 Generated with [Claude Code](https://claude.com/claude-code)+EOF+)"+```