Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: make plan check run scheduler HA compatible (#18765)

p0nyc6daba9dd695
diff --git a/backend/api/v1/plan_service.go b/backend/api/v1/plan_service.go
index 185b444886..9eb7193b8b 100644
--- a/backend/api/v1/plan_service.go
+++ b/backend/api/v1/plan_service.go
@@ -490,11 +490,11 @@ func (s *PlanService) CancelPlanCheckRun(ctx context.Context, request *connect.R
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("plan check run not found for plan %d", planUID))
}
- if planCheckRun.Status != store.PlanCheckRunStatusRunning {
- return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("plan check run is not running"))
+ if planCheckRun.Status != store.PlanCheckRunStatusRunning && planCheckRun.Status != store.PlanCheckRunStatusAvailable {
+ return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("plan check run is not running or available"))
}
- // Cancel the plan check run.
+ // Cancel in-flight plan check run if running.
if cancelFunc, ok := s.bus.RunningPlanCheckRunsCancelFunc.Load(planCheckRun.UID); ok {
cancelFunc.(context.CancelFunc)()
}
diff --git a/backend/component/bus/bus.go b/backend/component/bus/bus.go
index afa2ae7323..2d0cb4a6c5 100644
--- a/backend/component/bus/bus.go
+++ b/backend/component/bus/bus.go
@@ -16,8 +16,6 @@ type Bus struct {
// RunningTaskRunsCancelFunc is the cancelFunc of running taskruns.
RunningTaskRunsCancelFunc sync.Map // map[taskRunID]context.CancelFunc
- // RunningPlanChecks is the set of running plan checks.
- RunningPlanChecks sync.Map
// RunningPlanCheckRunsCancelFunc is the cancelFunc of running plan checks.
RunningPlanCheckRunsCancelFunc sync.Map // map[planCheckRunUID]context.CancelFunc
diff --git a/backend/migrator/migration/3.14/0022##plan_check_run_ha.sql b/backend/migrator/migration/3.14/0022##plan_check_run_ha.sql
new file mode 100644
index 0000000000..715b7cfca5
--- /dev/null
+++ b/backend/migrator/migration/3.14/0022##plan_check_run_ha.sql
@@ -0,0 +1,15 @@
+-- Add AVAILABLE status for HA-compatible plan check scheduling.
+-- Uses FOR UPDATE SKIP LOCKED pattern for atomic claiming.
+
+-- Update status constraint to include AVAILABLE
+ALTER TABLE plan_check_run
+ DROP CONSTRAINT plan_check_run_status_check,
+ ADD CONSTRAINT plan_check_run_status_check
+ CHECK (status IN ('AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED'));
+
+-- Convert existing RUNNING to AVAILABLE (will be re-claimed after deployment)
+UPDATE plan_check_run SET status = 'AVAILABLE' WHERE status = 'RUNNING';
+
+-- Update index to include AVAILABLE for efficient claiming
+DROP INDEX IF EXISTS idx_plan_check_run_active_status;
+CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status IN ('AVAILABLE', 'RUNNING');
diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sql
index 3a8abdd9e4..fca73f1a77 100644
--- a/backend/migrator/migration/LATEST.sql
+++ b/backend/migrator/migration/LATEST.sql
@@ -215,14 +215,14 @@ CREATE TABLE plan_check_run (
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
plan_id bigint NOT NULL REFERENCES plan(id),
- status text NOT NULL CHECK (status IN ('RUNNING', 'DONE', 'FAILED', 'CANCELED')),
+ status text NOT NULL CHECK (status IN ('AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED')),
-- Stored as PlanCheckRunResult (proto/store/store/plan_check_run.proto)
result jsonb NOT NULL DEFAULT '{}'
);
CREATE UNIQUE INDEX idx_plan_check_run_unique_plan_id ON plan_check_run(plan_id);
-CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status = 'RUNNING';
+CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status IN ('AVAILABLE', 'RUNNING');
ALTER SEQUENCE plan_check_run_id_seq RESTART WITH 101;
diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.go
index 636c0d226a..7028e63940 100644
--- a/backend/migrator/migrator_test.go
+++ b/backend/migrator/migrator_test.go
@@ -12,8 +12,8 @@ import (
func TestLatestVersion(t *testing.T) {
files, err := getSortedVersionedFiles()
require.NoError(t, err)
- require.Equal(t, semver.MustParse("3.14.21"), *files[len(files)-1].version)
- require.Equal(t, "migration/3.14/0021##remove_plan_check_run_config_payload.sql", files[len(files)-1].path)
+ require.Equal(t, semver.MustParse("3.14.22"), *files[len(files)-1].version)
+ require.Equal(t, "migration/3.14/0022##plan_check_run_ha.sql", files[len(files)-1].path)
}
func TestVersionUnique(t *testing.T) {
diff --git a/backend/runner/plancheck/scheduler.go b/backend/runner/plancheck/scheduler.go
index 7957dcf732..14ce31ca28 100644
--- a/backend/runner/plancheck/scheduler.go
+++ b/backend/runner/plancheck/scheduler.go
@@ -68,112 +68,96 @@ func (s *Scheduler) runOnce(ctx context.Context) {
}
}()
- planCheckRuns, err := s.store.ListPlanCheckRuns(ctx, &store.FindPlanCheckRunMessage{
- Status: &[]store.PlanCheckRunStatus{
- store.PlanCheckRunStatusRunning,
- },
- })
+ claimed, err := s.store.ClaimAvailablePlanCheckRuns(ctx)
if err != nil {
- slog.Error("failed to list running plan check runs", log.BBError(err))
+ slog.Error("failed to claim available plan check runs", log.BBError(err))
return
}
- for _, planCheckRun := range planCheckRuns {
- s.runPlanCheckRun(ctx, planCheckRun)
+ for _, c := range claimed {
+ go s.runPlanCheckRun(ctx, c.UID, c.PlanUID)
}
}
-func (s *Scheduler) runPlanCheckRun(ctx context.Context, planCheckRun *store.PlanCheckRunMessage) {
- // Skip the plan check run if it is already running.
- if _, ok := s.bus.RunningPlanChecks.Load(planCheckRun.UID); ok {
+func (s *Scheduler) runPlanCheckRun(ctx context.Context, uid int, planUID int64) {
+ ctxWithCancel, cancel := context.WithCancel(ctx)
+ defer cancel()
+ s.bus.RunningPlanCheckRunsCancelFunc.Store(uid, cancel)
+ defer s.bus.RunningPlanCheckRunsCancelFunc.Delete(uid)
+
+ // Fetch plan to derive check targets at runtime
+ plan, err := s.store.GetPlan(ctxWithCancel, &store.FindPlanMessage{UID: &planUID})
+ if err != nil {
+ s.markPlanCheckRunFailed(ctxWithCancel, uid, err.Error())
+ return
+ }
+ if plan == nil {
+ s.markPlanCheckRunFailed(ctxWithCancel, uid, "plan not found")
return
}
- s.bus.RunningPlanChecks.Store(planCheckRun.UID, true)
- go func() {
- defer func() {
- s.bus.RunningPlanChecks.Delete(planCheckRun.UID)
- s.bus.RunningPlanCheckRunsCancelFunc.Delete(planCheckRun.UID)
- }()
-
- ctxWithCancel, cancel := context.WithCancel(ctx)
- defer cancel()
- s.bus.RunningPlanCheckRunsCancelFunc.Store(planCheckRun.UID, cancel)
-
- // Fetch plan to derive check targets at runtime
- plan, err := s.store.GetPlan(ctxWithCancel, &store.FindPlanMessage{UID: &planCheckRun.PlanUID})
- if err != nil {
- s.markPlanCheckRunFailed(ctx, planCheckRun, err.Error())
- return
- }
- if plan == nil {
- s.markPlanCheckRunFailed(ctx, planCheckRun, "plan not found")
- return
- }
-
- project, err := s.store.GetProject(ctxWithCancel, &store.FindProjectMessage{ResourceID: &plan.ProjectID})
- if err != nil {
- s.markPlanCheckRunFailed(ctx, planCheckRun, err.Error())
- return
- }
- if project == nil {
- s.markPlanCheckRunFailed(ctx, planCheckRun, "project not found")
- return
- }
+ project, err := s.store.GetProject(ctxWithCancel, &store.FindProjectMessage{ResourceID: &plan.ProjectID})
+ if err != nil {
+ s.markPlanCheckRunFailed(ctxWithCancel, uid, err.Error())
+ return
+ }
+ if project == nil {
+ s.markPlanCheckRunFailed(ctxWithCancel, uid, "project not found")
+ return
+ }
- // Get database group if needed (for spec expansion)
- databaseGroup, err := s.getDatabaseGroupForPlan(ctxWithCancel, plan)
- if err != nil {
- s.markPlanCheckRunFailed(ctx, planCheckRun, err.Error())
- return
- }
+ // Get database group if needed (for spec expansion)
+ databaseGroup, err := s.getDatabaseGroupForPlan(ctxWithCancel, plan)
+ if err != nil {
+ s.markPlanCheckRunFailed(ctxWithCancel, uid, err.Error())
+ return
+ }
- // Derive check targets from plan
- targets, err := DeriveCheckTargets(project, plan, databaseGroup)
- if err != nil {
- s.markPlanCheckRunFailed(ctx, planCheckRun, err.Error())
- return
- }
+ // Derive check targets from plan
+ targets, err := DeriveCheckTargets(project, plan, databaseGroup)
+ if err != nil {
+ s.markPlanCheckRunFailed(ctxWithCancel, uid, err.Error())
+ return
+ }
- var results []*storepb.PlanCheckRunResult_Result
- for _, target := range targets {
- targetResults, targetErr := s.executor.RunForTarget(ctxWithCancel, target)
- if targetErr != nil {
- err = targetErr
- break
- }
- results = append(results, targetResults...)
+ var results []*storepb.PlanCheckRunResult_Result
+ for _, target := range targets {
+ targetResults, targetErr := s.executor.RunForTarget(ctxWithCancel, target)
+ if targetErr != nil {
+ err = targetErr
+ break
}
- if err != nil {
- if errors.Is(err, context.Canceled) {
- s.markPlanCheckRunCanceled(ctx, planCheckRun, err.Error())
- } else {
- s.markPlanCheckRunFailed(ctx, planCheckRun, err.Error())
- }
+ results = append(results, targetResults...)
+ }
+ if err != nil {
+ if errors.Is(err, context.Canceled) {
+ s.markPlanCheckRunCanceled(ctxWithCancel, uid, err.Error())
} else {
- s.markPlanCheckRunDone(ctx, planCheckRun, results)
+ s.markPlanCheckRunFailed(ctxWithCancel, uid, err.Error())
}
- }()
+ } else {
+ s.markPlanCheckRunDone(ctxWithCancel, uid, planUID, results)
+ }
}
-func (s *Scheduler) markPlanCheckRunDone(ctx context.Context, planCheckRun *store.PlanCheckRunMessage, results []*storepb.PlanCheckRunResult_Result) {
+func (s *Scheduler) markPlanCheckRunDone(ctx context.Context, uid int, planUID int64, results []*storepb.PlanCheckRunResult_Result) {
result := &storepb.PlanCheckRunResult{
Results: results,
}
if err := s.store.UpdatePlanCheckRun(ctx,
store.PlanCheckRunStatusDone,
result,
- planCheckRun.UID,
+ uid,
); err != nil {
slog.Error("failed to mark plan check run done", log.BBError(err))
return
}
// Auto-create rollout if plan checks pass
- issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PlanUID: &planCheckRun.PlanUID})
+ issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PlanUID: &planUID})
if err != nil {
slog.Error("failed to get issue for approval check after plan check",
- slog.Int("plan_id", int(planCheckRun.PlanUID)),
+ slog.Int("plan_id", int(planUID)),
log.BBError(err))
return
}
@@ -181,31 +165,31 @@ func (s *Scheduler) markPlanCheckRunDone(ctx context.Context, planCheckRun *stor
// Trigger approval finding
s.bus.ApprovalCheckChan <- int64(issue.UID)
// Trigger rollout creation (existing behavior)
- s.bus.RolloutCreationChan <- planCheckRun.PlanUID
+ s.bus.RolloutCreationChan <- planUID
}
}
-func (s *Scheduler) markPlanCheckRunFailed(ctx context.Context, planCheckRun *store.PlanCheckRunMessage, reason string) {
+func (s *Scheduler) markPlanCheckRunFailed(ctx context.Context, uid int, reason string) {
result := &storepb.PlanCheckRunResult{
Error: reason,
}
if err := s.store.UpdatePlanCheckRun(ctx,
store.PlanCheckRunStatusFailed,
result,
- planCheckRun.UID,
+ uid,
); err != nil {
slog.Error("failed to mark plan check run failed", log.BBError(err))
}
}
-func (s *Scheduler) markPlanCheckRunCanceled(ctx context.Context, planCheckRun *store.PlanCheckRunMessage, reason string) {
+func (s *Scheduler) markPlanCheckRunCanceled(ctx context.Context, uid int, reason string) {
result := &storepb.PlanCheckRunResult{
Error: reason,
}
if err := s.store.UpdatePlanCheckRun(ctx,
store.PlanCheckRunStatusCanceled,
result,
- planCheckRun.UID,
+ uid,
); err != nil {
slog.Error("failed to mark plan check run canceled", log.BBError(err))
}
diff --git a/backend/store/plan_check_run.go b/backend/store/plan_check_run.go
index c099e55e01..3174d125b5 100644
--- a/backend/store/plan_check_run.go
+++ b/backend/store/plan_check_run.go
@@ -16,6 +16,8 @@ import (
type PlanCheckRunStatus string
const (
+ // PlanCheckRunStatusAvailable is the plan check status for AVAILABLE.
+ PlanCheckRunStatusAvailable PlanCheckRunStatus = "AVAILABLE"
// PlanCheckRunStatusRunning is the plan check status for RUNNING.
PlanCheckRunStatusRunning PlanCheckRunStatus = "RUNNING"
// PlanCheckRunStatusDone is the plan check status for DONE.
@@ -47,6 +49,7 @@ type FindPlanCheckRunMessage struct {
}
// CreatePlanCheckRun creates or replaces the plan check run for a plan.
+// Always creates with AVAILABLE status for HA-safe scheduling.
func (s *Store) CreatePlanCheckRun(ctx context.Context, create *PlanCheckRunMessage) error {
result, err := protojson.Marshal(create.Result)
if err != nil {
@@ -61,7 +64,7 @@ func (s *Store) CreatePlanCheckRun(ctx context.Context, create *PlanCheckRunMess
result = EXCLUDED.result,
updated_at = now()
`
- if _, err := s.GetDB().ExecContext(ctx, query, create.PlanUID, create.Status, result); err != nil {
+ if _, err := s.GetDB().ExecContext(ctx, query, create.PlanUID, PlanCheckRunStatusAvailable, result); err != nil {
return errors.Wrapf(err, "failed to upsert plan check run")
}
return nil
@@ -187,3 +190,50 @@ func (s *Store) BatchCancelPlanCheckRuns(ctx context.Context, planCheckRunUIDs [
}
return nil
}
+
+// ClaimedPlanCheckRun represents a plan check run that was atomically claimed.
+type ClaimedPlanCheckRun struct {
+ UID int
+ PlanUID int64
+}
+
+// ClaimAvailablePlanCheckRuns atomically claims all AVAILABLE plan check runs by updating them to RUNNING
+// and returns the claimed UIDs. Uses FOR UPDATE SKIP LOCKED to allow concurrent schedulers to claim different runs.
+func (s *Store) ClaimAvailablePlanCheckRuns(ctx context.Context) ([]*ClaimedPlanCheckRun, error) {
+ q := qb.Q().Space(`
+ UPDATE plan_check_run
+ SET status = ?, updated_at = now()
+ WHERE id IN (
+ SELECT id FROM plan_check_run
+ WHERE status = ?
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING id, plan_id
+ `, PlanCheckRunStatusRunning, PlanCheckRunStatusAvailable)
+
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to build sql")
+ }
+
+ rows, err := s.GetDB().QueryContext(ctx, query, args...)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to claim plan check runs")
+ }
+ defer rows.Close()
+
+ var claimed []*ClaimedPlanCheckRun
+ for rows.Next() {
+ var c ClaimedPlanCheckRun
+ if err := rows.Scan(&c.UID, &c.PlanUID); err != nil {
+ return nil, err
+ }
+ claimed = append(claimed, &c)
+ }
+
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+
+ return claimed, nil
+}
diff --git a/docs/plans/2026-01-05-plan-check-run-ha-design.md b/docs/plans/2026-01-05-plan-check-run-ha-design.md
new file mode 100644
index 0000000000..7ed5ac07b9
--- /dev/null
+++ b/docs/plans/2026-01-05-plan-check-run-ha-design.md
@@ -0,0 +1,105 @@
+# Plan Check Run Scheduler HA Compatibility
+
+## Overview
+
+Make the plan check run scheduler HA (High Availability) compatible by following patterns established in the task run scheduler.
+
+## Current State
+
+The plan check scheduler is **not HA-compatible**:
+- Uses in-memory `sync.Map` to track running checks
+- Simple polling loop every 5 seconds
+- No database locking - relies on single-instance assumption
+- Status: `RUNNING → DONE/FAILED/CANCELED`
+
+## Target State
+
+Follow the task run scheduler HA pattern:
+- Use `FOR UPDATE SKIP LOCKED` to atomically claim work
+- Status state machine: `AVAILABLE → RUNNING → DONE/FAILED/CANCELED`
+- Multiple instances compete fairly via database queries
+- No leader election - peer-to-peer work queue pattern
+
+## Design
+
+### Schema Changes
+
+**Migration file:** `backend/migrator/migration/3.X/XXXX##plan_check_run_ha.sql`
+
+```sql
+-- Add AVAILABLE to status check constraint
+ALTER TABLE plan_check_run
+ DROP CONSTRAINT plan_check_run_status_check,
+ ADD CONSTRAINT plan_check_run_status_check
+ CHECK (status IN ('AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED'));
+
+-- Convert existing RUNNING to AVAILABLE (will be re-claimed)
+UPDATE plan_check_run SET status = 'AVAILABLE' WHERE status = 'RUNNING';
+
+-- Update index to include AVAILABLE for efficient claiming
+DROP INDEX IF EXISTS idx_plan_check_run_active_status;
+CREATE INDEX idx_plan_check_run_active_status
+ ON plan_check_run(status, id)
+ WHERE status IN ('AVAILABLE', 'RUNNING');
+```
+
+### Store Layer Changes
+
+**File:** `backend/store/plan_check_run.go`
+
+Add claiming function:
+
+```go
+// ClaimAvailablePlanCheckRuns atomically claims all AVAILABLE plan check runs.
+// Uses FOR UPDATE SKIP LOCKED for HA-safe concurrent claiming.
+func (s *Store) ClaimAvailablePlanCheckRuns(ctx context.Context) ([]*PlanCheckRunMessage, error) {
+ query := `
+ UPDATE plan_check_run
+ SET status = 'RUNNING', updated_at = now()
+ WHERE id IN (
+ SELECT id FROM plan_check_run
+ WHERE status = 'AVAILABLE'
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING id, plan_id
+ `
+ // Execute and return claimed runs
+}
+```
+
+Modify `CreatePlanCheckRun`: Change default status from `RUNNING` to `AVAILABLE`.
+
+### Scheduler Changes
+
+**File:** `backend/runner/plancheck/scheduler.go`
+
+1. **Remove in-memory tracking** - Delete `RunningPlanChecks` and `RunningPlanCheckRunsCancelFunc` sync.Maps
+
+2. **Update `runOnce()`** - Use atomic claiming instead of querying RUNNING status:
+ ```go
+ func (s *Scheduler) runOnce(ctx context.Context) {
+ claimed, err := s.store.ClaimAvailablePlanCheckRuns(ctx)
+ for _, planCheckRun := range claimed {
+ go s.runPlanCheckRun(ctx, planCheckRun)
+ }
+ }
+ ```
+
+3. **Simplify cancellation** - Update database status to `CANCELED` directly
+
+### API Changes
+
+No changes needed - store layer handles status transition.
+
+## Files to Modify
+
+| File | Change |
+|------|--------|
+| `backend/migrator/migration/3.X/XXXX##plan_check_run_ha.sql` | New migration |
+| `backend/migrator/migration/LATEST.sql` | Update schema |
+| `backend/store/plan_check_run.go` | Add claiming, change default status |
+| `backend/runner/plancheck/scheduler.go` | Remove sync.Map, use claiming |
+
+## Migration Strategy
+
+Existing `RUNNING` plan check runs are converted to `AVAILABLE` during migration. They will be re-claimed and re-executed by the scheduler after deployment.
diff --git a/docs/plans/2026-01-05-plan-check-run-ha.md b/docs/plans/2026-01-05-plan-check-run-ha.md
new file mode 100644
index 0000000000..b2cd80435d
--- /dev/null
+++ b/docs/plans/2026-01-05-plan-check-run-ha.md
@@ -0,0 +1,551 @@
+# Plan Check Run HA Implementation Plan
+
+> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
+
+**Goal:** Make the plan check run scheduler HA-compatible using database-level atomic claiming.
+
+**Architecture:** Add `AVAILABLE` status to plan check runs. Scheduler atomically claims `AVAILABLE` runs using `FOR UPDATE SKIP LOCKED`, transitioning them to `RUNNING`. Remove in-memory tracking from bus.
+
+**Tech Stack:** Go, PostgreSQL, protobuf
+
+---
+
+## Task 1: Add AVAILABLE status constant to store
+
+**Files:**
+- Modify: `backend/store/plan_check_run.go:18-27`
+
+**Step 1: Add the new status constant**
+
+In `backend/store/plan_check_run.go`, add `PlanCheckRunStatusAvailable` after line 18:
+
+```go
+const (
+ // PlanCheckRunStatusAvailable is the plan check status for AVAILABLE.
+ PlanCheckRunStatusAvailable PlanCheckRunStatus = "AVAILABLE"
+ // PlanCheckRunStatusRunning is the plan check status for RUNNING.
+ PlanCheckRunStatusRunning PlanCheckRunStatus = "RUNNING"
+ // PlanCheckRunStatusDone is the plan check status for DONE.
+ PlanCheckRunStatusDone PlanCheckRunStatus = "DONE"
+ // PlanCheckRunStatusFailed is the plan check status for FAILED.
+ PlanCheckRunStatusFailed PlanCheckRunStatus = "FAILED"
+ // PlanCheckRunStatusCanceled is the plan check status for CANCELED.
+ PlanCheckRunStatusCanceled PlanCheckRunStatus = "CANCELED"
+)
+```
+
+**Step 2: Verify build**
+
+Run: `go build ./backend/store/...`
+Expected: Build succeeds
+
+**Step 3: Commit**
+
+```bash
+but commit plan-check-ha -m "feat(store): add AVAILABLE status for plan check runs"
+```
+
+---
+
+## Task 2: Add ClaimAvailablePlanCheckRuns function to store
+
+**Files:**
+- Modify: `backend/store/plan_check_run.go`
+
+**Step 1: Add ClaimedPlanCheckRun struct and claiming function**
+
+Add after `BatchCancelPlanCheckRuns` function (after line 189):
+
+```go
+// ClaimedPlanCheckRun represents a plan check run that was atomically claimed.
+type ClaimedPlanCheckRun struct {
+ UID int
+ PlanUID int64
+}
+
+// ClaimAvailablePlanCheckRuns atomically claims all AVAILABLE plan check runs by updating them to RUNNING
+// and returns the claimed UIDs. Uses FOR UPDATE SKIP LOCKED to allow concurrent schedulers to claim different runs.
+func (s *Store) ClaimAvailablePlanCheckRuns(ctx context.Context) ([]*ClaimedPlanCheckRun, error) {
+ q := qb.Q().Space(`
+ UPDATE plan_check_run
+ SET status = ?, updated_at = now()
+ WHERE id IN (
+ SELECT id FROM plan_check_run
+ WHERE status = ?
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING id, plan_id
+ `, PlanCheckRunStatusRunning, PlanCheckRunStatusAvailable)
+
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to build sql")
+ }
+
+ rows, err := s.GetDB().QueryContext(ctx, query, args...)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to claim plan check runs")
+ }
+ defer rows.Close()
+
+ var claimed []*ClaimedPlanCheckRun
+ for rows.Next() {
+ var c ClaimedPlanCheckRun
+ if err := rows.Scan(&c.UID, &c.PlanUID); err != nil {
+ return nil, err
+ }
+ claimed = append(claimed, &c)
+ }
+
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+
+ return claimed, nil
+}
+```
+
+**Step 2: Verify build**
+
+Run: `go build ./backend/store/...`
+Expected: Build succeeds
+
+**Step 3: Commit**
+
+```bash
+but commit plan-check-ha -m "feat(store): add ClaimAvailablePlanCheckRuns for HA scheduling"
+```
+
+---
+
+## Task 3: Update CreatePlanCheckRun to use AVAILABLE status
+
+**Files:**
+- Modify: `backend/store/plan_check_run.go:50-68`
+
+**Step 1: Modify CreatePlanCheckRun to always use AVAILABLE**
+
+Replace the `CreatePlanCheckRun` function to ignore the passed status and always use AVAILABLE:
+
+```go
+// CreatePlanCheckRun creates or replaces the plan check run for a plan.
+// Always creates with AVAILABLE status for HA-safe scheduling.
+func (s *Store) CreatePlanCheckRun(ctx context.Context, create *PlanCheckRunMessage) error {
+ result, err := protojson.Marshal(create.Result)
+ if err != nil {
+ return errors.Wrapf(err, "failed to marshal result")
+ }
+
+ query := `
+ INSERT INTO plan_check_run (plan_id, status, result)
+ VALUES ($1, $2, $3)
+ ON CONFLICT (plan_id) DO UPDATE SET
+ status = EXCLUDED.status,
+ result = EXCLUDED.result,
+ updated_at = now()
+ `
+ if _, err := s.GetDB().ExecContext(ctx, query, create.PlanUID, PlanCheckRunStatusAvailable, result); err != nil {
+ return errors.Wrapf(err, "failed to upsert plan check run")
+ }
+ return nil
+}
+```
+
+**Step 2: Verify build**
+
+Run: `go build ./backend/store/...`
+Expected: Build succeeds
+
+**Step 3: Commit**
+
+```bash
+but commit plan-check-ha -m "feat(store): CreatePlanCheckRun always uses AVAILABLE status"
+```
+
+---
+
+## Task 4: Remove in-memory plan check tracking from bus
+
+**Files:**
+- Modify: `backend/component/bus/bus.go:19-22`
+
+**Step 1: Remove RunningPlanChecks and RunningPlanCheckRunsCancelFunc**
+
+Remove these two lines from the Bus struct:
+
+```go
+// RunningPlanChecks is the set of running plan checks.
+RunningPlanChecks sync.Map
+// RunningPlanCheckRunsCancelFunc is the cancelFunc of running plan checks.
+RunningPlanCheckRunsCancelFunc sync.Map // map[planCheckRunUID]context.CancelFunc
+```
+
+The Bus struct should now look like:
+
+```go
+// Bus is the message bus for all in-memory communication within the server.
+type Bus struct {
+ // ApprovalCheckChan signals when an issue needs approval template finding.
+ // Triggered by plan check completion, issue creation (if checks already done).
+ ApprovalCheckChan chan int64 // issue UID
+
+ TaskRunSchedulerInfo sync.Map // map[taskRunID]*storepb.SchedulerInfo
+
+ // RunningTaskRunsCancelFunc is the cancelFunc of running taskruns.
+ RunningTaskRunsCancelFunc sync.Map // map[taskRunID]context.CancelFunc
+
+ // PlanCheckTickleChan is the tickler for plan check scheduler.
+ PlanCheckTickleChan chan int
+ // TaskRunTickleChan is the tickler for task run scheduler.
+ TaskRunTickleChan chan int
+
+ // RolloutCreationChan is the channel for automatic rollout creation.
+ RolloutCreationChan chan int64
+
+ // PlanCompletionCheckChan signals when a plan might be complete (for PIPELINE_COMPLETED webhook).
+ PlanCompletionCheckChan chan int64
+}
+```
+
+**Step 2: Verify build fails (expected - scheduler still references these)**
+
+Run: `go build ./backend/...`
+Expected: Build fails with references to deleted fields
+
+**Step 3: Commit (partial - will fix in next task)**
+
+```bash
+but commit plan-check-ha -m "refactor(bus): remove in-memory plan check tracking"
+```
+
+---
+
+## Task 5: Update scheduler to use atomic claiming
+
+**Files:**
+- Modify: `backend/runner/plancheck/scheduler.go`
+
+**Step 1: Update runOnce to use claiming**
+
+Replace the `runOnce` function:
+
+```go
+func (s *Scheduler) runOnce(ctx context.Context) {
+ defer func() {
+ if r := recover(); r != nil {
+ err, ok := r.(error)
+ if !ok {
+ err = errors.Errorf("%v", r)
+ }
+ slog.Error("Plan check scheduler PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))
+ }
+ }()
+
+ claimed, err := s.store.ClaimAvailablePlanCheckRuns(ctx)
+ if err != nil {
+ slog.Error("failed to claim available plan check runs", log.BBError(err))
+ return
+ }
+
+ for _, c := range claimed {
+ go s.runPlanCheckRun(ctx, c.UID, c.PlanUID)
+ }
+}
+```
+
+**Step 2: Update runPlanCheckRun signature and remove in-memory tracking**
+
+Replace the `runPlanCheckRun` function:
+
+```go
+func (s *Scheduler) runPlanCheckRun(ctx context.Context, uid int, planUID int64) {
+ // Fetch plan to derive check targets at runtime
+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &planUID})
+ if err != nil {
+ s.markPlanCheckRunFailed(ctx, uid, planUID, err.Error())
+ return
+ }
+ if plan == nil {
+ s.markPlanCheckRunFailed(ctx, uid, planUID, "plan not found")
+ return
+ }
+
+ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})
+ if err != nil {
+ s.markPlanCheckRunFailed(ctx, uid, planUID, err.Error())
+ return
+ }
+ if project == nil {
+ s.markPlanCheckRunFailed(ctx, uid, planUID, "project not found")
+ return
+ }
+
+ // Get database group if needed (for spec expansion)
+ databaseGroup, err := s.getDatabaseGroupForPlan(ctx, plan)
+ if err != nil {
+ s.markPlanCheckRunFailed(ctx, uid, planUID, err.Error())
+ return
+ }
+
+ // Derive check targets from plan
+ targets, err := DeriveCheckTargets(project, plan, databaseGroup)
+ if err != nil {
+ s.markPlanCheckRunFailed(ctx, uid, planUID, err.Error())
+ return
+ }
+
+ var results []*storepb.PlanCheckRunResult_Result
+ for _, target := range targets {
+ targetResults, targetErr := s.executor.RunForTarget(ctx, target)
+ if targetErr != nil {
+ err = targetErr
+ break
+ }
+ results = append(results, targetResults...)
+ }
+ if err != nil {
+ if errors.Is(err, context.Canceled) {
+ s.markPlanCheckRunCanceled(ctx, uid, planUID, err.Error())
+ } else {
+ s.markPlanCheckRunFailed(ctx, uid, planUID, err.Error())
+ }
+ } else {
+ s.markPlanCheckRunDone(ctx, uid, planUID, results)
+ }
+}
+```
+
+**Step 3: Update helper functions to use uid and planUID parameters**
+
+Replace the three mark functions:
+
+```go
+func (s *Scheduler) markPlanCheckRunDone(ctx context.Context, uid int, planUID int64, results []*storepb.PlanCheckRunResult_Result) {
+ result := &storepb.PlanCheckRunResult{
+ Results: results,
+ }
+ if err := s.store.UpdatePlanCheckRun(ctx,
+ store.PlanCheckRunStatusDone,
+ result,
+ uid,
+ ); err != nil {
+ slog.Error("failed to mark plan check run done", log.BBError(err))
+ return
+ }
+
+ // Auto-create rollout if plan checks pass
+ issue, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PlanUID: &planUID})
+ if err != nil {
+ slog.Error("failed to get issue for approval check after plan check",
+ slog.Int("plan_id", int(planUID)),
+ log.BBError(err))
+ return
+ }
+ if issue != nil && issue.PlanUID != nil {
+ // Trigger approval finding
+ s.bus.ApprovalCheckChan <- int64(issue.UID)
+ // Trigger rollout creation (existing behavior)
+ s.bus.RolloutCreationChan <- planUID
+ }
+}
+
+func (s *Scheduler) markPlanCheckRunFailed(ctx context.Context, uid int, planUID int64, reason string) {
+ result := &storepb.PlanCheckRunResult{
+ Error: reason,
+ }
+ if err := s.store.UpdatePlanCheckRun(ctx,
+ store.PlanCheckRunStatusFailed,
+ result,
+ uid,
+ ); err != nil {
+ slog.Error("failed to mark plan check run failed", log.BBError(err))
+ }
+}
+
+func (s *Scheduler) markPlanCheckRunCanceled(ctx context.Context, uid int, planUID int64, reason string) {
+ result := &storepb.PlanCheckRunResult{
+ Error: reason,
+ }
+ if err := s.store.UpdatePlanCheckRun(ctx,
+ store.PlanCheckRunStatusCanceled,
+ result,
+ uid,
+ ); err != nil {
+ slog.Error("failed to mark plan check run canceled", log.BBError(err))
+ }
+}
+```
+
+**Step 4: Verify build**
+
+Run: `go build ./backend/...`
+Expected: Build succeeds
+
+**Step 5: Commit**
+
+```bash
+but commit plan-check-ha -m "refactor(plancheck): use atomic claiming instead of in-memory tracking"
+```
+
+---
+
+## Task 6: Add database migration
+
+**Files:**
+- Create: `backend/migrator/migration/3.14/0022##plan_check_run_ha.sql`
+- Modify: `backend/migrator/migration/LATEST.sql:218,225`
+
+**Step 1: Create migration file**
+
+Create `backend/migrator/migration/3.14/0022##plan_check_run_ha.sql`:
+
+```sql
+-- Add AVAILABLE status for HA-compatible plan check scheduling.
+-- Uses FOR UPDATE SKIP LOCKED pattern for atomic claiming.
+
+-- Update status constraint to include AVAILABLE
+ALTER TABLE plan_check_run
+ DROP CONSTRAINT plan_check_run_status_check,
+ ADD CONSTRAINT plan_check_run_status_check
+ CHECK (status IN ('AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED'));
+
+-- Convert existing RUNNING to AVAILABLE (will be re-claimed after deployment)
+UPDATE plan_check_run SET status = 'AVAILABLE' WHERE status = 'RUNNING';
+
+-- Update index to include AVAILABLE for efficient claiming
+DROP INDEX IF EXISTS idx_plan_check_run_active_status;
+CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status IN ('AVAILABLE', 'RUNNING');
+```
+
+**Step 2: Update LATEST.sql constraint**
+
+In `backend/migrator/migration/LATEST.sql`, change line 218 from:
+
+```sql
+ status text NOT NULL CHECK (status IN ('RUNNING', 'DONE', 'FAILED', 'CANCELED')),
+```
+
+to:
+
+```sql
+ status text NOT NULL CHECK (status IN ('AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED')),
+```
+
+**Step 3: Update LATEST.sql index**
+
+In `backend/migrator/migration/LATEST.sql`, change line 225 from:
+
+```sql
+CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status = 'RUNNING';
+```
+
+to:
+
+```sql
+CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERE status IN ('AVAILABLE', 'RUNNING');
+```
+
+**Step 4: Commit**
+
+```bash
+but commit plan-check-ha -m "chore(migration): add AVAILABLE status for plan check runs"
+```
+
+---
+
+## Task 7: Update migrator test version
+
+**Files:**
+- Modify: `backend/migrator/migrator_test.go`
+
+**Step 1: Find and update TestLatestVersion**
+
+Search for `TestLatestVersion` and update the migration count to include the new migration file.
+
+Run: `grep -n "TestLatestVersion\|3.14" backend/migrator/migrator_test.go`
+
+Update the `3.14` entry to include the new migration count (should be 22 now).
+
+**Step 2: Verify test**
+
+Run: `go test -v -count=1 github.com/bytebase/bytebase/backend/migrator -run ^TestLatestVersion$`
+Expected: Test passes
+
+**Step 3: Commit**
+
+```bash
+but commit plan-check-ha -m "test(migrator): update version for plan check run HA migration"
+```
+
+---
+
+## Task 8: Run linter and fix issues
+
+**Step 1: Run golangci-lint**
+
+Run: `golangci-lint run --allow-parallel-runners`
+
+**Step 2: Fix any issues reported**
+
+Common issues to watch for:
+- Unused parameters (prefix with `_`)
+- Import ordering
+
+**Step 3: Run lint again until clean**
+
+Run: `golangci-lint run --allow-parallel-runners`
+Expected: No issues
+
+**Step 4: Commit if any fixes**
+
+```bash
+but commit plan-check-ha -m "fix: address linter issues"
+```
+
+---
+
+## Task 9: Build and verify
+
+**Step 1: Full backend build**
+
+Run: `go build -ldflags "-w -s" -p=16 -o ./bytebase-build/bytebase ./backend/bin/server/main.go`
+Expected: Build succeeds
+
+**Step 2: Run related tests**
+
+Run: `go test -v -count=1 github.com/bytebase/bytebase/backend/store -run PlanCheck`
+Run: `go test -v -count=1 github.com/bytebase/bytebase/backend/runner/plancheck/...`
+
+**Step 3: Commit if any fixes needed**
+
+---
+
+## Task 10: Create PR
+
+**Step 1: Push branch**
+
+```bash
+but push plan-check-ha
+```
+
+**Step 2: Create PR**
+
+```bash
+gh pr create --base main --head plan-check-ha \
+ --title "feat: make plan check run scheduler HA compatible" \
+ --body "$(cat <<'EOF'
+## Summary
+- Add `AVAILABLE` status to plan check runs
+- Implement atomic claiming with `FOR UPDATE SKIP LOCKED`
+- Remove in-memory tracking from bus component
+- Follows the same HA pattern as task run scheduler
+
+## Test plan
+- [ ] Verify plan checks still execute correctly in single-instance mode
+- [ ] Verify migration applies cleanly
+- [ ] Verify existing RUNNING plan checks are re-executed after deployment
+
+🤖 Generated with [Claude Code](https://claude.com/claude-code)
+EOF
+)"
+```