Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: add timeout-based stale detection for plan check runs (#18843)
p0ny795d3a22e2dd
diff --git a/backend/runner/cleaner/data_cleaner.go b/backend/runner/cleaner/data_cleaner.goindex 6e9f36e301..64683977dc 100644--- a/backend/runner/cleaner/data_cleaner.go+++ b/backend/runner/cleaner/data_cleaner.go@@ -14,6 +14,7 @@ const (cleanupInterval = 1 * time.HourstaleDetectionInterval = 30 * time.SecondstalenessThreshold = 1 * time.Minute+ planCheckRunTimeout = 10 * time.MinuteheartbeatRetentionPeriod = 1 * time.HourexportArchiveRetentionPeriod = 24 * time.Houroauth2ClientRetentionPeriod = 30 * 24 * time.Hour // 30 days of inactivity@@ -48,6 +49,7 @@ func (c *DataCleaner) Run(ctx context.Context, wg *sync.WaitGroup) {// Run cleanup immediately on startupc.cleanup(ctx)c.detectStaleTaskRuns(ctx)+ c.detectStalePlanCheckRuns(ctx)for {select {@@ -55,6 +57,7 @@ func (c *DataCleaner) Run(ctx context.Context, wg *sync.WaitGroup) {c.cleanup(ctx)case <-staleTicker.C:c.detectStaleTaskRuns(ctx)+ c.detectStalePlanCheckRuns(ctx)case <-ctx.Done():return}@@ -79,6 +82,17 @@ func (c *DataCleaner) detectStaleTaskRuns(ctx context.Context) {}}+func (c *DataCleaner) detectStalePlanCheckRuns(ctx context.Context) {+ rowsAffected, err := c.store.FailStalePlanCheckRuns(ctx, planCheckRunTimeout)+ if err != nil {+ slog.Error("Failed to detect stale plan check runs", log.BBError(err))+ return+ }+ if rowsAffected > 0 {+ slog.Info("Marked stale plan check runs as failed", slog.Int64("count", rowsAffected))+ }+}+func (c *DataCleaner) cleanupStaleHeartbeats(ctx context.Context) {rowsAffected, err := c.store.DeleteStaleReplicaHeartbeats(ctx, heartbeatRetentionPeriod)if err != nil {diff --git a/backend/store/plan_check_run.go b/backend/store/plan_check_run.goindex 3174d125b5..80f11c3a22 100644--- a/backend/store/plan_check_run.go+++ b/backend/store/plan_check_run.go@@ -197,6 +197,32 @@ type ClaimedPlanCheckRun struct {PlanUID int64}+// FailStalePlanCheckRuns marks RUNNING plan check runs as FAILED if they have been running+// longer than the timeout threshold. Unlike task runs, plan check runs don't use heartbeat+// detection - they use a simple timeout since they have a bounded execution time.+func (s *Store) FailStalePlanCheckRuns(ctx context.Context, timeout time.Duration) (int64, error) {+ q := qb.Q().Space(`+ UPDATE plan_check_run+ SET status = ?,+ result = '{"results": [{"status": "ERROR", "title": "Plan check run timed out", "content": "The plan check run was abandoned because it exceeded the maximum execution time."}]}',+ updated_at = now()+ WHERE status = ?+ AND updated_at < now() - ?::INTERVAL+ `, PlanCheckRunStatusFailed, PlanCheckRunStatusRunning, timeout.String())++ query, args, err := q.ToSQL()+ if err != nil {+ return 0, errors.Wrapf(err, "failed to build sql")+ }++ result, err := s.GetDB().ExecContext(ctx, query, args...)+ if err != nil {+ return 0, errors.Wrapf(err, "failed to mark stale plan check runs as failed")+ }++ return result.RowsAffected()+}+// ClaimAvailablePlanCheckRuns atomically claims all AVAILABLE plan check runs by updating them to RUNNING// and returns the claimed UIDs. Uses FOR UPDATE SKIP LOCKED to allow concurrent schedulers to claim different runs.func (s *Store) ClaimAvailablePlanCheckRuns(ctx context.Context) ([]*ClaimedPlanCheckRun, error) {