Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: add timeout-based stale detection for plan check runs (#18843)

p0ny795d3a22e2dd
diff --git a/backend/runner/cleaner/data_cleaner.go b/backend/runner/cleaner/data_cleaner.go
index 6e9f36e301..64683977dc 100644
--- a/backend/runner/cleaner/data_cleaner.go
+++ b/backend/runner/cleaner/data_cleaner.go
@@ -14,6 +14,7 @@ const (
cleanupInterval = 1 * time.Hour
staleDetectionInterval = 30 * time.Second
stalenessThreshold = 1 * time.Minute
+ planCheckRunTimeout = 10 * time.Minute
heartbeatRetentionPeriod = 1 * time.Hour
exportArchiveRetentionPeriod = 24 * time.Hour
oauth2ClientRetentionPeriod = 30 * 24 * time.Hour // 30 days of inactivity
@@ -48,6 +49,7 @@ func (c *DataCleaner) Run(ctx context.Context, wg *sync.WaitGroup) {
// Run cleanup immediately on startup
c.cleanup(ctx)
c.detectStaleTaskRuns(ctx)
+ c.detectStalePlanCheckRuns(ctx)
for {
select {
@@ -55,6 +57,7 @@ func (c *DataCleaner) Run(ctx context.Context, wg *sync.WaitGroup) {
c.cleanup(ctx)
case <-staleTicker.C:
c.detectStaleTaskRuns(ctx)
+ c.detectStalePlanCheckRuns(ctx)
case <-ctx.Done():
return
}
@@ -79,6 +82,17 @@ func (c *DataCleaner) detectStaleTaskRuns(ctx context.Context) {
}
}
+func (c *DataCleaner) detectStalePlanCheckRuns(ctx context.Context) {
+ rowsAffected, err := c.store.FailStalePlanCheckRuns(ctx, planCheckRunTimeout)
+ if err != nil {
+ slog.Error("Failed to detect stale plan check runs", log.BBError(err))
+ return
+ }
+ if rowsAffected > 0 {
+ slog.Info("Marked stale plan check runs as failed", slog.Int64("count", rowsAffected))
+ }
+}
+
func (c *DataCleaner) cleanupStaleHeartbeats(ctx context.Context) {
rowsAffected, err := c.store.DeleteStaleReplicaHeartbeats(ctx, heartbeatRetentionPeriod)
if err != nil {
diff --git a/backend/store/plan_check_run.go b/backend/store/plan_check_run.go
index 3174d125b5..80f11c3a22 100644
--- a/backend/store/plan_check_run.go
+++ b/backend/store/plan_check_run.go
@@ -197,6 +197,32 @@ type ClaimedPlanCheckRun struct {
PlanUID int64
}
+// FailStalePlanCheckRuns marks RUNNING plan check runs as FAILED if they have been running
+// longer than the timeout threshold. Unlike task runs, plan check runs don't use heartbeat
+// detection - they use a simple timeout since they have a bounded execution time.
+func (s *Store) FailStalePlanCheckRuns(ctx context.Context, timeout time.Duration) (int64, error) {
+ q := qb.Q().Space(`
+ UPDATE plan_check_run
+ SET status = ?,
+ result = '{"results": [{"status": "ERROR", "title": "Plan check run timed out", "content": "The plan check run was abandoned because it exceeded the maximum execution time."}]}',
+ updated_at = now()
+ WHERE status = ?
+ AND updated_at < now() - ?::INTERVAL
+ `, PlanCheckRunStatusFailed, PlanCheckRunStatusRunning, timeout.String())
+
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to build sql")
+ }
+
+ result, err := s.GetDB().ExecContext(ctx, query, args...)
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to mark stale plan check runs as failed")
+ }
+
+ return result.RowsAffected()
+}
+
// ClaimAvailablePlanCheckRuns atomically claims all AVAILABLE plan check runs by updating them to RUNNING
// and returns the claimed UIDs. Uses FOR UPDATE SKIP LOCKED to allow concurrent schedulers to claim different runs.
func (s *Store) ClaimAvailablePlanCheckRuns(ctx context.Context) ([]*ClaimedPlanCheckRun, error) {