Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: use transaction advisory lock for pending scheduler (#19656)

p0ny1606cdc2293b
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.go
index 6dc0d4c625..826e44689d 100644
--- a/backend/runner/taskrun/pending_scheduler.go
+++ b/backend/runner/taskrun/pending_scheduler.go
@@ -2,6 +2,7 @@ package taskrun
import (
"context"
+ "database/sql"
"fmt"
"log/slog"
"sync"
@@ -45,21 +46,29 @@ func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.Wa
}
}
-func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {
+func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) (err error) {
+ startedAt := time.Now()
+ tx, err := s.store.GetDB().BeginTx(ctx, nil)
+ if err != nil {
+ return errors.Wrapf(err, "failed to begin pending scheduler transaction")
+ }
+ defer func() {
+ if rollbackErr := tx.Rollback(); rollbackErr != nil && !errors.Is(rollbackErr, sql.ErrTxDone) {
+ slog.Error("Failed to rollback pending scheduler transaction", log.BBError(rollbackErr))
+ }
+ }()
+
// Acquire cluster-wide mutex - only one replica runs at a time.
- lock, acquired, err := store.TryAdvisoryLock(ctx, s.store.GetDB(), store.AdvisoryLockKeyPendingScheduler)
+ acquired, err := store.TryAdvisoryXactLock(ctx, tx, store.AdvisoryLockKeyPendingScheduler)
if err != nil {
- return errors.Wrapf(err, "failed to acquire advisory lock")
+ return errors.Wrapf(err, "failed to acquire pending scheduler advisory lock")
}
if !acquired {
- // Another replica is running, skip this cycle.
+ slog.Debug("Pending scheduler advisory lock held by another replica, skipping",
+ slog.Duration("duration", time.Since(startedAt)),
+ )
return nil
}
- defer func() {
- if err := lock.Release(); err != nil {
- slog.Error("Failed to release pending scheduler advisory lock", log.BBError(err))
- }
- }()
taskRuns, err := s.store.ListTaskRunsByStatus(ctx, []storepb.TaskRun_Status{storepb.TaskRun_PENDING})
if err != nil {
@@ -74,10 +83,17 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {
for _, taskRun := range taskRuns {
if err := s.schedulePendingTaskRun(ctx, taskRun, sc); err != nil {
- slog.Error("failed to schedule pending task run", log.BBError(err))
+ slog.Error("failed to schedule pending task run",
+ slog.Int64("taskRunID", taskRun.ID),
+ log.BBError(err),
+ )
}
}
+ if err := tx.Commit(); err != nil {
+ return errors.Wrapf(err, "failed to commit pending scheduler transaction")
+ }
+
return nil
}
diff --git a/backend/store/advisory_lock.go b/backend/store/advisory_lock.go
index 42a076d8b6..f655708ca6 100644
--- a/backend/store/advisory_lock.go
+++ b/backend/store/advisory_lock.go
@@ -50,6 +50,17 @@ func TryAdvisoryLock(ctx context.Context, db *sql.DB, key AdvisoryLockKey) (*Adv
return &AdvisoryLock{conn: conn, key: key}, true, nil
}
+// TryAdvisoryXactLock attempts to acquire a transaction-level advisory lock.
+// Returns true if acquired, false if already held by another transaction.
+// The lock is automatically released when the transaction ends.
+func TryAdvisoryXactLock(ctx context.Context, tx *sql.Tx, key AdvisoryLockKey) (bool, error) {
+ var acquired bool
+ if err := tx.QueryRowContext(ctx, "SELECT pg_try_advisory_xact_lock($1)", int64(key)).Scan(&acquired); err != nil {
+ return false, err
+ }
+ return acquired, nil
+}
+
// Release releases the advisory lock and returns the connection to the pool.
// Uses context.Background() to ensure cleanup completes even if parent ctx is cancelled.
func (l *AdvisoryLock) Release() error {