Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: use transaction advisory lock for pending scheduler (#19656)
p0ny1606cdc2293b
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.goindex 6dc0d4c625..826e44689d 100644--- a/backend/runner/taskrun/pending_scheduler.go+++ b/backend/runner/taskrun/pending_scheduler.go@@ -2,6 +2,7 @@ package taskrunimport ("context"+ "database/sql""fmt""log/slog""sync"@@ -45,21 +46,29 @@ func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.Wa}}-func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {+func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) (err error) {+ startedAt := time.Now()+ tx, err := s.store.GetDB().BeginTx(ctx, nil)+ if err != nil {+ return errors.Wrapf(err, "failed to begin pending scheduler transaction")+ }+ defer func() {+ if rollbackErr := tx.Rollback(); rollbackErr != nil && !errors.Is(rollbackErr, sql.ErrTxDone) {+ slog.Error("Failed to rollback pending scheduler transaction", log.BBError(rollbackErr))+ }+ }()+// Acquire cluster-wide mutex - only one replica runs at a time.- lock, acquired, err := store.TryAdvisoryLock(ctx, s.store.GetDB(), store.AdvisoryLockKeyPendingScheduler)+ acquired, err := store.TryAdvisoryXactLock(ctx, tx, store.AdvisoryLockKeyPendingScheduler)if err != nil {- return errors.Wrapf(err, "failed to acquire advisory lock")+ return errors.Wrapf(err, "failed to acquire pending scheduler advisory lock")}if !acquired {- // Another replica is running, skip this cycle.+ slog.Debug("Pending scheduler advisory lock held by another replica, skipping",+ slog.Duration("duration", time.Since(startedAt)),+ )return nil}- defer func() {- if err := lock.Release(); err != nil {- slog.Error("Failed to release pending scheduler advisory lock", log.BBError(err))- }- }()taskRuns, err := s.store.ListTaskRunsByStatus(ctx, []storepb.TaskRun_Status{storepb.TaskRun_PENDING})if err != nil {@@ -74,10 +83,17 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {for _, taskRun := range taskRuns {if err := s.schedulePendingTaskRun(ctx, taskRun, sc); err != nil {- slog.Error("failed to schedule pending task run", log.BBError(err))+ slog.Error("failed to schedule pending task run",+ slog.Int64("taskRunID", taskRun.ID),+ log.BBError(err),+ )}}+ if err := tx.Commit(); err != nil {+ return errors.Wrapf(err, "failed to commit pending scheduler transaction")+ }+return nil}diff --git a/backend/store/advisory_lock.go b/backend/store/advisory_lock.goindex 42a076d8b6..f655708ca6 100644--- a/backend/store/advisory_lock.go+++ b/backend/store/advisory_lock.go@@ -50,6 +50,17 @@ func TryAdvisoryLock(ctx context.Context, db *sql.DB, key AdvisoryLockKey) (*Advreturn &AdvisoryLock{conn: conn, key: key}, true, nil}+// TryAdvisoryXactLock attempts to acquire a transaction-level advisory lock.+// Returns true if acquired, false if already held by another transaction.+// The lock is automatically released when the transaction ends.+func TryAdvisoryXactLock(ctx context.Context, tx *sql.Tx, key AdvisoryLockKey) (bool, error) {+ var acquired bool+ if err := tx.QueryRowContext(ctx, "SELECT pg_try_advisory_xact_lock($1)", int64(key)).Scan(&acquired); err != nil {+ return false, err+ }+ return acquired, nil+}+// Release releases the advisory lock and returns the connection to the pool.// Uses context.Background() to ensure cleanup completes even if parent ctx is cancelled.func (l *AdvisoryLock) Release() error {