Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor: unify advisory lock API and add HA support for schema syncer (#18835)

p0ny53dfbc6e48c8
diff --git a/backend/migrator/migrator.go b/backend/migrator/migrator.go
index 74bb263645..f2efa94c7a 100644
--- a/backend/migrator/migrator.go
+++ b/backend/migrator/migrator.go
@@ -16,6 +16,7 @@ import (
"github.com/lib/pq"
"github.com/pkg/errors"
+ "github.com/bytebase/bytebase/backend/common/log"
"github.com/bytebase/bytebase/backend/store"
)
@@ -42,11 +43,15 @@ var goMigrations = map[string]GoMigrationFunc{
func MigrateSchema(ctx context.Context, db *sql.DB) error {
// Acquire advisory lock to ensure only one replica runs migrations.
// This blocks until the lock is available.
- release, err := store.AcquireAdvisoryLock(ctx, db, store.AdvisoryLockKeyMigration)
+ lock, err := store.AcquireAdvisoryLock(ctx, db, store.AdvisoryLockKeyMigration)
if err != nil {
return errors.Wrap(err, "failed to acquire migration lock")
}
- defer release()
+ defer func() {
+ if err := lock.Release(); err != nil {
+ slog.Error("Failed to release migration advisory lock", log.BBError(err))
+ }
+ }()
files, err := getSortedVersionedFiles()
if err != nil {
diff --git a/backend/runner/schemasync/syncer.go b/backend/runner/schemasync/syncer.go
index 5db89b120d..16e9c21d13 100644
--- a/backend/runner/schemasync/syncer.go
+++ b/backend/runner/schemasync/syncer.go
@@ -127,6 +127,21 @@ func (s *Syncer) trySyncAll(ctx context.Context) {
}
}()
+ lock, acquired, err := store.TryAdvisoryLock(ctx, s.store.GetDB(), store.AdvisoryLockKeySchemaSyncer)
+ if err != nil {
+ slog.Error("Failed to acquire schema syncer advisory lock", log.BBError(err))
+ return
+ }
+ if !acquired {
+ slog.Debug("Schema syncer advisory lock held by another replica, skipping")
+ return
+ }
+ defer func() {
+ if err := lock.Release(); err != nil {
+ slog.Error("Failed to release schema syncer advisory lock", log.BBError(err))
+ }
+ }()
+
wp := pool.New().WithMaxGoroutines(MaximumOutstanding)
instances, err := s.store.ListInstances(ctx, &store.FindInstanceMessage{})
if err != nil {
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.go
index 52cf75c1aa..4d608b63ae 100644
--- a/backend/runner/taskrun/pending_scheduler.go
+++ b/backend/runner/taskrun/pending_scheduler.go
@@ -40,7 +40,7 @@ func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.Wa
func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {
// Acquire cluster-wide mutex - only one replica runs at a time.
- lock, acquired, err := s.store.TryAdvisoryLock(ctx, store.AdvisoryLockKeyPendingScheduler)
+ lock, acquired, err := store.TryAdvisoryLock(ctx, s.store.GetDB(), store.AdvisoryLockKeyPendingScheduler)
if err != nil {
return errors.Wrapf(err, "failed to acquire advisory lock")
}
@@ -48,7 +48,11 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {
// Another replica is running, skip this cycle.
return nil
}
- defer func() { _ = lock.Release() }()
+ defer func() {
+ if err := lock.Release(); err != nil {
+ slog.Error("Failed to release pending scheduler advisory lock", log.BBError(err))
+ }
+ }()
taskRuns, err := s.store.ListTaskRuns(ctx, &store.FindTaskRunMessage{
Status: &[]storepb.TaskRun_Status{storepb.TaskRun_PENDING},
@@ -57,7 +61,7 @@ func (s *Scheduler) schedulePendingTaskRuns(ctx context.Context) error {
return errors.Wrapf(err, "failed to list pending task runs")
}
- // Build context once per cycle
+ // Build context once per cycle.
sc, err := newSchedulingContext(ctx, s.store)
if err != nil {
return errors.Wrapf(err, "failed to create scheduling context")
diff --git a/backend/store/advisory_lock.go b/backend/store/advisory_lock.go
index edb6daa3cc..42a076d8b6 100644
--- a/backend/store/advisory_lock.go
+++ b/backend/store/advisory_lock.go
@@ -16,6 +16,9 @@ const (
// AdvisoryLockKeyMigration is used by the schema migrator to ensure only
// one replica runs database migrations at a time.
AdvisoryLockKeyMigration AdvisoryLockKey = 1002
+ // AdvisoryLockKeySchemaSyncer is used by the schema syncer to ensure only
+ // one replica runs periodic schema sync at a time.
+ AdvisoryLockKeySchemaSyncer AdvisoryLockKey = 1003
)
// AdvisoryLock holds a dedicated connection for a session-level advisory lock.
@@ -27,8 +30,8 @@ type AdvisoryLock struct {
// TryAdvisoryLock attempts to acquire a session-level advisory lock using a
// dedicated connection. Returns (lock, true) if acquired, (nil, false) if
// already held by another session. Caller must call lock.Release() when done.
-func (s *Store) TryAdvisoryLock(ctx context.Context, key AdvisoryLockKey) (*AdvisoryLock, bool, error) {
- conn, err := s.dbConnManager.GetDB().Conn(ctx)
+func TryAdvisoryLock(ctx context.Context, db *sql.DB, key AdvisoryLockKey) (*AdvisoryLock, bool, error) {
+ conn, err := db.Conn(ctx)
if err != nil {
return nil, false, err
}
@@ -48,6 +51,7 @@ func (s *Store) TryAdvisoryLock(ctx context.Context, key AdvisoryLockKey) (*Advi
}
// Release releases the advisory lock and returns the connection to the pool.
+// Uses context.Background() to ensure cleanup completes even if parent ctx is cancelled.
func (l *AdvisoryLock) Release() error {
if l.conn == nil {
return nil
@@ -58,9 +62,9 @@ func (l *AdvisoryLock) Release() error {
}
// AcquireAdvisoryLock acquires a session-level advisory lock, blocking until
-// the lock is available. Returns a release function that must be called when done.
+// the lock is available. Caller must call lock.Release() when done.
// This is useful for migrations where we want to wait rather than fail fast.
-func AcquireAdvisoryLock(ctx context.Context, db *sql.DB, key AdvisoryLockKey) (func(), error) {
+func AcquireAdvisoryLock(ctx context.Context, db *sql.DB, key AdvisoryLockKey) (*AdvisoryLock, error) {
conn, err := db.Conn(ctx)
if err != nil {
return nil, err
@@ -72,9 +76,5 @@ func AcquireAdvisoryLock(ctx context.Context, db *sql.DB, key AdvisoryLockKey) (
return nil, err
}
- release := func() {
- _, _ = conn.ExecContext(context.Background(), "SELECT pg_advisory_unlock($1)", int64(key))
- conn.Close()
- }
- return release, nil
+ return &AdvisoryLock{conn: conn, key: key}, nil
}