Skip to content

Agent Mesh

fix(controlplane): avoid racing key rotation across replicas

Antonio Ojeaa54d1e75d44e
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index 2a8ab291..3126fa51 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -277,6 +277,18 @@ func (s *Server) runKeyRotationLoop() {
for {
select {
case <-ticker.C:
+ // Replicas share one DB but tick independently; only the replica
+ // that wins this claim may rotate for the current window.
+ claimed, err := s.store.ClaimKeyRotation(s.ctx, time.Now(), s.config.KeyRotationInterval)
+ if err != nil {
+ logger.Errorf("Failed to claim key rotation window: %v", err)
+ continue
+ }
+ if !claimed {
+ logger.Debug("Skipping key rotation: another replica already claimed this window")
+ continue
+ }
+
logger.Info("Rotating Biscuit signing keys...")
newPub, newPriv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.go
index 5dbe2ccd..af46818c 100644
--- a/internal/storage/sql_store.go
+++ b/internal/storage/sql_store.go
@@ -312,6 +312,26 @@ var migrations = []migration{
`ALTER TABLE enrollment_requests ADD COLUMN region TEXT DEFAULT '' NOT NULL`,
},
},
+ {
+ // rotation_lock backs ClaimKeyRotation: a singleton row whose
+ // next_rotation_at only advances via a conditional UPDATE, so it
+ // doubles as a driver-agnostic mutex across control-plane replicas.
+ version: 6,
+ postgres: []string{
+ `CREATE TABLE IF NOT EXISTS rotation_lock (
+ id SMALLINT PRIMARY KEY,
+ next_rotation_at BIGINT NOT NULL
+ )`,
+ `INSERT INTO rotation_lock (id, next_rotation_at) VALUES (1, 0) ON CONFLICT (id) DO NOTHING`,
+ },
+ sqlite: []string{
+ `CREATE TABLE IF NOT EXISTS rotation_lock (
+ id INTEGER PRIMARY KEY,
+ next_rotation_at BIGINT NOT NULL
+ )`,
+ `INSERT OR IGNORE INTO rotation_lock (id, next_rotation_at) VALUES (1, 0)`,
+ },
+ },
}
func (s *SQLStore) initSchema() error {
@@ -492,6 +512,20 @@ func (s *SQLStore) GetAllValidKeys(ctx context.Context) ([]KeyPair, error) {
return keys, rows.Err()
}
+// ClaimKeyRotation implements Store.
+func (s *SQLStore) ClaimKeyRotation(ctx context.Context, now time.Time, interval time.Duration) (bool, error) {
+ query := s.rebind(`UPDATE rotation_lock SET next_rotation_at = ? WHERE id = 1 AND next_rotation_at <= ?`)
+ res, err := s.db.ExecContext(ctx, query, now.Add(interval).UnixMilli(), now.UnixMilli())
+ if err != nil {
+ return false, err
+ }
+ n, err := res.RowsAffected()
+ if err != nil {
+ return false, err
+ }
+ return n == 1, nil
+}
+
// RotateKeys implements Store.
func (s *SQLStore) RotateKeys(ctx context.Context, newPriv ed25519.PrivateKey, newPub ed25519.PublicKey, gracePeriod time.Duration) error {
tx, err := s.db.BeginTx(ctx, nil)
diff --git a/internal/storage/sql_store_test.go b/internal/storage/sql_store_test.go
index 6237a0c2..2c3b5f6a 100644
--- a/internal/storage/sql_store_test.go
+++ b/internal/storage/sql_store_test.go
@@ -22,6 +22,8 @@ import (
"os"
"path/filepath"
"reflect"
+ "sync"
+ "sync/atomic"
"testing"
"time"
@@ -139,6 +141,76 @@ func TestKeyRingOps(t *testing.T) {
}
}
+func TestClaimKeyRotation(t *testing.T) {
+ store := newTestStore(t)
+ defer func() { _ = store.Close() }()
+
+ ctx := context.Background()
+ t0 := time.Now()
+ interval := 24 * time.Hour
+
+ // Fresh state: the first replica to tick claims the window.
+ claimed, err := store.ClaimKeyRotation(ctx, t0, interval)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if !claimed {
+ t.Fatal("expected first claim to succeed")
+ }
+
+ // A second replica racing for the same window loses.
+ claimed, err = store.ClaimKeyRotation(ctx, t0.Add(time.Second), interval)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if claimed {
+ t.Fatal("expected concurrent claim in the same window to fail")
+ }
+
+ // Once the interval has elapsed, the next window can be claimed again.
+ claimed, err = store.ClaimKeyRotation(ctx, t0.Add(interval+time.Second), interval)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if !claimed {
+ t.Fatal("expected claim after the interval elapsed to succeed")
+ }
+}
+
+func TestClaimKeyRotationConcurrent(t *testing.T) {
+ store := newTestStore(t)
+ defer func() { _ = store.Close() }()
+
+ ctx := context.Background()
+ now := time.Now()
+ interval := 24 * time.Hour
+
+ // Simulate several control-plane replicas racing to claim the same
+ // rotation window concurrently; exactly one must win.
+ const replicas = 10
+ var wg sync.WaitGroup
+ var wins atomic.Int32
+ for i := 0; i < replicas; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ claimed, err := store.ClaimKeyRotation(ctx, now, interval)
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ return
+ }
+ if claimed {
+ wins.Add(1)
+ }
+ }()
+ }
+ wg.Wait()
+
+ if got := wins.Load(); got != 1 {
+ t.Fatalf("expected exactly 1 replica to win the claim, got %d", got)
+ }
+}
+
func TestNodeEnrollmentOps(t *testing.T) {
store := newTestStore(t)
defer func() { _ = store.Close() }()
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
index 252819c9..2685cbc7 100644
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -112,6 +112,13 @@ type Store interface {
// RotateKeys rotates the current key to a new key pair and sets the expiration of the old key.
RotateKeys(ctx context.Context, newPriv ed25519.PrivateKey, newPub ed25519.PublicKey, gracePeriod time.Duration) error
+ // ClaimKeyRotation atomically claims the next scheduled key-rotation
+ // window, so multiple control-plane replicas sharing one database rotate
+ // keys exactly once per interval instead of racing independently. It
+ // returns true if the caller won the claim (advancing the deadline by
+ // interval), false if another replica already claimed this window.
+ ClaimKeyRotation(ctx context.Context, now time.Time, interval time.Duration) (bool, error)
+
// SaveInitialKey sets the initial key pair if no keys exist yet.
SaveInitialKey(ctx context.Context, priv ed25519.PrivateKey, pub ed25519.PublicKey) error