Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
fix(controlplane): avoid racing key rotation across replicas
Antonio Ojeaa54d1e75d44e
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex 2a8ab291..3126fa51 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -277,6 +277,18 @@ func (s *Server) runKeyRotationLoop() {for {select {case <-ticker.C:+ // Replicas share one DB but tick independently; only the replica+ // that wins this claim may rotate for the current window.+ claimed, err := s.store.ClaimKeyRotation(s.ctx, time.Now(), s.config.KeyRotationInterval)+ if err != nil {+ logger.Errorf("Failed to claim key rotation window: %v", err)+ continue+ }+ if !claimed {+ logger.Debug("Skipping key rotation: another replica already claimed this window")+ continue+ }+logger.Info("Rotating Biscuit signing keys...")newPub, newPriv, err := ed25519.GenerateKey(rand.Reader)if err != nil {diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.goindex 5dbe2ccd..af46818c 100644--- a/internal/storage/sql_store.go+++ b/internal/storage/sql_store.go@@ -312,6 +312,26 @@ var migrations = []migration{`ALTER TABLE enrollment_requests ADD COLUMN region TEXT DEFAULT '' NOT NULL`,},},+ {+ // rotation_lock backs ClaimKeyRotation: a singleton row whose+ // next_rotation_at only advances via a conditional UPDATE, so it+ // doubles as a driver-agnostic mutex across control-plane replicas.+ version: 6,+ postgres: []string{+ `CREATE TABLE IF NOT EXISTS rotation_lock (+ id SMALLINT PRIMARY KEY,+ next_rotation_at BIGINT NOT NULL+ )`,+ `INSERT INTO rotation_lock (id, next_rotation_at) VALUES (1, 0) ON CONFLICT (id) DO NOTHING`,+ },+ sqlite: []string{+ `CREATE TABLE IF NOT EXISTS rotation_lock (+ id INTEGER PRIMARY KEY,+ next_rotation_at BIGINT NOT NULL+ )`,+ `INSERT OR IGNORE INTO rotation_lock (id, next_rotation_at) VALUES (1, 0)`,+ },+ },}func (s *SQLStore) initSchema() error {@@ -492,6 +512,20 @@ func (s *SQLStore) GetAllValidKeys(ctx context.Context) ([]KeyPair, error) {return keys, rows.Err()}+// ClaimKeyRotation implements Store.+func (s *SQLStore) ClaimKeyRotation(ctx context.Context, now time.Time, interval time.Duration) (bool, error) {+ query := s.rebind(`UPDATE rotation_lock SET next_rotation_at = ? WHERE id = 1 AND next_rotation_at <= ?`)+ res, err := s.db.ExecContext(ctx, query, now.Add(interval).UnixMilli(), now.UnixMilli())+ if err != nil {+ return false, err+ }+ n, err := res.RowsAffected()+ if err != nil {+ return false, err+ }+ return n == 1, nil+}+// RotateKeys implements Store.func (s *SQLStore) RotateKeys(ctx context.Context, newPriv ed25519.PrivateKey, newPub ed25519.PublicKey, gracePeriod time.Duration) error {tx, err := s.db.BeginTx(ctx, nil)diff --git a/internal/storage/sql_store_test.go b/internal/storage/sql_store_test.goindex 6237a0c2..2c3b5f6a 100644--- a/internal/storage/sql_store_test.go+++ b/internal/storage/sql_store_test.go@@ -22,6 +22,8 @@ import ("os""path/filepath""reflect"+ "sync"+ "sync/atomic""testing""time"@@ -139,6 +141,76 @@ func TestKeyRingOps(t *testing.T) {}}+func TestClaimKeyRotation(t *testing.T) {+ store := newTestStore(t)+ defer func() { _ = store.Close() }()++ ctx := context.Background()+ t0 := time.Now()+ interval := 24 * time.Hour++ // Fresh state: the first replica to tick claims the window.+ claimed, err := store.ClaimKeyRotation(ctx, t0, interval)+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if !claimed {+ t.Fatal("expected first claim to succeed")+ }++ // A second replica racing for the same window loses.+ claimed, err = store.ClaimKeyRotation(ctx, t0.Add(time.Second), interval)+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if claimed {+ t.Fatal("expected concurrent claim in the same window to fail")+ }++ // Once the interval has elapsed, the next window can be claimed again.+ claimed, err = store.ClaimKeyRotation(ctx, t0.Add(interval+time.Second), interval)+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if !claimed {+ t.Fatal("expected claim after the interval elapsed to succeed")+ }+}++func TestClaimKeyRotationConcurrent(t *testing.T) {+ store := newTestStore(t)+ defer func() { _ = store.Close() }()++ ctx := context.Background()+ now := time.Now()+ interval := 24 * time.Hour++ // Simulate several control-plane replicas racing to claim the same+ // rotation window concurrently; exactly one must win.+ const replicas = 10+ var wg sync.WaitGroup+ var wins atomic.Int32+ for i := 0; i < replicas; i++ {+ wg.Add(1)+ go func() {+ defer wg.Done()+ claimed, err := store.ClaimKeyRotation(ctx, now, interval)+ if err != nil {+ t.Errorf("unexpected error: %v", err)+ return+ }+ if claimed {+ wins.Add(1)+ }+ }()+ }+ wg.Wait()++ if got := wins.Load(); got != 1 {+ t.Fatalf("expected exactly 1 replica to win the claim, got %d", got)+ }+}+func TestNodeEnrollmentOps(t *testing.T) {store := newTestStore(t)defer func() { _ = store.Close() }()diff --git a/internal/storage/storage.go b/internal/storage/storage.goindex 252819c9..2685cbc7 100644--- a/internal/storage/storage.go+++ b/internal/storage/storage.go@@ -112,6 +112,13 @@ type Store interface {// RotateKeys rotates the current key to a new key pair and sets the expiration of the old key.RotateKeys(ctx context.Context, newPriv ed25519.PrivateKey, newPub ed25519.PublicKey, gracePeriod time.Duration) error+ // ClaimKeyRotation atomically claims the next scheduled key-rotation+ // window, so multiple control-plane replicas sharing one database rotate+ // keys exactly once per interval instead of racing independently. It+ // returns true if the caller won the claim (advancing the deadline by+ // interval), false if another replica already claimed this window.+ ClaimKeyRotation(ctx context.Context, now time.Time, interval time.Duration) (bool, error)+// SaveInitialKey sets the initial key pair if no keys exist yet.SaveInitialKey(ctx context.Context, priv ed25519.PrivateKey, pub ed25519.PublicKey) error