Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
controlplane: the key rotation deadline lives in the store, not in the process uptime
Antonio Ojea50c44fe93fd5
diff --git a/internal/controlplane/key_rotation_test.go b/internal/controlplane/key_rotation_test.gonew file mode 100644index 00000000..efb6fd4c--- /dev/null+++ b/internal/controlplane/key_rotation_test.go@@ -0,0 +1,97 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package controlplane++import (+ "context"+ "testing"+ "time"+)++func TestKeyRotationCheckInterval(t *testing.T) {+ for _, tc := range []struct {+ interval, want time.Duration+ }{+ {24 * time.Hour, time.Minute},+ {time.Hour, time.Minute},+ {5 * time.Minute, 30 * time.Second},+ {2 * time.Second, 200 * time.Millisecond},+ } {+ if got := keyRotationCheckInterval(tc.interval); got != tc.want {+ t.Errorf("keyRotationCheckInterval(%s) = %s, want %s", tc.interval, got, tc.want)+ }+ }+}++// The deadline is the store's, not the process's: a rotation that was due+// while no replica was up happens on the first check after a start, and a+// replica that restarts before the deadline does not push it back.+func TestRotateKeysIfDueFollowsTheStoredDeadline(t *testing.T) {+ store := newMetricsTestStore(t)+ ctx := context.Background()+ interval := 24 * time.Hour+ issuer, _ := startCustomMockOIDC(t)+ srv, err := NewServer(Options{+ DriverName: "sqlite", DataSourceName: "unused", OIDCIssuer: issuer,+ KeyRotationInterval: interval,+ }, store)+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = srv.Close() }()+ if err := srv.Init(); err != nil {+ t.Fatal(err)+ }+ keys := func() int {+ t.Helper()+ ks, err := store.GetAllValidKeys(ctx)+ if err != nil {+ t.Fatal(err)+ }+ return len(ks)+ }+ if got := keys(); got != 1 {+ t.Fatalf("after Init the keyring holds %d keys, want 1", got)+ }++ // Init scheduled the first rotation an interval from now; the loop it+ // started checks every minute, these checks are direct.+ next, err := store.NextKeyRotation(ctx)+ if err != nil {+ t.Fatal(err)+ }+ if next.IsZero() || time.Until(next) < interval-time.Minute {+ t.Fatalf("first rotation scheduled at %s, want about %s from now", next, interval)+ }++ srv.rotateKeysIfDue(time.Now())+ if got := keys(); got != 1 {+ t.Fatalf("a check before the deadline rotated: %d keys", got)+ }++ // Five days later, as after a week of restarts that never reached a tick.+ late := next.Add(5 * 24 * time.Hour)+ srv.rotateKeysIfDue(late)+ if got := keys(); got != 2 {+ t.Fatalf("a check past the deadline did not rotate: %d keys, want 2", got)+ }+ if next, _ = store.NextKeyRotation(ctx); !next.Equal(late.Add(interval).Truncate(time.Millisecond)) {+ t.Fatalf("deadline after the late rotation = %s, want %s", next, late.Add(interval))+ }+ srv.rotateKeysIfDue(late.Add(time.Minute))+ if got := keys(); got != 2 {+ t.Fatalf("a check right after a rotation rotated again: %d keys", got)+ }+}diff --git a/internal/controlplane/metrics.go b/internal/controlplane/metrics.goindex 29adf675..22a8483b 100644--- a/internal/controlplane/metrics.go+++ b/internal/controlplane/metrics.go@@ -113,6 +113,9 @@ type meshSnapshot struct {// the retiring one stops verifying; zero when there is none.signingKeys intretiringKeyExpiry time.Time+ // nextKeyRotation is when the store says the next rotation is due; a+ // value in the past is a rotation no replica has performed.+ nextKeyRotation time.Time// storageRows is the row count of every table the control plane keeps// bounded; a table that grows past its retention is a reaper that// stopped working.@@ -145,6 +148,7 @@ type meshStateCollector struct {meshPeersDesc *prometheus.DescsigningKeysDesc *prometheus.DescretiringKeyDesc *prometheus.Desc+ nextRotationDesc *prometheus.DescstorageRowsDesc *prometheus.DescscrapeOKDesc *prometheus.DescscrapeSampleDesc *prometheus.Desc@@ -200,6 +204,10 @@ func newMeshStateCollector(store storage.Store) *meshStateCollector {"agentmesh_control_plane_retiring_key_expiry_timestamp_seconds","Unix time the retiring signing key stops verifying; absent when no rotation is in its grace period",nil, nil),+ nextRotationDesc: prometheus.NewDesc(+ "agentmesh_control_plane_next_key_rotation_timestamp_seconds",+ "Unix time the next signing key rotation is due; in the past when it is overdue",+ nil, nil),storageRowsDesc: prometheus.NewDesc("agentmesh_control_plane_storage_rows","Rows in a control plane store table",@@ -219,7 +227,7 @@ func (c *meshStateCollector) Describe(ch chan<- *prometheus.Desc) {for _, d := range []*prometheus.Desc{c.nodesDesc, c.usersDesc, c.requestsDesc, c.tokensDesc, c.routersDesc,c.routerPeersDesc, c.routerDHTDesc, c.routerLeaseDesc, c.meshPeersDesc,- c.signingKeysDesc, c.retiringKeyDesc, c.storageRowsDesc, c.scrapeOKDesc, c.scrapeSampleDesc,+ c.signingKeysDesc, c.retiringKeyDesc, c.nextRotationDesc, c.storageRowsDesc, c.scrapeOKDesc, c.scrapeSampleDesc,} {ch <- d}@@ -266,6 +274,9 @@ func (c *meshStateCollector) Collect(ch chan<- prometheus.Metric) {if !s.retiringKeyExpiry.IsZero() {ch <- prometheus.MustNewConstMetric(c.retiringKeyDesc, prometheus.GaugeValue, float64(s.retiringKeyExpiry.Unix()))}+ if !s.nextKeyRotation.IsZero() {+ ch <- prometheus.MustNewConstMetric(c.nextRotationDesc, prometheus.GaugeValue, float64(s.nextKeyRotation.Unix()))+ }for table, n := range s.storageRows {ch <- prometheus.MustNewConstMetric(c.storageRowsDesc, prometheus.GaugeValue, float64(n), table)}@@ -351,6 +362,10 @@ func readMeshSnapshot(ctx context.Context, store storage.Store, now time.Time) (if err != nil {return snap, err}+ snap.nextKeyRotation, err = store.NextKeyRotation(ctx)+ if err != nil {+ return snap, err+ }return snap, nil}diff --git a/internal/controlplane/metrics_test.go b/internal/controlplane/metrics_test.goindex 099d97d7..d066d760 100644--- a/internal/controlplane/metrics_test.go+++ b/internal/controlplane/metrics_test.go@@ -344,6 +344,7 @@ func TestMetricsEndpointServesMeshStateAndRuntime(t *testing.T) {"agentmesh_control_plane_routers_active 0","agentmesh_control_plane_mesh_connected_peers 0",`agentmesh_control_plane_storage_rows{table="nodes"} 0`,+ "agentmesh_control_plane_next_key_rotation_timestamp_seconds ",`agentmesh_control_plane_storage_rows{table="enrollment_requests"} 0`,"agentmesh_control_plane_catalog_entries ",`agentmesh_control_plane_storage_rows_deleted_total{table="routers"} `,diff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex a535d4ea..9b4537a4 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -300,6 +300,19 @@ func (s *Server) Init() error {} else if err != nil {return fmt.Errorf("failed to query initial keyring status: %w", err)}+ // The rotation deadline lives in the store, shared by the replicas and+ // kept across restarts; a store that never had one gets its first an+ // interval from now.+ if interval := s.config.KeyRotationInterval; interval > 0 {+ first := time.Now().Add(interval)+ scheduled, err := s.store.ScheduleKeyRotationIfUnset(ctx, first)+ if err != nil {+ return fmt.Errorf("failed to schedule the first key rotation: %w", err)+ }+ if scheduled {+ logger.Infof("First key rotation scheduled at %s", first.UTC().Format(time.RFC3339))+ }+ }// Bootstrap Policy is now disabled; starting default closed.@@ -514,64 +527,80 @@ func (s *Server) getProviders() map[string]*oidc.Provider {return pCopy}+// keyRotationCheckInterval is how often a replica asks the store whether a+// rotation is due: a tenth of the interval, at most once a minute. The+// deadline itself lives in the store, so a replica that restarts more often+// than the interval still rotates on time.+func keyRotationCheckInterval(interval time.Duration) time.Duration {+ return min(time.Minute, interval/10)+}++// runKeyRotationLoop rotates the signing keys when the deadline the store+// holds (set by Init, advanced by each rotation) has passed. The check runs+// well within the interval, so a control plane that restarts more often+// than it rotates still rotates on time instead of never.func (s *Server) runKeyRotationLoop() {defer s.wg.Done()- if s.config.KeyRotationInterval <= 0 {+ interval := s.config.KeyRotationInterval+ if interval <= 0 {return}- ticker := time.NewTicker(s.config.KeyRotationInterval)+ ticker := time.NewTicker(keyRotationCheckInterval(interval))defer ticker.Stop()for {select {case <-ticker.C:- // Replicas share one DB but tick independently; only the replica- // that wins this claim may rotate for the current window.- now := time.Now()- claimed, err := s.store.ClaimKeyRotation(s.ctx, now, s.config.KeyRotationInterval)- if err != nil {- logger.Errorf("Failed to claim key rotation window: %v", err)- continue- }- if !claimed {- logger.Debug("Skipping key rotation: another replica already claimed this window")- continue- }-- logger.Info("Rotating Biscuit signing keys...")- newPub, newPriv, err := ed25519.GenerateKey(rand.Reader)- if err != nil {- logger.Errorf("Failed to generate key pair for rotation: %v", err)- // Give up the window so a retry isn't stuck waiting a full interval.- if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, s.config.KeyRotationInterval); relErr != nil {- logger.Errorf("Failed to release key rotation claim: %v", relErr)- }- continue- }- err = s.store.RotateKeys(s.ctx, newPriv, newPub, s.config.KeyGracePeriod)- if err != nil {- logger.Errorf("Failed to rotate keyring: %v", err)- // Same as above: don't let a failed rotation strand the mesh- // on unrotated keys until the next full interval.- if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, s.config.KeyRotationInterval); relErr != nil {- logger.Errorf("Failed to release key rotation claim: %v", relErr)- }- } else {- logger.Infof("Key rotation committed. New current public key: %s", hex.EncodeToString(newPub))- if _, oidcErr := s.RotateOIDCKey(s.config.KeyGracePeriod); oidcErr != nil {- logger.Warnf("Failed to rotate OIDC signing key: %v", oidcErr)- }- if err := s.getMeshAdapter().PublishEvent(s.ctx, api.MeshEvent_KEY_ROTATION, "", newPub); err != nil {- logger.Warnf("Failed to publish KEY_ROTATION event to mesh: %v", err)- }- }+ s.rotateKeysIfDue(time.Now())case <-s.ctx.Done():return}}}+// rotateKeysIfDue claims the rotation window and rotates when it is due.+// Replicas share one store and check independently; only the one that wins+// the claim rotates for this window.+func (s *Server) rotateKeysIfDue(now time.Time) {+ interval := s.config.KeyRotationInterval+ claimed, err := s.store.ClaimKeyRotation(s.ctx, now, interval)+ if err != nil {+ logger.Errorf("Failed to claim key rotation window: %v", err)+ return+ }+ if !claimed {+ return+ }++ logger.Info("Rotating Biscuit signing keys...")+ newPub, newPriv, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ logger.Errorf("Failed to generate key pair for rotation: %v", err)+ // Give up the window so a retry isn't stuck waiting a full interval.+ if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, interval); relErr != nil {+ logger.Errorf("Failed to release key rotation claim: %v", relErr)+ }+ return+ }+ if err := s.store.RotateKeys(s.ctx, newPriv, newPub, s.config.KeyGracePeriod); err != nil {+ logger.Errorf("Failed to rotate keyring: %v", err)+ // Same as above: don't let a failed rotation strand the mesh+ // on unrotated keys until the next full interval.+ if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, interval); relErr != nil {+ logger.Errorf("Failed to release key rotation claim: %v", relErr)+ }+ return+ }+ logger.Infof("Key rotation committed. New current public key: %s; next rotation at %s", hex.EncodeToString(newPub), now.Add(interval).UTC().Format(time.RFC3339))+ if _, oidcErr := s.RotateOIDCKey(s.config.KeyGracePeriod); oidcErr != nil {+ logger.Warnf("Failed to rotate OIDC signing key: %v", oidcErr)+ }+ if err := s.getMeshAdapter().PublishEvent(s.ctx, api.MeshEvent_KEY_ROTATION, "", newPub); err != nil {+ logger.Warnf("Failed to publish KEY_ROTATION event to mesh: %v", err)+ }+}+// HandleHealthz HTTP GET `/healthz`func (s *Server) HandleHealthz(w http.ResponseWriter, r *http.Request) {if r.Method != http.MethodGet {diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.goindex 6ca97f52..f7e75157 100644--- a/internal/storage/sql_store.go+++ b/internal/storage/sql_store.go@@ -857,6 +857,31 @@ func (s *SQLStore) ReleaseKeyRotationClaim(ctx context.Context, now time.Time, ireturn err}+// ScheduleKeyRotationIfUnset implements Store.+func (s *SQLStore) ScheduleKeyRotationIfUnset(ctx context.Context, at time.Time) (bool, error) {+ res, err := s.db.ExecContext(ctx, s.rebind(`UPDATE rotation_lock SET next_rotation_at = ? WHERE id = 1 AND next_rotation_at = 0`), at.UnixMilli())+ if err != nil {+ return false, err+ }+ n, err := res.RowsAffected()+ if err != nil {+ return false, err+ }+ return n == 1, nil+}++// NextKeyRotation implements Store.+func (s *SQLStore) NextKeyRotation(ctx context.Context) (time.Time, error) {+ var ms int64+ if err := s.db.QueryRowContext(ctx, `SELECT next_rotation_at FROM rotation_lock WHERE id = 1`).Scan(&ms); err != nil {+ return time.Time{}, err+ }+ if ms == 0 {+ return time.Time{}, nil+ }+ return time.UnixMilli(ms), nil+}+// RotateKeys implements Store.func (s *SQLStore) RotateKeys(ctx context.Context, newPriv ed25519.PrivateKey, newPub ed25519.PublicKey, gracePeriod time.Duration) error {tx, err := s.db.BeginTx(ctx, nil)diff --git a/internal/storage/sql_store_test.go b/internal/storage/sql_store_test.goindex 1158a029..7ffd234b 100644--- a/internal/storage/sql_store_test.go+++ b/internal/storage/sql_store_test.go@@ -225,6 +225,53 @@ func TestClaimKeyRotation(t *testing.T) {}}+func TestScheduleKeyRotationIfUnset(t *testing.T) {+ store := newTestStore(t)+ defer func() { _ = store.Close() }()+ ctx := context.Background()+ t0 := time.Now().Truncate(time.Millisecond)+ interval := 24 * time.Hour++ next, err := store.NextKeyRotation(ctx)+ if err != nil {+ t.Fatal(err)+ }+ if !next.IsZero() {+ t.Fatalf("fresh store has a rotation deadline %s, want none", next)+ }++ // The first replica to start schedules; a replica starting later, or+ // the same one restarting, leaves the deadline alone.+ scheduled, err := store.ScheduleKeyRotationIfUnset(ctx, t0.Add(interval))+ if err != nil {+ t.Fatal(err)+ }+ if !scheduled {+ t.Fatal("first schedule on a fresh store was not applied")+ }+ scheduled, err = store.ScheduleKeyRotationIfUnset(ctx, t0.Add(time.Hour))+ if err != nil {+ t.Fatal(err)+ }+ if scheduled {+ t.Fatal("a second schedule replaced the deadline; want it kept")+ }+ if next, _ = store.NextKeyRotation(ctx); !next.Equal(t0.Add(interval)) {+ t.Fatalf("deadline = %s, want %s", next, t0.Add(interval))+ }++ // Not due before the deadline, due at it; a claim moves it one interval on.+ if claimed, _ := store.ClaimKeyRotation(ctx, t0.Add(interval-time.Second), interval); claimed {+ t.Fatal("claim before the deadline succeeded")+ }+ if claimed, _ := store.ClaimKeyRotation(ctx, t0.Add(interval), interval); !claimed {+ t.Fatal("claim at the deadline failed")+ }+ if next, _ = store.NextKeyRotation(ctx); !next.Equal(t0.Add(2 * interval)) {+ t.Fatalf("deadline after the claim = %s, want %s", next, t0.Add(2*interval))+ }+}+func TestReleaseKeyRotationClaim(t *testing.T) {store := newTestStore(t)defer func() { _ = store.Close() }()diff --git a/internal/storage/storage.go b/internal/storage/storage.goindex 52ad4730..ca9095a1 100644--- a/internal/storage/storage.go+++ b/internal/storage/storage.go@@ -248,6 +248,15 @@ type Store interface {// the rotation that followed a successful claim failed.ReleaseKeyRotationClaim(ctx context.Context, now time.Time, interval time.Duration) error+ // ScheduleKeyRotationIfUnset sets the first rotation deadline of a store+ // that never had one, and reports whether it did. A store that already+ // carries a deadline, due or not, is left as it is.+ ScheduleKeyRotationIfUnset(ctx context.Context, at time.Time) (bool, error)++ // NextKeyRotation is the deadline ClaimKeyRotation will honour: the+ // instant the next rotation is due, zero when none was ever scheduled.+ NextKeyRotation(ctx context.Context) (time.Time, error)+// SaveInitialKey sets the initial key pair if no keys exist yet.SaveInitialKey(ctx context.Context, priv ed25519.PrivateKey, pub ed25519.PublicKey) errordiff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.mdindex b0d0d50f..c35086d3 100644--- a/site/content/docs/reference/control-plane.md+++ b/site/content/docs/reference/control-plane.md@@ -149,6 +149,13 @@ with `403`). The console uses these routes.- **Bans propagate in two ways.** `/info` lists banned peer IDs for nodes androuters that start or restart. A signed mesh event reaches the ones thatare already running.+- **The rotation deadline is in the database.** The first rotation is due+ one `--key-rotation-interval` after the first start, each rotation sets+ the next, and every replica checks the deadline at least once a minute.+ A control plane that restarts more often than it rotates still rotates on+ time, and one that was down when a rotation was due rotates on its first+ check. `agentmesh_control_plane_next_key_rotation_timestamp_seconds` is+ the deadline; a value in the past is a rotation that is not happening.- **Node records are garbage-collected.** A node whose session has expired isdeleted after `--node-retention`. A node enrolled with a bootstrap tokenhas no session; its record is deleted once it has not refreshed its