Skip to content

Agent Mesh

controlplane: the key rotation deadline lives in the store, not in the process uptime

Antonio Ojea50c44fe93fd5
diff --git a/internal/controlplane/key_rotation_test.go b/internal/controlplane/key_rotation_test.go
new file mode 100644
index 00000000..efb6fd4c
--- /dev/null
+++ b/internal/controlplane/key_rotation_test.go
@@ -0,0 +1,97 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+ "context"
+ "testing"
+ "time"
+)
+
+func TestKeyRotationCheckInterval(t *testing.T) {
+ for _, tc := range []struct {
+ interval, want time.Duration
+ }{
+ {24 * time.Hour, time.Minute},
+ {time.Hour, time.Minute},
+ {5 * time.Minute, 30 * time.Second},
+ {2 * time.Second, 200 * time.Millisecond},
+ } {
+ if got := keyRotationCheckInterval(tc.interval); got != tc.want {
+ t.Errorf("keyRotationCheckInterval(%s) = %s, want %s", tc.interval, got, tc.want)
+ }
+ }
+}
+
+// The deadline is the store's, not the process's: a rotation that was due
+// while no replica was up happens on the first check after a start, and a
+// replica that restarts before the deadline does not push it back.
+func TestRotateKeysIfDueFollowsTheStoredDeadline(t *testing.T) {
+ store := newMetricsTestStore(t)
+ ctx := context.Background()
+ interval := 24 * time.Hour
+ issuer, _ := startCustomMockOIDC(t)
+ srv, err := NewServer(Options{
+ DriverName: "sqlite", DataSourceName: "unused", OIDCIssuer: issuer,
+ KeyRotationInterval: interval,
+ }, store)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = srv.Close() }()
+ if err := srv.Init(); err != nil {
+ t.Fatal(err)
+ }
+ keys := func() int {
+ t.Helper()
+ ks, err := store.GetAllValidKeys(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return len(ks)
+ }
+ if got := keys(); got != 1 {
+ t.Fatalf("after Init the keyring holds %d keys, want 1", got)
+ }
+
+ // Init scheduled the first rotation an interval from now; the loop it
+ // started checks every minute, these checks are direct.
+ next, err := store.NextKeyRotation(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if next.IsZero() || time.Until(next) < interval-time.Minute {
+ t.Fatalf("first rotation scheduled at %s, want about %s from now", next, interval)
+ }
+
+ srv.rotateKeysIfDue(time.Now())
+ if got := keys(); got != 1 {
+ t.Fatalf("a check before the deadline rotated: %d keys", got)
+ }
+
+ // Five days later, as after a week of restarts that never reached a tick.
+ late := next.Add(5 * 24 * time.Hour)
+ srv.rotateKeysIfDue(late)
+ if got := keys(); got != 2 {
+ t.Fatalf("a check past the deadline did not rotate: %d keys, want 2", got)
+ }
+ if next, _ = store.NextKeyRotation(ctx); !next.Equal(late.Add(interval).Truncate(time.Millisecond)) {
+ t.Fatalf("deadline after the late rotation = %s, want %s", next, late.Add(interval))
+ }
+ srv.rotateKeysIfDue(late.Add(time.Minute))
+ if got := keys(); got != 2 {
+ t.Fatalf("a check right after a rotation rotated again: %d keys", got)
+ }
+}
diff --git a/internal/controlplane/metrics.go b/internal/controlplane/metrics.go
index 29adf675..22a8483b 100644
--- a/internal/controlplane/metrics.go
+++ b/internal/controlplane/metrics.go
@@ -113,6 +113,9 @@ type meshSnapshot struct {
// the retiring one stops verifying; zero when there is none.
signingKeys int
retiringKeyExpiry time.Time
+ // nextKeyRotation is when the store says the next rotation is due; a
+ // value in the past is a rotation no replica has performed.
+ nextKeyRotation time.Time
// storageRows is the row count of every table the control plane keeps
// bounded; a table that grows past its retention is a reaper that
// stopped working.
@@ -145,6 +148,7 @@ type meshStateCollector struct {
meshPeersDesc *prometheus.Desc
signingKeysDesc *prometheus.Desc
retiringKeyDesc *prometheus.Desc
+ nextRotationDesc *prometheus.Desc
storageRowsDesc *prometheus.Desc
scrapeOKDesc *prometheus.Desc
scrapeSampleDesc *prometheus.Desc
@@ -200,6 +204,10 @@ func newMeshStateCollector(store storage.Store) *meshStateCollector {
"agentmesh_control_plane_retiring_key_expiry_timestamp_seconds",
"Unix time the retiring signing key stops verifying; absent when no rotation is in its grace period",
nil, nil),
+ nextRotationDesc: prometheus.NewDesc(
+ "agentmesh_control_plane_next_key_rotation_timestamp_seconds",
+ "Unix time the next signing key rotation is due; in the past when it is overdue",
+ nil, nil),
storageRowsDesc: prometheus.NewDesc(
"agentmesh_control_plane_storage_rows",
"Rows in a control plane store table",
@@ -219,7 +227,7 @@ func (c *meshStateCollector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
c.nodesDesc, c.usersDesc, c.requestsDesc, c.tokensDesc, c.routersDesc,
c.routerPeersDesc, c.routerDHTDesc, c.routerLeaseDesc, c.meshPeersDesc,
- c.signingKeysDesc, c.retiringKeyDesc, c.storageRowsDesc, c.scrapeOKDesc, c.scrapeSampleDesc,
+ c.signingKeysDesc, c.retiringKeyDesc, c.nextRotationDesc, c.storageRowsDesc, c.scrapeOKDesc, c.scrapeSampleDesc,
} {
ch <- d
}
@@ -266,6 +274,9 @@ func (c *meshStateCollector) Collect(ch chan<- prometheus.Metric) {
if !s.retiringKeyExpiry.IsZero() {
ch <- prometheus.MustNewConstMetric(c.retiringKeyDesc, prometheus.GaugeValue, float64(s.retiringKeyExpiry.Unix()))
}
+ if !s.nextKeyRotation.IsZero() {
+ ch <- prometheus.MustNewConstMetric(c.nextRotationDesc, prometheus.GaugeValue, float64(s.nextKeyRotation.Unix()))
+ }
for table, n := range s.storageRows {
ch <- prometheus.MustNewConstMetric(c.storageRowsDesc, prometheus.GaugeValue, float64(n), table)
}
@@ -351,6 +362,10 @@ func readMeshSnapshot(ctx context.Context, store storage.Store, now time.Time) (
if err != nil {
return snap, err
}
+ snap.nextKeyRotation, err = store.NextKeyRotation(ctx)
+ if err != nil {
+ return snap, err
+ }
return snap, nil
}
diff --git a/internal/controlplane/metrics_test.go b/internal/controlplane/metrics_test.go
index 099d97d7..d066d760 100644
--- a/internal/controlplane/metrics_test.go
+++ b/internal/controlplane/metrics_test.go
@@ -344,6 +344,7 @@ func TestMetricsEndpointServesMeshStateAndRuntime(t *testing.T) {
"agentmesh_control_plane_routers_active 0",
"agentmesh_control_plane_mesh_connected_peers 0",
`agentmesh_control_plane_storage_rows{table="nodes"} 0`,
+ "agentmesh_control_plane_next_key_rotation_timestamp_seconds ",
`agentmesh_control_plane_storage_rows{table="enrollment_requests"} 0`,
"agentmesh_control_plane_catalog_entries ",
`agentmesh_control_plane_storage_rows_deleted_total{table="routers"} `,
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index a535d4ea..9b4537a4 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -300,6 +300,19 @@ func (s *Server) Init() error {
} else if err != nil {
return fmt.Errorf("failed to query initial keyring status: %w", err)
}
+ // The rotation deadline lives in the store, shared by the replicas and
+ // kept across restarts; a store that never had one gets its first an
+ // interval from now.
+ if interval := s.config.KeyRotationInterval; interval > 0 {
+ first := time.Now().Add(interval)
+ scheduled, err := s.store.ScheduleKeyRotationIfUnset(ctx, first)
+ if err != nil {
+ return fmt.Errorf("failed to schedule the first key rotation: %w", err)
+ }
+ if scheduled {
+ logger.Infof("First key rotation scheduled at %s", first.UTC().Format(time.RFC3339))
+ }
+ }
// Bootstrap Policy is now disabled; starting default closed.
@@ -514,64 +527,80 @@ func (s *Server) getProviders() map[string]*oidc.Provider {
return pCopy
}
+// keyRotationCheckInterval is how often a replica asks the store whether a
+// rotation is due: a tenth of the interval, at most once a minute. The
+// deadline itself lives in the store, so a replica that restarts more often
+// than the interval still rotates on time.
+func keyRotationCheckInterval(interval time.Duration) time.Duration {
+ return min(time.Minute, interval/10)
+}
+
+// runKeyRotationLoop rotates the signing keys when the deadline the store
+// holds (set by Init, advanced by each rotation) has passed. The check runs
+// well within the interval, so a control plane that restarts more often
+// than it rotates still rotates on time instead of never.
func (s *Server) runKeyRotationLoop() {
defer s.wg.Done()
- if s.config.KeyRotationInterval <= 0 {
+ interval := s.config.KeyRotationInterval
+ if interval <= 0 {
return
}
- ticker := time.NewTicker(s.config.KeyRotationInterval)
+ ticker := time.NewTicker(keyRotationCheckInterval(interval))
defer ticker.Stop()
for {
select {
case <-ticker.C:
- // Replicas share one DB but tick independently; only the replica
- // that wins this claim may rotate for the current window.
- now := time.Now()
- claimed, err := s.store.ClaimKeyRotation(s.ctx, now, s.config.KeyRotationInterval)
- if err != nil {
- logger.Errorf("Failed to claim key rotation window: %v", err)
- continue
- }
- if !claimed {
- logger.Debug("Skipping key rotation: another replica already claimed this window")
- continue
- }
-
- logger.Info("Rotating Biscuit signing keys...")
- newPub, newPriv, err := ed25519.GenerateKey(rand.Reader)
- if err != nil {
- logger.Errorf("Failed to generate key pair for rotation: %v", err)
- // Give up the window so a retry isn't stuck waiting a full interval.
- if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, s.config.KeyRotationInterval); relErr != nil {
- logger.Errorf("Failed to release key rotation claim: %v", relErr)
- }
- continue
- }
- err = s.store.RotateKeys(s.ctx, newPriv, newPub, s.config.KeyGracePeriod)
- if err != nil {
- logger.Errorf("Failed to rotate keyring: %v", err)
- // Same as above: don't let a failed rotation strand the mesh
- // on unrotated keys until the next full interval.
- if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, s.config.KeyRotationInterval); relErr != nil {
- logger.Errorf("Failed to release key rotation claim: %v", relErr)
- }
- } else {
- logger.Infof("Key rotation committed. New current public key: %s", hex.EncodeToString(newPub))
- if _, oidcErr := s.RotateOIDCKey(s.config.KeyGracePeriod); oidcErr != nil {
- logger.Warnf("Failed to rotate OIDC signing key: %v", oidcErr)
- }
- if err := s.getMeshAdapter().PublishEvent(s.ctx, api.MeshEvent_KEY_ROTATION, "", newPub); err != nil {
- logger.Warnf("Failed to publish KEY_ROTATION event to mesh: %v", err)
- }
- }
+ s.rotateKeysIfDue(time.Now())
case <-s.ctx.Done():
return
}
}
}
+// rotateKeysIfDue claims the rotation window and rotates when it is due.
+// Replicas share one store and check independently; only the one that wins
+// the claim rotates for this window.
+func (s *Server) rotateKeysIfDue(now time.Time) {
+ interval := s.config.KeyRotationInterval
+ claimed, err := s.store.ClaimKeyRotation(s.ctx, now, interval)
+ if err != nil {
+ logger.Errorf("Failed to claim key rotation window: %v", err)
+ return
+ }
+ if !claimed {
+ return
+ }
+
+ logger.Info("Rotating Biscuit signing keys...")
+ newPub, newPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ logger.Errorf("Failed to generate key pair for rotation: %v", err)
+ // Give up the window so a retry isn't stuck waiting a full interval.
+ if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, interval); relErr != nil {
+ logger.Errorf("Failed to release key rotation claim: %v", relErr)
+ }
+ return
+ }
+ if err := s.store.RotateKeys(s.ctx, newPriv, newPub, s.config.KeyGracePeriod); err != nil {
+ logger.Errorf("Failed to rotate keyring: %v", err)
+ // Same as above: don't let a failed rotation strand the mesh
+ // on unrotated keys until the next full interval.
+ if relErr := s.store.ReleaseKeyRotationClaim(s.ctx, now, interval); relErr != nil {
+ logger.Errorf("Failed to release key rotation claim: %v", relErr)
+ }
+ return
+ }
+ logger.Infof("Key rotation committed. New current public key: %s; next rotation at %s", hex.EncodeToString(newPub), now.Add(interval).UTC().Format(time.RFC3339))
+ if _, oidcErr := s.RotateOIDCKey(s.config.KeyGracePeriod); oidcErr != nil {
+ logger.Warnf("Failed to rotate OIDC signing key: %v", oidcErr)
+ }
+ if err := s.getMeshAdapter().PublishEvent(s.ctx, api.MeshEvent_KEY_ROTATION, "", newPub); err != nil {
+ logger.Warnf("Failed to publish KEY_ROTATION event to mesh: %v", err)
+ }
+}
+
// HandleHealthz HTTP GET `/healthz`
func (s *Server) HandleHealthz(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.go
index 6ca97f52..f7e75157 100644
--- a/internal/storage/sql_store.go
+++ b/internal/storage/sql_store.go
@@ -857,6 +857,31 @@ func (s *SQLStore) ReleaseKeyRotationClaim(ctx context.Context, now time.Time, i
return err
}
+// ScheduleKeyRotationIfUnset implements Store.
+func (s *SQLStore) ScheduleKeyRotationIfUnset(ctx context.Context, at time.Time) (bool, error) {
+ res, err := s.db.ExecContext(ctx, s.rebind(`UPDATE rotation_lock SET next_rotation_at = ? WHERE id = 1 AND next_rotation_at = 0`), at.UnixMilli())
+ if err != nil {
+ return false, err
+ }
+ n, err := res.RowsAffected()
+ if err != nil {
+ return false, err
+ }
+ return n == 1, nil
+}
+
+// NextKeyRotation implements Store.
+func (s *SQLStore) NextKeyRotation(ctx context.Context) (time.Time, error) {
+ var ms int64
+ if err := s.db.QueryRowContext(ctx, `SELECT next_rotation_at FROM rotation_lock WHERE id = 1`).Scan(&ms); err != nil {
+ return time.Time{}, err
+ }
+ if ms == 0 {
+ return time.Time{}, nil
+ }
+ return time.UnixMilli(ms), nil
+}
+
// RotateKeys implements Store.
func (s *SQLStore) RotateKeys(ctx context.Context, newPriv ed25519.PrivateKey, newPub ed25519.PublicKey, gracePeriod time.Duration) error {
tx, err := s.db.BeginTx(ctx, nil)
diff --git a/internal/storage/sql_store_test.go b/internal/storage/sql_store_test.go
index 1158a029..7ffd234b 100644
--- a/internal/storage/sql_store_test.go
+++ b/internal/storage/sql_store_test.go
@@ -225,6 +225,53 @@ func TestClaimKeyRotation(t *testing.T) {
}
}
+func TestScheduleKeyRotationIfUnset(t *testing.T) {
+ store := newTestStore(t)
+ defer func() { _ = store.Close() }()
+ ctx := context.Background()
+ t0 := time.Now().Truncate(time.Millisecond)
+ interval := 24 * time.Hour
+
+ next, err := store.NextKeyRotation(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !next.IsZero() {
+ t.Fatalf("fresh store has a rotation deadline %s, want none", next)
+ }
+
+ // The first replica to start schedules; a replica starting later, or
+ // the same one restarting, leaves the deadline alone.
+ scheduled, err := store.ScheduleKeyRotationIfUnset(ctx, t0.Add(interval))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !scheduled {
+ t.Fatal("first schedule on a fresh store was not applied")
+ }
+ scheduled, err = store.ScheduleKeyRotationIfUnset(ctx, t0.Add(time.Hour))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if scheduled {
+ t.Fatal("a second schedule replaced the deadline; want it kept")
+ }
+ if next, _ = store.NextKeyRotation(ctx); !next.Equal(t0.Add(interval)) {
+ t.Fatalf("deadline = %s, want %s", next, t0.Add(interval))
+ }
+
+ // Not due before the deadline, due at it; a claim moves it one interval on.
+ if claimed, _ := store.ClaimKeyRotation(ctx, t0.Add(interval-time.Second), interval); claimed {
+ t.Fatal("claim before the deadline succeeded")
+ }
+ if claimed, _ := store.ClaimKeyRotation(ctx, t0.Add(interval), interval); !claimed {
+ t.Fatal("claim at the deadline failed")
+ }
+ if next, _ = store.NextKeyRotation(ctx); !next.Equal(t0.Add(2 * interval)) {
+ t.Fatalf("deadline after the claim = %s, want %s", next, t0.Add(2*interval))
+ }
+}
+
func TestReleaseKeyRotationClaim(t *testing.T) {
store := newTestStore(t)
defer func() { _ = store.Close() }()
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
index 52ad4730..ca9095a1 100644
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -248,6 +248,15 @@ type Store interface {
// the rotation that followed a successful claim failed.
ReleaseKeyRotationClaim(ctx context.Context, now time.Time, interval time.Duration) error
+ // ScheduleKeyRotationIfUnset sets the first rotation deadline of a store
+ // that never had one, and reports whether it did. A store that already
+ // carries a deadline, due or not, is left as it is.
+ ScheduleKeyRotationIfUnset(ctx context.Context, at time.Time) (bool, error)
+
+ // NextKeyRotation is the deadline ClaimKeyRotation will honour: the
+ // instant the next rotation is due, zero when none was ever scheduled.
+ NextKeyRotation(ctx context.Context) (time.Time, error)
+
// SaveInitialKey sets the initial key pair if no keys exist yet.
SaveInitialKey(ctx context.Context, priv ed25519.PrivateKey, pub ed25519.PublicKey) error
diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.md
index b0d0d50f..c35086d3 100644
--- a/site/content/docs/reference/control-plane.md
+++ b/site/content/docs/reference/control-plane.md
@@ -149,6 +149,13 @@ with `403`). The console uses these routes.
- **Bans propagate in two ways.** `/info` lists banned peer IDs for nodes and
routers that start or restart. A signed mesh event reaches the ones that
are already running.
+- **The rotation deadline is in the database.** The first rotation is due
+ one `--key-rotation-interval` after the first start, each rotation sets
+ the next, and every replica checks the deadline at least once a minute.
+ A control plane that restarts more often than it rotates still rotates on
+ time, and one that was down when a rotation was due rotates on its first
+ check. `agentmesh_control_plane_next_key_rotation_timestamp_seconds` is
+ the deadline; a value in the past is a rotation that is not happening.
- **Node records are garbage-collected.** A node whose session has expired is
deleted after `--node-retention`. A node enrolled with a bootstrap token
has no session; its record is deleted once it has not refreshed its