Skip to content

Agent Mesh

router: one control plane sync loop, --control-plane-sync-interval

Antonio Ojeab9719e0b4e2c
diff --git a/cmd/agentmesh-one/main.go b/cmd/agentmesh-one/main.go
index 9247fb3c..0e93a76e 100644
--- a/cmd/agentmesh-one/main.go
+++ b/cmd/agentmesh-one/main.go
@@ -234,8 +234,7 @@ func main() {
rootCmd.Flags().BoolVar(&cpTunables.ManualEnrollment, "control-plane-manual-enrollment", false, "Queue bootstrap enrollments for admin approval instead of auto-approving")
// Embedded router tunables.
- rootCmd.Flags().DurationVar(&routerTunables.KeysSyncInterval, "router-keys-sync-interval", 0, "Biscuit public key refresh interval (0 keeps the component default)")
- rootCmd.Flags().DurationVar(&routerTunables.LeaseRenewInterval, "router-lease-renew-interval", 0, "Lease renewal interval (0 keeps the component default)")
+ rootCmd.Flags().DurationVar(&routerTunables.ControlPlaneSyncInterval, "router-control-plane-sync-interval", 0, "How often the router renews its lease and pulls keys, routers, bans and policy from the control plane (0 keeps the component default)")
rootCmd.Flags().IntVar(&routerTunables.LowWaterMark, "router-low-watermark", 0, "Connection manager low watermark (0 is 15% below the high watermark)")
rootCmd.Flags().IntVar(&routerTunables.HighWaterMark, "router-high-watermark", 0, "Connection manager high watermark (0 keeps the component default)")
rootCmd.Flags().IntVar(&routerTunables.ConnsPerSourceIP, "router-conns-per-source-ip", 0, "Per-source-IP connection budget (0 follows the high watermark; proxied peers share source IPs)")
diff --git a/cmd/agentmesh-router/main.go b/cmd/agentmesh-router/main.go
index ff801fb7..dc0f0962 100644
--- a/cmd/agentmesh-router/main.go
+++ b/cmd/agentmesh-router/main.go
@@ -51,32 +51,31 @@ func init() {
}
var (
- controlPlaneURL string
- insecureControlPlane bool
- listenAddrs []string
- externalAddrs []string
- keysSyncInterval time.Duration
- leaseRenewInterval time.Duration
- oidcToken string
- bootstrapToken string
- bootstrapTokenPath string
- jwtPath string
- keysPath string
- allowLoopback bool
- connsPerSourceIP int
- logLevel string
- logFormat string
- lowWaterMark int
- highWaterMark int
- metricsAddr string
- pprofAddr string
- relayLimitDuration time.Duration
- relayLimitData router.ByteSize
- relayMaxReservations int
- relayMaxCircuits int
- shutdownLeaseTTL time.Duration
- authDeadline time.Duration
- labelFlags []string
+ controlPlaneURL string
+ insecureControlPlane bool
+ listenAddrs []string
+ externalAddrs []string
+ controlPlaneSyncInterval time.Duration
+ oidcToken string
+ bootstrapToken string
+ bootstrapTokenPath string
+ jwtPath string
+ keysPath string
+ allowLoopback bool
+ connsPerSourceIP int
+ logLevel string
+ logFormat string
+ lowWaterMark int
+ highWaterMark int
+ metricsAddr string
+ pprofAddr string
+ relayLimitDuration time.Duration
+ relayLimitData router.ByteSize
+ relayMaxReservations int
+ relayMaxCircuits int
+ shutdownLeaseTTL time.Duration
+ authDeadline time.Duration
+ labelFlags []string
)
var logger = golog.Logger("agentmesh-router-cli")
@@ -94,8 +93,7 @@ func main() {
AllowInsecureControlPlane: insecureControlPlane,
ListenAddrs: listenAddrs,
ExternalAddrs: externalAddrs,
- KeysSyncInterval: keysSyncInterval,
- LeaseRenewInterval: leaseRenewInterval,
+ ControlPlaneSyncInterval: controlPlaneSyncInterval,
OIDCToken: oidcToken,
BootstrapToken: bootstrapToken,
BootstrapTokenPath: bootstrapTokenPath,
@@ -150,8 +148,7 @@ func main() {
rootCmd.Flags().BoolVar(&insecureControlPlane, "insecure-control-plane", false, "Accept a plaintext http:// control plane URL to a non-loopback host (whoever answers it becomes this router's trust root; only for networks you already trust)")
rootCmd.Flags().StringSliceVar(&listenAddrs, "listen", []string{"/ip4/0.0.0.0/tcp/5001", "/ip6/::/tcp/5001"}, "libp2p Listen Addresses")
rootCmd.Flags().StringSliceVar(&externalAddrs, "external-addr", []string{}, "External addresses to announce to control plane")
- rootCmd.Flags().DurationVar(&keysSyncInterval, "keys-sync-interval", 5*time.Minute, "Key synchronization polling interval")
- rootCmd.Flags().DurationVar(&leaseRenewInterval, "lease-renew-interval", 300*time.Second, "Lease renewal registration interval")
+ rootCmd.Flags().DurationVar(&controlPlaneSyncInterval, "control-plane-sync-interval", router.DefaultControlPlaneSyncInterval, "How often the lease is renewed and signing keys, the router list, bans and the mesh policy are pulled from the control plane; a mesh event pulls at once. Keep it well below the control plane's --lease-duration and --key-grace-period.")
rootCmd.Flags().StringVar(&oidcToken, "oidc-token", "", "OIDC ID token for enrollment")
rootCmd.Flags().StringVar(&bootstrapToken, "bootstrap-token", "", "Pre-shared bootstrap token for enrollment")
rootCmd.Flags().StringVar(&bootstrapTokenPath, "bootstrap-token-path", "", "Path to file containing bootstrap token for enrollment")
diff --git a/internal/router/config.go b/internal/router/config.go
index 915c72f3..382ee4eb 100644
--- a/internal/router/config.go
+++ b/internal/router/config.go
@@ -35,25 +35,33 @@ const DefaultShutdownLeaseTTL = 30 * time.Second
// less.
const DefaultAuthDeadline = time.Minute
+// DefaultControlPlaneSyncInterval is how often a router talks to the control
+// plane when nothing happens: the lease is renewed, and /info, /keys and
+// /policies re-read, every five minutes. What has to be quick does not wait
+// for it: a mesh event runs a pass at once, a new router reads /info on its
+// own first pass and handshakes with every listed router, and a lookup that
+// finds nothing re-reads /info (refederateIfStale).
+const DefaultControlPlaneSyncInterval = 5 * time.Minute
+
// Options holds configuration details for the agentmesh-router.
type Options struct {
- ControlPlaneURL string
- ListenAddrs []string
- ExternalAddrs []string
- KeysSyncInterval time.Duration
- PolicySyncInterval time.Duration
- LeaseRenewInterval time.Duration
- OIDCProvider string // For OIDC enrollment
- OIDCToken string // OIDC JWT token for enrollment
- BootstrapToken string // Pre-shared bootstrap token for enrollment
- BootstrapTokenPath string // File path containing pre-shared bootstrap token
- JWTPath string // File path containing OIDC JWT token
- KeysDBPath string // Path to save peer private key
- AllowLoopback bool
- BiscuitTimeout time.Duration
- LogVerbose bool
- LowWaterMark int
- HighWaterMark int
+ ControlPlaneURL string
+ ListenAddrs []string
+ ExternalAddrs []string
+ // ControlPlaneSyncInterval is how often the router's one pass over the
+ // control plane runs: lease, /info, /keys, /policies, credential expiry.
+ ControlPlaneSyncInterval time.Duration
+ OIDCProvider string // For OIDC enrollment
+ OIDCToken string // OIDC JWT token for enrollment
+ BootstrapToken string // Pre-shared bootstrap token for enrollment
+ BootstrapTokenPath string // File path containing pre-shared bootstrap token
+ JWTPath string // File path containing OIDC JWT token
+ KeysDBPath string // Path to save peer private key
+ AllowLoopback bool
+ BiscuitTimeout time.Duration
+ LogVerbose bool
+ LowWaterMark int
+ HighWaterMark int
// RelayLimitDuration / RelayLimitData cap each relayed connection; 0 is
// no limit, so Default() leaves them alone.
RelayLimitDuration time.Duration
@@ -119,14 +127,8 @@ func (o *Options) Default() {
if len(o.ListenAddrs) == 0 {
o.ListenAddrs = []string{"/ip4/0.0.0.0/tcp/5001", "/ip6/::/tcp/5001"}
}
- if o.KeysSyncInterval <= 0 {
- o.KeysSyncInterval = 5 * time.Minute
- }
- if o.PolicySyncInterval <= 0 {
- o.PolicySyncInterval = 15 * time.Minute
- }
- if o.LeaseRenewInterval <= 0 {
- o.LeaseRenewInterval = 30 * time.Second
+ if o.ControlPlaneSyncInterval <= 0 {
+ o.ControlPlaneSyncInterval = DefaultControlPlaneSyncInterval
}
if o.HighWaterMark <= 0 {
o.HighWaterMark = DefaultHighWaterMark
diff --git a/internal/router/router.go b/internal/router/router.go
index 20c39f18..d0821de1 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -379,8 +379,9 @@ type Router struct {
// only the last biscuit it issued.
refreshMu sync.Mutex
- keysSyncTrigger chan struct{}
- policySyncTrigger chan struct{}
+ // syncTrigger asks the control plane sync loop for a pass now; a mesh
+ // event sends on it.
+ syncTrigger chan struct{}
// Control contexts
ctx context.Context
@@ -415,12 +416,11 @@ func NewRouter(ctx context.Context, config Options) (*Router, error) {
}
return &Router{
- config: config,
- ctx: ctx,
- cancel: cancel,
- handshakeLimiter: handshakeLimiter,
- keysSyncTrigger: make(chan struct{}, 1),
- policySyncTrigger: make(chan struct{}, 1),
+ config: config,
+ ctx: ctx,
+ cancel: cancel,
+ handshakeLimiter: handshakeLimiter,
+ syncTrigger: make(chan struct{}, 1),
}, nil
}
@@ -599,12 +599,8 @@ func (r *Router) Start() (retErr error) {
})
// 5. Start background routines
- r.wg.Add(9)
- go r.runLeaseRenewalLoop()
- go r.runKeysSyncLoop()
- go r.runPolicySyncLoop()
- go r.runFederationLoop()
- go r.runBiscuitRenewalLoop()
+ r.wg.Add(5)
+ go r.runControlPlaneSyncLoop()
go r.runShedLoop()
go r.listenForControlPlaneEvents(r.ctx)
go func() {
@@ -1049,65 +1045,93 @@ func (r *Router) listenForControlPlaneEvents(ctx context.Context) {
}
}
case api.MeshEvent_KEY_ROTATION:
- logger.Infof("[Router Event] Received KEY_ROTATION event, triggering key sync")
- r.triggerKeysSync()
+ logger.Infof("[Router Event] Received KEY_ROTATION event, triggering control plane sync")
+ r.triggerSync()
case api.MeshEvent_POLICY_UPDATE:
- logger.Infof("[Router Event] Received POLICY_UPDATE event, triggering policy sync")
- r.triggerPolicySync()
+ logger.Infof("[Router Event] Received POLICY_UPDATE event, triggering control plane sync")
+ r.triggerSync()
}
}
}
-func (r *Router) triggerKeysSync() {
+func (r *Router) triggerSync() {
select {
- case r.keysSyncTrigger <- struct{}{}:
+ case r.syncTrigger <- struct{}{}:
default:
}
}
-func (r *Router) runKeysSyncLoop() {
+// runControlPlaneSyncLoop is the router's one conversation with the control
+// plane. Every ControlPlaneSyncInterval, and at once on a mesh event, a pass
+// re-reads /keys, renews the lease, re-reads /info (the routers to federate
+// with and the ban set), re-reads /policies and refreshes the router's own
+// credential when it nears expiry. Between passes, while a listed router is
+// not yet authenticated, the federation step alone is retried every
+// federationCatchUp, so a router that cannot be reached costs the control
+// plane nothing beyond the pass.
+func (r *Router) runControlPlaneSyncLoop() {
defer r.wg.Done()
- ticker := time.NewTicker(r.config.KeysSyncInterval)
- defer ticker.Stop()
-
+ timer := time.NewTimer(0)
+ defer timer.Stop()
+ var lastPass time.Time
for {
+ now := false
select {
- case <-ticker.C:
- case <-r.keysSyncTrigger:
+ case <-timer.C:
+ case <-r.syncTrigger:
+ now = true
+ if !timer.Stop() {
+ select {
+ case <-timer.C:
+ default:
+ }
+ }
case <-r.ctx.Done():
return
}
- if err := r.syncKeys(); err != nil {
- logger.Errorf("Failed keys sync: %v", err)
+ if now || time.Since(lastPass) >= r.config.ControlPlaneSyncInterval {
+ r.syncControlPlane()
+ lastPass = time.Now()
+ } else {
+ r.connectBootstrapRouters()
+ }
+ next := r.config.ControlPlaneSyncInterval - time.Since(lastPass)
+ if listed, authenticated := r.peerRouterCounts(); authenticated < listed && federationCatchUp < next {
+ next = federationCatchUp
}
+ timer.Reset(max(next, 0))
}
}
-func (r *Router) triggerPolicySync() {
- select {
- case r.policySyncTrigger <- struct{}{}:
- default:
+// syncControlPlane is one pass of the sync loop. Keys come first: a lease
+// or a policy read presented with a credential the control plane no longer
+// signs for is refused until the rotation is learned and the credential
+// refreshed (syncKeys).
+func (r *Router) syncControlPlane() {
+ if err := r.syncKeys(); err != nil {
+ logger.Errorf("Failed keys sync: %v", err)
+ }
+ r.renewLease()
+ r.connectBootstrapRouters()
+ if err := r.syncPolicy(); err != nil {
+ logger.Errorf("Failed policy sync: %v", err)
}
+ r.refreshExpiringCredential()
}
-// runPolicySyncLoop pulls the mesh policy at start, on each POLICY_UPDATE
-// event and every PolicySyncInterval: the event is best effort, the pull is
-// what bounds how long a policy change takes to reach this router.
-func (r *Router) runPolicySyncLoop() {
- defer r.wg.Done()
- ticker := time.NewTicker(r.config.PolicySyncInterval)
- defer ticker.Stop()
-
- for {
- if err := r.syncPolicy(); err != nil {
- logger.Errorf("Failed policy sync: %v", err)
- }
- select {
- case <-ticker.C:
- case <-r.policySyncTrigger:
- case <-r.ctx.Done():
- return
- }
+// refreshExpiringCredential refreshes the router's own credential once less
+// than a fifth of its lifetime is left, so a credential is never presented
+// close to its expiry.
+func (r *Router) refreshExpiringCredential() {
+ r.keysMu.RLock()
+ expiration := r.credential.expiration
+ r.keysMu.RUnlock()
+ if expiration.IsZero() || time.Until(expiration) >= api.BiscuitTokenTTL/5 {
+ return
+ }
+ logger.Infof("Router biscuit expiring in %v, triggering proactive refresh...", time.Until(expiration))
+ if err := r.RefreshEnrollment(r.ctx); err != nil {
+ logger.Errorf("Proactive router biscuit refresh failed: %v", err)
}
}
@@ -1157,24 +1181,6 @@ func (r *Router) meshPolicyRules() []biscuit.Rule {
return nil
}
-func (r *Router) runLeaseRenewalLoop() {
- defer r.wg.Done()
- ticker := time.NewTicker(r.config.LeaseRenewInterval)
- defer ticker.Stop()
-
- // Initial renewal after host is online
- r.renewLease()
-
- for {
- select {
- case <-ticker.C:
- r.renewLease()
- case <-r.ctx.Done():
- return
- }
- }
-}
-
// renewLease extends the lease for the control plane's duration. Once the
// router is shutting down it does nothing: the withdrawal sent by Close is
// the last word, and a renewal racing it would re-advertise a router that
@@ -1414,35 +1420,12 @@ func (r *Router) reconcileBannedPeers(peerIDs []string, fetchedAt time.Time) {
}
}
-// federationInterval is how often a router reads the control plane's list
-// of routers and authenticates with the ones it has not; federationCatchUp
-// is the interval while the list names a router it has not reached yet,
-// which it does at start, when the other routers started at the same
-// moment and leased after this one's first pass, and whenever a router
-// joins.
-const (
- federationInterval = 30 * time.Second
- federationCatchUp = 2 * time.Second
-)
-
-func (r *Router) runFederationLoop() {
- defer r.wg.Done()
- timer := time.NewTimer(0)
- defer timer.Stop()
- for {
- select {
- case <-timer.C:
- r.connectBootstrapRouters()
- next := federationInterval
- if listed, authenticated := r.peerRouterCounts(); authenticated < listed {
- next = federationCatchUp
- }
- timer.Reset(next)
- case <-r.ctx.Done():
- return
- }
- }
-}
+// federationCatchUp is how often the sync loop retries the federation step
+// alone while the control plane's list names a router this one has not
+// authenticated with yet, which it does at start, when the other routers
+// started at the same moment and leased after this one's first pass, and
+// whenever a router joins.
+const federationCatchUp = 2 * time.Second
func (r *Router) connectBootstrapRouters() {
// Taken before the request: anything banned after this point cannot be
@@ -2017,27 +2000,3 @@ func (r *Router) verifyOwnBiscuit(token []byte, peerID peer.ID) error {
}
return identity.VerifyBiscuitRole(token, key, r.config.RequiredRole, r.config.BiscuitTimeout)
}
-
-func (r *Router) runBiscuitRenewalLoop() {
- defer r.wg.Done()
- ticker := time.NewTicker(api.TokenRefreshCheckInterval)
- defer ticker.Stop()
-
- for {
- select {
- case <-ticker.C:
- r.keysMu.RLock()
- expiration := r.credential.expiration
- r.keysMu.RUnlock()
-
- if !expiration.IsZero() && time.Until(expiration) < api.BiscuitTokenTTL/5 { // 80% elapsed lifespan of 24h (remaining time < 20% of TTL)
- logger.Infof("Router biscuit expiring in %v, triggering proactive refresh...", time.Until(expiration))
- if err := r.RefreshEnrollment(r.ctx); err != nil {
- logger.Errorf("Proactive router biscuit refresh failed: %v", err)
- }
- }
- case <-r.ctx.Done():
- return
- }
- }
-}
diff --git a/internal/router/router_test.go b/internal/router/router_test.go
index 279b42c5..874a156a 100644
--- a/internal/router/router_test.go
+++ b/internal/router/router_test.go
@@ -184,14 +184,13 @@ func TestRouterIntegration(t *testing.T) {
// 1. Create and Start Router
rOpts := Options{
- ControlPlaneURL: cpURL,
- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
- KeysSyncInterval: 2 * time.Second,
- LeaseRenewInterval: 2 * time.Second,
- OIDCToken: routerJWT,
- KeysDBPath: routerKeyPath,
- AllowLoopback: true,
- BiscuitTimeout: 1 * time.Second,
+ ControlPlaneURL: cpURL,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ ControlPlaneSyncInterval: 2 * time.Second,
+ OIDCToken: routerJWT,
+ KeysDBPath: routerKeyPath,
+ AllowLoopback: true,
+ BiscuitTimeout: 1 * time.Second,
}
r, err := NewRouter(context.Background(), rOpts)
@@ -354,14 +353,13 @@ func TestRouterFederation(t *testing.T) {
r1KeyPath := filepath.Join(tempDir, "router1.key")
r1, err := NewRouter(context.Background(), Options{
- ControlPlaneURL: cpURL,
- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
- KeysSyncInterval: 2 * time.Second,
- LeaseRenewInterval: 2 * time.Second,
- OIDCToken: router1JWT,
- KeysDBPath: r1KeyPath,
- AllowLoopback: true,
- BiscuitTimeout: 1 * time.Second,
+ ControlPlaneURL: cpURL,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ ControlPlaneSyncInterval: 2 * time.Second,
+ OIDCToken: router1JWT,
+ KeysDBPath: r1KeyPath,
+ AllowLoopback: true,
+ BiscuitTimeout: 1 * time.Second,
})
if err != nil {
t.Fatal(err)
@@ -373,14 +371,13 @@ func TestRouterFederation(t *testing.T) {
r2KeyPath := filepath.Join(tempDir, "router2.key")
r2, err := NewRouter(context.Background(), Options{
- ControlPlaneURL: cpURL,
- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
- KeysSyncInterval: 2 * time.Second,
- LeaseRenewInterval: 2 * time.Second,
- OIDCToken: router2JWT,
- KeysDBPath: r2KeyPath,
- AllowLoopback: true,
- BiscuitTimeout: 1 * time.Second,
+ ControlPlaneURL: cpURL,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ ControlPlaneSyncInterval: 2 * time.Second,
+ OIDCToken: router2JWT,
+ KeysDBPath: r2KeyPath,
+ AllowLoopback: true,
+ BiscuitTimeout: 1 * time.Second,
})
if err != nil {
t.Fatal(err)
@@ -443,14 +440,13 @@ func TestRouterProactiveTokenRefresh(t *testing.T) {
})
rOpts := Options{
- ControlPlaneURL: cpURL,
- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
- KeysSyncInterval: 20 * time.Second,
- LeaseRenewInterval: 20 * time.Second,
- OIDCToken: routerJWT,
- KeysDBPath: routerKeyPath,
- AllowLoopback: true,
- BiscuitTimeout: 1 * time.Second,
+ ControlPlaneURL: cpURL,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ ControlPlaneSyncInterval: 20 * time.Second,
+ OIDCToken: routerJWT,
+ KeysDBPath: routerKeyPath,
+ AllowLoopback: true,
+ BiscuitTimeout: 1 * time.Second,
}
r, err := NewRouter(context.Background(), rOpts)
@@ -504,14 +500,13 @@ func TestRouterLeaseRenewalReEnrollOn401(t *testing.T) {
})
rOpts := Options{
- ControlPlaneURL: cpURL,
- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
- KeysSyncInterval: 20 * time.Second,
- LeaseRenewInterval: 20 * time.Second,
- OIDCToken: routerJWT,
- KeysDBPath: routerKeyPath,
- AllowLoopback: true,
- BiscuitTimeout: 1 * time.Second,
+ ControlPlaneURL: cpURL,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ ControlPlaneSyncInterval: 20 * time.Second,
+ OIDCToken: routerJWT,
+ KeysDBPath: routerKeyPath,
+ AllowLoopback: true,
+ BiscuitTimeout: 1 * time.Second,
}
r, err := NewRouter(context.Background(), rOpts)
@@ -705,14 +700,13 @@ func TestRouterProactiveRefreshReEnrollOn401(t *testing.T) {
})
rOpts := Options{
- ControlPlaneURL: cpURL,
- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
- KeysSyncInterval: 20 * time.Second,
- LeaseRenewInterval: 20 * time.Second,
- OIDCToken: routerJWT,
- KeysDBPath: routerKeyPath,
- AllowLoopback: true,
- BiscuitTimeout: 1 * time.Second,
+ ControlPlaneURL: cpURL,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ ControlPlaneSyncInterval: 20 * time.Second,
+ OIDCToken: routerJWT,
+ KeysDBPath: routerKeyPath,
+ AllowLoopback: true,
+ BiscuitTimeout: 1 * time.Second,
}
r, err := NewRouter(context.Background(), rOpts)
@@ -764,14 +758,13 @@ func TestRouterGossipSubBannedEvent(t *testing.T) {
})
rOpts := Options{
- ControlPlaneURL: cpURL,
- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
- KeysSyncInterval: 20 * time.Second,
- LeaseRenewInterval: 20 * time.Second,
- OIDCToken: routerJWT,
- KeysDBPath: routerKeyPath,
- AllowLoopback: true,
- BiscuitTimeout: 1 * time.Second,
+ ControlPlaneURL: cpURL,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ ControlPlaneSyncInterval: 20 * time.Second,
+ OIDCToken: routerJWT,
+ KeysDBPath: routerKeyPath,
+ AllowLoopback: true,
+ BiscuitTimeout: 1 * time.Second,
}
r, err := NewRouter(context.Background(), rOpts)
@@ -783,6 +776,17 @@ func TestRouterGossipSubBannedEvent(t *testing.T) {
}
defer func() { _ = r.Close() }()
+ // The control plane never learns of this ban, so the event must land
+ // after the first sync pass read /info: a pass that fetched after the
+ // event would read the ban's absence as an unban (reconcileBannedPeers),
+ // which is right for a real ban the control plane recorded first.
+ for deadline := time.Now().Add(5 * time.Second); r.lastFederation.Load() == 0; {
+ if time.Now().After(deadline) {
+ t.Fatal("the router's first sync pass did not read /info")
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+
privNode, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)
if err != nil {
t.Fatalf("failed to generate node keypair: %v", err)
@@ -1326,14 +1330,13 @@ func TestRouterRefreshEnrollmentReattestsJWT(t *testing.T) {
}
r, err := NewRouter(context.Background(), Options{
- ControlPlaneURL: cpURL,
- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
- KeysSyncInterval: time.Hour,
- LeaseRenewInterval: time.Hour,
- JWTPath: jwtPath,
- KeysDBPath: filepath.Join(tempDir, "router.key"),
- AllowLoopback: true,
- BiscuitTimeout: time.Second,
+ ControlPlaneURL: cpURL,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ ControlPlaneSyncInterval: time.Hour,
+ JWTPath: jwtPath,
+ KeysDBPath: filepath.Join(tempDir, "router.key"),
+ AllowLoopback: true,
+ BiscuitTimeout: time.Second,
})
if err != nil {
t.Fatalf("NewRouter: %v", err)
diff --git a/internal/router/trust_test.go b/internal/router/trust_test.go
index 63a47915..dbb51eba 100644
--- a/internal/router/trust_test.go
+++ b/internal/router/trust_test.go
@@ -163,7 +163,13 @@ func TestSyncKeysRequiresTrustedSignature(t *testing.T) {
})
}
-func TestKeysSyncLoopCoalescesTriggers(t *testing.T) {
+// TestControlPlaneSyncLoopCoalescesTriggers pins two properties of the one
+// control plane loop: a burst of mesh events while a pass is in flight costs
+// one follow-up pass, and readers of the trust set never wait on the control
+// plane. The fake control plane holds /keys until released and answers the
+// rest of the pass with empty documents; the router is not enrolled, so the
+// lease and policy steps log and move on.
+func TestControlPlaneSyncLoopCoalescesTriggers(t *testing.T) {
publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
@@ -179,6 +185,10 @@ func TestKeysSyncLoopCoalescesTriggers(t *testing.T) {
requests := make(chan struct{}, 4)
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+ if request.URL.Path != "/keys" {
+ w.Header().Set("Content-Type", "application/x-protobuf")
+ return
+ }
requests <- struct{}{}
select {
case <-release:
@@ -187,13 +197,13 @@ func TestKeysSyncLoopCoalescesTriggers(t *testing.T) {
}
}))
t.Cleanup(server.Close)
- router, err := NewRouter(context.Background(), Options{ControlPlaneURL: server.URL, KeysSyncInterval: time.Hour})
+ router, err := NewRouter(context.Background(), Options{ControlPlaneURL: server.URL, ControlPlaneSyncInterval: time.Hour})
if err != nil {
t.Fatal(err)
}
router.trustedPublicKeys = []ed25519.PublicKey{publicKey}
router.wg.Add(1)
- go router.runKeysSyncLoop()
+ go router.runControlPlaneSyncLoop()
t.Cleanup(func() {
router.cancel()
router.wg.Wait()
@@ -206,25 +216,26 @@ func TestKeysSyncLoopCoalescesTriggers(t *testing.T) {
t.Fatal(message)
}
}
- router.triggerKeysSync()
- await(requests, "key sync did not start")
+ // The loop's first pass runs at start.
+ await(requests, "the first pass did not start")
notified := make(chan struct{})
go func() {
for range 10 {
- router.triggerKeysSync()
+ router.triggerSync()
}
router.getTrustedPublicKeys()
close(notified)
}()
await(notified, "notifications or key readers blocked on the control-plane request")
- if pending := len(router.keysSyncTrigger); pending != 1 {
+ if pending := len(router.syncTrigger); pending != 1 {
t.Fatalf("pending triggers = %d, want one coalesced follow-up", pending)
}
release <- struct{}{}
- await(requests, "pending key sync was lost")
- if pending := len(router.keysSyncTrigger); pending != 0 {
- t.Fatalf("pending triggers = %d after follow-up", pending)
+ await(requests, "the follow-up pass was lost")
+ if pending := len(router.syncTrigger); pending != 0 {
+ t.Fatalf("pending triggers = %d after the follow-up", pending)
}
+ release <- struct{}{}
}
func TestSyncKeysRefreshesOnRotation(t *testing.T) {
diff --git a/internal/standalone/standalone.go b/internal/standalone/standalone.go
index 82277fca..e5e5d105 100644
--- a/internal/standalone/standalone.go
+++ b/internal/standalone/standalone.go
@@ -144,10 +144,9 @@ type ControlPlaneTunables struct {
// RouterTunables are the embedded router's operator knobs.
type RouterTunables struct {
- // KeysSyncInterval is how often biscuit public keys are refreshed.
- KeysSyncInterval time.Duration
- // LeaseRenewInterval is how often the router renews its lease.
- LeaseRenewInterval time.Duration
+ // ControlPlaneSyncInterval is how often the router renews its lease and
+ // pulls keys, routers, bans and policy from the control plane.
+ ControlPlaneSyncInterval time.Duration
// LowWaterMark / HighWaterMark bound the connection manager.
LowWaterMark int
HighWaterMark int
@@ -367,20 +366,19 @@ func (s *Server) Start(ctx context.Context) (retErr error) {
}
rtr, err := router.NewRouter(ctx, router.Options{
- ControlPlaneURL: "http://" + cp.Addr(),
- ListenAddrs: append([]string{wsAddr}, s.opts.P2PListen...),
- ExternalAddrs: externalAddrs,
- AllowLoopback: !s.opts.Router.DisallowLoopback,
- KeysDBPath: routerKeyPath,
- BootstrapToken: routerToken,
- KeysSyncInterval: s.opts.Router.KeysSyncInterval,
- LeaseRenewInterval: s.opts.Router.LeaseRenewInterval,
- LowWaterMark: s.opts.Router.LowWaterMark,
- HighWaterMark: s.opts.Router.HighWaterMark,
- RelayLimitDuration: s.opts.Router.RelayLimitDuration,
- RelayLimitData: int64(s.opts.Router.RelayLimitData),
- ShutdownLeaseTTL: s.opts.Router.ShutdownLeaseTTL,
- AuthDeadline: s.opts.Router.AuthDeadline,
+ ControlPlaneURL: "http://" + cp.Addr(),
+ ListenAddrs: append([]string{wsAddr}, s.opts.P2PListen...),
+ ExternalAddrs: externalAddrs,
+ AllowLoopback: !s.opts.Router.DisallowLoopback,
+ KeysDBPath: routerKeyPath,
+ BootstrapToken: routerToken,
+ ControlPlaneSyncInterval: s.opts.Router.ControlPlaneSyncInterval,
+ LowWaterMark: s.opts.Router.LowWaterMark,
+ HighWaterMark: s.opts.Router.HighWaterMark,
+ RelayLimitDuration: s.opts.Router.RelayLimitDuration,
+ RelayLimitData: int64(s.opts.Router.RelayLimitData),
+ ShutdownLeaseTTL: s.opts.Router.ShutdownLeaseTTL,
+ AuthDeadline: s.opts.Router.AuthDeadline,
// Single-port deployments typically sit behind a TLS-terminating
// proxy (Cloud Run, L7 LBs) or NAT where every peer shares a few
// source IPs; libp2p's default 8-conns-per-IP cap would throttle
diff --git a/site/content/docs/concepts/identity.md b/site/content/docs/concepts/identity.md
index 328f31ab..4fb15ace 100644
--- a/site/content/docs/concepts/identity.md
+++ b/site/content/docs/concepts/identity.md
@@ -201,10 +201,10 @@ any user action.
The control plane rotates its Biscuit Ed25519 signing key every
`--key-rotation-interval` (24 hours by default). The previous key stays valid
for `--key-grace-period` (1 hour by default) and is then retired. `GET /keys`
-returns the current set of keys, signed by each key in the set. Routers poll
-it every `--keys-sync-interval`; nodes fetch it at enrollment and then every
-`--control-plane-sync-interval`, together with the ban set, revocation list,
-and the mesh policy. A rotation event only brings the next pull forward. Both
+returns the current set of keys, signed by each key in the set. Routers and
+nodes fetch it at enrollment and then every `--control-plane-sync-interval`,
+together with the ban set, revocation list, and the mesh policy. A rotation
+event only brings the next pull forward. Both
accept a new set only if one of its signatures verifies under a key they
already trust. The first key comes from enrollment, and each later key is
vouched for by the key it replaces.
diff --git a/site/content/docs/reference/agentmesh-one.md b/site/content/docs/reference/agentmesh-one.md
index c47cc539..0e3f572f 100644
--- a/site/content/docs/reference/agentmesh-one.md
+++ b/site/content/docs/reference/agentmesh-one.md
@@ -65,8 +65,7 @@ of zero leaves the default of that component unchanged.
| `--control-plane-biscuit-ttl` | `--biscuit-ttl` |
| `--control-plane-workload-session-ttl` | `--workload-session-ttl` |
| `--control-plane-manual-enrollment` | The opposite of `--auto-approve-enrollment`: queue bootstrap enrollments for approval. |
-| `--router-keys-sync-interval` | `agentmesh-router --keys-sync-interval` |
-| `--router-lease-renew-interval` | `--lease-renew-interval` |
+| `--router-control-plane-sync-interval` | `agentmesh-router --control-plane-sync-interval` |
| `--router-low-watermark`, `--router-high-watermark` | `--low-watermark`, `--high-watermark` |
| `--router-conns-per-source-ip` | `--conns-per-source-ip`. `0` follows the high watermark, because peers behind a proxy share source IPs. |
| `--router-relay-limit-duration`, `--router-relay-limit-data` | `--relay-limit-duration`, `--relay-limit-data`. `0` keeps `1h` and no data limit, so set a long duration such as `24h` instead of an unlimited one. |
diff --git a/site/content/docs/reference/policy.md b/site/content/docs/reference/policy.md
index ea14a783..fd3ae7ba 100644
--- a/site/content/docs/reference/policy.md
+++ b/site/content/docs/reference/policy.md
@@ -125,8 +125,8 @@ A change to `allowed_publish` takes effect without a new credential. The
grant is also rendered into the policy text as
`granted_publish_*(...) <- role("<role>")` (and `<- label(...)` for
`served_by`), and every verifier adds that text to its authorizer: the
-router, a node and an SDK member at their next sync, all of them at once on
-the `POLICY_UPDATE` event
+router at its next `--control-plane-sync-interval` pass, a node and an SDK
+member at their next sync, all of them at once on the `POLICY_UPDATE` event
the control plane publishes. A node whose policy text changed announces its
services again, so a record the router refused under the previous policy is
in the directory within the sync interval; until then the router answers
@@ -355,8 +355,8 @@ identity's roles from the bindings and mints into the credential one `role()`
fact per role and the compiled `granted_*` facts of every role. The control
plane also renders the policy as Datalog rules served as `datalog_rules` at
`GET /policies`. Nodes, routers and SDK members fetch this text
-(`--control-plane-sync-interval`, 15 minutes) and add it to their authorizer
-as it arrives.
+(`--control-plane-sync-interval`, 15 minutes for a node, 5 for a router) and
+add it to their authorizer as it arrives.
At the destination node, in this order:
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index 390bb4f1..479850df 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -38,8 +38,7 @@ replica count.
| `--listen` | `/ip4/0.0.0.0/tcp/5001`, `/ip6/::/tcp/5001` | libp2p listen addresses. Repeatable. Add `/ip4/0.0.0.0/udp/5001/quic-v1` for QUIC. |
| `--external-addr` | none | Addresses to announce instead of the detected ones, such as `/dnsaddr/bootstrap.example.com` or `/ip4/<public-ip>/tcp/5001`. Repeatable. |
| `--keys-path` | `router.key` | The router's private key. It fixes the peer ID across restarts and belongs on persistent storage. |
-| `--keys-sync-interval` | `5m` | How often `/keys` is polled for signing-key rotations. |
-| `--lease-renew-interval` | `300s` | How often the lease is renewed. Must be well below the control plane's `--lease-duration`. |
+| `--control-plane-sync-interval` | `5m` | How often the router's one pass over the control plane runs: it renews the lease, re-reads `/info`, `/keys` and `/policies`, and refreshes its own credential when it nears expiry. A mesh event runs a pass at once. Must be well below the control plane's `--lease-duration` and `--key-grace-period`. |
| `--allow-loopback` | `false` | Announce and accept loopback and link-local addresses. For a router and nodes on one host. |
| `--conns-per-source-ip` | a quarter of `--high-watermark` (`1000`) | Inbound connections accepted per source address. One address can hold at most this share of the router's connection budget, so filling a router takes at least four addresses. Members behind a NAT or a cluster's SNAT share one address and each holds one connection per router, two while enrolling. |
| `--low-watermark`, `--high-watermark` | 15% below the high mark (`3400`), `4000` | How many members the router holds: peers that passed the handshake, other routers aside. At the high mark the router sends members away (see below) until it is back at the low mark, so the gap between the two is what one shedding moves; the default moves 15%. The same numbers bound connections: above the high mark the connection manager closes connections, oldest first among those that carry no session, then members; connections to the other routers are never closed. A connection without a session lasts at most `--auth-deadline`; the connection manager leaves a connection alone while it is younger than its grace period, and libp2p's connection cap, twice the high mark, is what bounds the total (see the [scalability guide](../../guides/scalability/#what-bounds-the-mesh)). |
@@ -60,8 +59,13 @@ replica count.
2. Starts the libp2p host, the DHT it shares with the other routers, the
directory it serves to its members, the relay service and GossipSub.
3. Registers a lease at `POST /routers/lease` with its announced addresses
- and renews it every `--lease-renew-interval`. The control plane lists the
- router on `/info` while the lease is live.
+ and renews it every `--control-plane-sync-interval`, in the same pass
+ that re-reads `/keys`, `/info` (the routers to federate with and the ban
+ set) and `/policies` (the mesh policy, which decides what a member may
+ publish). A `KEY_ROTATION` or `POLICY_UPDATE` event runs the pass at
+ once; while the list names a router this one has not authenticated with,
+ the federation step alone is retried every 2 seconds. The control plane
+ lists the router on `/info` while the lease is live.
4. Runs the mutual credential handshake on every inbound connection and
refuses peers whose credential does not verify and peers on the ban list.
Until a peer passes it, the router serves it nothing but the handshake:
@@ -87,7 +91,8 @@ replica count.
a key or for a peer; the router answers from its own records and from
the routers closest to the key, asked at once. A member keeps no DHT
and connects to no router it does not hold.
-6. Refreshes its own credential before it expires, like a node.
+6. Refreshes its own credential before it expires, like a node, at the
+ pass that finds less than a fifth of its lifetime left.
7. Every 5 seconds, compares the members it holds, the peers that passed
the handshake, with `--high-watermark`. At the mark it picks members at
random, as many as take it back to `--low-watermark`, and sends each a
diff --git a/tests/integration/failover_test.go b/tests/integration/failover_test.go
index 83fe69ff..10abe592 100644
--- a/tests/integration/failover_test.go
+++ b/tests/integration/failover_test.go
@@ -149,7 +149,7 @@ roles: []
"--keys-path", filepath.Join(tmpDir, "router_keysA.db"),
"--allow-loopback",
"--oidc-token", routerJWT,
- "--lease-renew-interval", "250ms",
+ "--control-plane-sync-interval", "250ms",
)
if err := cmdRouterA.Start(); err != nil {
t.Fatalf("failed to start Router A: %v", err)
@@ -182,7 +182,7 @@ roles: []
"--keys-path", filepath.Join(tmpDir, "router_keysB.db"),
"--allow-loopback",
"--oidc-token", routerJWT,
- "--lease-renew-interval", "250ms",
+ "--control-plane-sync-interval", "250ms",
)
if err := cmdRouterB.Start(); err != nil {
t.Fatalf("failed to start Router B: %v", err)
diff --git a/tests/integration/minimal_helpers_test.go b/tests/integration/minimal_helpers_test.go
index 3f54ddeb..cbdfa765 100644
--- a/tests/integration/minimal_helpers_test.go
+++ b/tests/integration/minimal_helpers_test.go
@@ -471,7 +471,7 @@ func startRouter(t *testing.T, tmpDir string, cpPort int, mintToken func(map[str
"--oidc-token", routerJWT,
// Each renewal carries the router's connected peers, which is how
// tests see a node reach the mesh through the control plane.
- "--lease-renew-interval", "250ms",
+ "--control-plane-sync-interval", "250ms",
)
routerCmd.Stdout = os.Stdout
routerCmd.Stderr = os.Stderr
diff --git a/tests/integration/readmission_test.go b/tests/integration/readmission_test.go
index 53181521..ed84d62a 100644
--- a/tests/integration/readmission_test.go
+++ b/tests/integration/readmission_test.go
@@ -159,14 +159,13 @@ func startReadmissionMesh(t *testing.T, ctx context.Context) *readmissionMesh {
cpPort, _ := strconv.Atoi(portStr)
rtr, err := router.NewRouter(ctx, router.Options{
- ControlPlaneURL: baseURL,
- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
- KeysDBPath: filepath.Join(tmp, "router.key"),
- OIDCToken: mintToken(map[string]interface{}{"sub": "router", "groups": []string{"routers"}, "roles": []string{api.RoleRouter}}),
- AllowLoopback: true,
- KeysSyncInterval: 20 * time.Second,
- LeaseRenewInterval: 500 * time.Millisecond,
- BiscuitTimeout: time.Second,
+ ControlPlaneURL: baseURL,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ KeysDBPath: filepath.Join(tmp, "router.key"),
+ OIDCToken: mintToken(map[string]interface{}{"sub": "router", "groups": []string{"routers"}, "roles": []string{api.RoleRouter}}),
+ AllowLoopback: true,
+ ControlPlaneSyncInterval: 500 * time.Millisecond,
+ BiscuitTimeout: time.Second,
})
if err != nil {
t.Fatalf("failed to create router: %v", err)
diff --git a/tests/integration/revocation_test.go b/tests/integration/revocation_test.go
index b667cea5..0f3e7932 100644
--- a/tests/integration/revocation_test.go
+++ b/tests/integration/revocation_test.go
@@ -97,8 +97,7 @@ roles:
"--keys-path", filepath.Join(tmpDir, "router-keys.db"),
"--allow-loopback",
"--oidc-token", routerJWT,
- "--lease-renew-interval", "2s",
- "--keys-sync-interval", "1s",
+ "--control-plane-sync-interval", "1s",
)
routerCmd.Stdout = os.Stdout
routerCmd.Stderr = os.Stderr
diff --git a/tests/integration/rotation_test.go b/tests/integration/rotation_test.go
index 9fd42c28..59c4344b 100644
--- a/tests/integration/rotation_test.go
+++ b/tests/integration/rotation_test.go
@@ -100,8 +100,7 @@ roles:
"--keys-path", filepath.Join(tmpDir, "router-keys.db"),
"--allow-loopback",
"--oidc-token", routerJWT,
- "--lease-renew-interval", "1s",
- "--keys-sync-interval", "500ms",
+ "--control-plane-sync-interval", "500ms",
)
routerCmd.Stdout = os.Stdout
routerCmd.Stderr = os.Stderr
diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.go
index 6440e066..1eed597b 100644
--- a/tests/integration/router_attachment_test.go
+++ b/tests/integration/router_attachment_test.go
@@ -108,7 +108,7 @@ func startLabelledRouter(t *testing.T, routerBin, tmpDir, name string, cpPort in
"--keys-path", keysPath,
"--allow-loopback",
"--oidc-token", routerJWT,
- "--lease-renew-interval", "250ms",
+ "--control-plane-sync-interval", "250ms",
"--metrics-addr", metricsAddr,
}
for _, l := range labels {
diff --git a/tests/integration/router_shutdown_test.go b/tests/integration/router_shutdown_test.go
index 006a5812..9d39a7bc 100644
--- a/tests/integration/router_shutdown_test.go
+++ b/tests/integration/router_shutdown_test.go
@@ -73,7 +73,7 @@ func TestRouterGracefulShutdownWithdrawsLease(t *testing.T) {
"--oidc-token", routerJWT,
// Long enough that only a withdrawal, never a renewal, changes
// the active set during the test.
- "--lease-renew-interval", "1m",
+ "--control-plane-sync-interval", "1m",
"--shutdown-lease-ttl", shutdownTTL,
)
if err := cmd.Start(); err != nil {