Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router: one control plane sync loop, --control-plane-sync-interval
Antonio Ojeab9719e0b4e2c
diff --git a/cmd/agentmesh-one/main.go b/cmd/agentmesh-one/main.goindex 9247fb3c..0e93a76e 100644--- a/cmd/agentmesh-one/main.go+++ b/cmd/agentmesh-one/main.go@@ -234,8 +234,7 @@ func main() {rootCmd.Flags().BoolVar(&cpTunables.ManualEnrollment, "control-plane-manual-enrollment", false, "Queue bootstrap enrollments for admin approval instead of auto-approving")// Embedded router tunables.- rootCmd.Flags().DurationVar(&routerTunables.KeysSyncInterval, "router-keys-sync-interval", 0, "Biscuit public key refresh interval (0 keeps the component default)")- rootCmd.Flags().DurationVar(&routerTunables.LeaseRenewInterval, "router-lease-renew-interval", 0, "Lease renewal interval (0 keeps the component default)")+ rootCmd.Flags().DurationVar(&routerTunables.ControlPlaneSyncInterval, "router-control-plane-sync-interval", 0, "How often the router renews its lease and pulls keys, routers, bans and policy from the control plane (0 keeps the component default)")rootCmd.Flags().IntVar(&routerTunables.LowWaterMark, "router-low-watermark", 0, "Connection manager low watermark (0 is 15% below the high watermark)")rootCmd.Flags().IntVar(&routerTunables.HighWaterMark, "router-high-watermark", 0, "Connection manager high watermark (0 keeps the component default)")rootCmd.Flags().IntVar(&routerTunables.ConnsPerSourceIP, "router-conns-per-source-ip", 0, "Per-source-IP connection budget (0 follows the high watermark; proxied peers share source IPs)")diff --git a/cmd/agentmesh-router/main.go b/cmd/agentmesh-router/main.goindex ff801fb7..dc0f0962 100644--- a/cmd/agentmesh-router/main.go+++ b/cmd/agentmesh-router/main.go@@ -51,32 +51,31 @@ func init() {}var (- controlPlaneURL string- insecureControlPlane bool- listenAddrs []string- externalAddrs []string- keysSyncInterval time.Duration- leaseRenewInterval time.Duration- oidcToken string- bootstrapToken string- bootstrapTokenPath string- jwtPath string- keysPath string- allowLoopback bool- connsPerSourceIP int- logLevel string- logFormat string- lowWaterMark int- highWaterMark int- metricsAddr string- pprofAddr string- relayLimitDuration time.Duration- relayLimitData router.ByteSize- relayMaxReservations int- relayMaxCircuits int- shutdownLeaseTTL time.Duration- authDeadline time.Duration- labelFlags []string+ controlPlaneURL string+ insecureControlPlane bool+ listenAddrs []string+ externalAddrs []string+ controlPlaneSyncInterval time.Duration+ oidcToken string+ bootstrapToken string+ bootstrapTokenPath string+ jwtPath string+ keysPath string+ allowLoopback bool+ connsPerSourceIP int+ logLevel string+ logFormat string+ lowWaterMark int+ highWaterMark int+ metricsAddr string+ pprofAddr string+ relayLimitDuration time.Duration+ relayLimitData router.ByteSize+ relayMaxReservations int+ relayMaxCircuits int+ shutdownLeaseTTL time.Duration+ authDeadline time.Duration+ labelFlags []string)var logger = golog.Logger("agentmesh-router-cli")@@ -94,8 +93,7 @@ func main() {AllowInsecureControlPlane: insecureControlPlane,ListenAddrs: listenAddrs,ExternalAddrs: externalAddrs,- KeysSyncInterval: keysSyncInterval,- LeaseRenewInterval: leaseRenewInterval,+ ControlPlaneSyncInterval: controlPlaneSyncInterval,OIDCToken: oidcToken,BootstrapToken: bootstrapToken,BootstrapTokenPath: bootstrapTokenPath,@@ -150,8 +148,7 @@ func main() {rootCmd.Flags().BoolVar(&insecureControlPlane, "insecure-control-plane", false, "Accept a plaintext http:// control plane URL to a non-loopback host (whoever answers it becomes this router's trust root; only for networks you already trust)")rootCmd.Flags().StringSliceVar(&listenAddrs, "listen", []string{"/ip4/0.0.0.0/tcp/5001", "/ip6/::/tcp/5001"}, "libp2p Listen Addresses")rootCmd.Flags().StringSliceVar(&externalAddrs, "external-addr", []string{}, "External addresses to announce to control plane")- rootCmd.Flags().DurationVar(&keysSyncInterval, "keys-sync-interval", 5*time.Minute, "Key synchronization polling interval")- rootCmd.Flags().DurationVar(&leaseRenewInterval, "lease-renew-interval", 300*time.Second, "Lease renewal registration interval")+ rootCmd.Flags().DurationVar(&controlPlaneSyncInterval, "control-plane-sync-interval", router.DefaultControlPlaneSyncInterval, "How often the lease is renewed and signing keys, the router list, bans and the mesh policy are pulled from the control plane; a mesh event pulls at once. Keep it well below the control plane's --lease-duration and --key-grace-period.")rootCmd.Flags().StringVar(&oidcToken, "oidc-token", "", "OIDC ID token for enrollment")rootCmd.Flags().StringVar(&bootstrapToken, "bootstrap-token", "", "Pre-shared bootstrap token for enrollment")rootCmd.Flags().StringVar(&bootstrapTokenPath, "bootstrap-token-path", "", "Path to file containing bootstrap token for enrollment")diff --git a/internal/router/config.go b/internal/router/config.goindex 915c72f3..382ee4eb 100644--- a/internal/router/config.go+++ b/internal/router/config.go@@ -35,25 +35,33 @@ const DefaultShutdownLeaseTTL = 30 * time.Second// less.const DefaultAuthDeadline = time.Minute+// DefaultControlPlaneSyncInterval is how often a router talks to the control+// plane when nothing happens: the lease is renewed, and /info, /keys and+// /policies re-read, every five minutes. What has to be quick does not wait+// for it: a mesh event runs a pass at once, a new router reads /info on its+// own first pass and handshakes with every listed router, and a lookup that+// finds nothing re-reads /info (refederateIfStale).+const DefaultControlPlaneSyncInterval = 5 * time.Minute+// Options holds configuration details for the agentmesh-router.type Options struct {- ControlPlaneURL string- ListenAddrs []string- ExternalAddrs []string- KeysSyncInterval time.Duration- PolicySyncInterval time.Duration- LeaseRenewInterval time.Duration- OIDCProvider string // For OIDC enrollment- OIDCToken string // OIDC JWT token for enrollment- BootstrapToken string // Pre-shared bootstrap token for enrollment- BootstrapTokenPath string // File path containing pre-shared bootstrap token- JWTPath string // File path containing OIDC JWT token- KeysDBPath string // Path to save peer private key- AllowLoopback bool- BiscuitTimeout time.Duration- LogVerbose bool- LowWaterMark int- HighWaterMark int+ ControlPlaneURL string+ ListenAddrs []string+ ExternalAddrs []string+ // ControlPlaneSyncInterval is how often the router's one pass over the+ // control plane runs: lease, /info, /keys, /policies, credential expiry.+ ControlPlaneSyncInterval time.Duration+ OIDCProvider string // For OIDC enrollment+ OIDCToken string // OIDC JWT token for enrollment+ BootstrapToken string // Pre-shared bootstrap token for enrollment+ BootstrapTokenPath string // File path containing pre-shared bootstrap token+ JWTPath string // File path containing OIDC JWT token+ KeysDBPath string // Path to save peer private key+ AllowLoopback bool+ BiscuitTimeout time.Duration+ LogVerbose bool+ LowWaterMark int+ HighWaterMark int// RelayLimitDuration / RelayLimitData cap each relayed connection; 0 is// no limit, so Default() leaves them alone.RelayLimitDuration time.Duration@@ -119,14 +127,8 @@ func (o *Options) Default() {if len(o.ListenAddrs) == 0 {o.ListenAddrs = []string{"/ip4/0.0.0.0/tcp/5001", "/ip6/::/tcp/5001"}}- if o.KeysSyncInterval <= 0 {- o.KeysSyncInterval = 5 * time.Minute- }- if o.PolicySyncInterval <= 0 {- o.PolicySyncInterval = 15 * time.Minute- }- if o.LeaseRenewInterval <= 0 {- o.LeaseRenewInterval = 30 * time.Second+ if o.ControlPlaneSyncInterval <= 0 {+ o.ControlPlaneSyncInterval = DefaultControlPlaneSyncInterval}if o.HighWaterMark <= 0 {o.HighWaterMark = DefaultHighWaterMarkdiff --git a/internal/router/router.go b/internal/router/router.goindex 20c39f18..d0821de1 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -379,8 +379,9 @@ type Router struct {// only the last biscuit it issued.refreshMu sync.Mutex- keysSyncTrigger chan struct{}- policySyncTrigger chan struct{}+ // syncTrigger asks the control plane sync loop for a pass now; a mesh+ // event sends on it.+ syncTrigger chan struct{}// Control contextsctx context.Context@@ -415,12 +416,11 @@ func NewRouter(ctx context.Context, config Options) (*Router, error) {}return &Router{- config: config,- ctx: ctx,- cancel: cancel,- handshakeLimiter: handshakeLimiter,- keysSyncTrigger: make(chan struct{}, 1),- policySyncTrigger: make(chan struct{}, 1),+ config: config,+ ctx: ctx,+ cancel: cancel,+ handshakeLimiter: handshakeLimiter,+ syncTrigger: make(chan struct{}, 1),}, nil}@@ -599,12 +599,8 @@ func (r *Router) Start() (retErr error) {})// 5. Start background routines- r.wg.Add(9)- go r.runLeaseRenewalLoop()- go r.runKeysSyncLoop()- go r.runPolicySyncLoop()- go r.runFederationLoop()- go r.runBiscuitRenewalLoop()+ r.wg.Add(5)+ go r.runControlPlaneSyncLoop()go r.runShedLoop()go r.listenForControlPlaneEvents(r.ctx)go func() {@@ -1049,65 +1045,93 @@ func (r *Router) listenForControlPlaneEvents(ctx context.Context) {}}case api.MeshEvent_KEY_ROTATION:- logger.Infof("[Router Event] Received KEY_ROTATION event, triggering key sync")- r.triggerKeysSync()+ logger.Infof("[Router Event] Received KEY_ROTATION event, triggering control plane sync")+ r.triggerSync()case api.MeshEvent_POLICY_UPDATE:- logger.Infof("[Router Event] Received POLICY_UPDATE event, triggering policy sync")- r.triggerPolicySync()+ logger.Infof("[Router Event] Received POLICY_UPDATE event, triggering control plane sync")+ r.triggerSync()}}}-func (r *Router) triggerKeysSync() {+func (r *Router) triggerSync() {select {- case r.keysSyncTrigger <- struct{}{}:+ case r.syncTrigger <- struct{}{}:default:}}-func (r *Router) runKeysSyncLoop() {+// runControlPlaneSyncLoop is the router's one conversation with the control+// plane. Every ControlPlaneSyncInterval, and at once on a mesh event, a pass+// re-reads /keys, renews the lease, re-reads /info (the routers to federate+// with and the ban set), re-reads /policies and refreshes the router's own+// credential when it nears expiry. Between passes, while a listed router is+// not yet authenticated, the federation step alone is retried every+// federationCatchUp, so a router that cannot be reached costs the control+// plane nothing beyond the pass.+func (r *Router) runControlPlaneSyncLoop() {defer r.wg.Done()- ticker := time.NewTicker(r.config.KeysSyncInterval)- defer ticker.Stop()-+ timer := time.NewTimer(0)+ defer timer.Stop()+ var lastPass time.Timefor {+ now := falseselect {- case <-ticker.C:- case <-r.keysSyncTrigger:+ case <-timer.C:+ case <-r.syncTrigger:+ now = true+ if !timer.Stop() {+ select {+ case <-timer.C:+ default:+ }+ }case <-r.ctx.Done():return}- if err := r.syncKeys(); err != nil {- logger.Errorf("Failed keys sync: %v", err)+ if now || time.Since(lastPass) >= r.config.ControlPlaneSyncInterval {+ r.syncControlPlane()+ lastPass = time.Now()+ } else {+ r.connectBootstrapRouters()+ }+ next := r.config.ControlPlaneSyncInterval - time.Since(lastPass)+ if listed, authenticated := r.peerRouterCounts(); authenticated < listed && federationCatchUp < next {+ next = federationCatchUp}+ timer.Reset(max(next, 0))}}-func (r *Router) triggerPolicySync() {- select {- case r.policySyncTrigger <- struct{}{}:- default:+// syncControlPlane is one pass of the sync loop. Keys come first: a lease+// or a policy read presented with a credential the control plane no longer+// signs for is refused until the rotation is learned and the credential+// refreshed (syncKeys).+func (r *Router) syncControlPlane() {+ if err := r.syncKeys(); err != nil {+ logger.Errorf("Failed keys sync: %v", err)+ }+ r.renewLease()+ r.connectBootstrapRouters()+ if err := r.syncPolicy(); err != nil {+ logger.Errorf("Failed policy sync: %v", err)}+ r.refreshExpiringCredential()}-// runPolicySyncLoop pulls the mesh policy at start, on each POLICY_UPDATE-// event and every PolicySyncInterval: the event is best effort, the pull is-// what bounds how long a policy change takes to reach this router.-func (r *Router) runPolicySyncLoop() {- defer r.wg.Done()- ticker := time.NewTicker(r.config.PolicySyncInterval)- defer ticker.Stop()-- for {- if err := r.syncPolicy(); err != nil {- logger.Errorf("Failed policy sync: %v", err)- }- select {- case <-ticker.C:- case <-r.policySyncTrigger:- case <-r.ctx.Done():- return- }+// refreshExpiringCredential refreshes the router's own credential once less+// than a fifth of its lifetime is left, so a credential is never presented+// close to its expiry.+func (r *Router) refreshExpiringCredential() {+ r.keysMu.RLock()+ expiration := r.credential.expiration+ r.keysMu.RUnlock()+ if expiration.IsZero() || time.Until(expiration) >= api.BiscuitTokenTTL/5 {+ return+ }+ logger.Infof("Router biscuit expiring in %v, triggering proactive refresh...", time.Until(expiration))+ if err := r.RefreshEnrollment(r.ctx); err != nil {+ logger.Errorf("Proactive router biscuit refresh failed: %v", err)}}@@ -1157,24 +1181,6 @@ func (r *Router) meshPolicyRules() []biscuit.Rule {return nil}-func (r *Router) runLeaseRenewalLoop() {- defer r.wg.Done()- ticker := time.NewTicker(r.config.LeaseRenewInterval)- defer ticker.Stop()-- // Initial renewal after host is online- r.renewLease()-- for {- select {- case <-ticker.C:- r.renewLease()- case <-r.ctx.Done():- return- }- }-}-// renewLease extends the lease for the control plane's duration. Once the// router is shutting down it does nothing: the withdrawal sent by Close is// the last word, and a renewal racing it would re-advertise a router that@@ -1414,35 +1420,12 @@ func (r *Router) reconcileBannedPeers(peerIDs []string, fetchedAt time.Time) {}}-// federationInterval is how often a router reads the control plane's list-// of routers and authenticates with the ones it has not; federationCatchUp-// is the interval while the list names a router it has not reached yet,-// which it does at start, when the other routers started at the same-// moment and leased after this one's first pass, and whenever a router-// joins.-const (- federationInterval = 30 * time.Second- federationCatchUp = 2 * time.Second-)--func (r *Router) runFederationLoop() {- defer r.wg.Done()- timer := time.NewTimer(0)- defer timer.Stop()- for {- select {- case <-timer.C:- r.connectBootstrapRouters()- next := federationInterval- if listed, authenticated := r.peerRouterCounts(); authenticated < listed {- next = federationCatchUp- }- timer.Reset(next)- case <-r.ctx.Done():- return- }- }-}+// federationCatchUp is how often the sync loop retries the federation step+// alone while the control plane's list names a router this one has not+// authenticated with yet, which it does at start, when the other routers+// started at the same moment and leased after this one's first pass, and+// whenever a router joins.+const federationCatchUp = 2 * time.Secondfunc (r *Router) connectBootstrapRouters() {// Taken before the request: anything banned after this point cannot be@@ -2017,27 +2000,3 @@ func (r *Router) verifyOwnBiscuit(token []byte, peerID peer.ID) error {}return identity.VerifyBiscuitRole(token, key, r.config.RequiredRole, r.config.BiscuitTimeout)}--func (r *Router) runBiscuitRenewalLoop() {- defer r.wg.Done()- ticker := time.NewTicker(api.TokenRefreshCheckInterval)- defer ticker.Stop()-- for {- select {- case <-ticker.C:- r.keysMu.RLock()- expiration := r.credential.expiration- r.keysMu.RUnlock()-- if !expiration.IsZero() && time.Until(expiration) < api.BiscuitTokenTTL/5 { // 80% elapsed lifespan of 24h (remaining time < 20% of TTL)- logger.Infof("Router biscuit expiring in %v, triggering proactive refresh...", time.Until(expiration))- if err := r.RefreshEnrollment(r.ctx); err != nil {- logger.Errorf("Proactive router biscuit refresh failed: %v", err)- }- }- case <-r.ctx.Done():- return- }- }-}diff --git a/internal/router/router_test.go b/internal/router/router_test.goindex 279b42c5..874a156a 100644--- a/internal/router/router_test.go+++ b/internal/router/router_test.go@@ -184,14 +184,13 @@ func TestRouterIntegration(t *testing.T) {// 1. Create and Start RouterrOpts := Options{- ControlPlaneURL: cpURL,- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},- KeysSyncInterval: 2 * time.Second,- LeaseRenewInterval: 2 * time.Second,- OIDCToken: routerJWT,- KeysDBPath: routerKeyPath,- AllowLoopback: true,- BiscuitTimeout: 1 * time.Second,+ ControlPlaneURL: cpURL,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ ControlPlaneSyncInterval: 2 * time.Second,+ OIDCToken: routerJWT,+ KeysDBPath: routerKeyPath,+ AllowLoopback: true,+ BiscuitTimeout: 1 * time.Second,}r, err := NewRouter(context.Background(), rOpts)@@ -354,14 +353,13 @@ func TestRouterFederation(t *testing.T) {r1KeyPath := filepath.Join(tempDir, "router1.key")r1, err := NewRouter(context.Background(), Options{- ControlPlaneURL: cpURL,- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},- KeysSyncInterval: 2 * time.Second,- LeaseRenewInterval: 2 * time.Second,- OIDCToken: router1JWT,- KeysDBPath: r1KeyPath,- AllowLoopback: true,- BiscuitTimeout: 1 * time.Second,+ ControlPlaneURL: cpURL,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ ControlPlaneSyncInterval: 2 * time.Second,+ OIDCToken: router1JWT,+ KeysDBPath: r1KeyPath,+ AllowLoopback: true,+ BiscuitTimeout: 1 * time.Second,})if err != nil {t.Fatal(err)@@ -373,14 +371,13 @@ func TestRouterFederation(t *testing.T) {r2KeyPath := filepath.Join(tempDir, "router2.key")r2, err := NewRouter(context.Background(), Options{- ControlPlaneURL: cpURL,- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},- KeysSyncInterval: 2 * time.Second,- LeaseRenewInterval: 2 * time.Second,- OIDCToken: router2JWT,- KeysDBPath: r2KeyPath,- AllowLoopback: true,- BiscuitTimeout: 1 * time.Second,+ ControlPlaneURL: cpURL,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ ControlPlaneSyncInterval: 2 * time.Second,+ OIDCToken: router2JWT,+ KeysDBPath: r2KeyPath,+ AllowLoopback: true,+ BiscuitTimeout: 1 * time.Second,})if err != nil {t.Fatal(err)@@ -443,14 +440,13 @@ func TestRouterProactiveTokenRefresh(t *testing.T) {})rOpts := Options{- ControlPlaneURL: cpURL,- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},- KeysSyncInterval: 20 * time.Second,- LeaseRenewInterval: 20 * time.Second,- OIDCToken: routerJWT,- KeysDBPath: routerKeyPath,- AllowLoopback: true,- BiscuitTimeout: 1 * time.Second,+ ControlPlaneURL: cpURL,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ ControlPlaneSyncInterval: 20 * time.Second,+ OIDCToken: routerJWT,+ KeysDBPath: routerKeyPath,+ AllowLoopback: true,+ BiscuitTimeout: 1 * time.Second,}r, err := NewRouter(context.Background(), rOpts)@@ -504,14 +500,13 @@ func TestRouterLeaseRenewalReEnrollOn401(t *testing.T) {})rOpts := Options{- ControlPlaneURL: cpURL,- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},- KeysSyncInterval: 20 * time.Second,- LeaseRenewInterval: 20 * time.Second,- OIDCToken: routerJWT,- KeysDBPath: routerKeyPath,- AllowLoopback: true,- BiscuitTimeout: 1 * time.Second,+ ControlPlaneURL: cpURL,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ ControlPlaneSyncInterval: 20 * time.Second,+ OIDCToken: routerJWT,+ KeysDBPath: routerKeyPath,+ AllowLoopback: true,+ BiscuitTimeout: 1 * time.Second,}r, err := NewRouter(context.Background(), rOpts)@@ -705,14 +700,13 @@ func TestRouterProactiveRefreshReEnrollOn401(t *testing.T) {})rOpts := Options{- ControlPlaneURL: cpURL,- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},- KeysSyncInterval: 20 * time.Second,- LeaseRenewInterval: 20 * time.Second,- OIDCToken: routerJWT,- KeysDBPath: routerKeyPath,- AllowLoopback: true,- BiscuitTimeout: 1 * time.Second,+ ControlPlaneURL: cpURL,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ ControlPlaneSyncInterval: 20 * time.Second,+ OIDCToken: routerJWT,+ KeysDBPath: routerKeyPath,+ AllowLoopback: true,+ BiscuitTimeout: 1 * time.Second,}r, err := NewRouter(context.Background(), rOpts)@@ -764,14 +758,13 @@ func TestRouterGossipSubBannedEvent(t *testing.T) {})rOpts := Options{- ControlPlaneURL: cpURL,- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},- KeysSyncInterval: 20 * time.Second,- LeaseRenewInterval: 20 * time.Second,- OIDCToken: routerJWT,- KeysDBPath: routerKeyPath,- AllowLoopback: true,- BiscuitTimeout: 1 * time.Second,+ ControlPlaneURL: cpURL,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ ControlPlaneSyncInterval: 20 * time.Second,+ OIDCToken: routerJWT,+ KeysDBPath: routerKeyPath,+ AllowLoopback: true,+ BiscuitTimeout: 1 * time.Second,}r, err := NewRouter(context.Background(), rOpts)@@ -783,6 +776,17 @@ func TestRouterGossipSubBannedEvent(t *testing.T) {}defer func() { _ = r.Close() }()+ // The control plane never learns of this ban, so the event must land+ // after the first sync pass read /info: a pass that fetched after the+ // event would read the ban's absence as an unban (reconcileBannedPeers),+ // which is right for a real ban the control plane recorded first.+ for deadline := time.Now().Add(5 * time.Second); r.lastFederation.Load() == 0; {+ if time.Now().After(deadline) {+ t.Fatal("the router's first sync pass did not read /info")+ }+ time.Sleep(20 * time.Millisecond)+ }+privNode, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)if err != nil {t.Fatalf("failed to generate node keypair: %v", err)@@ -1326,14 +1330,13 @@ func TestRouterRefreshEnrollmentReattestsJWT(t *testing.T) {}r, err := NewRouter(context.Background(), Options{- ControlPlaneURL: cpURL,- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},- KeysSyncInterval: time.Hour,- LeaseRenewInterval: time.Hour,- JWTPath: jwtPath,- KeysDBPath: filepath.Join(tempDir, "router.key"),- AllowLoopback: true,- BiscuitTimeout: time.Second,+ ControlPlaneURL: cpURL,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ ControlPlaneSyncInterval: time.Hour,+ JWTPath: jwtPath,+ KeysDBPath: filepath.Join(tempDir, "router.key"),+ AllowLoopback: true,+ BiscuitTimeout: time.Second,})if err != nil {t.Fatalf("NewRouter: %v", err)diff --git a/internal/router/trust_test.go b/internal/router/trust_test.goindex 63a47915..dbb51eba 100644--- a/internal/router/trust_test.go+++ b/internal/router/trust_test.go@@ -163,7 +163,13 @@ func TestSyncKeysRequiresTrustedSignature(t *testing.T) {})}-func TestKeysSyncLoopCoalescesTriggers(t *testing.T) {+// TestControlPlaneSyncLoopCoalescesTriggers pins two properties of the one+// control plane loop: a burst of mesh events while a pass is in flight costs+// one follow-up pass, and readers of the trust set never wait on the control+// plane. The fake control plane holds /keys until released and answers the+// rest of the pass with empty documents; the router is not enrolled, so the+// lease and policy steps log and move on.+func TestControlPlaneSyncLoopCoalescesTriggers(t *testing.T) {publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader)if err != nil {t.Fatal(err)@@ -179,6 +185,10 @@ func TestKeysSyncLoopCoalescesTriggers(t *testing.T) {requests := make(chan struct{}, 4)release := make(chan struct{})server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {+ if request.URL.Path != "/keys" {+ w.Header().Set("Content-Type", "application/x-protobuf")+ return+ }requests <- struct{}{}select {case <-release:@@ -187,13 +197,13 @@ func TestKeysSyncLoopCoalescesTriggers(t *testing.T) {}}))t.Cleanup(server.Close)- router, err := NewRouter(context.Background(), Options{ControlPlaneURL: server.URL, KeysSyncInterval: time.Hour})+ router, err := NewRouter(context.Background(), Options{ControlPlaneURL: server.URL, ControlPlaneSyncInterval: time.Hour})if err != nil {t.Fatal(err)}router.trustedPublicKeys = []ed25519.PublicKey{publicKey}router.wg.Add(1)- go router.runKeysSyncLoop()+ go router.runControlPlaneSyncLoop()t.Cleanup(func() {router.cancel()router.wg.Wait()@@ -206,25 +216,26 @@ func TestKeysSyncLoopCoalescesTriggers(t *testing.T) {t.Fatal(message)}}- router.triggerKeysSync()- await(requests, "key sync did not start")+ // The loop's first pass runs at start.+ await(requests, "the first pass did not start")notified := make(chan struct{})go func() {for range 10 {- router.triggerKeysSync()+ router.triggerSync()}router.getTrustedPublicKeys()close(notified)}()await(notified, "notifications or key readers blocked on the control-plane request")- if pending := len(router.keysSyncTrigger); pending != 1 {+ if pending := len(router.syncTrigger); pending != 1 {t.Fatalf("pending triggers = %d, want one coalesced follow-up", pending)}release <- struct{}{}- await(requests, "pending key sync was lost")- if pending := len(router.keysSyncTrigger); pending != 0 {- t.Fatalf("pending triggers = %d after follow-up", pending)+ await(requests, "the follow-up pass was lost")+ if pending := len(router.syncTrigger); pending != 0 {+ t.Fatalf("pending triggers = %d after the follow-up", pending)}+ release <- struct{}{}}func TestSyncKeysRefreshesOnRotation(t *testing.T) {diff --git a/internal/standalone/standalone.go b/internal/standalone/standalone.goindex 82277fca..e5e5d105 100644--- a/internal/standalone/standalone.go+++ b/internal/standalone/standalone.go@@ -144,10 +144,9 @@ type ControlPlaneTunables struct {// RouterTunables are the embedded router's operator knobs.type RouterTunables struct {- // KeysSyncInterval is how often biscuit public keys are refreshed.- KeysSyncInterval time.Duration- // LeaseRenewInterval is how often the router renews its lease.- LeaseRenewInterval time.Duration+ // ControlPlaneSyncInterval is how often the router renews its lease and+ // pulls keys, routers, bans and policy from the control plane.+ ControlPlaneSyncInterval time.Duration// LowWaterMark / HighWaterMark bound the connection manager.LowWaterMark intHighWaterMark int@@ -367,20 +366,19 @@ func (s *Server) Start(ctx context.Context) (retErr error) {}rtr, err := router.NewRouter(ctx, router.Options{- ControlPlaneURL: "http://" + cp.Addr(),- ListenAddrs: append([]string{wsAddr}, s.opts.P2PListen...),- ExternalAddrs: externalAddrs,- AllowLoopback: !s.opts.Router.DisallowLoopback,- KeysDBPath: routerKeyPath,- BootstrapToken: routerToken,- KeysSyncInterval: s.opts.Router.KeysSyncInterval,- LeaseRenewInterval: s.opts.Router.LeaseRenewInterval,- LowWaterMark: s.opts.Router.LowWaterMark,- HighWaterMark: s.opts.Router.HighWaterMark,- RelayLimitDuration: s.opts.Router.RelayLimitDuration,- RelayLimitData: int64(s.opts.Router.RelayLimitData),- ShutdownLeaseTTL: s.opts.Router.ShutdownLeaseTTL,- AuthDeadline: s.opts.Router.AuthDeadline,+ ControlPlaneURL: "http://" + cp.Addr(),+ ListenAddrs: append([]string{wsAddr}, s.opts.P2PListen...),+ ExternalAddrs: externalAddrs,+ AllowLoopback: !s.opts.Router.DisallowLoopback,+ KeysDBPath: routerKeyPath,+ BootstrapToken: routerToken,+ ControlPlaneSyncInterval: s.opts.Router.ControlPlaneSyncInterval,+ LowWaterMark: s.opts.Router.LowWaterMark,+ HighWaterMark: s.opts.Router.HighWaterMark,+ RelayLimitDuration: s.opts.Router.RelayLimitDuration,+ RelayLimitData: int64(s.opts.Router.RelayLimitData),+ ShutdownLeaseTTL: s.opts.Router.ShutdownLeaseTTL,+ AuthDeadline: s.opts.Router.AuthDeadline,// Single-port deployments typically sit behind a TLS-terminating// proxy (Cloud Run, L7 LBs) or NAT where every peer shares a few// source IPs; libp2p's default 8-conns-per-IP cap would throttlediff --git a/site/content/docs/concepts/identity.md b/site/content/docs/concepts/identity.mdindex 328f31ab..4fb15ace 100644--- a/site/content/docs/concepts/identity.md+++ b/site/content/docs/concepts/identity.md@@ -201,10 +201,10 @@ any user action.The control plane rotates its Biscuit Ed25519 signing key every`--key-rotation-interval` (24 hours by default). The previous key stays validfor `--key-grace-period` (1 hour by default) and is then retired. `GET /keys`-returns the current set of keys, signed by each key in the set. Routers poll-it every `--keys-sync-interval`; nodes fetch it at enrollment and then every-`--control-plane-sync-interval`, together with the ban set, revocation list,-and the mesh policy. A rotation event only brings the next pull forward. Both+returns the current set of keys, signed by each key in the set. Routers and+nodes fetch it at enrollment and then every `--control-plane-sync-interval`,+together with the ban set, revocation list, and the mesh policy. A rotation+event only brings the next pull forward. Bothaccept a new set only if one of its signatures verifies under a key theyalready trust. The first key comes from enrollment, and each later key isvouched for by the key it replaces.diff --git a/site/content/docs/reference/agentmesh-one.md b/site/content/docs/reference/agentmesh-one.mdindex c47cc539..0e3f572f 100644--- a/site/content/docs/reference/agentmesh-one.md+++ b/site/content/docs/reference/agentmesh-one.md@@ -65,8 +65,7 @@ of zero leaves the default of that component unchanged.| `--control-plane-biscuit-ttl` | `--biscuit-ttl` || `--control-plane-workload-session-ttl` | `--workload-session-ttl` || `--control-plane-manual-enrollment` | The opposite of `--auto-approve-enrollment`: queue bootstrap enrollments for approval. |-| `--router-keys-sync-interval` | `agentmesh-router --keys-sync-interval` |-| `--router-lease-renew-interval` | `--lease-renew-interval` |+| `--router-control-plane-sync-interval` | `agentmesh-router --control-plane-sync-interval` || `--router-low-watermark`, `--router-high-watermark` | `--low-watermark`, `--high-watermark` || `--router-conns-per-source-ip` | `--conns-per-source-ip`. `0` follows the high watermark, because peers behind a proxy share source IPs. || `--router-relay-limit-duration`, `--router-relay-limit-data` | `--relay-limit-duration`, `--relay-limit-data`. `0` keeps `1h` and no data limit, so set a long duration such as `24h` instead of an unlimited one. |diff --git a/site/content/docs/reference/policy.md b/site/content/docs/reference/policy.mdindex ea14a783..fd3ae7ba 100644--- a/site/content/docs/reference/policy.md+++ b/site/content/docs/reference/policy.md@@ -125,8 +125,8 @@ A change to `allowed_publish` takes effect without a new credential. Thegrant is also rendered into the policy text as`granted_publish_*(...) <- role("<role>")` (and `<- label(...)` for`served_by`), and every verifier adds that text to its authorizer: the-router, a node and an SDK member at their next sync, all of them at once on-the `POLICY_UPDATE` event+router at its next `--control-plane-sync-interval` pass, a node and an SDK+member at their next sync, all of them at once on the `POLICY_UPDATE` eventthe control plane publishes. A node whose policy text changed announces itsservices again, so a record the router refused under the previous policy isin the directory within the sync interval; until then the router answers@@ -355,8 +355,8 @@ identity's roles from the bindings and mints into the credential one `role()`fact per role and the compiled `granted_*` facts of every role. The controlplane also renders the policy as Datalog rules served as `datalog_rules` at`GET /policies`. Nodes, routers and SDK members fetch this text-(`--control-plane-sync-interval`, 15 minutes) and add it to their authorizer-as it arrives.+(`--control-plane-sync-interval`, 15 minutes for a node, 5 for a router) and+add it to their authorizer as it arrives.At the destination node, in this order:diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.mdindex 390bb4f1..479850df 100644--- a/site/content/docs/reference/router.md+++ b/site/content/docs/reference/router.md@@ -38,8 +38,7 @@ replica count.| `--listen` | `/ip4/0.0.0.0/tcp/5001`, `/ip6/::/tcp/5001` | libp2p listen addresses. Repeatable. Add `/ip4/0.0.0.0/udp/5001/quic-v1` for QUIC. || `--external-addr` | none | Addresses to announce instead of the detected ones, such as `/dnsaddr/bootstrap.example.com` or `/ip4/<public-ip>/tcp/5001`. Repeatable. || `--keys-path` | `router.key` | The router's private key. It fixes the peer ID across restarts and belongs on persistent storage. |-| `--keys-sync-interval` | `5m` | How often `/keys` is polled for signing-key rotations. |-| `--lease-renew-interval` | `300s` | How often the lease is renewed. Must be well below the control plane's `--lease-duration`. |+| `--control-plane-sync-interval` | `5m` | How often the router's one pass over the control plane runs: it renews the lease, re-reads `/info`, `/keys` and `/policies`, and refreshes its own credential when it nears expiry. A mesh event runs a pass at once. Must be well below the control plane's `--lease-duration` and `--key-grace-period`. || `--allow-loopback` | `false` | Announce and accept loopback and link-local addresses. For a router and nodes on one host. || `--conns-per-source-ip` | a quarter of `--high-watermark` (`1000`) | Inbound connections accepted per source address. One address can hold at most this share of the router's connection budget, so filling a router takes at least four addresses. Members behind a NAT or a cluster's SNAT share one address and each holds one connection per router, two while enrolling. || `--low-watermark`, `--high-watermark` | 15% below the high mark (`3400`), `4000` | How many members the router holds: peers that passed the handshake, other routers aside. At the high mark the router sends members away (see below) until it is back at the low mark, so the gap between the two is what one shedding moves; the default moves 15%. The same numbers bound connections: above the high mark the connection manager closes connections, oldest first among those that carry no session, then members; connections to the other routers are never closed. A connection without a session lasts at most `--auth-deadline`; the connection manager leaves a connection alone while it is younger than its grace period, and libp2p's connection cap, twice the high mark, is what bounds the total (see the [scalability guide](../../guides/scalability/#what-bounds-the-mesh)). |@@ -60,8 +59,13 @@ replica count.2. Starts the libp2p host, the DHT it shares with the other routers, thedirectory it serves to its members, the relay service and GossipSub.3. Registers a lease at `POST /routers/lease` with its announced addresses- and renews it every `--lease-renew-interval`. The control plane lists the- router on `/info` while the lease is live.+ and renews it every `--control-plane-sync-interval`, in the same pass+ that re-reads `/keys`, `/info` (the routers to federate with and the ban+ set) and `/policies` (the mesh policy, which decides what a member may+ publish). A `KEY_ROTATION` or `POLICY_UPDATE` event runs the pass at+ once; while the list names a router this one has not authenticated with,+ the federation step alone is retried every 2 seconds. The control plane+ lists the router on `/info` while the lease is live.4. Runs the mutual credential handshake on every inbound connection andrefuses peers whose credential does not verify and peers on the ban list.Until a peer passes it, the router serves it nothing but the handshake:@@ -87,7 +91,8 @@ replica count.a key or for a peer; the router answers from its own records and fromthe routers closest to the key, asked at once. A member keeps no DHTand connects to no router it does not hold.-6. Refreshes its own credential before it expires, like a node.+6. Refreshes its own credential before it expires, like a node, at the+ pass that finds less than a fifth of its lifetime left.7. Every 5 seconds, compares the members it holds, the peers that passedthe handshake, with `--high-watermark`. At the mark it picks members atrandom, as many as take it back to `--low-watermark`, and sends each adiff --git a/tests/integration/failover_test.go b/tests/integration/failover_test.goindex 83fe69ff..10abe592 100644--- a/tests/integration/failover_test.go+++ b/tests/integration/failover_test.go@@ -149,7 +149,7 @@ roles: []"--keys-path", filepath.Join(tmpDir, "router_keysA.db"),"--allow-loopback","--oidc-token", routerJWT,- "--lease-renew-interval", "250ms",+ "--control-plane-sync-interval", "250ms",)if err := cmdRouterA.Start(); err != nil {t.Fatalf("failed to start Router A: %v", err)@@ -182,7 +182,7 @@ roles: []"--keys-path", filepath.Join(tmpDir, "router_keysB.db"),"--allow-loopback","--oidc-token", routerJWT,- "--lease-renew-interval", "250ms",+ "--control-plane-sync-interval", "250ms",)if err := cmdRouterB.Start(); err != nil {t.Fatalf("failed to start Router B: %v", err)diff --git a/tests/integration/minimal_helpers_test.go b/tests/integration/minimal_helpers_test.goindex 3f54ddeb..cbdfa765 100644--- a/tests/integration/minimal_helpers_test.go+++ b/tests/integration/minimal_helpers_test.go@@ -471,7 +471,7 @@ func startRouter(t *testing.T, tmpDir string, cpPort int, mintToken func(map[str"--oidc-token", routerJWT,// Each renewal carries the router's connected peers, which is how// tests see a node reach the mesh through the control plane.- "--lease-renew-interval", "250ms",+ "--control-plane-sync-interval", "250ms",)routerCmd.Stdout = os.StdoutrouterCmd.Stderr = os.Stderrdiff --git a/tests/integration/readmission_test.go b/tests/integration/readmission_test.goindex 53181521..ed84d62a 100644--- a/tests/integration/readmission_test.go+++ b/tests/integration/readmission_test.go@@ -159,14 +159,13 @@ func startReadmissionMesh(t *testing.T, ctx context.Context) *readmissionMesh {cpPort, _ := strconv.Atoi(portStr)rtr, err := router.NewRouter(ctx, router.Options{- ControlPlaneURL: baseURL,- ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},- KeysDBPath: filepath.Join(tmp, "router.key"),- OIDCToken: mintToken(map[string]interface{}{"sub": "router", "groups": []string{"routers"}, "roles": []string{api.RoleRouter}}),- AllowLoopback: true,- KeysSyncInterval: 20 * time.Second,- LeaseRenewInterval: 500 * time.Millisecond,- BiscuitTimeout: time.Second,+ ControlPlaneURL: baseURL,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ KeysDBPath: filepath.Join(tmp, "router.key"),+ OIDCToken: mintToken(map[string]interface{}{"sub": "router", "groups": []string{"routers"}, "roles": []string{api.RoleRouter}}),+ AllowLoopback: true,+ ControlPlaneSyncInterval: 500 * time.Millisecond,+ BiscuitTimeout: time.Second,})if err != nil {t.Fatalf("failed to create router: %v", err)diff --git a/tests/integration/revocation_test.go b/tests/integration/revocation_test.goindex b667cea5..0f3e7932 100644--- a/tests/integration/revocation_test.go+++ b/tests/integration/revocation_test.go@@ -97,8 +97,7 @@ roles:"--keys-path", filepath.Join(tmpDir, "router-keys.db"),"--allow-loopback","--oidc-token", routerJWT,- "--lease-renew-interval", "2s",- "--keys-sync-interval", "1s",+ "--control-plane-sync-interval", "1s",)routerCmd.Stdout = os.StdoutrouterCmd.Stderr = os.Stderrdiff --git a/tests/integration/rotation_test.go b/tests/integration/rotation_test.goindex 9fd42c28..59c4344b 100644--- a/tests/integration/rotation_test.go+++ b/tests/integration/rotation_test.go@@ -100,8 +100,7 @@ roles:"--keys-path", filepath.Join(tmpDir, "router-keys.db"),"--allow-loopback","--oidc-token", routerJWT,- "--lease-renew-interval", "1s",- "--keys-sync-interval", "500ms",+ "--control-plane-sync-interval", "500ms",)routerCmd.Stdout = os.StdoutrouterCmd.Stderr = os.Stderrdiff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.goindex 6440e066..1eed597b 100644--- a/tests/integration/router_attachment_test.go+++ b/tests/integration/router_attachment_test.go@@ -108,7 +108,7 @@ func startLabelledRouter(t *testing.T, routerBin, tmpDir, name string, cpPort in"--keys-path", keysPath,"--allow-loopback","--oidc-token", routerJWT,- "--lease-renew-interval", "250ms",+ "--control-plane-sync-interval", "250ms","--metrics-addr", metricsAddr,}for _, l := range labels {diff --git a/tests/integration/router_shutdown_test.go b/tests/integration/router_shutdown_test.goindex 006a5812..9d39a7bc 100644--- a/tests/integration/router_shutdown_test.go+++ b/tests/integration/router_shutdown_test.go@@ -73,7 +73,7 @@ func TestRouterGracefulShutdownWithdrawsLease(t *testing.T) {"--oidc-token", routerJWT,// Long enough that only a withdrawal, never a renewal, changes// the active set during the test.- "--lease-renew-interval", "1m",+ "--control-plane-sync-interval", "1m","--shutdown-lease-ttl", shutdownTTL,)if err := cmd.Start(); err != nil {