Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: pull keys, bans and policy from the control plane in one loop
Antonio Ojea1b4cb4e35d17
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.goindex 9bed6b41..1ff5d893 100644--- a/cmd/sam-node/main.go+++ b/cmd/sam-node/main.go@@ -53,56 +53,56 @@ func init() {}var (- controlPlaneAddr string- joinFlag bool- jwtFlag string- jwtPathFlag string- bootstrapTokenFlag string- clientIDFlag string- clientSecretFlag string- controlPlanePublicKeyFlag string- bindAddrFlag string- metricsAddrFlag string- socketPathFlag string- meshFlag string- discoveryIntervalFlag string- listenAddrs []string- enableRelayFlag bool- configFile string- oidcIssuerFlag string- deviceAuthURLFlag string- audienceFlag string- dataDirFlag string- headlessFlag bool- authModeFlag string- insecureControlPlaneFlag bool- daemonizeFlag bool- resetAllFlag bool- assumeYesFlag bool- offlineAccessFlag bool- logLevelFlag string- keyGracePeriodFlag time.Duration- allowLoopbackFlag bool- announcePrivateFlag bool- monitorBootstrapFlag time.Duration- monitorCheckIntervalFlag time.Duration- autoRelayMinIntervalFlag time.Duration- autoRelayBootDelayFlag time.Duration- autoRelayBackoffFlag time.Duration- routerConnectTimeoutFlag time.Duration- apiTokenFlag string- apiTokenPathFlag string- bootstrapTokenPathFlag string- clientSecretPathFlag string- tlsCertFlag string- tlsKeyFlag string- tlsCAFlag string- dhtProviderAddrTTLFlag time.Duration- dhtMaxRecordAgeFlag time.Duration- dhtLookupLimitFlag int- discoveryConcurrencyFlag int- backendProbeTimeoutFlag time.Duration- policySyncIntervalFlag time.Duration+ controlPlaneAddr string+ joinFlag bool+ jwtFlag string+ jwtPathFlag string+ bootstrapTokenFlag string+ clientIDFlag string+ clientSecretFlag string+ controlPlanePublicKeyFlag string+ bindAddrFlag string+ metricsAddrFlag string+ socketPathFlag string+ meshFlag string+ discoveryIntervalFlag string+ listenAddrs []string+ enableRelayFlag bool+ configFile string+ oidcIssuerFlag string+ deviceAuthURLFlag string+ audienceFlag string+ dataDirFlag string+ headlessFlag bool+ authModeFlag string+ insecureControlPlaneFlag bool+ daemonizeFlag bool+ resetAllFlag bool+ assumeYesFlag bool+ offlineAccessFlag bool+ logLevelFlag string+ keyGracePeriodFlag time.Duration+ allowLoopbackFlag bool+ announcePrivateFlag bool+ monitorBootstrapFlag time.Duration+ monitorCheckIntervalFlag time.Duration+ autoRelayMinIntervalFlag time.Duration+ autoRelayBootDelayFlag time.Duration+ autoRelayBackoffFlag time.Duration+ routerConnectTimeoutFlag time.Duration+ apiTokenFlag string+ apiTokenPathFlag string+ bootstrapTokenPathFlag string+ clientSecretPathFlag string+ tlsCertFlag string+ tlsKeyFlag string+ tlsCAFlag string+ dhtProviderAddrTTLFlag time.Duration+ dhtMaxRecordAgeFlag time.Duration+ dhtLookupLimitFlag int+ discoveryConcurrencyFlag int+ backendProbeTimeoutFlag time.Duration+ controlPlaneSyncIntervalFlag time.Duration)var logger = golog.Logger("sam-node-cli")@@ -453,32 +453,32 @@ func main() {}priv := node.GetOrGenerateKey(store)meshNode, err = node.NewSamNode(node.Options{- PrivKey: priv,- ControlPlanePubKey: controlPlanePubKey,- RouterAddrs: routerAddrs,- Store: store,- BannedPeerIDs: bannedPeerIDs,- MeshID: meshFlag,- DiscoveryInterval: discoveryIntervalFlag,- ListenAddrs: listenAddrs,- EnableRelay: enableRelayFlag,- NodeConfig: nodeConfig,- KeyGracePeriod: keyGracePeriodFlag,- AllowLoopback: allowLoopbackFlag,- AnnouncePrivateAddrs: &announcePrivateFlag,- MonitorBootstrap: monitorBootstrapFlag,- MonitorInterval: monitorCheckIntervalFlag,- AutoRelayMinInterval: autoRelayMinIntervalFlag,- AutoRelayBootDelay: autoRelayBootDelayFlag,- AutoRelayBackoff: autoRelayBackoffFlag,- RouterConnectTimeout: routerConnectTimeoutFlag,- RequiredRole: api.RoleNode,- PolicySyncInterval: policySyncIntervalFlag,- DHTProviderAddrTTL: dhtProviderAddrTTLFlag,- DHTMaxRecordAge: dhtMaxRecordAgeFlag,- DHTLookupLimit: dhtLookupLimitFlag,- DiscoveryConcurrency: discoveryConcurrencyFlag,- BackendProbeTimeout: backendProbeTimeoutFlag,+ PrivKey: priv,+ ControlPlanePubKey: controlPlanePubKey,+ RouterAddrs: routerAddrs,+ Store: store,+ BannedPeerIDs: bannedPeerIDs,+ MeshID: meshFlag,+ DiscoveryInterval: discoveryIntervalFlag,+ ListenAddrs: listenAddrs,+ EnableRelay: enableRelayFlag,+ NodeConfig: nodeConfig,+ KeyGracePeriod: keyGracePeriodFlag,+ AllowLoopback: allowLoopbackFlag,+ AnnouncePrivateAddrs: &announcePrivateFlag,+ MonitorBootstrap: monitorBootstrapFlag,+ MonitorInterval: monitorCheckIntervalFlag,+ AutoRelayMinInterval: autoRelayMinIntervalFlag,+ AutoRelayBootDelay: autoRelayBootDelayFlag,+ AutoRelayBackoff: autoRelayBackoffFlag,+ RouterConnectTimeout: routerConnectTimeoutFlag,+ RequiredRole: api.RoleNode,+ ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,+ DHTProviderAddrTTL: dhtProviderAddrTTLFlag,+ DHTMaxRecordAge: dhtMaxRecordAgeFlag,+ DHTLookupLimit: dhtLookupLimitFlag,+ DiscoveryConcurrency: discoveryConcurrencyFlag,+ BackendProbeTimeout: backendProbeTimeoutFlag,})if err != nil {logger.Fatalf("Failed to initialize mesh node: %v", err)@@ -522,31 +522,31 @@ func main() {priv := node.GetOrGenerateKey(store)enrollCtx, enrollCancel := context.WithCancel(context.Background())meshNode, err = node.NewSamNode(node.Options{- PrivKey: priv,- RouterAddrs: initRouterAddrs,- Store: store,- BannedPeerIDs: bannedPeerIDs,- MeshID: meshFlag,- DiscoveryInterval: discoveryIntervalFlag,- ListenAddrs: listenAddrs,- EnableRelay: enableRelayFlag,- NodeConfig: nodeConfig,- KeyGracePeriod: keyGracePeriodFlag,- AllowLoopback: allowLoopbackFlag,- AnnouncePrivateAddrs: &announcePrivateFlag,- MonitorBootstrap: monitorBootstrapFlag,- MonitorInterval: monitorCheckIntervalFlag,- AutoRelayMinInterval: autoRelayMinIntervalFlag,- AutoRelayBootDelay: autoRelayBootDelayFlag,- AutoRelayBackoff: autoRelayBackoffFlag,- RouterConnectTimeout: routerConnectTimeoutFlag,- RequiredRole: api.RoleNode,- PolicySyncInterval: policySyncIntervalFlag,- DHTProviderAddrTTL: dhtProviderAddrTTLFlag,- DHTMaxRecordAge: dhtMaxRecordAgeFlag,- DHTLookupLimit: dhtLookupLimitFlag,- DiscoveryConcurrency: discoveryConcurrencyFlag,- BackendProbeTimeout: backendProbeTimeoutFlag,+ PrivKey: priv,+ RouterAddrs: initRouterAddrs,+ Store: store,+ BannedPeerIDs: bannedPeerIDs,+ MeshID: meshFlag,+ DiscoveryInterval: discoveryIntervalFlag,+ ListenAddrs: listenAddrs,+ EnableRelay: enableRelayFlag,+ NodeConfig: nodeConfig,+ KeyGracePeriod: keyGracePeriodFlag,+ AllowLoopback: allowLoopbackFlag,+ AnnouncePrivateAddrs: &announcePrivateFlag,+ MonitorBootstrap: monitorBootstrapFlag,+ MonitorInterval: monitorCheckIntervalFlag,+ AutoRelayMinInterval: autoRelayMinIntervalFlag,+ AutoRelayBootDelay: autoRelayBootDelayFlag,+ AutoRelayBackoff: autoRelayBackoffFlag,+ RouterConnectTimeout: routerConnectTimeoutFlag,+ RequiredRole: api.RoleNode,+ ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,+ DHTProviderAddrTTL: dhtProviderAddrTTLFlag,+ DHTMaxRecordAge: dhtMaxRecordAgeFlag,+ DHTLookupLimit: dhtLookupLimitFlag,+ DiscoveryConcurrency: discoveryConcurrencyFlag,+ BackendProbeTimeout: backendProbeTimeoutFlag,})if err != nil {enrollCancel()@@ -592,28 +592,28 @@ func main() {logger.Debugf("listenAddrs: %v, allowLoopback: %v", listenAddrs, allowLoopbackFlag)meshNode, err = node.NewSamNode(node.Options{- PrivKey: priv,- ControlPlanePubKey: controlPlanePubKey,- RouterAddrs: newRouterAddrs,- Store: store,- BannedPeerIDs: bannedPeerIDs,- MeshID: meshFlag,- DiscoveryInterval: discoveryIntervalFlag,- ListenAddrs: listenAddrs,- EnableRelay: enableRelayFlag,- NodeConfig: nodeConfig,- KeyGracePeriod: keyGracePeriodFlag,- AllowLoopback: allowLoopbackFlag,- AnnouncePrivateAddrs: &announcePrivateFlag,- MonitorBootstrap: monitorBootstrapFlag,- MonitorInterval: monitorCheckIntervalFlag,- AutoRelayMinInterval: autoRelayMinIntervalFlag,- AutoRelayBootDelay: autoRelayBootDelayFlag,- AutoRelayBackoff: autoRelayBackoffFlag,- RouterConnectTimeout: routerConnectTimeoutFlag,- RequiredRole: api.RoleNode,- PolicySyncInterval: policySyncIntervalFlag,- BackendProbeTimeout: backendProbeTimeoutFlag,+ PrivKey: priv,+ ControlPlanePubKey: controlPlanePubKey,+ RouterAddrs: newRouterAddrs,+ Store: store,+ BannedPeerIDs: bannedPeerIDs,+ MeshID: meshFlag,+ DiscoveryInterval: discoveryIntervalFlag,+ ListenAddrs: listenAddrs,+ EnableRelay: enableRelayFlag,+ NodeConfig: nodeConfig,+ KeyGracePeriod: keyGracePeriodFlag,+ AllowLoopback: allowLoopbackFlag,+ AnnouncePrivateAddrs: &announcePrivateFlag,+ MonitorBootstrap: monitorBootstrapFlag,+ MonitorInterval: monitorCheckIntervalFlag,+ AutoRelayMinInterval: autoRelayMinIntervalFlag,+ AutoRelayBootDelay: autoRelayBootDelayFlag,+ AutoRelayBackoff: autoRelayBackoffFlag,+ RouterConnectTimeout: routerConnectTimeoutFlag,+ RequiredRole: api.RoleNode,+ ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,+ BackendProbeTimeout: backendProbeTimeoutFlag,})if err != nil {logger.Fatalf("Failed to initialize node after enrollment: %v", err)@@ -736,25 +736,25 @@ func main() {priv := node.GetOrGenerateKey(store)meshNode, err := node.NewSamNode(node.Options{- PrivKey: priv,- RouterAddrs: initRouterAddrs,- Store: store,- MeshID: meshFlag,- DiscoveryInterval: discoveryIntervalFlag,- ListenAddrs: []string{"/ip4/0.0.0.0/udp/0/quic-v1", "/ip4/0.0.0.0/tcp/0"},- EnableRelay: enableRelayFlag,- NodeConfig: nodeConfig,- KeyGracePeriod: keyGracePeriodFlag,- AllowLoopback: allowLoopbackFlag,- AnnouncePrivateAddrs: &announcePrivateFlag,- MonitorBootstrap: 2 * time.Minute,- MonitorInterval: 1 * time.Minute,- AutoRelayMinInterval: 30 * time.Second,- AutoRelayBootDelay: 0 * time.Second,- AutoRelayBackoff: 3 * time.Second,- RouterConnectTimeout: routerConnectTimeoutFlag,- RequiredRole: api.RoleNode,- PolicySyncInterval: policySyncIntervalFlag,+ PrivKey: priv,+ RouterAddrs: initRouterAddrs,+ Store: store,+ MeshID: meshFlag,+ DiscoveryInterval: discoveryIntervalFlag,+ ListenAddrs: []string{"/ip4/0.0.0.0/udp/0/quic-v1", "/ip4/0.0.0.0/tcp/0"},+ EnableRelay: enableRelayFlag,+ NodeConfig: nodeConfig,+ KeyGracePeriod: keyGracePeriodFlag,+ AllowLoopback: allowLoopbackFlag,+ AnnouncePrivateAddrs: &announcePrivateFlag,+ MonitorBootstrap: 2 * time.Minute,+ MonitorInterval: 1 * time.Minute,+ AutoRelayMinInterval: 30 * time.Second,+ AutoRelayBootDelay: 0 * time.Second,+ AutoRelayBackoff: 3 * time.Second,+ RouterConnectTimeout: routerConnectTimeoutFlag,+ RequiredRole: api.RoleNode,+ ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,})if err != nil {logger.Fatalf("Failed to initialize node for enrollment: %v", err)@@ -875,7 +875,7 @@ func main() {runCmd.Flags().DurationVar(&dhtMaxRecordAgeFlag, "dht-max-record-age", 0, "Maximum age for DHT records (0s uses library default)")runCmd.Flags().IntVar(&dhtLookupLimitFlag, "dht-lookup-limit", 0, "Maximum number of providers to query from the DHT (0 uses default 20)")runCmd.Flags().IntVar(&discoveryConcurrencyFlag, "discovery-concurrency", 0, "Max concurrent catalog fetches during discovery (0 uses default 10)")- runCmd.Flags().DurationVar(&policySyncIntervalFlag, "policy-sync-interval", 1*time.Hour, "Interval for syncing mesh policy from the control plane")+ runCmd.Flags().DurationVar(&controlPlaneSyncIntervalFlag, "control-plane-sync-interval", node.DefaultControlPlaneSyncInterval, "How often signing keys, bans, router addresses and mesh policy are pulled from the control plane")runCmd.Flags().DurationVar(&backendProbeTimeoutFlag, "backend-probe-timeout", 0, "Timeout for probing a command-spawned service backend before advertising it (0 uses default 2s); raise this for backends with slower cold-start times")rootCmd.PersistentFlags().StringVar(&controlPlaneAddr, "control-plane", "", "Control plane URL")rootCmd.PersistentFlags().BoolVar(&insecureControlPlaneFlag, "insecure-control-plane", false, "Accept a plaintext http:// control plane URL to a non-loopback host (whoever answers it becomes this node's trust root; only for networks you already trust)")diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.gonew file mode 100644index 00000000..7316e620--- /dev/null+++ b/internal/node/controlplane_sync.go@@ -0,0 +1,226 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "context"+ "errors"+ "fmt"+ "math/rand"+ "time"++ "github.com/libp2p/go-libp2p/core/peer"+)++// The node reads three things from the control plane while it runs: the+// signing keys it verifies every credential against (/keys), the ban set and+// router addresses (/info), and the mesh policy (/policies). They are pulled+// together, by one loop, on one interval. Gossip events are notifications+// that bring the next pull forward; they are never the only way state+// arrives, because a once-published event with no replay is missed by any+// node that was down, partitioned, or simply enrolled later.++// syncControlPlane is the node's single pull from the control plane. Each+// part is attempted even when another fails, so a policy outage does not+// stop a key rotation from landing; the errors are reported together.+func (n *SamNode) syncControlPlane(ctx context.Context) error {+ if n.Store == nil {+ return errors.New("node has no store")+ }+ controlPlaneURL, err := n.Store.LoadControlPlaneURL()+ if err != nil || controlPlaneURL == "" {+ return errors.New("control plane URL not found in store")+ }+ var errs []error+ if err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {+ errs = append(errs, fmt.Errorf("keys: %w", err))+ }+ if err := n.syncMeshInfo(ctx, controlPlaneURL); err != nil {+ errs = append(errs, fmt.Errorf("info: %w", err))+ }+ if err := n.syncMeshPolicy(ctx); err != nil {+ errs = append(errs, fmt.Errorf("policy: %w", err))+ }+ return errors.Join(errs...)+}++// syncTrustedKeys replaces the trust set with the control plane's current+// /keys answer and persists it. Enrollment hands out only the newest key, so+// this is how a node learns the key still in its rotation grace period (which+// routers and peers may still be signed by) and any successor it did not hear+// about. Verification against the keys already trusted keeps whoever answers+// the URL from becoming the trust root; an empty answer is a failure so the+// set is never wiped.+func (n *SamNode) syncTrustedKeys(ctx context.Context, controlPlaneURL string) error {+ n.keysMu.RLock()+ existing := append([]TrustedKey(nil), n.trustedKeys...)+ n.keysMu.RUnlock()+ if len(existing) == 0 {+ return errors.New("no trusted control plane keys to verify /keys against")+ }+ keys, err := FetchControlPlaneKeys(ctx, controlPlaneURL, publicKeysOf(existing))+ if err != nil {+ return err+ }+ if len(keys) == 0 {+ return errors.New("/keys returned no keys")+ }+ merged := mergeTrustedKeys(existing, keys, time.Now())+ n.keysMu.Lock()+ n.trustedKeys = merged+ snapshot := append([]TrustedKey(nil), merged...)+ n.keysMu.Unlock()+ n.persistTrustedKeys(snapshot)+ logger.Debugf("Synced %d valid control plane keys", len(merged))+ return nil+}++// syncMeshInfo reads /info: the router addresses are persisted for the next+// start, and the ban set is reconciled against the revocation cache, which is+// how a running node learns a ban or an unban it got no event for.+func (n *SamNode) syncMeshInfo(ctx context.Context, controlPlaneURL string) error {+ // Taken before the request: a ban recorded after this instant cannot be+ // in the answer, so its absence must not be read as an unban.+ fetchedAt := time.Now()+ info, err := FetchControlPlaneInfo(ctx, controlPlaneURL)+ if err != nil {+ return err+ }+ if len(info.RouterAddresses) > 0 {+ pubKey, _, loadErr := n.Store.LoadMeshConfig()+ switch {+ case loadErr != nil:+ logger.Warnf("Failed to load mesh config; not persisting router addresses: %v", loadErr)+ case len(pubKey) > 0:+ if saveErr := n.Store.SaveMeshConfig(pubKey, info.RouterAddresses); saveErr != nil {+ logger.Warnf("Failed to persist router addresses: %v", saveErr)+ }+ }+ }+ n.reconcileBannedPeers(info.GetBannedPeerIds(), fetchedAt)+ return nil+}++// reconcileBannedPeers makes the revocation cache match the control plane's+// ban set as of fetchedAt. Bans recorded at or after fetchedAt are kept even+// when absent from the answer: the answer predates them and cannot speak to+// them (see the router's reconcileBannedPeers for the same rule).+func (n *SamNode) reconcileBannedPeers(peerIDs []string, fetchedAt time.Time) {+ if n.revokedPeers == nil {+ return+ }+ banned := make(map[string]peer.ID, len(peerIDs))+ for _, id := range peerIDs {+ p, err := peer.Decode(id)+ if err != nil {+ logger.Warnf("Ignoring undecodable banned peer ID %q from the control plane: %v", id, err)+ continue+ }+ banned[p.String()] = p+ }+ fetchedAtMs := fetchedAt.UnixMilli()+ for _, key := range n.revokedPeers.Keys() {+ if _, still := banned[key]; still {+ continue+ }+ if bannedAt, ok := n.revokedPeers.Peek(key); ok && bannedAt >= fetchedAtMs {+ continue+ }+ logger.Infof("Peer %s is no longer banned by the control plane; lifting the local ban", key)+ n.revokedPeers.Remove(key)+ }+ for key, p := range banned {+ if n.revokedPeers.Contains(key) {+ continue+ }+ logger.Infow("peer banned by the control plane", "event", meshEventBanned, "peer", key)+ n.banPeer(p, fetchedAtMs)+ }+}++// banPeer records the ban and evicts the peer: the cache entry is what the+// gater and the auth path consult, and dropping the admission is what stops+// the relay ACL from honouring a session that was already established.+func (n *SamNode) banPeer(p peer.ID, bannedAtMs int64) {+ if n.revokedPeers != nil {+ n.revokedPeers.Add(p.String(), bannedAtMs)+ }+ n.authPeers.Delete(p)+ if n.Host != nil {+ _ = n.Host.Network().ClosePeer(p)+ }+}++// triggerControlPlaneSync asks the sync loop to run soon, after a random+// delay of up to the configured jitter so a fleet told at once does not hit+// the control plane at once. A pending request is not duplicated.+func (n *SamNode) triggerControlPlaneSync() {+ select {+ case n.controlPlaneSyncTrigger <- struct{}{}:+ default:+ }+}++// startControlPlaneSyncLoop pulls once shortly after start, then every+// interval and whenever triggered. Failures are logged at Warn once and at+// Debug while they persist, with an Info on recovery.+func (n *SamNode) startControlPlaneSyncLoop(ctx context.Context, interval time.Duration) {+ if interval <= 0 || n.Store == nil {+ return+ }+ go func() {+ timer := time.NewTimer(2 * time.Second)+ defer timer.Stop()+ failures := 0+ for {+ select {+ case <-ctx.Done():+ return+ case <-timer.C:+ case <-n.controlPlaneSyncTrigger:+ if !timer.Stop() {+ select {+ case <-timer.C:+ default:+ }+ }+ jitter := n.config.ControlPlaneSyncJitter+ if jitter > 0 {+ select {+ case <-ctx.Done():+ return+ case <-time.After(time.Duration(rand.Int63n(int64(jitter)))):+ }+ }+ }+ timer.Reset(interval)++ err := n.syncControlPlane(ctx)+ switch {+ case err != nil && failures == 0:+ logger.Warnf("Control plane sync failed: %v", err)+ case err != nil:+ logger.Debugf("Control plane sync still failing (%d consecutive): %v", failures+1, err)+ case failures > 0:+ logger.Infof("Control plane sync recovered after %d failures", failures)+ }+ if err != nil {+ failures+++ } else {+ failures = 0+ }+ }+ }()+}diff --git a/internal/node/controlplane_test.go b/internal/node/controlplane_test.goindex d40c13ed..a45b3812 100644--- a/internal/node/controlplane_test.go+++ b/internal/node/controlplane_test.go@@ -28,6 +28,8 @@ import ("time""github.com/google/sam/api"+ lru "github.com/hashicorp/golang-lru/v2"+ "github.com/libp2p/go-libp2p/core/peer""google.golang.org/protobuf/proto")@@ -60,6 +62,263 @@ func TestMergeTrustedKeys(t *testing.T) {}}+// mustGenerateKey is an ed25519 key pair or a failed test.+func mustGenerateKey(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) {+ t.Helper()+ pub, priv, err := ed25519.GenerateKey(nil)+ if err != nil {+ t.Fatalf("ed25519.GenerateKey: %v", err)+ }+ return pub, priv+}++// keysServer serves /keys with the given set, signed by the matching private+// keys; a nil signer leaves the answer unsigned.+func keysServer(t *testing.T, pubs []ed25519.PublicKey, privs []ed25519.PrivateKey) *httptest.Server {+ t.Helper()+ return httptest.NewServer(keysHandler(t, pubs, privs))+}++func keysHandler(t *testing.T, pubs []ed25519.PublicKey, privs []ed25519.PrivateKey) http.HandlerFunc {+ t.Helper()+ return func(w http.ResponseWriter, r *http.Request) {+ resp := &api.KeysResponse{}+ for _, p := range pubs {+ resp.PublicKeys = append(resp.PublicKeys, p)+ }+ if privs != nil {+ if err := api.SignKeysResponse(resp, privs, time.Now()); err != nil {+ t.Errorf("SignKeysResponse: %v", err)+ http.Error(w, err.Error(), http.StatusInternalServerError)+ return+ }+ }+ protoHandler(t, resp)(w, r)+ }+}++// protoHandler answers with msg; a marshal failure fails the test rather+// than handing the client an empty body it might accept.+func protoHandler(t *testing.T, msg proto.Message) http.HandlerFunc {+ t.Helper()+ return func(w http.ResponseWriter, r *http.Request) {+ data, err := proto.Marshal(msg)+ if err != nil {+ t.Errorf("proto.Marshal(%T): %v", msg, err)+ http.Error(w, err.Error(), http.StatusInternalServerError)+ return+ }+ w.Header().Set("Content-Type", "application/x-protobuf")+ if _, err := w.Write(data); err != nil {+ t.Errorf("write %T: %v", msg, err)+ }+ }+}++func TestSyncTrustedKeys(t *testing.T) {+ retiredPub, retiredPriv := mustGenerateKey(t)+ currentPub, currentPriv := mustGenerateKey(t)+ strangerPub, strangerPriv := mustGenerateKey(t)++ newNode := func(t *testing.T) *SamNode {+ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = store.Close() })+ return &SamNode{Store: store, trustedKeys: []TrustedKey{{Key: currentPub, ReceivedAt: time.Now()}}}+ }++ t.Run("adopts the set a trusted key vouches for", func(t *testing.T) {+ srv := keysServer(t, []ed25519.PublicKey{retiredPub, currentPub}, []ed25519.PrivateKey{retiredPriv, currentPriv})+ defer srv.Close()+ n := newNode(t)+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil {+ t.Fatalf("syncTrustedKeys: %v", err)+ }+ if len(n.trustedKeys) != 2 || !containsTrustedKey(n.trustedKeys, retiredPub) || !containsTrustedKey(n.trustedKeys, currentPub) {+ t.Errorf("trust set = %d keys, want retired and current", len(n.trustedKeys))+ }+ stored, err := n.Store.LoadTrustedKeys()+ if err != nil {+ t.Fatal(err)+ }+ if len(stored) != 2 {+ t.Errorf("persisted %d keys, want 2", len(stored))+ }+ })++ t.Run("rejects a set no trusted key signed", func(t *testing.T) {+ srv := keysServer(t, []ed25519.PublicKey{strangerPub}, []ed25519.PrivateKey{strangerPriv})+ defer srv.Close()+ n := newNode(t)+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {+ t.Fatal("a /keys answer signed only by an unknown key must not be adopted")+ }+ if len(n.trustedKeys) != 1 || !n.trustedKeys[0].Key.Equal(currentPub) {+ t.Errorf("trust set changed on a rejected answer: %d keys", len(n.trustedKeys))+ }+ })++ t.Run("rejects an unsigned set", func(t *testing.T) {+ srv := keysServer(t, []ed25519.PublicKey{currentPub, strangerPub}, nil)+ defer srv.Close()+ n := newNode(t)+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {+ t.Fatal("an unsigned /keys answer must not be adopted")+ }+ if containsTrustedKey(n.trustedKeys, strangerPub) {+ t.Error("unsigned answer widened the trust set")+ }+ })++ t.Run("nothing trusted yet is an error, not a wipe", func(t *testing.T) {+ srv := keysServer(t, []ed25519.PublicKey{currentPub}, []ed25519.PrivateKey{currentPriv})+ defer srv.Close()+ n := newNode(t)+ n.trustedKeys = nil+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {+ t.Fatal("with no trusted key there is nothing to verify /keys against")+ }+ })+}++// A ban the control plane holds is applied, one it no longer holds is lifted,+// and one recorded after the answer was requested is left alone: the answer+// predates it and cannot speak to it.+func TestReconcileBannedPeers(t *testing.T) {+ stillBanned := randomPeerID(t)+ unbanned := randomPeerID(t)+ newlyBanned := randomPeerID(t)+ bannedAfterFetch := randomPeerID(t)++ cache, err := lru.New[string, int64](10)+ if err != nil {+ t.Fatal(err)+ }+ n := &SamNode{revokedPeers: cache}+ fetchedAt := time.Now()+ n.revokedPeers.Add(stillBanned.String(), fetchedAt.Add(-time.Hour).UnixMilli())+ n.revokedPeers.Add(unbanned.String(), fetchedAt.Add(-time.Hour).UnixMilli())+ n.revokedPeers.Add(bannedAfterFetch.String(), fetchedAt.UnixMilli())++ n.reconcileBannedPeers([]string{stillBanned.String(), newlyBanned.String(), "not-a-peer-id"}, fetchedAt)++ for _, want := range []peer.ID{stillBanned, newlyBanned, bannedAfterFetch} {+ if !n.revokedPeers.Contains(want.String()) {+ t.Errorf("%s must be banned after reconciliation", want)+ }+ }+ if n.revokedPeers.Contains(unbanned.String()) {+ t.Error("a peer absent from the control plane's ban set must be unbanned")+ }+}++// One pull brings keys, bans and policy together, and a failing part does+// not stop the others from landing.+func TestSyncControlPlane(t *testing.T) {+ oldPub, oldPriv := mustGenerateKey(t)+ newPub, newPriv := mustGenerateKey(t)+ banned := randomPeerID(t)++ mux := http.NewServeMux()+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{oldPub, newPub}, []ed25519.PrivateKey{oldPriv, newPriv}))+ mux.HandleFunc("/info", protoHandler(t, &api.ControlPlaneInfoResponse{+ RouterAddresses: []string{"/ip4/10.0.0.1/tcp/4501/p2p/" + randomPeerID(t).String()},+ BannedPeerIds: []string{banned.String()},+ }))+ mux.HandleFunc("/policies", func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, "policy store down", http.StatusInternalServerError)+ })+ srv := httptest.NewServer(mux)+ defer srv.Close()++ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = store.Close() }()+ if err := store.SaveControlPlaneURL(srv.URL); err != nil {+ t.Fatal(err)+ }+ if err := store.SaveMeshConfig(oldPub, nil); err != nil {+ t.Fatal(err)+ }+ cache, err := lru.New[string, int64](10)+ if err != nil {+ t.Fatal(err)+ }+ n := &SamNode{Store: store, revokedPeers: cache, trustedKeys: []TrustedKey{{Key: oldPub, ReceivedAt: time.Now()}}}+ n.SetIdentityCache([]byte("identity"))++ err = n.syncControlPlane(context.Background())+ if err == nil || !strings.Contains(err.Error(), "policy") {+ t.Fatalf("syncControlPlane = %v, want the policy failure reported", err)+ }+ if !containsTrustedKey(n.trustedKeys, newPub) {+ t.Error("rotated key not learned although /keys answered")+ }+ if !n.revokedPeers.Contains(banned.String()) {+ t.Error("ban not applied although /info answered")+ }+ _, addrs, err := store.LoadMeshConfig()+ if err != nil {+ t.Fatalf("LoadMeshConfig: %v", err)+ }+ if len(addrs) != 1 {+ t.Errorf("router addresses not persisted: got %v, want 1", addrs)+ }+}++// The loop is what turns a missed gossip event into a delay rather than a+// permanent split: a running node picks the successor key up on its own, and+// a trigger brings the next pull forward.+func TestControlPlaneSyncLoop(t *testing.T) {+ oldPub, oldPriv := mustGenerateKey(t)+ newPub, newPriv := mustGenerateKey(t)++ mux := http.NewServeMux()+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{oldPub, newPub}, []ed25519.PrivateKey{oldPriv, newPriv}))+ mux.HandleFunc("/info", protoHandler(t, &api.ControlPlaneInfoResponse{}))+ mux.HandleFunc("/policies", protoHandler(t, &api.PolicyConfigGetResponse{}))+ srv := httptest.NewServer(mux)+ defer srv.Close()++ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = store.Close() }()+ if err := store.SaveControlPlaneURL(srv.URL); err != nil {+ t.Fatal(err)+ }++ n := &SamNode{+ Store: store,+ trustedKeys: []TrustedKey{{Key: oldPub, ReceivedAt: time.Now()}},+ controlPlaneSyncTrigger: make(chan struct{}, 1),+ config: Options{ControlPlaneSyncJitter: time.Millisecond},+ }+ n.SetIdentityCache([]byte("identity"))+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ // A long interval: only the trigger can make the first pull happen in time.+ n.startControlPlaneSyncLoop(ctx, time.Hour)+ n.triggerControlPlaneSync()++ deadline := time.Now().Add(5 * time.Second)+ for time.Now().Before(deadline) {+ n.keysMu.RLock()+ learned := containsTrustedKey(n.trustedKeys, newPub)+ n.keysMu.RUnlock()+ if learned {+ return+ }+ time.Sleep(20 * time.Millisecond)+ }+ t.Fatal("triggered control plane sync never picked up the rotated key")+}+func TestFetchControlPlaneInfo(t *testing.T) {expectedInfo := &api.ControlPlaneInfoResponse{RouterAddresses: []string{"/ip4/127.0.0.1/tcp/4001"},diff --git a/internal/node/enroll.go b/internal/node/enroll.goindex bbed85eb..b1f3efd7 100644--- a/internal/node/enroll.go+++ b/internal/node/enroll.go@@ -72,6 +72,19 @@ func (n *SamNode) Enroll(ctx context.Context, controlPlaneURL string, jwt stringreturn n.connectToRouters(ctx, enrollResp.RouterAddresses)}+// adoptEnrolledKeys widens the trust set from the single key an enrollment+// response carries to every key the control plane currently signs with.+// The router handshake that follows verifies the router's own biscuit, and+// right after a rotation that is still signed by the retiring key; with only+// the newest key loaded the node would refuse every router. Best effort: a+// control plane too old to serve /keys still enrolls with one key.+func (n *SamNode) adoptEnrolledKeys(ctx context.Context, controlPlaneURL string, newest []byte) {+ n.addTrustedKey(ed25519.PublicKey(newest))+ if err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {+ logger.Warnf("Could not fetch the control plane's full key set after enrollment (continuing with the enrollment key only): %v", err)+ }+}+// enrollHTTP performs the HTTP half of enrollment for an explicit peer// identity, so it can run before the libp2p host exists (startup recovery).// privKey signs the proof-of-possession challenge; peerID must be its own.@@ -123,7 +136,12 @@ func (n *SamNode) enrollHTTP(ctx context.Context, controlPlaneURL, jwt string, p}}()- return n.processEnrollResponse(resp)+ enrollResp, err := n.processEnrollResponse(resp)+ if err != nil {+ return nil, err+ }+ n.adoptEnrolledKeys(ctx, controlPlaneURL, enrollResp.ControlPlanePublicKey)+ return enrollResp, nil}// ReEnrollWithRefreshToken re-enrolls over HTTP with a JWT from the stored@@ -191,8 +209,6 @@ func (n *SamNode) processEnrollResponse(resp *http.Response) (*api.EnrollResponsreturn nil, fmt.Errorf("failed to save mesh config: %v", err)}- n.addTrustedKey(ed25519.PublicKey(enrollResp.ControlPlanePublicKey))-return &enrollResp, nil}@@ -387,7 +403,7 @@ func (n *SamNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, breturn fmt.Errorf("failed to save mesh config: %v", err)}- n.addTrustedKey(ed25519.PublicKey(enrollResp.ControlPlanePublicKey))+ n.adoptEnrolledKeys(ctx, controlPlaneURL, enrollResp.ControlPlanePublicKey)// Connect and Auth to router after enrollment to join the meshif len(enrollResp.RouterAddresses) == 0 {diff --git a/internal/node/enroll_test.go b/internal/node/enroll_test.goindex d104d72a..1facf273 100644--- a/internal/node/enroll_test.go+++ b/internal/node/enroll_test.go@@ -532,6 +532,90 @@ func TestEnroll_InvalidControlPlanePublicKeySize(t *testing.T) {}}+// TestEnrollTrustsRouterSignedByGraceKey is the hub testnet outage of+// 2026-09-20: the control plane had rotated its signing key, the routers+// still presented biscuits signed by the retiring key (legitimately in its+// grace period), and the enroll response carries only the newest key. A+// fresh node must fetch the full set from /keys before the router handshake+// or it refuses every router and enrollment fails.+func TestEnrollTrustsRouterSignedByGraceKey(t *testing.T) {+ retiredPub, retiredPriv := mustGenerateKey(t)+ currentPub, currentPriv := mustGenerateKey(t)++ routerAddr := startMockRouterWithKey(t, retiredPriv, api.RoleRouter)++ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = store.Close() }()+ privKey := GetOrGenerateKey(store)++ mux := http.NewServeMux()+ mux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {+ body, err := io.ReadAll(r.Body)+ if err != nil {+ t.Errorf("read /register body: %v", err)+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }+ var req api.EnrollRequest+ if err := proto.Unmarshal(body, &req); err != nil {+ t.Errorf("decode /register body: %v", err)+ http.Error(w, "bad request", http.StatusBadRequest)+ return+ }+ peerID, err := peer.Decode(req.PeerId)+ if err != nil {+ t.Errorf("decode enrolling peer id %q: %v", req.PeerId, err)+ http.Error(w, "bad peer id", http.StatusBadRequest)+ return+ }+ protoHandler(t, &api.EnrollResponse{+ BiscuitToken: mintRoleBiscuit(t, currentPriv, peerID, api.RoleNode),+ ControlPlanePublicKey: currentPub,+ RouterAddresses: []string{routerAddr},+ Expiration: time.Now().Add(24 * time.Hour).Unix(),+ })(w, r)+ })+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{retiredPub, currentPub}, []ed25519.PrivateKey{retiredPriv, currentPriv}))+ cpSrv := httptest.NewServer(mux)+ defer cpSrv.Close()++ node, err := NewSamNode(Options{+ PrivKey: privKey,+ Store: store,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ AllowLoopback: true,+ })+ if err != nil {+ t.Fatal(err)+ }+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)+ defer cancel()+ if err := node.Start(ctx); err != nil {+ t.Fatalf("Start: %v", err)+ }+ defer func() { _ = node.Host.Close() }()++ if err := node.Enroll(ctx, cpSrv.URL, "dummy-jwt"); err != nil {+ t.Fatalf("Enroll must accept a router signed by the key still in grace: %v", err)+ }++ for _, want := range []ed25519.PublicKey{retiredPub, currentPub} {+ if !containsTrustedKey(node.trustedKeys, want) {+ t.Errorf("trust set %d keys, missing %x", len(node.trustedKeys), want)+ }+ }+ stored, err := store.LoadTrustedKeys()+ if err != nil {+ t.Fatal(err)+ }+ if len(stored) != 2 {+ t.Errorf("persisted trust set has %d keys, want both", len(stored))+ }+}+func TestProcessEnrollResponse_Errors(t *testing.T) {dir := t.TempDir()store, err := NewStore(dir)diff --git a/internal/node/node.go b/internal/node/node.goindex b80e2ae9..d00b10b6 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -89,6 +89,11 @@ const (// Key pruningKeyPruningInterval = 1 * time.Hour+ // DefaultControlPlaneSyncInterval matches the router's --keys-sync-interval:+ // well inside the control plane's default 1h key grace period, so a node+ // has several chances to learn a successor key while its predecessor can+ // still vouch for it.+ DefaultControlPlaneSyncInterval = 5 * time.Minute// Reprovide intervalReprovideInterval = 5 * time.Minute@@ -172,13 +177,14 @@ type SamNode struct {BoundSocketPath stringAllowLoopback bool- authSuccess chan struct{}- authOnce sync.Once- currentRelays []peer.AddrInfo- reprovideTrigger chan struct{}- BiscuitTimeout time.Duration- cachedIdentity atomic.Value- logger *golog.ZapEventLogger+ authSuccess chan struct{}+ authOnce sync.Once+ currentRelays []peer.AddrInfo+ reprovideTrigger chan struct{}+ controlPlaneSyncTrigger chan struct{}+ BiscuitTimeout time.Duration+ cachedIdentity atomic.Value+ logger *golog.ZapEventLogger// metricsRegistry holds this node's state collector; see metricsHandler.metricsOnce sync.Once@@ -285,17 +291,18 @@ func NewSamNode(cfg Options) (*SamNode, error) {}node := &SamNode{- config: cfg,- Store: cfg.Store,- trustedKeys: trustedKeys,- peerLastEventTime: make(map[string]int64),- authenticatedRouters: make(map[peer.ID]bool),- nodeConfig: cfg.NodeConfig,- AllowLoopback: cfg.AllowLoopback,- authSuccess: make(chan struct{}),- reprovideTrigger: make(chan struct{}, 1),- BiscuitTimeout: cfg.BiscuitTimeout,- logger: golog.Logger("sam-node"),+ config: cfg,+ Store: cfg.Store,+ trustedKeys: trustedKeys,+ peerLastEventTime: make(map[string]int64),+ authenticatedRouters: make(map[peer.ID]bool),+ nodeConfig: cfg.NodeConfig,+ AllowLoopback: cfg.AllowLoopback,+ authSuccess: make(chan struct{}),+ reprovideTrigger: make(chan struct{}, 1),+ controlPlaneSyncTrigger: make(chan struct{}, 1),+ BiscuitTimeout: cfg.BiscuitTimeout,+ logger: golog.Logger("sam-node"),}var err error@@ -660,8 +667,9 @@ func (n *SamNode) Start(ctx context.Context) error {// Periodically and on-demand reprovide registered services to the DHTn.startReprovideLoop(ctx, ReprovideInterval)- // Periodically sync mesh policy- n.startPolicySyncLoop(ctx, n.config.PolicySyncInterval)+ // Everything read from the control plane (keys, bans, router addresses,+ // policy) in one loop; gossip events only bring the next pull forward.+ n.startControlPlaneSyncLoop(ctx, n.config.ControlPlaneSyncInterval)// Periodically self-report local services to the control plane, so an// admin can see mesh-wide service topology.@@ -1337,23 +1345,7 @@ func (n *SamNode) listenForControlPlaneEvents(ctx context.Context) {n.handleKeyRotationEvent(&event)case api.MeshEvent_POLICY_UPDATE:logger.Infow("[Mesh Event] policy update received, triggering sync", "event", meshEventPolicyUpdate, "peer", msg.ReceivedFrom.String())- go func() {- maxJitter := n.config.PolicySyncJitter- if maxJitter <= 0 {- maxJitter = 10 * time.Second- }- jitter := time.Duration(rand.Int63n(int64(maxJitter)))- timer := time.NewTimer(jitter)- defer timer.Stop()- select {- case <-timer.C:- case <-ctx.Done():- return- }- if err := n.syncMeshPolicy(ctx); err != nil {- logger.Warnf("Failed to sync mesh policy after event: %v", err)- }- }()+ n.triggerControlPlaneSync()}}}@@ -1379,18 +1371,9 @@ func (n *SamNode) handleBannedEvent(event *api.MeshEvent) {n.mu.Unlock()logger.Infow("[Mesh Event] peer banned", "event", meshEventBanned, "peer", canonicalID)-- if n.revokedPeers != nil {- n.revokedPeers.Add(canonicalID, event.Timestamp)- }- // Drop any prior admission, otherwise the relay ACL keeps honouring it.- // The cache entry above is not written to disk: a restarted node picks the- // ban back up from the control plane's ban set in /info (see- // SyncMeshConfig), which is also how an unban reaches it.- n.authPeers.Delete(p)- if n.Host != nil {- _ = n.Host.Network().ClosePeer(p)- }+ // Not written to disk: a restarted node picks the ban back up from the+ // control plane's ban set in /info, which is also how an unban reaches it.+ n.banPeer(p, event.Timestamp)}func containsTrustedKey(keys []TrustedKey, key ed25519.PublicKey) bool {@@ -2368,35 +2351,3 @@ func (n *SamNode) startCatalogReportLoop(ctx context.Context, initialDelay, inte}}()}--func (n *SamNode) startPolicySyncLoop(ctx context.Context, interval time.Duration) {- if interval <= 0 {- interval = 1 * time.Hour- }-- go func() {- // Run initial sync after a short delay- select {- case <-ctx.Done():- return- case <-time.After(2 * time.Second):- if err := n.syncMeshPolicy(ctx); err != nil {- logger.Warnf("Initial mesh policy sync failed: %v", err)- }- }-- ticker := time.NewTicker(interval)- defer ticker.Stop()-- for {- select {- case <-ctx.Done():- return- case <-ticker.C:- if err := n.syncMeshPolicy(ctx); err != nil {- logger.Warnf("Periodic mesh policy sync failed: %v", err)- }- }- }- }()-}diff --git a/internal/node/options.go b/internal/node/options.goindex a1564b1e..3c25bbbc 100644--- a/internal/node/options.go+++ b/internal/node/options.go@@ -74,10 +74,13 @@ type Options struct {DiscoveryConcurrency int// RequiredRole restricts enrollment and startup to only accept tokens containing this role.RequiredRole string- // PolicySyncInterval specifies how often the node syncs the mesh policy from the control plane.- PolicySyncInterval time.Duration- // PolicySyncJitter specifies the maximum jitter delay when scheduling policy syncs on event broadcasts.- PolicySyncJitter time.Duration+ // ControlPlaneSyncInterval is how often the node pulls what it reads from+ // the control plane: signing keys, ban set and router addresses, mesh+ // policy. Zero uses the default; negative disables the loop.+ ControlPlaneSyncInterval time.Duration+ // ControlPlaneSyncJitter is the maximum random delay before a sync that a+ // gossip event asked for, so a fleet told at once does not pull at once.+ ControlPlaneSyncJitter time.Duration// BackendProbeTimeout bounds how long a command-spawned service backend// (sam-node.yaml's `command`, spawned as a local subprocess) is given to// answer before the service is registered but withheld from@@ -143,8 +146,8 @@ func (o *Options) Default() {if o.RequiredRole == "" {o.RequiredRole = api.RoleNode}- if o.PolicySyncInterval == 0 {- o.PolicySyncInterval = 1 * time.Hour+ if o.ControlPlaneSyncInterval == 0 {+ o.ControlPlaneSyncInterval = DefaultControlPlaneSyncInterval}if o.CatalogReportInterval <= 0 {o.CatalogReportInterval = 1 * time.Minute@@ -152,8 +155,8 @@ func (o *Options) Default() {if o.CatalogReportInitialDelay <= 0 {o.CatalogReportInitialDelay = 5 * time.Second}- if o.PolicySyncJitter <= 0 {- o.PolicySyncJitter = 10 * time.Second+ if o.ControlPlaneSyncJitter <= 0 {+ o.ControlPlaneSyncJitter = 10 * time.Second}if o.NodeConfig == nil {o.NodeConfig = &NodeConfigComplete{}diff --git a/site/content/docs/concepts/authorization.md b/site/content/docs/concepts/authorization.mdindex 3094f740..37542e19 100644--- a/site/content/docs/concepts/authorization.md+++ b/site/content/docs/concepts/authorization.md@@ -93,8 +93,8 @@ becomes `granted_service_exact("mcp", "calculator")`, `mcp://*` becomes`granted_service_suffix("mcp", ".internal")`. Wildcards keep their dot, so`*.acme.example` matches `svc.acme.example` but not `evil-acme.example`.-Nodes also fetch the policy themselves, every `--policy-sync-interval` (one-hour by default), and compile it into rules such as+Nodes also fetch the policy themselves, every `--control-plane-sync-interval`+(five minutes by default), and compile it into rules such as`role("developer") <- group("eng")` and`granted_service_exact("mcp", "calculator") <- role("developer")`. When anode verifies a credential, these rules run against the identity facts in it.diff --git a/site/content/docs/concepts/identity.md b/site/content/docs/concepts/identity.mdindex 1b9a6c6d..a31b3f6f 100644--- a/site/content/docs/concepts/identity.md+++ b/site/content/docs/concepts/identity.md@@ -129,10 +129,12 @@ The control plane rotates its signing key every `--key-rotation-interval`(24 hours by default). The previous key stays valid for `--key-grace-period`(1 hour by default) and is then retired. `GET /keys` returns the current setof keys, signed by each key in the set. Routers poll it every-`--keys-sync-interval`. Nodes fetch it at start and when they receive a-rotation event. Both accept a new set only if one of its signatures verifies-under a key they already trust. The first key comes from enrollment, and-each later key is vouched for by the key it replaces.+`--keys-sync-interval`; nodes fetch it at enrollment and then every+`--control-plane-sync-interval`, together with the ban set and the mesh+policy. A rotation event only brings the next pull forward. Both accept a+new set only if one of its signatures verifies under a key they already+trust. The first key comes from enrollment, and each later key is vouched+for by the key it replaces.Nobody can verify a credential signed by a retired key, including thecontrol plane. A node that was offline for a whole grace period thereforediff --git a/site/content/docs/reference/policy.md b/site/content/docs/reference/policy.mdindex 6f92e32e..82a38ea6 100644--- a/site/content/docs/reference/policy.md+++ b/site/content/docs/reference/policy.md@@ -145,7 +145,7 @@ must be on the role itself.At enrollment and at refresh, the control plane resolves the identity's rolesfrom the bindings and mints into the credential one `role()` fact per roleand the compiled `granted_*` facts of every role. Nodes also fetch the policy-(`--policy-sync-interval`, 1 hour) and compile it into rules such as+(`--control-plane-sync-interval`, 5 minutes) and compile it into rules such as`role("developer") <- group("engineering")` and`granted_service_exact("mcp","code-reviewer") <- role("developer")`. Theserules run at verification time. Additions therefore reach nodes within thediff --git a/site/content/docs/reference/sam-node.md b/site/content/docs/reference/sam-node.mdindex 9bd3bd9e..2668ebe7 100644--- a/site/content/docs/reference/sam-node.md+++ b/site/content/docs/reference/sam-node.md@@ -111,7 +111,7 @@ enroll.| `--discovery-concurrency` | `10` | Concurrent catalog fetches during discovery. || `--dht-provider-addr-ttl`, `--dht-max-record-age` | library defaults | DHT record lifetimes. || `--backend-probe-timeout` | `2s` | How long a service backend may take to answer before the node declines to advertise it. Raise it for subprocesses that start slowly. |-| `--policy-sync-interval` | `1h` | How often the mesh policy is fetched again from the control plane. |+| `--control-plane-sync-interval` | `5m` | How often signing keys, the ban set, router addresses and the mesh policy are pulled from the control plane. || `--key-grace-period` | `24h` | How long a rotated-out control plane key is still accepted for verifying peers. || `--monitor-bootstrap` | `2m` | Delay before the router-connection monitor starts. || `--monitor-interval` | `1m` | How often the monitor checks that a router is connected. |