Skip to content

Agent Mesh

node: pull keys, bans and policy from the control plane in one loop

Antonio Ojea1b4cb4e35d17
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index 9bed6b41..1ff5d893 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -53,56 +53,56 @@ func init() {
}
var (
- controlPlaneAddr string
- joinFlag bool
- jwtFlag string
- jwtPathFlag string
- bootstrapTokenFlag string
- clientIDFlag string
- clientSecretFlag string
- controlPlanePublicKeyFlag string
- bindAddrFlag string
- metricsAddrFlag string
- socketPathFlag string
- meshFlag string
- discoveryIntervalFlag string
- listenAddrs []string
- enableRelayFlag bool
- configFile string
- oidcIssuerFlag string
- deviceAuthURLFlag string
- audienceFlag string
- dataDirFlag string
- headlessFlag bool
- authModeFlag string
- insecureControlPlaneFlag bool
- daemonizeFlag bool
- resetAllFlag bool
- assumeYesFlag bool
- offlineAccessFlag bool
- logLevelFlag string
- keyGracePeriodFlag time.Duration
- allowLoopbackFlag bool
- announcePrivateFlag bool
- monitorBootstrapFlag time.Duration
- monitorCheckIntervalFlag time.Duration
- autoRelayMinIntervalFlag time.Duration
- autoRelayBootDelayFlag time.Duration
- autoRelayBackoffFlag time.Duration
- routerConnectTimeoutFlag time.Duration
- apiTokenFlag string
- apiTokenPathFlag string
- bootstrapTokenPathFlag string
- clientSecretPathFlag string
- tlsCertFlag string
- tlsKeyFlag string
- tlsCAFlag string
- dhtProviderAddrTTLFlag time.Duration
- dhtMaxRecordAgeFlag time.Duration
- dhtLookupLimitFlag int
- discoveryConcurrencyFlag int
- backendProbeTimeoutFlag time.Duration
- policySyncIntervalFlag time.Duration
+ controlPlaneAddr string
+ joinFlag bool
+ jwtFlag string
+ jwtPathFlag string
+ bootstrapTokenFlag string
+ clientIDFlag string
+ clientSecretFlag string
+ controlPlanePublicKeyFlag string
+ bindAddrFlag string
+ metricsAddrFlag string
+ socketPathFlag string
+ meshFlag string
+ discoveryIntervalFlag string
+ listenAddrs []string
+ enableRelayFlag bool
+ configFile string
+ oidcIssuerFlag string
+ deviceAuthURLFlag string
+ audienceFlag string
+ dataDirFlag string
+ headlessFlag bool
+ authModeFlag string
+ insecureControlPlaneFlag bool
+ daemonizeFlag bool
+ resetAllFlag bool
+ assumeYesFlag bool
+ offlineAccessFlag bool
+ logLevelFlag string
+ keyGracePeriodFlag time.Duration
+ allowLoopbackFlag bool
+ announcePrivateFlag bool
+ monitorBootstrapFlag time.Duration
+ monitorCheckIntervalFlag time.Duration
+ autoRelayMinIntervalFlag time.Duration
+ autoRelayBootDelayFlag time.Duration
+ autoRelayBackoffFlag time.Duration
+ routerConnectTimeoutFlag time.Duration
+ apiTokenFlag string
+ apiTokenPathFlag string
+ bootstrapTokenPathFlag string
+ clientSecretPathFlag string
+ tlsCertFlag string
+ tlsKeyFlag string
+ tlsCAFlag string
+ dhtProviderAddrTTLFlag time.Duration
+ dhtMaxRecordAgeFlag time.Duration
+ dhtLookupLimitFlag int
+ discoveryConcurrencyFlag int
+ backendProbeTimeoutFlag time.Duration
+ controlPlaneSyncIntervalFlag time.Duration
)
var logger = golog.Logger("sam-node-cli")
@@ -453,32 +453,32 @@ func main() {
}
priv := node.GetOrGenerateKey(store)
meshNode, err = node.NewSamNode(node.Options{
- PrivKey: priv,
- ControlPlanePubKey: controlPlanePubKey,
- RouterAddrs: routerAddrs,
- Store: store,
- BannedPeerIDs: bannedPeerIDs,
- MeshID: meshFlag,
- DiscoveryInterval: discoveryIntervalFlag,
- ListenAddrs: listenAddrs,
- EnableRelay: enableRelayFlag,
- NodeConfig: nodeConfig,
- KeyGracePeriod: keyGracePeriodFlag,
- AllowLoopback: allowLoopbackFlag,
- AnnouncePrivateAddrs: &announcePrivateFlag,
- MonitorBootstrap: monitorBootstrapFlag,
- MonitorInterval: monitorCheckIntervalFlag,
- AutoRelayMinInterval: autoRelayMinIntervalFlag,
- AutoRelayBootDelay: autoRelayBootDelayFlag,
- AutoRelayBackoff: autoRelayBackoffFlag,
- RouterConnectTimeout: routerConnectTimeoutFlag,
- RequiredRole: api.RoleNode,
- PolicySyncInterval: policySyncIntervalFlag,
- DHTProviderAddrTTL: dhtProviderAddrTTLFlag,
- DHTMaxRecordAge: dhtMaxRecordAgeFlag,
- DHTLookupLimit: dhtLookupLimitFlag,
- DiscoveryConcurrency: discoveryConcurrencyFlag,
- BackendProbeTimeout: backendProbeTimeoutFlag,
+ PrivKey: priv,
+ ControlPlanePubKey: controlPlanePubKey,
+ RouterAddrs: routerAddrs,
+ Store: store,
+ BannedPeerIDs: bannedPeerIDs,
+ MeshID: meshFlag,
+ DiscoveryInterval: discoveryIntervalFlag,
+ ListenAddrs: listenAddrs,
+ EnableRelay: enableRelayFlag,
+ NodeConfig: nodeConfig,
+ KeyGracePeriod: keyGracePeriodFlag,
+ AllowLoopback: allowLoopbackFlag,
+ AnnouncePrivateAddrs: &announcePrivateFlag,
+ MonitorBootstrap: monitorBootstrapFlag,
+ MonitorInterval: monitorCheckIntervalFlag,
+ AutoRelayMinInterval: autoRelayMinIntervalFlag,
+ AutoRelayBootDelay: autoRelayBootDelayFlag,
+ AutoRelayBackoff: autoRelayBackoffFlag,
+ RouterConnectTimeout: routerConnectTimeoutFlag,
+ RequiredRole: api.RoleNode,
+ ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,
+ DHTProviderAddrTTL: dhtProviderAddrTTLFlag,
+ DHTMaxRecordAge: dhtMaxRecordAgeFlag,
+ DHTLookupLimit: dhtLookupLimitFlag,
+ DiscoveryConcurrency: discoveryConcurrencyFlag,
+ BackendProbeTimeout: backendProbeTimeoutFlag,
})
if err != nil {
logger.Fatalf("Failed to initialize mesh node: %v", err)
@@ -522,31 +522,31 @@ func main() {
priv := node.GetOrGenerateKey(store)
enrollCtx, enrollCancel := context.WithCancel(context.Background())
meshNode, err = node.NewSamNode(node.Options{
- PrivKey: priv,
- RouterAddrs: initRouterAddrs,
- Store: store,
- BannedPeerIDs: bannedPeerIDs,
- MeshID: meshFlag,
- DiscoveryInterval: discoveryIntervalFlag,
- ListenAddrs: listenAddrs,
- EnableRelay: enableRelayFlag,
- NodeConfig: nodeConfig,
- KeyGracePeriod: keyGracePeriodFlag,
- AllowLoopback: allowLoopbackFlag,
- AnnouncePrivateAddrs: &announcePrivateFlag,
- MonitorBootstrap: monitorBootstrapFlag,
- MonitorInterval: monitorCheckIntervalFlag,
- AutoRelayMinInterval: autoRelayMinIntervalFlag,
- AutoRelayBootDelay: autoRelayBootDelayFlag,
- AutoRelayBackoff: autoRelayBackoffFlag,
- RouterConnectTimeout: routerConnectTimeoutFlag,
- RequiredRole: api.RoleNode,
- PolicySyncInterval: policySyncIntervalFlag,
- DHTProviderAddrTTL: dhtProviderAddrTTLFlag,
- DHTMaxRecordAge: dhtMaxRecordAgeFlag,
- DHTLookupLimit: dhtLookupLimitFlag,
- DiscoveryConcurrency: discoveryConcurrencyFlag,
- BackendProbeTimeout: backendProbeTimeoutFlag,
+ PrivKey: priv,
+ RouterAddrs: initRouterAddrs,
+ Store: store,
+ BannedPeerIDs: bannedPeerIDs,
+ MeshID: meshFlag,
+ DiscoveryInterval: discoveryIntervalFlag,
+ ListenAddrs: listenAddrs,
+ EnableRelay: enableRelayFlag,
+ NodeConfig: nodeConfig,
+ KeyGracePeriod: keyGracePeriodFlag,
+ AllowLoopback: allowLoopbackFlag,
+ AnnouncePrivateAddrs: &announcePrivateFlag,
+ MonitorBootstrap: monitorBootstrapFlag,
+ MonitorInterval: monitorCheckIntervalFlag,
+ AutoRelayMinInterval: autoRelayMinIntervalFlag,
+ AutoRelayBootDelay: autoRelayBootDelayFlag,
+ AutoRelayBackoff: autoRelayBackoffFlag,
+ RouterConnectTimeout: routerConnectTimeoutFlag,
+ RequiredRole: api.RoleNode,
+ ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,
+ DHTProviderAddrTTL: dhtProviderAddrTTLFlag,
+ DHTMaxRecordAge: dhtMaxRecordAgeFlag,
+ DHTLookupLimit: dhtLookupLimitFlag,
+ DiscoveryConcurrency: discoveryConcurrencyFlag,
+ BackendProbeTimeout: backendProbeTimeoutFlag,
})
if err != nil {
enrollCancel()
@@ -592,28 +592,28 @@ func main() {
logger.Debugf("listenAddrs: %v, allowLoopback: %v", listenAddrs, allowLoopbackFlag)
meshNode, err = node.NewSamNode(node.Options{
- PrivKey: priv,
- ControlPlanePubKey: controlPlanePubKey,
- RouterAddrs: newRouterAddrs,
- Store: store,
- BannedPeerIDs: bannedPeerIDs,
- MeshID: meshFlag,
- DiscoveryInterval: discoveryIntervalFlag,
- ListenAddrs: listenAddrs,
- EnableRelay: enableRelayFlag,
- NodeConfig: nodeConfig,
- KeyGracePeriod: keyGracePeriodFlag,
- AllowLoopback: allowLoopbackFlag,
- AnnouncePrivateAddrs: &announcePrivateFlag,
- MonitorBootstrap: monitorBootstrapFlag,
- MonitorInterval: monitorCheckIntervalFlag,
- AutoRelayMinInterval: autoRelayMinIntervalFlag,
- AutoRelayBootDelay: autoRelayBootDelayFlag,
- AutoRelayBackoff: autoRelayBackoffFlag,
- RouterConnectTimeout: routerConnectTimeoutFlag,
- RequiredRole: api.RoleNode,
- PolicySyncInterval: policySyncIntervalFlag,
- BackendProbeTimeout: backendProbeTimeoutFlag,
+ PrivKey: priv,
+ ControlPlanePubKey: controlPlanePubKey,
+ RouterAddrs: newRouterAddrs,
+ Store: store,
+ BannedPeerIDs: bannedPeerIDs,
+ MeshID: meshFlag,
+ DiscoveryInterval: discoveryIntervalFlag,
+ ListenAddrs: listenAddrs,
+ EnableRelay: enableRelayFlag,
+ NodeConfig: nodeConfig,
+ KeyGracePeriod: keyGracePeriodFlag,
+ AllowLoopback: allowLoopbackFlag,
+ AnnouncePrivateAddrs: &announcePrivateFlag,
+ MonitorBootstrap: monitorBootstrapFlag,
+ MonitorInterval: monitorCheckIntervalFlag,
+ AutoRelayMinInterval: autoRelayMinIntervalFlag,
+ AutoRelayBootDelay: autoRelayBootDelayFlag,
+ AutoRelayBackoff: autoRelayBackoffFlag,
+ RouterConnectTimeout: routerConnectTimeoutFlag,
+ RequiredRole: api.RoleNode,
+ ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,
+ BackendProbeTimeout: backendProbeTimeoutFlag,
})
if err != nil {
logger.Fatalf("Failed to initialize node after enrollment: %v", err)
@@ -736,25 +736,25 @@ func main() {
priv := node.GetOrGenerateKey(store)
meshNode, err := node.NewSamNode(node.Options{
- PrivKey: priv,
- RouterAddrs: initRouterAddrs,
- Store: store,
- MeshID: meshFlag,
- DiscoveryInterval: discoveryIntervalFlag,
- ListenAddrs: []string{"/ip4/0.0.0.0/udp/0/quic-v1", "/ip4/0.0.0.0/tcp/0"},
- EnableRelay: enableRelayFlag,
- NodeConfig: nodeConfig,
- KeyGracePeriod: keyGracePeriodFlag,
- AllowLoopback: allowLoopbackFlag,
- AnnouncePrivateAddrs: &announcePrivateFlag,
- MonitorBootstrap: 2 * time.Minute,
- MonitorInterval: 1 * time.Minute,
- AutoRelayMinInterval: 30 * time.Second,
- AutoRelayBootDelay: 0 * time.Second,
- AutoRelayBackoff: 3 * time.Second,
- RouterConnectTimeout: routerConnectTimeoutFlag,
- RequiredRole: api.RoleNode,
- PolicySyncInterval: policySyncIntervalFlag,
+ PrivKey: priv,
+ RouterAddrs: initRouterAddrs,
+ Store: store,
+ MeshID: meshFlag,
+ DiscoveryInterval: discoveryIntervalFlag,
+ ListenAddrs: []string{"/ip4/0.0.0.0/udp/0/quic-v1", "/ip4/0.0.0.0/tcp/0"},
+ EnableRelay: enableRelayFlag,
+ NodeConfig: nodeConfig,
+ KeyGracePeriod: keyGracePeriodFlag,
+ AllowLoopback: allowLoopbackFlag,
+ AnnouncePrivateAddrs: &announcePrivateFlag,
+ MonitorBootstrap: 2 * time.Minute,
+ MonitorInterval: 1 * time.Minute,
+ AutoRelayMinInterval: 30 * time.Second,
+ AutoRelayBootDelay: 0 * time.Second,
+ AutoRelayBackoff: 3 * time.Second,
+ RouterConnectTimeout: routerConnectTimeoutFlag,
+ RequiredRole: api.RoleNode,
+ ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,
})
if err != nil {
logger.Fatalf("Failed to initialize node for enrollment: %v", err)
@@ -875,7 +875,7 @@ func main() {
runCmd.Flags().DurationVar(&dhtMaxRecordAgeFlag, "dht-max-record-age", 0, "Maximum age for DHT records (0s uses library default)")
runCmd.Flags().IntVar(&dhtLookupLimitFlag, "dht-lookup-limit", 0, "Maximum number of providers to query from the DHT (0 uses default 20)")
runCmd.Flags().IntVar(&discoveryConcurrencyFlag, "discovery-concurrency", 0, "Max concurrent catalog fetches during discovery (0 uses default 10)")
- runCmd.Flags().DurationVar(&policySyncIntervalFlag, "policy-sync-interval", 1*time.Hour, "Interval for syncing mesh policy from the control plane")
+ runCmd.Flags().DurationVar(&controlPlaneSyncIntervalFlag, "control-plane-sync-interval", node.DefaultControlPlaneSyncInterval, "How often signing keys, bans, router addresses and mesh policy are pulled from the control plane")
runCmd.Flags().DurationVar(&backendProbeTimeoutFlag, "backend-probe-timeout", 0, "Timeout for probing a command-spawned service backend before advertising it (0 uses default 2s); raise this for backends with slower cold-start times")
rootCmd.PersistentFlags().StringVar(&controlPlaneAddr, "control-plane", "", "Control plane URL")
rootCmd.PersistentFlags().BoolVar(&insecureControlPlaneFlag, "insecure-control-plane", false, "Accept a plaintext http:// control plane URL to a non-loopback host (whoever answers it becomes this node's trust root; only for networks you already trust)")
diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.go
new file mode 100644
index 00000000..7316e620
--- /dev/null
+++ b/internal/node/controlplane_sync.go
@@ -0,0 +1,226 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "math/rand"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+// The node reads three things from the control plane while it runs: the
+// signing keys it verifies every credential against (/keys), the ban set and
+// router addresses (/info), and the mesh policy (/policies). They are pulled
+// together, by one loop, on one interval. Gossip events are notifications
+// that bring the next pull forward; they are never the only way state
+// arrives, because a once-published event with no replay is missed by any
+// node that was down, partitioned, or simply enrolled later.
+
+// syncControlPlane is the node's single pull from the control plane. Each
+// part is attempted even when another fails, so a policy outage does not
+// stop a key rotation from landing; the errors are reported together.
+func (n *SamNode) syncControlPlane(ctx context.Context) error {
+ if n.Store == nil {
+ return errors.New("node has no store")
+ }
+ controlPlaneURL, err := n.Store.LoadControlPlaneURL()
+ if err != nil || controlPlaneURL == "" {
+ return errors.New("control plane URL not found in store")
+ }
+ var errs []error
+ if err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {
+ errs = append(errs, fmt.Errorf("keys: %w", err))
+ }
+ if err := n.syncMeshInfo(ctx, controlPlaneURL); err != nil {
+ errs = append(errs, fmt.Errorf("info: %w", err))
+ }
+ if err := n.syncMeshPolicy(ctx); err != nil {
+ errs = append(errs, fmt.Errorf("policy: %w", err))
+ }
+ return errors.Join(errs...)
+}
+
+// syncTrustedKeys replaces the trust set with the control plane's current
+// /keys answer and persists it. Enrollment hands out only the newest key, so
+// this is how a node learns the key still in its rotation grace period (which
+// routers and peers may still be signed by) and any successor it did not hear
+// about. Verification against the keys already trusted keeps whoever answers
+// the URL from becoming the trust root; an empty answer is a failure so the
+// set is never wiped.
+func (n *SamNode) syncTrustedKeys(ctx context.Context, controlPlaneURL string) error {
+ n.keysMu.RLock()
+ existing := append([]TrustedKey(nil), n.trustedKeys...)
+ n.keysMu.RUnlock()
+ if len(existing) == 0 {
+ return errors.New("no trusted control plane keys to verify /keys against")
+ }
+ keys, err := FetchControlPlaneKeys(ctx, controlPlaneURL, publicKeysOf(existing))
+ if err != nil {
+ return err
+ }
+ if len(keys) == 0 {
+ return errors.New("/keys returned no keys")
+ }
+ merged := mergeTrustedKeys(existing, keys, time.Now())
+ n.keysMu.Lock()
+ n.trustedKeys = merged
+ snapshot := append([]TrustedKey(nil), merged...)
+ n.keysMu.Unlock()
+ n.persistTrustedKeys(snapshot)
+ logger.Debugf("Synced %d valid control plane keys", len(merged))
+ return nil
+}
+
+// syncMeshInfo reads /info: the router addresses are persisted for the next
+// start, and the ban set is reconciled against the revocation cache, which is
+// how a running node learns a ban or an unban it got no event for.
+func (n *SamNode) syncMeshInfo(ctx context.Context, controlPlaneURL string) error {
+ // Taken before the request: a ban recorded after this instant cannot be
+ // in the answer, so its absence must not be read as an unban.
+ fetchedAt := time.Now()
+ info, err := FetchControlPlaneInfo(ctx, controlPlaneURL)
+ if err != nil {
+ return err
+ }
+ if len(info.RouterAddresses) > 0 {
+ pubKey, _, loadErr := n.Store.LoadMeshConfig()
+ switch {
+ case loadErr != nil:
+ logger.Warnf("Failed to load mesh config; not persisting router addresses: %v", loadErr)
+ case len(pubKey) > 0:
+ if saveErr := n.Store.SaveMeshConfig(pubKey, info.RouterAddresses); saveErr != nil {
+ logger.Warnf("Failed to persist router addresses: %v", saveErr)
+ }
+ }
+ }
+ n.reconcileBannedPeers(info.GetBannedPeerIds(), fetchedAt)
+ return nil
+}
+
+// reconcileBannedPeers makes the revocation cache match the control plane's
+// ban set as of fetchedAt. Bans recorded at or after fetchedAt are kept even
+// when absent from the answer: the answer predates them and cannot speak to
+// them (see the router's reconcileBannedPeers for the same rule).
+func (n *SamNode) reconcileBannedPeers(peerIDs []string, fetchedAt time.Time) {
+ if n.revokedPeers == nil {
+ return
+ }
+ banned := make(map[string]peer.ID, len(peerIDs))
+ for _, id := range peerIDs {
+ p, err := peer.Decode(id)
+ if err != nil {
+ logger.Warnf("Ignoring undecodable banned peer ID %q from the control plane: %v", id, err)
+ continue
+ }
+ banned[p.String()] = p
+ }
+ fetchedAtMs := fetchedAt.UnixMilli()
+ for _, key := range n.revokedPeers.Keys() {
+ if _, still := banned[key]; still {
+ continue
+ }
+ if bannedAt, ok := n.revokedPeers.Peek(key); ok && bannedAt >= fetchedAtMs {
+ continue
+ }
+ logger.Infof("Peer %s is no longer banned by the control plane; lifting the local ban", key)
+ n.revokedPeers.Remove(key)
+ }
+ for key, p := range banned {
+ if n.revokedPeers.Contains(key) {
+ continue
+ }
+ logger.Infow("peer banned by the control plane", "event", meshEventBanned, "peer", key)
+ n.banPeer(p, fetchedAtMs)
+ }
+}
+
+// banPeer records the ban and evicts the peer: the cache entry is what the
+// gater and the auth path consult, and dropping the admission is what stops
+// the relay ACL from honouring a session that was already established.
+func (n *SamNode) banPeer(p peer.ID, bannedAtMs int64) {
+ if n.revokedPeers != nil {
+ n.revokedPeers.Add(p.String(), bannedAtMs)
+ }
+ n.authPeers.Delete(p)
+ if n.Host != nil {
+ _ = n.Host.Network().ClosePeer(p)
+ }
+}
+
+// triggerControlPlaneSync asks the sync loop to run soon, after a random
+// delay of up to the configured jitter so a fleet told at once does not hit
+// the control plane at once. A pending request is not duplicated.
+func (n *SamNode) triggerControlPlaneSync() {
+ select {
+ case n.controlPlaneSyncTrigger <- struct{}{}:
+ default:
+ }
+}
+
+// startControlPlaneSyncLoop pulls once shortly after start, then every
+// interval and whenever triggered. Failures are logged at Warn once and at
+// Debug while they persist, with an Info on recovery.
+func (n *SamNode) startControlPlaneSyncLoop(ctx context.Context, interval time.Duration) {
+ if interval <= 0 || n.Store == nil {
+ return
+ }
+ go func() {
+ timer := time.NewTimer(2 * time.Second)
+ defer timer.Stop()
+ failures := 0
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-timer.C:
+ case <-n.controlPlaneSyncTrigger:
+ if !timer.Stop() {
+ select {
+ case <-timer.C:
+ default:
+ }
+ }
+ jitter := n.config.ControlPlaneSyncJitter
+ if jitter > 0 {
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(time.Duration(rand.Int63n(int64(jitter)))):
+ }
+ }
+ }
+ timer.Reset(interval)
+
+ err := n.syncControlPlane(ctx)
+ switch {
+ case err != nil && failures == 0:
+ logger.Warnf("Control plane sync failed: %v", err)
+ case err != nil:
+ logger.Debugf("Control plane sync still failing (%d consecutive): %v", failures+1, err)
+ case failures > 0:
+ logger.Infof("Control plane sync recovered after %d failures", failures)
+ }
+ if err != nil {
+ failures++
+ } else {
+ failures = 0
+ }
+ }
+ }()
+}
diff --git a/internal/node/controlplane_test.go b/internal/node/controlplane_test.go
index d40c13ed..a45b3812 100644
--- a/internal/node/controlplane_test.go
+++ b/internal/node/controlplane_test.go
@@ -28,6 +28,8 @@ import (
"time"
"github.com/google/sam/api"
+ lru "github.com/hashicorp/golang-lru/v2"
+ "github.com/libp2p/go-libp2p/core/peer"
"google.golang.org/protobuf/proto"
)
@@ -60,6 +62,263 @@ func TestMergeTrustedKeys(t *testing.T) {
}
}
+// mustGenerateKey is an ed25519 key pair or a failed test.
+func mustGenerateKey(t *testing.T) (ed25519.PublicKey, ed25519.PrivateKey) {
+ t.Helper()
+ pub, priv, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatalf("ed25519.GenerateKey: %v", err)
+ }
+ return pub, priv
+}
+
+// keysServer serves /keys with the given set, signed by the matching private
+// keys; a nil signer leaves the answer unsigned.
+func keysServer(t *testing.T, pubs []ed25519.PublicKey, privs []ed25519.PrivateKey) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(keysHandler(t, pubs, privs))
+}
+
+func keysHandler(t *testing.T, pubs []ed25519.PublicKey, privs []ed25519.PrivateKey) http.HandlerFunc {
+ t.Helper()
+ return func(w http.ResponseWriter, r *http.Request) {
+ resp := &api.KeysResponse{}
+ for _, p := range pubs {
+ resp.PublicKeys = append(resp.PublicKeys, p)
+ }
+ if privs != nil {
+ if err := api.SignKeysResponse(resp, privs, time.Now()); err != nil {
+ t.Errorf("SignKeysResponse: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ }
+ protoHandler(t, resp)(w, r)
+ }
+}
+
+// protoHandler answers with msg; a marshal failure fails the test rather
+// than handing the client an empty body it might accept.
+func protoHandler(t *testing.T, msg proto.Message) http.HandlerFunc {
+ t.Helper()
+ return func(w http.ResponseWriter, r *http.Request) {
+ data, err := proto.Marshal(msg)
+ if err != nil {
+ t.Errorf("proto.Marshal(%T): %v", msg, err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ w.Header().Set("Content-Type", "application/x-protobuf")
+ if _, err := w.Write(data); err != nil {
+ t.Errorf("write %T: %v", msg, err)
+ }
+ }
+}
+
+func TestSyncTrustedKeys(t *testing.T) {
+ retiredPub, retiredPriv := mustGenerateKey(t)
+ currentPub, currentPriv := mustGenerateKey(t)
+ strangerPub, strangerPriv := mustGenerateKey(t)
+
+ newNode := func(t *testing.T) *SamNode {
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = store.Close() })
+ return &SamNode{Store: store, trustedKeys: []TrustedKey{{Key: currentPub, ReceivedAt: time.Now()}}}
+ }
+
+ t.Run("adopts the set a trusted key vouches for", func(t *testing.T) {
+ srv := keysServer(t, []ed25519.PublicKey{retiredPub, currentPub}, []ed25519.PrivateKey{retiredPriv, currentPriv})
+ defer srv.Close()
+ n := newNode(t)
+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil {
+ t.Fatalf("syncTrustedKeys: %v", err)
+ }
+ if len(n.trustedKeys) != 2 || !containsTrustedKey(n.trustedKeys, retiredPub) || !containsTrustedKey(n.trustedKeys, currentPub) {
+ t.Errorf("trust set = %d keys, want retired and current", len(n.trustedKeys))
+ }
+ stored, err := n.Store.LoadTrustedKeys()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(stored) != 2 {
+ t.Errorf("persisted %d keys, want 2", len(stored))
+ }
+ })
+
+ t.Run("rejects a set no trusted key signed", func(t *testing.T) {
+ srv := keysServer(t, []ed25519.PublicKey{strangerPub}, []ed25519.PrivateKey{strangerPriv})
+ defer srv.Close()
+ n := newNode(t)
+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {
+ t.Fatal("a /keys answer signed only by an unknown key must not be adopted")
+ }
+ if len(n.trustedKeys) != 1 || !n.trustedKeys[0].Key.Equal(currentPub) {
+ t.Errorf("trust set changed on a rejected answer: %d keys", len(n.trustedKeys))
+ }
+ })
+
+ t.Run("rejects an unsigned set", func(t *testing.T) {
+ srv := keysServer(t, []ed25519.PublicKey{currentPub, strangerPub}, nil)
+ defer srv.Close()
+ n := newNode(t)
+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {
+ t.Fatal("an unsigned /keys answer must not be adopted")
+ }
+ if containsTrustedKey(n.trustedKeys, strangerPub) {
+ t.Error("unsigned answer widened the trust set")
+ }
+ })
+
+ t.Run("nothing trusted yet is an error, not a wipe", func(t *testing.T) {
+ srv := keysServer(t, []ed25519.PublicKey{currentPub}, []ed25519.PrivateKey{currentPriv})
+ defer srv.Close()
+ n := newNode(t)
+ n.trustedKeys = nil
+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {
+ t.Fatal("with no trusted key there is nothing to verify /keys against")
+ }
+ })
+}
+
+// A ban the control plane holds is applied, one it no longer holds is lifted,
+// and one recorded after the answer was requested is left alone: the answer
+// predates it and cannot speak to it.
+func TestReconcileBannedPeers(t *testing.T) {
+ stillBanned := randomPeerID(t)
+ unbanned := randomPeerID(t)
+ newlyBanned := randomPeerID(t)
+ bannedAfterFetch := randomPeerID(t)
+
+ cache, err := lru.New[string, int64](10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ n := &SamNode{revokedPeers: cache}
+ fetchedAt := time.Now()
+ n.revokedPeers.Add(stillBanned.String(), fetchedAt.Add(-time.Hour).UnixMilli())
+ n.revokedPeers.Add(unbanned.String(), fetchedAt.Add(-time.Hour).UnixMilli())
+ n.revokedPeers.Add(bannedAfterFetch.String(), fetchedAt.UnixMilli())
+
+ n.reconcileBannedPeers([]string{stillBanned.String(), newlyBanned.String(), "not-a-peer-id"}, fetchedAt)
+
+ for _, want := range []peer.ID{stillBanned, newlyBanned, bannedAfterFetch} {
+ if !n.revokedPeers.Contains(want.String()) {
+ t.Errorf("%s must be banned after reconciliation", want)
+ }
+ }
+ if n.revokedPeers.Contains(unbanned.String()) {
+ t.Error("a peer absent from the control plane's ban set must be unbanned")
+ }
+}
+
+// One pull brings keys, bans and policy together, and a failing part does
+// not stop the others from landing.
+func TestSyncControlPlane(t *testing.T) {
+ oldPub, oldPriv := mustGenerateKey(t)
+ newPub, newPriv := mustGenerateKey(t)
+ banned := randomPeerID(t)
+
+ mux := http.NewServeMux()
+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{oldPub, newPub}, []ed25519.PrivateKey{oldPriv, newPriv}))
+ mux.HandleFunc("/info", protoHandler(t, &api.ControlPlaneInfoResponse{
+ RouterAddresses: []string{"/ip4/10.0.0.1/tcp/4501/p2p/" + randomPeerID(t).String()},
+ BannedPeerIds: []string{banned.String()},
+ }))
+ mux.HandleFunc("/policies", func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "policy store down", http.StatusInternalServerError)
+ })
+ srv := httptest.NewServer(mux)
+ defer srv.Close()
+
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = store.Close() }()
+ if err := store.SaveControlPlaneURL(srv.URL); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SaveMeshConfig(oldPub, nil); err != nil {
+ t.Fatal(err)
+ }
+ cache, err := lru.New[string, int64](10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ n := &SamNode{Store: store, revokedPeers: cache, trustedKeys: []TrustedKey{{Key: oldPub, ReceivedAt: time.Now()}}}
+ n.SetIdentityCache([]byte("identity"))
+
+ err = n.syncControlPlane(context.Background())
+ if err == nil || !strings.Contains(err.Error(), "policy") {
+ t.Fatalf("syncControlPlane = %v, want the policy failure reported", err)
+ }
+ if !containsTrustedKey(n.trustedKeys, newPub) {
+ t.Error("rotated key not learned although /keys answered")
+ }
+ if !n.revokedPeers.Contains(banned.String()) {
+ t.Error("ban not applied although /info answered")
+ }
+ _, addrs, err := store.LoadMeshConfig()
+ if err != nil {
+ t.Fatalf("LoadMeshConfig: %v", err)
+ }
+ if len(addrs) != 1 {
+ t.Errorf("router addresses not persisted: got %v, want 1", addrs)
+ }
+}
+
+// The loop is what turns a missed gossip event into a delay rather than a
+// permanent split: a running node picks the successor key up on its own, and
+// a trigger brings the next pull forward.
+func TestControlPlaneSyncLoop(t *testing.T) {
+ oldPub, oldPriv := mustGenerateKey(t)
+ newPub, newPriv := mustGenerateKey(t)
+
+ mux := http.NewServeMux()
+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{oldPub, newPub}, []ed25519.PrivateKey{oldPriv, newPriv}))
+ mux.HandleFunc("/info", protoHandler(t, &api.ControlPlaneInfoResponse{}))
+ mux.HandleFunc("/policies", protoHandler(t, &api.PolicyConfigGetResponse{}))
+ srv := httptest.NewServer(mux)
+ defer srv.Close()
+
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = store.Close() }()
+ if err := store.SaveControlPlaneURL(srv.URL); err != nil {
+ t.Fatal(err)
+ }
+
+ n := &SamNode{
+ Store: store,
+ trustedKeys: []TrustedKey{{Key: oldPub, ReceivedAt: time.Now()}},
+ controlPlaneSyncTrigger: make(chan struct{}, 1),
+ config: Options{ControlPlaneSyncJitter: time.Millisecond},
+ }
+ n.SetIdentityCache([]byte("identity"))
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ // A long interval: only the trigger can make the first pull happen in time.
+ n.startControlPlaneSyncLoop(ctx, time.Hour)
+ n.triggerControlPlaneSync()
+
+ deadline := time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadline) {
+ n.keysMu.RLock()
+ learned := containsTrustedKey(n.trustedKeys, newPub)
+ n.keysMu.RUnlock()
+ if learned {
+ return
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ t.Fatal("triggered control plane sync never picked up the rotated key")
+}
+
func TestFetchControlPlaneInfo(t *testing.T) {
expectedInfo := &api.ControlPlaneInfoResponse{
RouterAddresses: []string{"/ip4/127.0.0.1/tcp/4001"},
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index bbed85eb..b1f3efd7 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -72,6 +72,19 @@ func (n *SamNode) Enroll(ctx context.Context, controlPlaneURL string, jwt string
return n.connectToRouters(ctx, enrollResp.RouterAddresses)
}
+// adoptEnrolledKeys widens the trust set from the single key an enrollment
+// response carries to every key the control plane currently signs with.
+// The router handshake that follows verifies the router's own biscuit, and
+// right after a rotation that is still signed by the retiring key; with only
+// the newest key loaded the node would refuse every router. Best effort: a
+// control plane too old to serve /keys still enrolls with one key.
+func (n *SamNode) adoptEnrolledKeys(ctx context.Context, controlPlaneURL string, newest []byte) {
+ n.addTrustedKey(ed25519.PublicKey(newest))
+ if err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {
+ logger.Warnf("Could not fetch the control plane's full key set after enrollment (continuing with the enrollment key only): %v", err)
+ }
+}
+
// enrollHTTP performs the HTTP half of enrollment for an explicit peer
// identity, so it can run before the libp2p host exists (startup recovery).
// privKey signs the proof-of-possession challenge; peerID must be its own.
@@ -123,7 +136,12 @@ func (n *SamNode) enrollHTTP(ctx context.Context, controlPlaneURL, jwt string, p
}
}()
- return n.processEnrollResponse(resp)
+ enrollResp, err := n.processEnrollResponse(resp)
+ if err != nil {
+ return nil, err
+ }
+ n.adoptEnrolledKeys(ctx, controlPlaneURL, enrollResp.ControlPlanePublicKey)
+ return enrollResp, nil
}
// ReEnrollWithRefreshToken re-enrolls over HTTP with a JWT from the stored
@@ -191,8 +209,6 @@ func (n *SamNode) processEnrollResponse(resp *http.Response) (*api.EnrollRespons
return nil, fmt.Errorf("failed to save mesh config: %v", err)
}
- n.addTrustedKey(ed25519.PublicKey(enrollResp.ControlPlanePublicKey))
-
return &enrollResp, nil
}
@@ -387,7 +403,7 @@ func (n *SamNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, b
return fmt.Errorf("failed to save mesh config: %v", err)
}
- n.addTrustedKey(ed25519.PublicKey(enrollResp.ControlPlanePublicKey))
+ n.adoptEnrolledKeys(ctx, controlPlaneURL, enrollResp.ControlPlanePublicKey)
// Connect and Auth to router after enrollment to join the mesh
if len(enrollResp.RouterAddresses) == 0 {
diff --git a/internal/node/enroll_test.go b/internal/node/enroll_test.go
index d104d72a..1facf273 100644
--- a/internal/node/enroll_test.go
+++ b/internal/node/enroll_test.go
@@ -532,6 +532,90 @@ func TestEnroll_InvalidControlPlanePublicKeySize(t *testing.T) {
}
}
+// TestEnrollTrustsRouterSignedByGraceKey is the hub testnet outage of
+// 2026-09-20: the control plane had rotated its signing key, the routers
+// still presented biscuits signed by the retiring key (legitimately in its
+// grace period), and the enroll response carries only the newest key. A
+// fresh node must fetch the full set from /keys before the router handshake
+// or it refuses every router and enrollment fails.
+func TestEnrollTrustsRouterSignedByGraceKey(t *testing.T) {
+ retiredPub, retiredPriv := mustGenerateKey(t)
+ currentPub, currentPriv := mustGenerateKey(t)
+
+ routerAddr := startMockRouterWithKey(t, retiredPriv, api.RoleRouter)
+
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = store.Close() }()
+ privKey := GetOrGenerateKey(store)
+
+ mux := http.NewServeMux()
+ mux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {
+ body, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Errorf("read /register body: %v", err)
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ var req api.EnrollRequest
+ if err := proto.Unmarshal(body, &req); err != nil {
+ t.Errorf("decode /register body: %v", err)
+ http.Error(w, "bad request", http.StatusBadRequest)
+ return
+ }
+ peerID, err := peer.Decode(req.PeerId)
+ if err != nil {
+ t.Errorf("decode enrolling peer id %q: %v", req.PeerId, err)
+ http.Error(w, "bad peer id", http.StatusBadRequest)
+ return
+ }
+ protoHandler(t, &api.EnrollResponse{
+ BiscuitToken: mintRoleBiscuit(t, currentPriv, peerID, api.RoleNode),
+ ControlPlanePublicKey: currentPub,
+ RouterAddresses: []string{routerAddr},
+ Expiration: time.Now().Add(24 * time.Hour).Unix(),
+ })(w, r)
+ })
+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{retiredPub, currentPub}, []ed25519.PrivateKey{retiredPriv, currentPriv}))
+ cpSrv := httptest.NewServer(mux)
+ defer cpSrv.Close()
+
+ node, err := NewSamNode(Options{
+ PrivKey: privKey,
+ Store: store,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ AllowLoopback: true,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+ if err := node.Start(ctx); err != nil {
+ t.Fatalf("Start: %v", err)
+ }
+ defer func() { _ = node.Host.Close() }()
+
+ if err := node.Enroll(ctx, cpSrv.URL, "dummy-jwt"); err != nil {
+ t.Fatalf("Enroll must accept a router signed by the key still in grace: %v", err)
+ }
+
+ for _, want := range []ed25519.PublicKey{retiredPub, currentPub} {
+ if !containsTrustedKey(node.trustedKeys, want) {
+ t.Errorf("trust set %d keys, missing %x", len(node.trustedKeys), want)
+ }
+ }
+ stored, err := store.LoadTrustedKeys()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(stored) != 2 {
+ t.Errorf("persisted trust set has %d keys, want both", len(stored))
+ }
+}
+
func TestProcessEnrollResponse_Errors(t *testing.T) {
dir := t.TempDir()
store, err := NewStore(dir)
diff --git a/internal/node/node.go b/internal/node/node.go
index b80e2ae9..d00b10b6 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -89,6 +89,11 @@ const (
// Key pruning
KeyPruningInterval = 1 * time.Hour
+ // DefaultControlPlaneSyncInterval matches the router's --keys-sync-interval:
+ // well inside the control plane's default 1h key grace period, so a node
+ // has several chances to learn a successor key while its predecessor can
+ // still vouch for it.
+ DefaultControlPlaneSyncInterval = 5 * time.Minute
// Reprovide interval
ReprovideInterval = 5 * time.Minute
@@ -172,13 +177,14 @@ type SamNode struct {
BoundSocketPath string
AllowLoopback bool
- authSuccess chan struct{}
- authOnce sync.Once
- currentRelays []peer.AddrInfo
- reprovideTrigger chan struct{}
- BiscuitTimeout time.Duration
- cachedIdentity atomic.Value
- logger *golog.ZapEventLogger
+ authSuccess chan struct{}
+ authOnce sync.Once
+ currentRelays []peer.AddrInfo
+ reprovideTrigger chan struct{}
+ controlPlaneSyncTrigger chan struct{}
+ BiscuitTimeout time.Duration
+ cachedIdentity atomic.Value
+ logger *golog.ZapEventLogger
// metricsRegistry holds this node's state collector; see metricsHandler.
metricsOnce sync.Once
@@ -285,17 +291,18 @@ func NewSamNode(cfg Options) (*SamNode, error) {
}
node := &SamNode{
- config: cfg,
- Store: cfg.Store,
- trustedKeys: trustedKeys,
- peerLastEventTime: make(map[string]int64),
- authenticatedRouters: make(map[peer.ID]bool),
- nodeConfig: cfg.NodeConfig,
- AllowLoopback: cfg.AllowLoopback,
- authSuccess: make(chan struct{}),
- reprovideTrigger: make(chan struct{}, 1),
- BiscuitTimeout: cfg.BiscuitTimeout,
- logger: golog.Logger("sam-node"),
+ config: cfg,
+ Store: cfg.Store,
+ trustedKeys: trustedKeys,
+ peerLastEventTime: make(map[string]int64),
+ authenticatedRouters: make(map[peer.ID]bool),
+ nodeConfig: cfg.NodeConfig,
+ AllowLoopback: cfg.AllowLoopback,
+ authSuccess: make(chan struct{}),
+ reprovideTrigger: make(chan struct{}, 1),
+ controlPlaneSyncTrigger: make(chan struct{}, 1),
+ BiscuitTimeout: cfg.BiscuitTimeout,
+ logger: golog.Logger("sam-node"),
}
var err error
@@ -660,8 +667,9 @@ func (n *SamNode) Start(ctx context.Context) error {
// Periodically and on-demand reprovide registered services to the DHT
n.startReprovideLoop(ctx, ReprovideInterval)
- // Periodically sync mesh policy
- n.startPolicySyncLoop(ctx, n.config.PolicySyncInterval)
+ // Everything read from the control plane (keys, bans, router addresses,
+ // policy) in one loop; gossip events only bring the next pull forward.
+ n.startControlPlaneSyncLoop(ctx, n.config.ControlPlaneSyncInterval)
// Periodically self-report local services to the control plane, so an
// admin can see mesh-wide service topology.
@@ -1337,23 +1345,7 @@ func (n *SamNode) listenForControlPlaneEvents(ctx context.Context) {
n.handleKeyRotationEvent(&event)
case api.MeshEvent_POLICY_UPDATE:
logger.Infow("[Mesh Event] policy update received, triggering sync", "event", meshEventPolicyUpdate, "peer", msg.ReceivedFrom.String())
- go func() {
- maxJitter := n.config.PolicySyncJitter
- if maxJitter <= 0 {
- maxJitter = 10 * time.Second
- }
- jitter := time.Duration(rand.Int63n(int64(maxJitter)))
- timer := time.NewTimer(jitter)
- defer timer.Stop()
- select {
- case <-timer.C:
- case <-ctx.Done():
- return
- }
- if err := n.syncMeshPolicy(ctx); err != nil {
- logger.Warnf("Failed to sync mesh policy after event: %v", err)
- }
- }()
+ n.triggerControlPlaneSync()
}
}
}
@@ -1379,18 +1371,9 @@ func (n *SamNode) handleBannedEvent(event *api.MeshEvent) {
n.mu.Unlock()
logger.Infow("[Mesh Event] peer banned", "event", meshEventBanned, "peer", canonicalID)
-
- if n.revokedPeers != nil {
- n.revokedPeers.Add(canonicalID, event.Timestamp)
- }
- // Drop any prior admission, otherwise the relay ACL keeps honouring it.
- // The cache entry above is not written to disk: a restarted node picks the
- // ban back up from the control plane's ban set in /info (see
- // SyncMeshConfig), which is also how an unban reaches it.
- n.authPeers.Delete(p)
- if n.Host != nil {
- _ = n.Host.Network().ClosePeer(p)
- }
+ // Not written to disk: a restarted node picks the ban back up from the
+ // control plane's ban set in /info, which is also how an unban reaches it.
+ n.banPeer(p, event.Timestamp)
}
func containsTrustedKey(keys []TrustedKey, key ed25519.PublicKey) bool {
@@ -2368,35 +2351,3 @@ func (n *SamNode) startCatalogReportLoop(ctx context.Context, initialDelay, inte
}
}()
}
-
-func (n *SamNode) startPolicySyncLoop(ctx context.Context, interval time.Duration) {
- if interval <= 0 {
- interval = 1 * time.Hour
- }
-
- go func() {
- // Run initial sync after a short delay
- select {
- case <-ctx.Done():
- return
- case <-time.After(2 * time.Second):
- if err := n.syncMeshPolicy(ctx); err != nil {
- logger.Warnf("Initial mesh policy sync failed: %v", err)
- }
- }
-
- ticker := time.NewTicker(interval)
- defer ticker.Stop()
-
- for {
- select {
- case <-ctx.Done():
- return
- case <-ticker.C:
- if err := n.syncMeshPolicy(ctx); err != nil {
- logger.Warnf("Periodic mesh policy sync failed: %v", err)
- }
- }
- }
- }()
-}
diff --git a/internal/node/options.go b/internal/node/options.go
index a1564b1e..3c25bbbc 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -74,10 +74,13 @@ type Options struct {
DiscoveryConcurrency int
// RequiredRole restricts enrollment and startup to only accept tokens containing this role.
RequiredRole string
- // PolicySyncInterval specifies how often the node syncs the mesh policy from the control plane.
- PolicySyncInterval time.Duration
- // PolicySyncJitter specifies the maximum jitter delay when scheduling policy syncs on event broadcasts.
- PolicySyncJitter time.Duration
+ // ControlPlaneSyncInterval is how often the node pulls what it reads from
+ // the control plane: signing keys, ban set and router addresses, mesh
+ // policy. Zero uses the default; negative disables the loop.
+ ControlPlaneSyncInterval time.Duration
+ // ControlPlaneSyncJitter is the maximum random delay before a sync that a
+ // gossip event asked for, so a fleet told at once does not pull at once.
+ ControlPlaneSyncJitter time.Duration
// BackendProbeTimeout bounds how long a command-spawned service backend
// (sam-node.yaml's `command`, spawned as a local subprocess) is given to
// answer before the service is registered but withheld from
@@ -143,8 +146,8 @@ func (o *Options) Default() {
if o.RequiredRole == "" {
o.RequiredRole = api.RoleNode
}
- if o.PolicySyncInterval == 0 {
- o.PolicySyncInterval = 1 * time.Hour
+ if o.ControlPlaneSyncInterval == 0 {
+ o.ControlPlaneSyncInterval = DefaultControlPlaneSyncInterval
}
if o.CatalogReportInterval <= 0 {
o.CatalogReportInterval = 1 * time.Minute
@@ -152,8 +155,8 @@ func (o *Options) Default() {
if o.CatalogReportInitialDelay <= 0 {
o.CatalogReportInitialDelay = 5 * time.Second
}
- if o.PolicySyncJitter <= 0 {
- o.PolicySyncJitter = 10 * time.Second
+ if o.ControlPlaneSyncJitter <= 0 {
+ o.ControlPlaneSyncJitter = 10 * time.Second
}
if o.NodeConfig == nil {
o.NodeConfig = &NodeConfigComplete{}
diff --git a/site/content/docs/concepts/authorization.md b/site/content/docs/concepts/authorization.md
index 3094f740..37542e19 100644
--- a/site/content/docs/concepts/authorization.md
+++ b/site/content/docs/concepts/authorization.md
@@ -93,8 +93,8 @@ becomes `granted_service_exact("mcp", "calculator")`, `mcp://*` becomes
`granted_service_suffix("mcp", ".internal")`. Wildcards keep their dot, so
`*.acme.example` matches `svc.acme.example` but not `evil-acme.example`.
-Nodes also fetch the policy themselves, every `--policy-sync-interval` (one
-hour by default), and compile it into rules such as
+Nodes also fetch the policy themselves, every `--control-plane-sync-interval`
+(five minutes by default), and compile it into rules such as
`role("developer") <- group("eng")` and
`granted_service_exact("mcp", "calculator") <- role("developer")`. When a
node verifies a credential, these rules run against the identity facts in it.
diff --git a/site/content/docs/concepts/identity.md b/site/content/docs/concepts/identity.md
index 1b9a6c6d..a31b3f6f 100644
--- a/site/content/docs/concepts/identity.md
+++ b/site/content/docs/concepts/identity.md
@@ -129,10 +129,12 @@ The control plane rotates its signing key every `--key-rotation-interval`
(24 hours by default). The previous key stays valid for `--key-grace-period`
(1 hour by default) and is then retired. `GET /keys` returns the current set
of keys, signed by each key in the set. Routers poll it every
-`--keys-sync-interval`. Nodes fetch it at start and when they receive a
-rotation event. Both accept a new set only if one of its signatures verifies
-under a key they already trust. The first key comes from enrollment, and
-each later key is vouched for by the key it replaces.
+`--keys-sync-interval`; nodes fetch it at enrollment and then every
+`--control-plane-sync-interval`, together with the ban set and the mesh
+policy. A rotation event only brings the next pull forward. Both accept a
+new set only if one of its signatures verifies under a key they already
+trust. The first key comes from enrollment, and each later key is vouched
+for by the key it replaces.
Nobody can verify a credential signed by a retired key, including the
control plane. A node that was offline for a whole grace period therefore
diff --git a/site/content/docs/reference/policy.md b/site/content/docs/reference/policy.md
index 6f92e32e..82a38ea6 100644
--- a/site/content/docs/reference/policy.md
+++ b/site/content/docs/reference/policy.md
@@ -145,7 +145,7 @@ must be on the role itself.
At enrollment and at refresh, the control plane resolves the identity's roles
from the bindings and mints into the credential one `role()` fact per role
and the compiled `granted_*` facts of every role. Nodes also fetch the policy
-(`--policy-sync-interval`, 1 hour) and compile it into rules such as
+(`--control-plane-sync-interval`, 5 minutes) and compile it into rules such as
`role("developer") <- group("engineering")` and
`granted_service_exact("mcp","code-reviewer") <- role("developer")`. These
rules run at verification time. Additions therefore reach nodes within the
diff --git a/site/content/docs/reference/sam-node.md b/site/content/docs/reference/sam-node.md
index 9bd3bd9e..2668ebe7 100644
--- a/site/content/docs/reference/sam-node.md
+++ b/site/content/docs/reference/sam-node.md
@@ -111,7 +111,7 @@ enroll.
| `--discovery-concurrency` | `10` | Concurrent catalog fetches during discovery. |
| `--dht-provider-addr-ttl`, `--dht-max-record-age` | library defaults | DHT record lifetimes. |
| `--backend-probe-timeout` | `2s` | How long a service backend may take to answer before the node declines to advertise it. Raise it for subprocesses that start slowly. |
-| `--policy-sync-interval` | `1h` | How often the mesh policy is fetched again from the control plane. |
+| `--control-plane-sync-interval` | `5m` | How often signing keys, the ban set, router addresses and the mesh policy are pulled from the control plane. |
| `--key-grace-period` | `24h` | How long a rotated-out control plane key is still accepted for verifying peers. |
| `--monitor-bootstrap` | `2m` | Delay before the router-connection monitor starts. |
| `--monitor-interval` | `1m` | How often the monitor checks that a router is connected. |