Skip to content

Agent Mesh

node: sync every 15m by default, spread the fleet's pulls

Antonio Ojeaa36c65af54f0
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index 1ff5d893..ce988382 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -875,7 +875,7 @@ func main() {
runCmd.Flags().DurationVar(&dhtMaxRecordAgeFlag, "dht-max-record-age", 0, "Maximum age for DHT records (0s uses library default)")
runCmd.Flags().IntVar(&dhtLookupLimitFlag, "dht-lookup-limit", 0, "Maximum number of providers to query from the DHT (0 uses default 20)")
runCmd.Flags().IntVar(&discoveryConcurrencyFlag, "discovery-concurrency", 0, "Max concurrent catalog fetches during discovery (0 uses default 10)")
- runCmd.Flags().DurationVar(&controlPlaneSyncIntervalFlag, "control-plane-sync-interval", node.DefaultControlPlaneSyncInterval, "How often signing keys, bans, router addresses and mesh policy are pulled from the control plane")
+ runCmd.Flags().DurationVar(&controlPlaneSyncIntervalFlag, "control-plane-sync-interval", node.DefaultControlPlaneSyncInterval, "How often signing keys, bans, router addresses and mesh policy are pulled from the control plane. Keep it well below the control plane's --key-grace-period; raise it on large meshes.")
runCmd.Flags().DurationVar(&backendProbeTimeoutFlag, "backend-probe-timeout", 0, "Timeout for probing a command-spawned service backend before advertising it (0 uses default 2s); raise this for backends with slower cold-start times")
rootCmd.PersistentFlags().StringVar(&controlPlaneAddr, "control-plane", "", "Control plane URL")
rootCmd.PersistentFlags().BoolVar(&insecureControlPlaneFlag, "insecure-control-plane", false, "Accept a plaintext http:// control plane URL to a non-loopback host (whoever answers it becomes this node's trust root; only for networks you already trust)")
diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.go
index 7316e620..619e52d2 100644
--- a/internal/node/controlplane_sync.go
+++ b/internal/node/controlplane_sync.go
@@ -174,8 +174,11 @@ func (n *SamNode) triggerControlPlaneSync() {
}
// startControlPlaneSyncLoop pulls once shortly after start, then every
-// interval and whenever triggered. Failures are logged at Warn once and at
-// Debug while they persist, with an Info on recovery.
+// interval and whenever triggered. Each periodic wait is stretched by up to a
+// tenth of the interval and each trigger delayed by up to the configured
+// jitter, so a fleet started or notified together does not pull together.
+// Failures are logged at Warn once and at Debug while they persist, with an
+// Info on recovery.
func (n *SamNode) startControlPlaneSyncLoop(ctx context.Context, interval time.Duration) {
if interval <= 0 || n.Store == nil {
return
@@ -196,8 +199,7 @@ func (n *SamNode) startControlPlaneSyncLoop(ctx context.Context, interval time.D
default:
}
}
- jitter := n.config.ControlPlaneSyncJitter
- if jitter > 0 {
+ if jitter := n.config.ControlPlaneSyncJitter; jitter > 0 {
select {
case <-ctx.Done():
return
@@ -205,7 +207,7 @@ func (n *SamNode) startControlPlaneSyncLoop(ctx context.Context, interval time.D
}
}
}
- timer.Reset(interval)
+ timer.Reset(interval + time.Duration(rand.Int63n(int64(interval/10)+1)))
err := n.syncControlPlane(ctx)
switch {
diff --git a/internal/node/node.go b/internal/node/node.go
index d00b10b6..9c4109fb 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -89,11 +89,14 @@ const (
// Key pruning
KeyPruningInterval = 1 * time.Hour
- // DefaultControlPlaneSyncInterval matches the router's --keys-sync-interval:
- // well inside the control plane's default 1h key grace period, so a node
- // has several chances to learn a successor key while its predecessor can
- // still vouch for it.
- DefaultControlPlaneSyncInterval = 5 * time.Minute
+ // DefaultControlPlaneSyncInterval bounds how long a running node can lag
+ // the control plane on keys, bans and policy when no gossip event reaches
+ // it. It must sit well inside the control plane's --key-grace-period
+ // (1h by default): a successor key can only be adopted while the key it
+ // replaces still vouches for it, and one attempt per window is a race.
+ // Four attempts per default window is the trade against load on large
+ // meshes, where an operator raising the grace period can raise this too.
+ DefaultControlPlaneSyncInterval = 15 * time.Minute
// Reprovide interval
ReprovideInterval = 5 * time.Minute
diff --git a/internal/node/options.go b/internal/node/options.go
index 3c25bbbc..40e7ef79 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -80,6 +80,9 @@ type Options struct {
ControlPlaneSyncInterval time.Duration
// ControlPlaneSyncJitter is the maximum random delay before a sync that a
// gossip event asked for, so a fleet told at once does not pull at once.
+ // Zero uses a tenth of ControlPlaneSyncInterval: the spread has to grow
+ // with the fleet's cadence, or a policy update on a large mesh is a
+ // stampede.
ControlPlaneSyncJitter time.Duration
// BackendProbeTimeout bounds how long a command-spawned service backend
// (sam-node.yaml's `command`, spawned as a local subprocess) is given to
@@ -155,8 +158,8 @@ func (o *Options) Default() {
if o.CatalogReportInitialDelay <= 0 {
o.CatalogReportInitialDelay = 5 * time.Second
}
- if o.ControlPlaneSyncJitter <= 0 {
- o.ControlPlaneSyncJitter = 10 * time.Second
+ if o.ControlPlaneSyncJitter <= 0 && o.ControlPlaneSyncInterval > 0 {
+ o.ControlPlaneSyncJitter = o.ControlPlaneSyncInterval / 10
}
if o.NodeConfig == nil {
o.NodeConfig = &NodeConfigComplete{}
diff --git a/site/content/docs/concepts/authorization.md b/site/content/docs/concepts/authorization.md
index 37542e19..6de4c2ad 100644
--- a/site/content/docs/concepts/authorization.md
+++ b/site/content/docs/concepts/authorization.md
@@ -94,7 +94,8 @@ becomes `granted_service_exact("mcp", "calculator")`, `mcp://*` becomes
`*.acme.example` matches `svc.acme.example` but not `evil-acme.example`.
Nodes also fetch the policy themselves, every `--control-plane-sync-interval`
-(five minutes by default), and compile it into rules such as
+(15 minutes by default, sooner when a policy update event reaches them), and
+compile it into rules such as
`role("developer") <- group("eng")` and
`granted_service_exact("mcp", "calculator") <- role("developer")`. When a
node verifies a credential, these rules run against the identity facts in it.
diff --git a/site/content/docs/reference/policy.md b/site/content/docs/reference/policy.md
index 82a38ea6..cc752b4f 100644
--- a/site/content/docs/reference/policy.md
+++ b/site/content/docs/reference/policy.md
@@ -145,7 +145,7 @@ must be on the role itself.
At enrollment and at refresh, the control plane resolves the identity's roles
from the bindings and mints into the credential one `role()` fact per role
and the compiled `granted_*` facts of every role. Nodes also fetch the policy
-(`--control-plane-sync-interval`, 5 minutes) and compile it into rules such as
+(`--control-plane-sync-interval`, 15 minutes) and compile it into rules such as
`role("developer") <- group("engineering")` and
`granted_service_exact("mcp","code-reviewer") <- role("developer")`. These
rules run at verification time. Additions therefore reach nodes within the
diff --git a/site/content/docs/reference/sam-node.md b/site/content/docs/reference/sam-node.md
index 2668ebe7..b6c93bb6 100644
--- a/site/content/docs/reference/sam-node.md
+++ b/site/content/docs/reference/sam-node.md
@@ -111,7 +111,7 @@ enroll.
| `--discovery-concurrency` | `10` | Concurrent catalog fetches during discovery. |
| `--dht-provider-addr-ttl`, `--dht-max-record-age` | library defaults | DHT record lifetimes. |
| `--backend-probe-timeout` | `2s` | How long a service backend may take to answer before the node declines to advertise it. Raise it for subprocesses that start slowly. |
-| `--control-plane-sync-interval` | `5m` | How often signing keys, the ban set, router addresses and the mesh policy are pulled from the control plane. |
+| `--control-plane-sync-interval` | `15m` | How often signing keys, the ban set, router addresses and the mesh policy are pulled from the control plane. Keep it well below the control plane's `--key-grace-period`; raise it on large meshes. |
| `--key-grace-period` | `24h` | How long a rotated-out control plane key is still accepted for verifying peers. |
| `--monitor-bootstrap` | `2m` | Delay before the router-connection monitor starts. |
| `--monitor-interval` | `1m` | How often the monitor checks that a router is connected. |