Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router: derive the per-source-IP connection cap from the high watermark
Antonio Ojeaafb61d125526
diff --git a/cmd/sam-router/main.go b/cmd/sam-router/main.goindex 1ac8dd50..4d578aad 100644--- a/cmd/sam-router/main.go+++ b/cmd/sam-router/main.go@@ -125,7 +125,7 @@ func main() {rootCmd.Flags().StringVar(&jwtPath, "jwt-path", "", "Path to file containing OIDC JWT token")rootCmd.Flags().StringVar(&keysPath, "keys-path", "router.key", "Path to save/load persistent private key")rootCmd.Flags().BoolVar(&allowLoopback, "allow-loopback", false, "Allow loopback and link-local addresses for discovery")- rootCmd.Flags().IntVar(&connsPerSourceIP, "conns-per-source-ip", 0, "Max inbound connections per source IP (0 keeps libp2p's default of 8); raise behind TLS-terminating proxies or NAT where many peers share source IPs")+ rootCmd.Flags().IntVar(&connsPerSourceIP, "conns-per-source-ip", 0, "Max inbound connections per source IP; 0 is a quarter of --high-watermark, so filling the router takes at least four addresses. Raise it where more members share one address")rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")rootCmd.Flags().DurationVar(&dhtProviderAddrTTL, "dht-provider-addr-ttl", router.DefaultDHTProviderAddrTTL, "How long a DHT provider record lives after its last announcement")rootCmd.Flags().DurationVar(&dhtMaxRecordAge, "dht-max-record-age", 0, "Maximum age for DHT records (0s uses library default)")diff --git a/internal/router/config.go b/internal/router/config.goindex 074c5063..1b3c93ad 100644--- a/internal/router/config.go+++ b/internal/router/config.go@@ -55,9 +55,15 @@ type Options struct {// HTTP requests arriving on the router's WebSocket listen addrs, letting a// single port carry both libp2p and REST traffic (single-port mode).HTTPFallbackHandler http.Handler- // ConnsPerSourceIP overrides libp2p's per-source-IP inbound connection- // cap (default: 8) when > 0. Raise it when the listener sits behind a- // TLS-terminating proxy or NAT, where many peers share a few source IPs.+ // ConnsPerSourceIP is the inbound connection cap per source address,+ // and scales the per-address connection rate limit with it. Zero+ // derives it from HighWaterMark: one address may hold at most a+ // quarter of the router's inbound budget, so filling a router takes+ // at least four addresses. libp2p's own default is 8, sized for a+ // public DHT where one IP is one peer; members of a mesh share+ // addresses (pods behind a node's SNAT, a cluster behind a NAT, an+ // office), and the handshake, not this cap, is what keeps strangers+ // out.ConnsPerSourceIP int// AllowInsecureControlPlane accepts a plaintext http:// ControlPlaneURL// to a non-loopback host. Off by default: whoever answers that URL is@@ -89,6 +95,9 @@ func (o *Options) Default() {if o.HighWaterMark <= 0 {o.HighWaterMark = 4000}+ if o.ConnsPerSourceIP <= 0 {+ o.ConnsPerSourceIP = DefaultConnsPerSourceIP(o.HighWaterMark)+ }if o.KeysDBPath == "" {o.KeysDBPath = "router.key"}diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.goindex 446b8115..29877713 100644--- a/internal/router/metrics_test.go+++ b/internal/router/metrics_test.go@@ -99,7 +99,8 @@ func TestRouterMetricsListener(t *testing.T) {for _, want := range []string{"sam_router_ready 1",`sam_router_info{peer_id="` + host.ID().String() + `"} 1`,- "sam_router_conns_per_source_ip_limit 8",+ // The derived cap: a quarter of the default high watermark.+ "sam_router_conns_per_source_ip_limit 1000","sam_router_inbound_connections_refused_total","sam_router_authenticated_peers 2","sam_router_banned_peers 1",diff --git a/internal/router/resource_observer.go b/internal/router/resource_observer.goindex 5cc3e030..4960e118 100644--- a/internal/router/resource_observer.go+++ b/internal/router/resource_observer.go@@ -137,8 +137,15 @@ var connsPerSourceIPDesc = prometheus.NewDesc("Inbound connections allowed from one source IP",nil, nil)-// defaultConnsPerSourceIP is libp2p's cap when --conns-per-source-ip is unset.-const defaultConnsPerSourceIP = 8+// connsPerSourceIPShare is the fraction of the inbound connection budget one+// source address may hold when no cap is configured.+const connsPerSourceIPShare = 4++// DefaultConnsPerSourceIP is the per-source-address cap derived from a+// connection manager high watermark.+func DefaultConnsPerSourceIP(highWaterMark int) int {+ return max(highWaterMark/connsPerSourceIPShare, 1)+}var _ connmgr.GetConnLimiter = (*observedResourceManager)(nil)@@ -147,7 +154,7 @@ func effectiveConnsPerSourceIP(configured int) int {if configured > 0 {return configured}- return defaultConnsPerSourceIP+ return DefaultConnsPerSourceIP(DefaultHighWaterMark)}// perPeerLogLimiter admits one log line per peer per interval. Entries olderdiff --git a/internal/router/resource_observer_test.go b/internal/router/resource_observer_test.goindex 866e77b9..a8d2c47d 100644--- a/internal/router/resource_observer_test.go+++ b/internal/router/resource_observer_test.go@@ -139,8 +139,8 @@ func TestPerPeerLogLimiter(t *testing.T) {}func TestEffectiveConnsPerSourceIP(t *testing.T) {- if got := effectiveConnsPerSourceIP(0); got != defaultConnsPerSourceIP {- t.Errorf("unset = %d, want libp2p's default %d", got, defaultConnsPerSourceIP)+ if got := effectiveConnsPerSourceIP(0); got != DefaultHighWaterMark/4 {+ t.Errorf("unset = %d, want a quarter of the default high watermark, %d", got, DefaultHighWaterMark/4)}if got := effectiveConnsPerSourceIP(64); got != 64 {t.Errorf("configured = %d, want 64", got)diff --git a/internal/router/router.go b/internal/router/router.goindex d7045fbc..86ac4c10 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -323,9 +323,9 @@ func (r *Router) transportOptions() libp2p.Option {// per-source-IP inbound connection cap, the per-subnet connection rate limit// and the system/transient scopes scaled to carry limit connections; see// Options.ConnsPerSourceIP. All three matter behind a proxy or NAT: every-// peer shares a few source IPs, so the default per-IP cap (8), the default-// per-IP rate (0.2 conns/s, burst 16) and the small transient scope each-// take down the whole listener under normal reconnect churn.+// peer shares a few source IPs, so libp2p's per-IP cap (8), its per-IP rate+// (0.2 conns/s, burst 16) and the small transient scope each take down the+// whole listener under normal reconnect churn.func perIPConnResourceManager(limit int) (network.ResourceManager, error) {limits := rcmgr.DefaultLimitslibp2p.SetDefaultServiceLimits(&limits)@@ -375,14 +375,6 @@ func perIPConnResourceManager(limit int) (network.ResourceManager, error) {)}-// defaultResourceManager is what libp2p installs when none is given, built-// here so it can be wrapped.-func defaultResourceManager() (network.ResourceManager, error) {- limits := rcmgr.DefaultLimits- libp2p.SetDefaultServiceLimits(&limits)- return rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(limits.AutoScale()))-}-// Start performs enrollment, syncs keys, launches libp2p host, and starts tasks.func (r *Router) Start() (retErr error) {defer func() {@@ -466,14 +458,8 @@ func (r *Router) Start() (retErr error) {}// Every refusal the resource manager makes is counted and logged with- // its source, whether the per-IP cap is the configured one or libp2p's- // default; a refused dialer only sees its connection close before TLS.- var mgr network.ResourceManager- if r.config.ConnsPerSourceIP > 0 {- mgr, err = perIPConnResourceManager(r.config.ConnsPerSourceIP)- } else {- mgr, err = defaultResourceManager()- }+ // its source; a refused dialer only sees its connection close before TLS.+ mgr, err := perIPConnResourceManager(r.config.ConnsPerSourceIP)if err != nil {return fmt.Errorf("failed to create resource manager: %w", err)}diff --git a/internal/router/router_test.go b/internal/router/router_test.goindex cc1755a9..6d25bdfd 100644--- a/internal/router/router_test.go+++ b/internal/router/router_test.go@@ -799,12 +799,26 @@ func TestRouterConnectionManagerWatermarks(t *testing.T) {if opts.LowWaterMark != 500 || opts.HighWaterMark != 1500 {t.Fatalf("expected explicit watermarks to be preserved, got Low: %d, High: %d", opts.LowWaterMark, opts.HighWaterMark)}+ // The per-address cap follows the budget it is a share of: libp2p's 8+ // assumes one peer per IP, which a cluster behind SNAT or a NAT is not.+ if opts.ConnsPerSourceIP != 375 {+ t.Fatalf("ConnsPerSourceIP = %d, want a quarter of the high watermark, 375", opts.ConnsPerSourceIP)+ }defaultOpts := Options{}defaultOpts.Default()if defaultOpts.LowWaterMark != 1000 || defaultOpts.HighWaterMark != 4000 {t.Fatalf("expected default watermarks 1000/4000, got Low: %d, High: %d", defaultOpts.LowWaterMark, defaultOpts.HighWaterMark)}+ if defaultOpts.ConnsPerSourceIP != 1000 {+ t.Fatalf("default ConnsPerSourceIP = %d, want 1000", defaultOpts.ConnsPerSourceIP)+ }++ pinned := Options{ConnsPerSourceIP: 8}+ pinned.Default()+ if pinned.ConnsPerSourceIP != 8 {+ t.Fatalf("explicit ConnsPerSourceIP = %d, want it kept at 8", pinned.ConnsPerSourceIP)+ }}func TestRouterDefaultBiscuitTimeout(t *testing.T) {diff --git a/site/content/docs/concepts/networking.md b/site/content/docs/concepts/networking.mdindex d6f28237..9675a5a2 100644--- a/site/content/docs/concepts/networking.md+++ b/site/content/docs/concepts/networking.md@@ -133,8 +133,10 @@ forwards it.## Resource limitsNodes rate-limit authentication attempts per peer and cap the size of every-frame and body they read. Routers cap inbound connections per source IP-(`--conns-per-source-ip`) and total connections (`--low-watermark`,-`--high-watermark`). A router behind a TLS-terminating proxy, or behind a NAT-that puts many peers on one address, needs a higher per-source limit.-`sam-one` raises it automatically for its embedded router.+frame and body they read. Routers cap inbound connections per source address+(`--conns-per-source-ip`, by default a quarter of the connection budget) and+total connections (`--low-watermark`, `--high-watermark`). Members behind a+NAT or a cluster's SNAT share one address, and the handshake, not the+per-address cap, is what keeps strangers out; the cap bounds how much of a+router one address can occupy. `sam-one` sets it to the whole budget for its+embedded router, which sits behind a proxy and sees one address.diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.mdindex 96c0bdb9..70add597 100644--- a/site/content/docs/reference/router.md+++ b/site/content/docs/reference/router.md@@ -40,7 +40,7 @@ replica count.| `--keys-sync-interval` | `5m` | How often `/keys` is polled for signing-key rotations. || `--lease-renew-interval` | `300s` | How often the lease is renewed. Must be well below the control plane's `--lease-duration`. || `--allow-loopback` | `false` | Announce and accept loopback and link-local addresses. For a router and nodes on one host. |-| `--conns-per-source-ip` | `8` (libp2p default) | Inbound connections accepted per source IP. Raise it behind a TLS-terminating proxy or a NAT that puts many peers on one address. |+| `--conns-per-source-ip` | a quarter of `--high-watermark` (`1000`) | Inbound connections accepted per source address. One address can hold at most this share of the router's connection budget, so filling a router takes at least four addresses. Members behind a NAT or a cluster's SNAT share one address and each holds one connection per router, two while enrolling. || `--low-watermark`, `--high-watermark` | `1000`, `4000` | Connection manager limits. Above the high mark, connections are trimmed down to the low mark. || `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. || `--dht-max-record-age` | library default | DHT value record lifetime. |