Skip to content

Agent Mesh

router: derive the per-source-IP connection cap from the high watermark

Antonio Ojeaafb61d125526
diff --git a/cmd/sam-router/main.go b/cmd/sam-router/main.go
index 1ac8dd50..4d578aad 100644
--- a/cmd/sam-router/main.go
+++ b/cmd/sam-router/main.go
@@ -125,7 +125,7 @@ func main() {
rootCmd.Flags().StringVar(&jwtPath, "jwt-path", "", "Path to file containing OIDC JWT token")
rootCmd.Flags().StringVar(&keysPath, "keys-path", "router.key", "Path to save/load persistent private key")
rootCmd.Flags().BoolVar(&allowLoopback, "allow-loopback", false, "Allow loopback and link-local addresses for discovery")
- rootCmd.Flags().IntVar(&connsPerSourceIP, "conns-per-source-ip", 0, "Max inbound connections per source IP (0 keeps libp2p's default of 8); raise behind TLS-terminating proxies or NAT where many peers share source IPs")
+ rootCmd.Flags().IntVar(&connsPerSourceIP, "conns-per-source-ip", 0, "Max inbound connections per source IP; 0 is a quarter of --high-watermark, so filling the router takes at least four addresses. Raise it where more members share one address")
rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")
rootCmd.Flags().DurationVar(&dhtProviderAddrTTL, "dht-provider-addr-ttl", router.DefaultDHTProviderAddrTTL, "How long a DHT provider record lives after its last announcement")
rootCmd.Flags().DurationVar(&dhtMaxRecordAge, "dht-max-record-age", 0, "Maximum age for DHT records (0s uses library default)")
diff --git a/internal/router/config.go b/internal/router/config.go
index 074c5063..1b3c93ad 100644
--- a/internal/router/config.go
+++ b/internal/router/config.go
@@ -55,9 +55,15 @@ type Options struct {
// HTTP requests arriving on the router's WebSocket listen addrs, letting a
// single port carry both libp2p and REST traffic (single-port mode).
HTTPFallbackHandler http.Handler
- // ConnsPerSourceIP overrides libp2p's per-source-IP inbound connection
- // cap (default: 8) when > 0. Raise it when the listener sits behind a
- // TLS-terminating proxy or NAT, where many peers share a few source IPs.
+ // ConnsPerSourceIP is the inbound connection cap per source address,
+ // and scales the per-address connection rate limit with it. Zero
+ // derives it from HighWaterMark: one address may hold at most a
+ // quarter of the router's inbound budget, so filling a router takes
+ // at least four addresses. libp2p's own default is 8, sized for a
+ // public DHT where one IP is one peer; members of a mesh share
+ // addresses (pods behind a node's SNAT, a cluster behind a NAT, an
+ // office), and the handshake, not this cap, is what keeps strangers
+ // out.
ConnsPerSourceIP int
// AllowInsecureControlPlane accepts a plaintext http:// ControlPlaneURL
// to a non-loopback host. Off by default: whoever answers that URL is
@@ -89,6 +95,9 @@ func (o *Options) Default() {
if o.HighWaterMark <= 0 {
o.HighWaterMark = 4000
}
+ if o.ConnsPerSourceIP <= 0 {
+ o.ConnsPerSourceIP = DefaultConnsPerSourceIP(o.HighWaterMark)
+ }
if o.KeysDBPath == "" {
o.KeysDBPath = "router.key"
}
diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.go
index 446b8115..29877713 100644
--- a/internal/router/metrics_test.go
+++ b/internal/router/metrics_test.go
@@ -99,7 +99,8 @@ func TestRouterMetricsListener(t *testing.T) {
for _, want := range []string{
"sam_router_ready 1",
`sam_router_info{peer_id="` + host.ID().String() + `"} 1`,
- "sam_router_conns_per_source_ip_limit 8",
+ // The derived cap: a quarter of the default high watermark.
+ "sam_router_conns_per_source_ip_limit 1000",
"sam_router_inbound_connections_refused_total",
"sam_router_authenticated_peers 2",
"sam_router_banned_peers 1",
diff --git a/internal/router/resource_observer.go b/internal/router/resource_observer.go
index 5cc3e030..4960e118 100644
--- a/internal/router/resource_observer.go
+++ b/internal/router/resource_observer.go
@@ -137,8 +137,15 @@ var connsPerSourceIPDesc = prometheus.NewDesc(
"Inbound connections allowed from one source IP",
nil, nil)
-// defaultConnsPerSourceIP is libp2p's cap when --conns-per-source-ip is unset.
-const defaultConnsPerSourceIP = 8
+// connsPerSourceIPShare is the fraction of the inbound connection budget one
+// source address may hold when no cap is configured.
+const connsPerSourceIPShare = 4
+
+// DefaultConnsPerSourceIP is the per-source-address cap derived from a
+// connection manager high watermark.
+func DefaultConnsPerSourceIP(highWaterMark int) int {
+ return max(highWaterMark/connsPerSourceIPShare, 1)
+}
var _ connmgr.GetConnLimiter = (*observedResourceManager)(nil)
@@ -147,7 +154,7 @@ func effectiveConnsPerSourceIP(configured int) int {
if configured > 0 {
return configured
}
- return defaultConnsPerSourceIP
+ return DefaultConnsPerSourceIP(DefaultHighWaterMark)
}
// perPeerLogLimiter admits one log line per peer per interval. Entries older
diff --git a/internal/router/resource_observer_test.go b/internal/router/resource_observer_test.go
index 866e77b9..a8d2c47d 100644
--- a/internal/router/resource_observer_test.go
+++ b/internal/router/resource_observer_test.go
@@ -139,8 +139,8 @@ func TestPerPeerLogLimiter(t *testing.T) {
}
func TestEffectiveConnsPerSourceIP(t *testing.T) {
- if got := effectiveConnsPerSourceIP(0); got != defaultConnsPerSourceIP {
- t.Errorf("unset = %d, want libp2p's default %d", got, defaultConnsPerSourceIP)
+ if got := effectiveConnsPerSourceIP(0); got != DefaultHighWaterMark/4 {
+ t.Errorf("unset = %d, want a quarter of the default high watermark, %d", got, DefaultHighWaterMark/4)
}
if got := effectiveConnsPerSourceIP(64); got != 64 {
t.Errorf("configured = %d, want 64", got)
diff --git a/internal/router/router.go b/internal/router/router.go
index d7045fbc..86ac4c10 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -323,9 +323,9 @@ func (r *Router) transportOptions() libp2p.Option {
// per-source-IP inbound connection cap, the per-subnet connection rate limit
// and the system/transient scopes scaled to carry limit connections; see
// Options.ConnsPerSourceIP. All three matter behind a proxy or NAT: every
-// peer shares a few source IPs, so the default per-IP cap (8), the default
-// per-IP rate (0.2 conns/s, burst 16) and the small transient scope each
-// take down the whole listener under normal reconnect churn.
+// peer shares a few source IPs, so libp2p's per-IP cap (8), its per-IP rate
+// (0.2 conns/s, burst 16) and the small transient scope each take down the
+// whole listener under normal reconnect churn.
func perIPConnResourceManager(limit int) (network.ResourceManager, error) {
limits := rcmgr.DefaultLimits
libp2p.SetDefaultServiceLimits(&limits)
@@ -375,14 +375,6 @@ func perIPConnResourceManager(limit int) (network.ResourceManager, error) {
)
}
-// defaultResourceManager is what libp2p installs when none is given, built
-// here so it can be wrapped.
-func defaultResourceManager() (network.ResourceManager, error) {
- limits := rcmgr.DefaultLimits
- libp2p.SetDefaultServiceLimits(&limits)
- return rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(limits.AutoScale()))
-}
-
// Start performs enrollment, syncs keys, launches libp2p host, and starts tasks.
func (r *Router) Start() (retErr error) {
defer func() {
@@ -466,14 +458,8 @@ func (r *Router) Start() (retErr error) {
}
// Every refusal the resource manager makes is counted and logged with
- // its source, whether the per-IP cap is the configured one or libp2p's
- // default; a refused dialer only sees its connection close before TLS.
- var mgr network.ResourceManager
- if r.config.ConnsPerSourceIP > 0 {
- mgr, err = perIPConnResourceManager(r.config.ConnsPerSourceIP)
- } else {
- mgr, err = defaultResourceManager()
- }
+ // its source; a refused dialer only sees its connection close before TLS.
+ mgr, err := perIPConnResourceManager(r.config.ConnsPerSourceIP)
if err != nil {
return fmt.Errorf("failed to create resource manager: %w", err)
}
diff --git a/internal/router/router_test.go b/internal/router/router_test.go
index cc1755a9..6d25bdfd 100644
--- a/internal/router/router_test.go
+++ b/internal/router/router_test.go
@@ -799,12 +799,26 @@ func TestRouterConnectionManagerWatermarks(t *testing.T) {
if opts.LowWaterMark != 500 || opts.HighWaterMark != 1500 {
t.Fatalf("expected explicit watermarks to be preserved, got Low: %d, High: %d", opts.LowWaterMark, opts.HighWaterMark)
}
+ // The per-address cap follows the budget it is a share of: libp2p's 8
+ // assumes one peer per IP, which a cluster behind SNAT or a NAT is not.
+ if opts.ConnsPerSourceIP != 375 {
+ t.Fatalf("ConnsPerSourceIP = %d, want a quarter of the high watermark, 375", opts.ConnsPerSourceIP)
+ }
defaultOpts := Options{}
defaultOpts.Default()
if defaultOpts.LowWaterMark != 1000 || defaultOpts.HighWaterMark != 4000 {
t.Fatalf("expected default watermarks 1000/4000, got Low: %d, High: %d", defaultOpts.LowWaterMark, defaultOpts.HighWaterMark)
}
+ if defaultOpts.ConnsPerSourceIP != 1000 {
+ t.Fatalf("default ConnsPerSourceIP = %d, want 1000", defaultOpts.ConnsPerSourceIP)
+ }
+
+ pinned := Options{ConnsPerSourceIP: 8}
+ pinned.Default()
+ if pinned.ConnsPerSourceIP != 8 {
+ t.Fatalf("explicit ConnsPerSourceIP = %d, want it kept at 8", pinned.ConnsPerSourceIP)
+ }
}
func TestRouterDefaultBiscuitTimeout(t *testing.T) {
diff --git a/site/content/docs/concepts/networking.md b/site/content/docs/concepts/networking.md
index d6f28237..9675a5a2 100644
--- a/site/content/docs/concepts/networking.md
+++ b/site/content/docs/concepts/networking.md
@@ -133,8 +133,10 @@ forwards it.
## Resource limits
Nodes rate-limit authentication attempts per peer and cap the size of every
-frame and body they read. Routers cap inbound connections per source IP
-(`--conns-per-source-ip`) and total connections (`--low-watermark`,
-`--high-watermark`). A router behind a TLS-terminating proxy, or behind a NAT
-that puts many peers on one address, needs a higher per-source limit.
-`sam-one` raises it automatically for its embedded router.
+frame and body they read. Routers cap inbound connections per source address
+(`--conns-per-source-ip`, by default a quarter of the connection budget) and
+total connections (`--low-watermark`, `--high-watermark`). Members behind a
+NAT or a cluster's SNAT share one address, and the handshake, not the
+per-address cap, is what keeps strangers out; the cap bounds how much of a
+router one address can occupy. `sam-one` sets it to the whole budget for its
+embedded router, which sits behind a proxy and sees one address.
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index 96c0bdb9..70add597 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -40,7 +40,7 @@ replica count.
| `--keys-sync-interval` | `5m` | How often `/keys` is polled for signing-key rotations. |
| `--lease-renew-interval` | `300s` | How often the lease is renewed. Must be well below the control plane's `--lease-duration`. |
| `--allow-loopback` | `false` | Announce and accept loopback and link-local addresses. For a router and nodes on one host. |
-| `--conns-per-source-ip` | `8` (libp2p default) | Inbound connections accepted per source IP. Raise it behind a TLS-terminating proxy or a NAT that puts many peers on one address. |
+| `--conns-per-source-ip` | a quarter of `--high-watermark` (`1000`) | Inbound connections accepted per source address. One address can hold at most this share of the router's connection budget, so filling a router takes at least four addresses. Members behind a NAT or a cluster's SNAT share one address and each holds one connection per router, two while enrolling. |
| `--low-watermark`, `--high-watermark` | `1000`, `4000` | Connection manager limits. Above the high mark, connections are trimmed down to the low mark. |
| `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. |
| `--dht-max-record-age` | library default | DHT value record lifetime. |