Skip to content

Agent Mesh

router: tunable per-source-IP connection budget for proxied listeners

Antonio Ojea71ea2ae5666c
diff --git a/cmd/sam-router/main.go b/cmd/sam-router/main.go
index 80a678e3..acb4d4df 100644
--- a/cmd/sam-router/main.go
+++ b/cmd/sam-router/main.go
@@ -38,6 +38,7 @@ var (
jwtPath string
keysPath string
allowLoopback bool
+ connsPerSourceIP int
logLevel string
dhtProviderAddrTTL time.Duration
dhtMaxRecordAge time.Duration
@@ -76,6 +77,7 @@ func main() {
JWTPath: jwtPath,
KeysDBPath: keysPath,
AllowLoopback: allowLoopback,
+ ConnsPerSourceIP: connsPerSourceIP,
DHTProviderAddrTTL: dhtProviderAddrTTL,
DHTMaxRecordAge: dhtMaxRecordAge,
LowWaterMark: lowWaterMark,
@@ -111,6 +113,7 @@ func main() {
rootCmd.Flags().StringVar(&jwtPath, "jwt-path", "", "Path to file containing OIDC JWT token")
rootCmd.Flags().StringVar(&keysPath, "keys-path", "router.key", "Path to save/load persistent private key")
rootCmd.Flags().BoolVar(&allowLoopback, "allow-loopback", false, "Allow loopback and link-local addresses for discovery")
+ rootCmd.Flags().IntVar(&connsPerSourceIP, "conns-per-source-ip", 0, "Max inbound connections per source IP (0 keeps libp2p's default of 8); raise behind TLS-terminating proxies or NAT where many peers share source IPs")
rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")
rootCmd.Flags().DurationVar(&dhtProviderAddrTTL, "dht-provider-addr-ttl", 0, "Time-To-Live for DHT provider addresses (0s uses library default)")
rootCmd.Flags().DurationVar(&dhtMaxRecordAge, "dht-max-record-age", 0, "Maximum age for DHT records (0s uses library default)")
diff --git a/internal/router/config.go b/internal/router/config.go
index b74fbefc..73ef9327 100644
--- a/internal/router/config.go
+++ b/internal/router/config.go
@@ -49,6 +49,10 @@ type Options struct {
// HTTP requests arriving on the router's WebSocket listen addrs, letting a
// single port carry both libp2p and REST traffic (single-port mode).
HTTPFallbackHandler http.Handler
+ // ConnsPerSourceIP overrides libp2p's per-source-IP inbound connection
+ // cap (default: 8) when > 0. Raise it when the listener sits behind a
+ // TLS-terminating proxy or NAT, where many peers share a few source IPs.
+ ConnsPerSourceIP int
}
// Default sets default values for options.
diff --git a/internal/router/router.go b/internal/router/router.go
index 6a9e6522..ed28cae0 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -24,6 +24,7 @@ import (
"io"
"net"
"net/http"
+ "net/netip"
"net/url"
"os"
"path/filepath"
@@ -45,6 +46,7 @@ import (
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
+ rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
"github.com/libp2p/go-libp2p/p2p/net/connmgr"
"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls"
@@ -53,6 +55,7 @@ import (
libp2pwebrtc "github.com/libp2p/go-libp2p/p2p/transport/webrtc"
ws "github.com/libp2p/go-libp2p/p2p/transport/websocket"
"github.com/libp2p/go-libp2p/p2p/transport/webtransport"
+ xrate "github.com/libp2p/go-libp2p/x/rate"
"github.com/libp2p/go-msgio"
"github.com/multiformats/go-multiaddr"
madns "github.com/multiformats/go-multiaddr-dns"
@@ -159,6 +162,61 @@ func (r *Router) transportOptions() libp2p.Option {
)
}
+// perIPConnResourceManager mirrors libp2p's default resource manager with the
+// per-source-IP inbound connection cap, the per-subnet connection rate limit
+// and the system/transient scopes scaled to carry limit connections; see
+// Options.ConnsPerSourceIP. All three matter behind a proxy or NAT: every
+// peer shares a few source IPs, so the default per-IP cap (8), the default
+// per-IP rate (0.2 conns/s, burst 16) and the small transient scope each
+// take down the whole listener under normal reconnect churn.
+func perIPConnResourceManager(limit int) (network.ResourceManager, error) {
+ limits := rcmgr.DefaultLimits
+ libp2p.SetDefaultServiceLimits(&limits)
+ scaled := limits.AutoScale()
+
+ overrides := rcmgr.PartialLimitConfig{
+ System: rcmgr.ResourceLimits{
+ Conns: rcmgr.LimitVal(2 * limit),
+ ConnsInbound: rcmgr.LimitVal(limit),
+ FD: rcmgr.LimitVal(2 * limit),
+ },
+ Transient: rcmgr.ResourceLimits{
+ Conns: rcmgr.LimitVal(limit),
+ ConnsInbound: rcmgr.LimitVal(limit),
+ FD: rcmgr.LimitVal(limit),
+ },
+ }
+
+ // Same shape as the rcmgr default limiter (loopback exempt, no global
+ // cap), with the per-subnet budget scaled by limit relative to the
+ // default per-IP cap of 8.
+ scale := float64(limit) / 8
+ connRateLimiter := &xrate.Limiter{
+ NetworkPrefixLimits: []xrate.PrefixLimit{
+ {Prefix: netip.MustParsePrefix("127.0.0.0/8"), Limit: xrate.Limit{}},
+ {Prefix: netip.MustParsePrefix("::1/128"), Limit: xrate.Limit{}},
+ },
+ SubnetRateLimiter: xrate.SubnetLimiter{
+ IPv4SubnetLimits: []xrate.SubnetLimit{
+ {PrefixLength: 32, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * limit}},
+ },
+ IPv6SubnetLimits: []xrate.SubnetLimit{
+ {PrefixLength: 56, Limit: xrate.Limit{RPS: 0.2 * scale, Burst: 2 * limit}},
+ },
+ GracePeriod: time.Minute,
+ },
+ }
+
+ return rcmgr.NewResourceManager(
+ rcmgr.NewFixedLimiter(overrides.Build(scaled)),
+ rcmgr.WithLimitPerSubnet(
+ []rcmgr.ConnLimitPerSubnet{{PrefixLength: 32, ConnCount: limit}},
+ []rcmgr.ConnLimitPerSubnet{{PrefixLength: 56, ConnCount: limit}},
+ ),
+ rcmgr.WithConnRateLimiters(connRateLimiter),
+ )
+}
+
// Start performs enrollment, syncs keys, launches libp2p host, and starts tasks.
func (r *Router) Start() error {
// 1. Load or Generate persistent identity key
@@ -222,6 +280,14 @@ func (r *Router) Start() error {
}),
}
+ if r.config.ConnsPerSourceIP > 0 {
+ mgr, err := perIPConnResourceManager(r.config.ConnsPerSourceIP)
+ if err != nil {
+ return fmt.Errorf("failed to create resource manager: %w", err)
+ }
+ p2pOpts = append(p2pOpts, libp2p.ResourceManager(mgr))
+ }
+
hostNode, err := libp2p.New(p2pOpts...)
if err != nil {
return err
diff --git a/internal/standalone/standalone.go b/internal/standalone/standalone.go
index 8b556a04..df9aba75 100644
--- a/internal/standalone/standalone.go
+++ b/internal/standalone/standalone.go
@@ -260,12 +260,18 @@ func (s *Server) Start(ctx context.Context) error {
}
rtr, err := router.NewRouter(ctx, router.Options{
- ControlPlaneURL: "http://" + cp.Addr(),
- ListenAddrs: append([]string{wsAddr}, s.opts.P2PListen...),
- ExternalAddrs: externalAddrs,
- AllowLoopback: true,
- KeysDBPath: filepath.Join(s.opts.DataDir, routerKeyFile),
- BootstrapToken: routerToken,
+ ControlPlaneURL: "http://" + cp.Addr(),
+ ListenAddrs: append([]string{wsAddr}, s.opts.P2PListen...),
+ ExternalAddrs: externalAddrs,
+ AllowLoopback: true,
+ KeysDBPath: filepath.Join(s.opts.DataDir, routerKeyFile),
+ BootstrapToken: routerToken,
+ // Single-port deployments typically sit behind a TLS-terminating
+ // proxy (Cloud Run, L7 LBs) or NAT where every peer shares a few
+ // source IPs; libp2p's default 8-conns-per-IP cap would throttle
+ // the whole listener. Matches the conn manager's high watermark so
+ // the global cap, not the per-IP one, is what binds.
+ ConnsPerSourceIP: router.DefaultHighWaterMark,
HTTPFallbackHandler: mux,
})
if err != nil {