Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router: a trim sheds 15% of the connections, not three quarters
Antonio Ojea589520a2eeb7
diff --git a/cmd/agentmesh-one/main.go b/cmd/agentmesh-one/main.goindex a3846d3c..2e18b6a0 100644--- a/cmd/agentmesh-one/main.go+++ b/cmd/agentmesh-one/main.go@@ -248,7 +248,7 @@ func main() {// Embedded router tunables.rootCmd.Flags().DurationVar(&routerTunables.KeysSyncInterval, "router-keys-sync-interval", 0, "Biscuit public key refresh interval (0 keeps the component default)")rootCmd.Flags().DurationVar(&routerTunables.LeaseRenewInterval, "router-lease-renew-interval", 0, "Lease renewal interval (0 keeps the component default)")- rootCmd.Flags().IntVar(&routerTunables.LowWaterMark, "router-low-watermark", 0, "Connection manager low watermark (0 keeps the component default)")+ rootCmd.Flags().IntVar(&routerTunables.LowWaterMark, "router-low-watermark", 0, "Connection manager low watermark (0 is 15% below the high watermark)")rootCmd.Flags().IntVar(&routerTunables.HighWaterMark, "router-high-watermark", 0, "Connection manager high watermark (0 keeps the component default)")rootCmd.Flags().IntVar(&routerTunables.ConnsPerSourceIP, "router-conns-per-source-ip", 0, "Per-source-IP connection budget (0 follows the high watermark; proxied peers share source IPs)")rootCmd.Flags().DurationVar(&routerTunables.DHTProviderAddrTTL, "router-dht-provider-addr-ttl", 0, "How long a DHT provider record lives after its last announcement (0 keeps the component default of 15m)")diff --git a/cmd/agentmesh-router/main.go b/cmd/agentmesh-router/main.goindex 73687476..2991ef09 100644--- a/cmd/agentmesh-router/main.go+++ b/cmd/agentmesh-router/main.go@@ -136,8 +136,8 @@ func main() {rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")rootCmd.Flags().DurationVar(&dhtProviderAddrTTL, "dht-provider-addr-ttl", router.DefaultDHTProviderAddrTTL, "How long a DHT provider record lives after its last announcement")rootCmd.Flags().DurationVar(&dhtMaxRecordAge, "dht-max-record-age", 0, "Maximum age for DHT records (0s uses library default)")- rootCmd.Flags().IntVar(&lowWaterMark, "low-watermark", 1000, "Connection manager low watermark limit")- rootCmd.Flags().IntVar(&highWaterMark, "high-watermark", 4000, "Connection manager high watermark limit")+ rootCmd.Flags().IntVar(&lowWaterMark, "low-watermark", 0, "Connections the router trims down to once it is over --high-watermark; 0 is 15% below the high watermark, so a trim sheds that share and no more")+ rootCmd.Flags().IntVar(&highWaterMark, "high-watermark", router.DefaultHighWaterMark, "Connections the router accepts before it trims; every member holds one, so this is the size of the mesh")rootCmd.Flags().StringVar(&metricsAddr, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address (e.g. 0.0.0.0:9090); unauthenticated, off by default")rootCmd.Flags().DurationVar(&relayLimitDuration, "relay-limit-duration", router.DefaultRelayLimitDuration, "Lifetime of each relayed connection (0 = no limit)")rootCmd.Flags().Var(&relayLimitData, "relay-limit-data", "Bytes relayed per direction on each relayed connection, e.g. 512MiB (0 = no limit)")diff --git a/internal/router/config.go b/internal/router/config.goindex 2434200f..d1bd66ea 100644--- a/internal/router/config.go+++ b/internal/router/config.go@@ -107,11 +107,11 @@ func (o *Options) Default() {if o.LeaseRenewInterval <= 0 {o.LeaseRenewInterval = 30 * time.Second}- if o.LowWaterMark <= 0 {- o.LowWaterMark = 1000- }if o.HighWaterMark <= 0 {- o.HighWaterMark = 4000+ o.HighWaterMark = DefaultHighWaterMark+ }+ if o.LowWaterMark <= 0 {+ o.LowWaterMark = DefaultLowWaterMark(o.HighWaterMark)}if o.ConnsPerSourceIP <= 0 {o.ConnsPerSourceIP = DefaultConnsPerSourceIP(o.HighWaterMark)@@ -141,5 +141,8 @@ func (o *Options) Validate() error {if o.ShutdownLeaseTTL < 0 {return fmt.Errorf("ShutdownLeaseTTL must not be negative, got %s", o.ShutdownLeaseTTL)}+ if o.LowWaterMark >= o.HighWaterMark {+ return fmt.Errorf("LowWaterMark (%d) must be below HighWaterMark (%d)", o.LowWaterMark, o.HighWaterMark)+ }return api.ValidateControlPlaneTransport(o.ControlPlaneURL, o.AllowInsecureControlPlane)}diff --git a/internal/router/router.go b/internal/router/router.goindex a4c8a299..43250f69 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -66,10 +66,21 @@ import (var logger = golog.Logger("agentmesh-router")+// DefaultLowWaterMark is the low watermark for a high one: TrimShare below+// it, so a trim sheds that share of the connections and no more.+func DefaultLowWaterMark(highWaterMark int) int {+ return max(1, int(float64(highWaterMark)*(1-TrimShare)))+}+const (- DefaultLowWaterMark = 1000DefaultHighWaterMark = 4000- ConnGracePeriod = 1 * time.Minute+ // TrimShare is the share of its connections a router sheds when it+ // crosses the high watermark: the low watermark it trims down to is+ // the high one less this. libp2p's own default trims from 4000 to 1000,+ // which on a hub whose every connection is a member session is most+ // of the mesh at once.+ TrimShare = 0.15+ ConnGracePeriod = 1 * time.Minute// DefaultDHTProviderAddrTTL is three of a node's 5-minute reprovides: a// node that is gone drops out of discovery within it, one late reprovide// does not. The library default (48h) keeps every past identity listed.@@ -359,15 +370,7 @@ func (r *Router) Start() (retErr error) {}// 4. Initialize libp2p host- low := r.config.LowWaterMark- if low <= 0 {- low = DefaultLowWaterMark- }- high := r.config.HighWaterMark- if high <= 0 {- high = DefaultHighWaterMark- }- cm, err := connmgr.NewConnManager(low, high, connmgr.WithGracePeriod(ConnGracePeriod))+ cm, err := connmgr.NewConnManager(r.config.LowWaterMark, r.config.HighWaterMark, connmgr.WithGracePeriod(ConnGracePeriod))if err != nil {return fmt.Errorf("failed to create connection manager: %w", err)}diff --git a/internal/router/router_test.go b/internal/router/router_test.goindex a9d2a699..1ea44f9a 100644--- a/internal/router/router_test.go+++ b/internal/router/router_test.go@@ -897,13 +897,27 @@ func TestRouterConnectionManagerWatermarks(t *testing.T) {defaultOpts := Options{}defaultOpts.Default()- if defaultOpts.LowWaterMark != 1000 || defaultOpts.HighWaterMark != 4000 {- t.Fatalf("expected default watermarks 1000/4000, got Low: %d, High: %d", defaultOpts.LowWaterMark, defaultOpts.HighWaterMark)+ // A trim sheds TrimShare of the connections: on a hub every connection+ // is a member session, and libp2p's own 4000 down to 1000 was most of+ // the mesh at once.+ if defaultOpts.LowWaterMark != 3400 || defaultOpts.HighWaterMark != 4000 {+ t.Fatalf("expected default watermarks 3400/4000, got Low: %d, High: %d", defaultOpts.LowWaterMark, defaultOpts.HighWaterMark)}if defaultOpts.ConnsPerSourceIP != 1000 {t.Fatalf("default ConnsPerSourceIP = %d, want 1000", defaultOpts.ConnsPerSourceIP)}+ // The low mark follows a high mark the operator sets.+ highOnly := Options{HighWaterMark: 10000}+ highOnly.Default()+ if highOnly.LowWaterMark != 8500 {+ t.Fatalf("LowWaterMark for a high of 10000 = %d, want 8500", highOnly.LowWaterMark)+ }+ inverted := Options{ControlPlaneURL: "http://127.0.0.1:8080", LowWaterMark: 4000, HighWaterMark: 3000}+ if err := inverted.Validate(); err == nil || !strings.Contains(err.Error(), "LowWaterMark") {+ t.Fatalf("a low mark above the high one: err = %v, want rejection", err)+ }+pinned := Options{ConnsPerSourceIP: 8}pinned.Default()if pinned.ConnsPerSourceIP != 8 {diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.mdindex 8e2e5ac9..6ed7fa2f 100644--- a/site/content/docs/reference/router.md+++ b/site/content/docs/reference/router.md@@ -41,7 +41,7 @@ replica count.| `--lease-renew-interval` | `300s` | How often the lease is renewed. Must be well below the control plane's `--lease-duration`. || `--allow-loopback` | `false` | Announce and accept loopback and link-local addresses. For a router and nodes on one host. || `--conns-per-source-ip` | a quarter of `--high-watermark` (`1000`) | Inbound connections accepted per source address. One address can hold at most this share of the router's connection budget, so filling a router takes at least four addresses. Members behind a NAT or a cluster's SNAT share one address and each holds one connection per router, two while enrolling. |-| `--low-watermark`, `--high-watermark` | `1000`, `4000` | Connection manager limits. Above the high mark, connections are trimmed down to the low mark. |+| `--low-watermark`, `--high-watermark` | 15% below the high mark (`3400`), `4000` | Connection manager limits. Every member holds one connection, so the high mark is the size of the mesh. Above it, the router closes connections down to the low mark, so the gap between the two is what one trim sheds; the default sheds 15%. || `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. || `--dht-max-record-age` | library default | DHT value record lifetime. || `--relay-limit-duration`, `--relay-limit-data` | `1h`, `0` | Caps on each relayed connection: lifetime, and bytes per direction (`512MiB`, `1GB`). The relay cuts the connection when either is reached. `0` means no limit. |