Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router: size the relay budget to the mesh
Antonio Ojea261b436c776f
diff --git a/cmd/sam-router/main.go b/cmd/sam-router/main.goindex 7f2cd1cf..4b3b6df8 100644--- a/cmd/sam-router/main.go+++ b/cmd/sam-router/main.go@@ -16,6 +16,7 @@ package mainimport ("context"+ "fmt""os""os/signal""syscall"@@ -49,6 +50,8 @@ var (metricsAddr stringrelayLimitDuration time.DurationrelayLimitData router.ByteSize+ relayMaxReservations int+ relayMaxCircuits intshutdownLeaseTTL time.Duration)@@ -93,6 +96,8 @@ func main() {MetricsAddr: metricsAddr,RelayLimitDuration: relayLimitDuration,RelayLimitData: int64(relayLimitData),+ RelayMaxReservations: relayMaxReservations,+ RelayMaxCircuits: relayMaxCircuits,ShutdownLeaseTTL: shutdownLeaseTTL,}@@ -136,6 +141,8 @@ func main() {rootCmd.Flags().StringVar(&metricsAddr, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address (e.g. 0.0.0.0:9090); unauthenticated, off by default")rootCmd.Flags().DurationVar(&relayLimitDuration, "relay-limit-duration", router.DefaultRelayLimitDuration, "Lifetime of each relayed connection (0 = no limit)")rootCmd.Flags().Var(&relayLimitData, "relay-limit-data", "Bytes relayed per direction on each relayed connection, e.g. 512MiB (0 = no limit)")+ rootCmd.Flags().IntVar(&relayMaxReservations, "relay-max-reservations", 0, "Members that may hold a relay reservation, which is what makes a member reachable through this router; 0 is --high-watermark")+ rootCmd.Flags().IntVar(&relayMaxCircuits, "relay-max-circuits", 0, fmt.Sprintf("Relayed connections one peer may hold at once through this router, as caller or as destination; 0 is %d", router.DefaultRelayMaxCircuits))rootCmd.Flags().DurationVar(&shutdownLeaseTTL, "shutdown-lease-ttl", router.DefaultShutdownLeaseTTL, "Sent in this router's last lease when it stops: how long it expects to be away, which is how long the control plane keeps listing it. 0 sends nothing and the lease expires on the control plane's schedule")ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)diff --git a/internal/router/config.go b/internal/router/config.goindex 3a539471..1d0b6f3f 100644--- a/internal/router/config.go+++ b/internal/router/config.go@@ -54,6 +54,13 @@ type Options struct {// no limit, so Default() leaves them alone.RelayLimitDuration time.DurationRelayLimitData int64+ // RelayMaxReservations is how many members may hold a relay reservation,+ // which is what makes a member reachable through this router. Zero is+ // HighWaterMark: every connected member may be reachable.+ RelayMaxReservations int+ // RelayMaxCircuits is how many relayed connections one peer may hold at+ // once through this router. Zero is DefaultRelayMaxCircuits.+ RelayMaxCircuits int// ShutdownLeaseTTL is the lease the router asks for as it stops: how// long the control plane keeps listing it, which is the time the router// expects to be away. Zero sends no last lease and the entry expires on@@ -109,6 +116,12 @@ func (o *Options) Default() {if o.ConnsPerSourceIP <= 0 {o.ConnsPerSourceIP = DefaultConnsPerSourceIP(o.HighWaterMark)}+ if o.RelayMaxReservations <= 0 {+ o.RelayMaxReservations = o.HighWaterMark+ }+ if o.RelayMaxCircuits <= 0 {+ o.RelayMaxCircuits = DefaultRelayMaxCircuits+ }if o.KeysDBPath == "" {o.KeysDBPath = "router.key"}diff --git a/internal/router/relay_limit.go b/internal/router/relay_limit.goindex 49c1dce5..70286e9b 100644--- a/internal/router/relay_limit.go+++ b/internal/router/relay_limit.go@@ -26,6 +26,29 @@ import (// DefaultRelayLimitDuration is how long a relayed connection lives unless the operator sets it.const DefaultRelayLimitDuration = time.Hour+// DefaultRelayMaxCircuits is how many relayed connections one peer may hold+// open through this router at once, as source or as destination. libp2p's+// own default is 16, sized for a peer that occasionally reaches a friend+// behind a NAT; a service that is reachable only through the router serves+// one caller per circuit, and a popular one serves hundreds.+const DefaultRelayMaxCircuits = 1024++// relayResources sizes the relay to the mesh. libp2p's defaults assume a+// public DHT: 128 reservations in all, 32 per autonomous system and 8 per+// address, so a fleet in one cloud region had its 33rd member unreachable.+// A reservation is what makes a member reachable through the router, so the+// budget follows the connection budget: every connected member may hold+// one, as many per address as may connect from one, and the per-ASN cap is+// not applied, since a whole cloud is one ASN.+func relayResources(maxReservations, maxCircuits, perIP int) relay.Resources {+ res := relay.DefaultResources()+ res.MaxReservations = maxReservations+ res.MaxReservationsPerASN = maxReservations+ res.MaxReservationsPerIP = perIP+ res.MaxCircuits = maxCircuits+ return res+}+// relayLimit maps operator limits (0 = none) onto go-libp2p, where only a nil// limit is unlimited and a zero field cuts every circuit at once.func relayLimit(duration time.Duration, data int64) *relay.RelayLimit {diff --git a/internal/router/router.go b/internal/router/router.goindex 4d672284..49ed49ef 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -497,6 +497,7 @@ func (r *Router) Start() (retErr error) {// Setup Relay_, err = relay.New(hostNode, relay.WithACL(&relayACL{r: r}),+ relay.WithResources(relayResources(r.config.RelayMaxReservations, r.config.RelayMaxCircuits, r.config.ConnsPerSourceIP)),relay.WithLimit(relayLimit(r.config.RelayLimitDuration, r.config.RelayLimitData)))if err != nil {return errdiff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.mdindex 67ed70d0..62fa1e80 100644--- a/site/content/docs/reference/router.md+++ b/site/content/docs/reference/router.md@@ -45,6 +45,8 @@ replica count.| `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. || `--dht-max-record-age` | library default | DHT value record lifetime. || `--relay-limit-duration`, `--relay-limit-data` | `1h`, `0` | Caps on each relayed connection: lifetime, and bytes per direction (`512MiB`, `1GB`). The relay cuts the connection when either is reached. `0` means no limit. |+| `--relay-max-reservations` | `--high-watermark` (`4000`) | Members that may hold a relay reservation on this router at once, which is how many members it can reach on behalf of callers. Per source address the cap is `--conns-per-source-ip`. |+| `--relay-max-circuits` | `1024` | Relayed connections one peer may hold at once through this router, as caller or as destination. A service that many members call at once needs this many open circuits. || `--shutdown-lease-ttl` | `30s` | Sent in the router's last lease when it stops: how long it expects to be away, which is how long the control plane keeps listing it. The default covers a pod that restarts in place. A router that is being removed for good can send `1s`. `0` sends nothing and the lease expires on the control plane's schedule. || `--metrics-addr` | off | Serve `/metrics`, `/healthz` and `/readyz` without authentication on this address. `/readyz` returns `200` once the router is enrolled and the libp2p host is up. Keep this address separate from the libp2p ports and inside the cluster. || `--log-level` | `info` | `debug`, `info`, `warn`, `error`. `LOG_FORMAT=json` selects JSON output. |